FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_mmap.c
1 /*-
2 * Copyright (c) 1988 University of Utah.
3 * Copyright (c) 1991, 1993
4 * The Regents of the University of California. All rights reserved.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * the Systems Programming Group of the University of Utah Computer
8 * Science Department.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 4. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$
35 *
36 * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94
37 */
38
39 /*
40 * Mapped file (mmap) interface to VM
41 */
42
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD: releng/7.3/sys/vm/vm_mmap.c 196782 2009-09-03 14:23:50Z jhb $");
45
46 #include "opt_compat.h"
47 #include "opt_hwpmc_hooks.h"
48 #include "opt_mac.h"
49
50 #include <sys/param.h>
51 #include <sys/systm.h>
52 #include <sys/kernel.h>
53 #include <sys/lock.h>
54 #include <sys/mutex.h>
55 #include <sys/sysproto.h>
56 #include <sys/filedesc.h>
57 #include <sys/priv.h>
58 #include <sys/proc.h>
59 #include <sys/resource.h>
60 #include <sys/resourcevar.h>
61 #include <sys/vnode.h>
62 #include <sys/fcntl.h>
63 #include <sys/file.h>
64 #include <sys/mman.h>
65 #include <sys/mount.h>
66 #include <sys/conf.h>
67 #include <sys/stat.h>
68 #include <sys/vmmeter.h>
69 #include <sys/sysctl.h>
70
71 #include <security/mac/mac_framework.h>
72
73 #include <vm/vm.h>
74 #include <vm/vm_param.h>
75 #include <vm/pmap.h>
76 #include <vm/vm_map.h>
77 #include <vm/vm_object.h>
78 #include <vm/vm_page.h>
79 #include <vm/vm_pager.h>
80 #include <vm/vm_pageout.h>
81 #include <vm/vm_extern.h>
82 #include <vm/vm_page.h>
83 #include <vm/vm_kern.h>
84
85 #ifdef HWPMC_HOOKS
86 #include <sys/pmckern.h>
87 #endif
88
89 #ifndef _SYS_SYSPROTO_H_
90 struct sbrk_args {
91 int incr;
92 };
93 #endif
94
95 static int max_proc_mmap;
96 SYSCTL_INT(_vm, OID_AUTO, max_proc_mmap, CTLFLAG_RW, &max_proc_mmap, 0, "");
97
98 /*
99 * Set the maximum number of vm_map_entry structures per process. Roughly
100 * speaking vm_map_entry structures are tiny, so allowing them to eat 1/100
101 * of our KVM malloc space still results in generous limits. We want a
102 * default that is good enough to prevent the kernel running out of resources
103 * if attacked from compromised user account but generous enough such that
104 * multi-threaded processes are not unduly inconvenienced.
105 */
106 static void vmmapentry_rsrc_init(void *);
107 SYSINIT(vmmersrc, SI_SUB_KVM_RSRC, SI_ORDER_FIRST, vmmapentry_rsrc_init,
108 NULL);
109
110 static void
111 vmmapentry_rsrc_init(dummy)
112 void *dummy;
113 {
114 max_proc_mmap = vm_kmem_size / sizeof(struct vm_map_entry);
115 max_proc_mmap /= 100;
116 }
117
118 static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
119 int *, struct vnode *, vm_ooffset_t *, vm_object_t *);
120 static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
121 int *, struct cdev *, vm_ooffset_t *, vm_object_t *);
122
123 /*
124 * MPSAFE
125 */
126 /* ARGSUSED */
127 int
128 sbrk(td, uap)
129 struct thread *td;
130 struct sbrk_args *uap;
131 {
132 /* Not yet implemented */
133 return (EOPNOTSUPP);
134 }
135
136 #ifndef _SYS_SYSPROTO_H_
137 struct sstk_args {
138 int incr;
139 };
140 #endif
141
142 /*
143 * MPSAFE
144 */
145 /* ARGSUSED */
146 int
147 sstk(td, uap)
148 struct thread *td;
149 struct sstk_args *uap;
150 {
151 /* Not yet implemented */
152 return (EOPNOTSUPP);
153 }
154
155 #if defined(COMPAT_43)
156 #ifndef _SYS_SYSPROTO_H_
157 struct getpagesize_args {
158 int dummy;
159 };
160 #endif
161
162 /* ARGSUSED */
163 int
164 ogetpagesize(td, uap)
165 struct thread *td;
166 struct getpagesize_args *uap;
167 {
168 /* MP SAFE */
169 td->td_retval[0] = PAGE_SIZE;
170 return (0);
171 }
172 #endif /* COMPAT_43 */
173
174
175 /*
176 * Memory Map (mmap) system call. Note that the file offset
177 * and address are allowed to be NOT page aligned, though if
178 * the MAP_FIXED flag it set, both must have the same remainder
179 * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not
180 * page-aligned, the actual mapping starts at trunc_page(addr)
181 * and the return value is adjusted up by the page offset.
182 *
183 * Generally speaking, only character devices which are themselves
184 * memory-based, such as a video framebuffer, can be mmap'd. Otherwise
185 * there would be no cache coherency between a descriptor and a VM mapping
186 * both to the same character device.
187 *
188 * Block devices can be mmap'd no matter what they represent. Cache coherency
189 * is maintained as long as you do not write directly to the underlying
190 * character device.
191 */
192 #ifndef _SYS_SYSPROTO_H_
193 struct mmap_args {
194 void *addr;
195 size_t len;
196 int prot;
197 int flags;
198 int fd;
199 long pad;
200 off_t pos;
201 };
202 #endif
203
204 /*
205 * MPSAFE
206 */
207 int
208 mmap(td, uap)
209 struct thread *td;
210 struct mmap_args *uap;
211 {
212 #ifdef HWPMC_HOOKS
213 struct pmckern_map_in pkm;
214 #endif
215 struct file *fp;
216 struct vnode *vp;
217 vm_offset_t addr;
218 vm_size_t size, pageoff;
219 vm_prot_t prot, maxprot;
220 void *handle;
221 objtype_t handle_type;
222 int flags, error;
223 off_t pos;
224 struct vmspace *vms = td->td_proc->p_vmspace;
225
226 addr = (vm_offset_t) uap->addr;
227 size = uap->len;
228 prot = uap->prot & VM_PROT_ALL;
229 flags = uap->flags;
230 pos = uap->pos;
231
232 fp = NULL;
233 /* make sure mapping fits into numeric range etc */
234 if ((ssize_t) uap->len < 0 ||
235 ((flags & MAP_ANON) && uap->fd != -1))
236 return (EINVAL);
237
238 if (flags & MAP_STACK) {
239 if ((uap->fd != -1) ||
240 ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE)))
241 return (EINVAL);
242 flags |= MAP_ANON;
243 pos = 0;
244 }
245
246 /*
247 * Align the file position to a page boundary,
248 * and save its page offset component.
249 */
250 pageoff = (pos & PAGE_MASK);
251 pos -= pageoff;
252
253 /* Adjust size for rounding (on both ends). */
254 size += pageoff; /* low end... */
255 size = (vm_size_t) round_page(size); /* hi end */
256
257 /*
258 * Check for illegal addresses. Watch out for address wrap... Note
259 * that VM_*_ADDRESS are not constants due to casts (argh).
260 */
261 if (flags & MAP_FIXED) {
262 /*
263 * The specified address must have the same remainder
264 * as the file offset taken modulo PAGE_SIZE, so it
265 * should be aligned after adjustment by pageoff.
266 */
267 addr -= pageoff;
268 if (addr & PAGE_MASK)
269 return (EINVAL);
270 /* Address range must be all in user VM space. */
271 if (addr < vm_map_min(&vms->vm_map) ||
272 addr + size > vm_map_max(&vms->vm_map))
273 return (EINVAL);
274 if (addr + size < addr)
275 return (EINVAL);
276 } else {
277 /*
278 * XXX for non-fixed mappings where no hint is provided or
279 * the hint would fall in the potential heap space,
280 * place it after the end of the largest possible heap.
281 *
282 * There should really be a pmap call to determine a reasonable
283 * location.
284 */
285 PROC_LOCK(td->td_proc);
286 if (addr == 0 ||
287 (addr >= round_page((vm_offset_t)vms->vm_taddr) &&
288 addr < round_page((vm_offset_t)vms->vm_daddr +
289 lim_max(td->td_proc, RLIMIT_DATA))))
290 addr = round_page((vm_offset_t)vms->vm_daddr +
291 lim_max(td->td_proc, RLIMIT_DATA));
292 PROC_UNLOCK(td->td_proc);
293 }
294 if (flags & MAP_ANON) {
295 /*
296 * Mapping blank space is trivial.
297 */
298 handle = NULL;
299 handle_type = OBJT_DEFAULT;
300 maxprot = VM_PROT_ALL;
301 pos = 0;
302 } else {
303 /*
304 * Mapping file, get fp for validation. Obtain vnode and make
305 * sure it is of appropriate type.
306 * don't let the descriptor disappear on us if we block
307 */
308 if ((error = fget(td, uap->fd, &fp)) != 0)
309 goto done;
310 if (fp->f_type != DTYPE_VNODE) {
311 error = ENODEV;
312 goto done;
313 }
314 /*
315 * POSIX shared-memory objects are defined to have
316 * kernel persistence, and are not defined to support
317 * read(2)/write(2) -- or even open(2). Thus, we can
318 * use MAP_ASYNC to trade on-disk coherence for speed.
319 * The shm_open(3) library routine turns on the FPOSIXSHM
320 * flag to request this behavior.
321 */
322 if (fp->f_flag & FPOSIXSHM)
323 flags |= MAP_NOSYNC;
324 vp = fp->f_vnode;
325 /*
326 * Ensure that file and memory protections are
327 * compatible. Note that we only worry about
328 * writability if mapping is shared; in this case,
329 * current and max prot are dictated by the open file.
330 * XXX use the vnode instead? Problem is: what
331 * credentials do we use for determination? What if
332 * proc does a setuid?
333 */
334 if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC)
335 maxprot = VM_PROT_NONE;
336 else
337 maxprot = VM_PROT_EXECUTE;
338 if (fp->f_flag & FREAD) {
339 maxprot |= VM_PROT_READ;
340 } else if (prot & PROT_READ) {
341 error = EACCES;
342 goto done;
343 }
344 /*
345 * If we are sharing potential changes (either via
346 * MAP_SHARED or via the implicit sharing of character
347 * device mappings), and we are trying to get write
348 * permission although we opened it without asking
349 * for it, bail out.
350 */
351 if ((flags & MAP_SHARED) != 0) {
352 if ((fp->f_flag & FWRITE) != 0) {
353 maxprot |= VM_PROT_WRITE;
354 } else if ((prot & PROT_WRITE) != 0) {
355 error = EACCES;
356 goto done;
357 }
358 } else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) {
359 maxprot |= VM_PROT_WRITE;
360 }
361 handle = (void *)vp;
362 handle_type = OBJT_VNODE;
363 }
364
365 /*
366 * Do not allow more then a certain number of vm_map_entry structures
367 * per process. Scale with the number of rforks sharing the map
368 * to make the limit reasonable for threads.
369 */
370 if (max_proc_mmap &&
371 vms->vm_map.nentries >= max_proc_mmap * vms->vm_refcnt) {
372 error = ENOMEM;
373 goto done;
374 }
375
376 td->td_fpop = fp;
377 error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot,
378 flags, handle_type, handle, pos);
379 td->td_fpop = NULL;
380 #ifdef HWPMC_HOOKS
381 /* inform hwpmc(4) if an executable is being mapped */
382 if (error == 0 && handle_type == OBJT_VNODE &&
383 (prot & PROT_EXEC)) {
384 pkm.pm_file = handle;
385 pkm.pm_address = (uintptr_t) addr;
386 PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm);
387 }
388 #endif
389 if (error == 0)
390 td->td_retval[0] = (register_t) (addr + pageoff);
391 done:
392 if (fp)
393 fdrop(fp, td);
394
395 return (error);
396 }
397
398 int
399 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap)
400 {
401 struct mmap_args oargs;
402
403 oargs.addr = uap->addr;
404 oargs.len = uap->len;
405 oargs.prot = uap->prot;
406 oargs.flags = uap->flags;
407 oargs.fd = uap->fd;
408 oargs.pos = uap->pos;
409 return (mmap(td, &oargs));
410 }
411
412 #ifdef COMPAT_43
413 #ifndef _SYS_SYSPROTO_H_
414 struct ommap_args {
415 caddr_t addr;
416 int len;
417 int prot;
418 int flags;
419 int fd;
420 long pos;
421 };
422 #endif
423 int
424 ommap(td, uap)
425 struct thread *td;
426 struct ommap_args *uap;
427 {
428 struct mmap_args nargs;
429 static const char cvtbsdprot[8] = {
430 0,
431 PROT_EXEC,
432 PROT_WRITE,
433 PROT_EXEC | PROT_WRITE,
434 PROT_READ,
435 PROT_EXEC | PROT_READ,
436 PROT_WRITE | PROT_READ,
437 PROT_EXEC | PROT_WRITE | PROT_READ,
438 };
439
440 #define OMAP_ANON 0x0002
441 #define OMAP_COPY 0x0020
442 #define OMAP_SHARED 0x0010
443 #define OMAP_FIXED 0x0100
444
445 nargs.addr = uap->addr;
446 nargs.len = uap->len;
447 nargs.prot = cvtbsdprot[uap->prot & 0x7];
448 nargs.flags = 0;
449 if (uap->flags & OMAP_ANON)
450 nargs.flags |= MAP_ANON;
451 if (uap->flags & OMAP_COPY)
452 nargs.flags |= MAP_COPY;
453 if (uap->flags & OMAP_SHARED)
454 nargs.flags |= MAP_SHARED;
455 else
456 nargs.flags |= MAP_PRIVATE;
457 if (uap->flags & OMAP_FIXED)
458 nargs.flags |= MAP_FIXED;
459 nargs.fd = uap->fd;
460 nargs.pos = uap->pos;
461 return (mmap(td, &nargs));
462 }
463 #endif /* COMPAT_43 */
464
465
466 #ifndef _SYS_SYSPROTO_H_
467 struct msync_args {
468 void *addr;
469 size_t len;
470 int flags;
471 };
472 #endif
473 /*
474 * MPSAFE
475 */
476 int
477 msync(td, uap)
478 struct thread *td;
479 struct msync_args *uap;
480 {
481 vm_offset_t addr;
482 vm_size_t size, pageoff;
483 int flags;
484 vm_map_t map;
485 int rv;
486
487 addr = (vm_offset_t) uap->addr;
488 size = uap->len;
489 flags = uap->flags;
490
491 pageoff = (addr & PAGE_MASK);
492 addr -= pageoff;
493 size += pageoff;
494 size = (vm_size_t) round_page(size);
495 if (addr + size < addr)
496 return (EINVAL);
497
498 if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE))
499 return (EINVAL);
500
501 map = &td->td_proc->p_vmspace->vm_map;
502
503 /*
504 * Clean the pages and interpret the return value.
505 */
506 rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0,
507 (flags & MS_INVALIDATE) != 0);
508 switch (rv) {
509 case KERN_SUCCESS:
510 return (0);
511 case KERN_INVALID_ADDRESS:
512 return (EINVAL); /* Sun returns ENOMEM? */
513 case KERN_INVALID_ARGUMENT:
514 return (EBUSY);
515 default:
516 return (EINVAL);
517 }
518 }
519
520 #ifndef _SYS_SYSPROTO_H_
521 struct munmap_args {
522 void *addr;
523 size_t len;
524 };
525 #endif
526 /*
527 * MPSAFE
528 */
529 int
530 munmap(td, uap)
531 struct thread *td;
532 struct munmap_args *uap;
533 {
534 #ifdef HWPMC_HOOKS
535 struct pmckern_map_out pkm;
536 vm_map_entry_t entry;
537 #endif
538 vm_offset_t addr;
539 vm_size_t size, pageoff;
540 vm_map_t map;
541
542 addr = (vm_offset_t) uap->addr;
543 size = uap->len;
544 if (size == 0)
545 return (EINVAL);
546
547 pageoff = (addr & PAGE_MASK);
548 addr -= pageoff;
549 size += pageoff;
550 size = (vm_size_t) round_page(size);
551 if (addr + size < addr)
552 return (EINVAL);
553
554 /*
555 * Check for illegal addresses. Watch out for address wrap...
556 */
557 map = &td->td_proc->p_vmspace->vm_map;
558 if (addr < vm_map_min(map) || addr + size > vm_map_max(map))
559 return (EINVAL);
560 vm_map_lock(map);
561 #ifdef HWPMC_HOOKS
562 /*
563 * Inform hwpmc if the address range being unmapped contains
564 * an executable region.
565 */
566 if (vm_map_lookup_entry(map, addr, &entry)) {
567 for (;
568 entry != &map->header && entry->start < addr + size;
569 entry = entry->next) {
570 if (vm_map_check_protection(map, entry->start,
571 entry->end, VM_PROT_EXECUTE) == TRUE) {
572 pkm.pm_address = (uintptr_t) addr;
573 pkm.pm_size = (size_t) size;
574 PMC_CALL_HOOK(td, PMC_FN_MUNMAP,
575 (void *) &pkm);
576 break;
577 }
578 }
579 }
580 #endif
581 /* returns nothing but KERN_SUCCESS anyway */
582 vm_map_delete(map, addr, addr + size);
583 vm_map_unlock(map);
584 return (0);
585 }
586
587 #ifndef _SYS_SYSPROTO_H_
588 struct mprotect_args {
589 const void *addr;
590 size_t len;
591 int prot;
592 };
593 #endif
594 /*
595 * MPSAFE
596 */
597 int
598 mprotect(td, uap)
599 struct thread *td;
600 struct mprotect_args *uap;
601 {
602 vm_offset_t addr;
603 vm_size_t size, pageoff;
604 vm_prot_t prot;
605
606 addr = (vm_offset_t) uap->addr;
607 size = uap->len;
608 prot = uap->prot & VM_PROT_ALL;
609 #if defined(VM_PROT_READ_IS_EXEC)
610 if (prot & VM_PROT_READ)
611 prot |= VM_PROT_EXECUTE;
612 #endif
613
614 pageoff = (addr & PAGE_MASK);
615 addr -= pageoff;
616 size += pageoff;
617 size = (vm_size_t) round_page(size);
618 if (addr + size < addr)
619 return (EINVAL);
620
621 switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr,
622 addr + size, prot, FALSE)) {
623 case KERN_SUCCESS:
624 return (0);
625 case KERN_PROTECTION_FAILURE:
626 return (EACCES);
627 }
628 return (EINVAL);
629 }
630
631 #ifndef _SYS_SYSPROTO_H_
632 struct minherit_args {
633 void *addr;
634 size_t len;
635 int inherit;
636 };
637 #endif
638 /*
639 * MPSAFE
640 */
641 int
642 minherit(td, uap)
643 struct thread *td;
644 struct minherit_args *uap;
645 {
646 vm_offset_t addr;
647 vm_size_t size, pageoff;
648 vm_inherit_t inherit;
649
650 addr = (vm_offset_t)uap->addr;
651 size = uap->len;
652 inherit = uap->inherit;
653
654 pageoff = (addr & PAGE_MASK);
655 addr -= pageoff;
656 size += pageoff;
657 size = (vm_size_t) round_page(size);
658 if (addr + size < addr)
659 return (EINVAL);
660
661 switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr,
662 addr + size, inherit)) {
663 case KERN_SUCCESS:
664 return (0);
665 case KERN_PROTECTION_FAILURE:
666 return (EACCES);
667 }
668 return (EINVAL);
669 }
670
671 #ifndef _SYS_SYSPROTO_H_
672 struct madvise_args {
673 void *addr;
674 size_t len;
675 int behav;
676 };
677 #endif
678
679 /*
680 * MPSAFE
681 */
682 /* ARGSUSED */
683 int
684 madvise(td, uap)
685 struct thread *td;
686 struct madvise_args *uap;
687 {
688 vm_offset_t start, end;
689 vm_map_t map;
690 struct proc *p;
691 int error;
692
693 /*
694 * Check for our special case, advising the swap pager we are
695 * "immortal."
696 */
697 if (uap->behav == MADV_PROTECT) {
698 error = priv_check(td, PRIV_VM_MADV_PROTECT);
699 if (error == 0) {
700 p = td->td_proc;
701 PROC_LOCK(p);
702 p->p_flag |= P_PROTECTED;
703 PROC_UNLOCK(p);
704 }
705 return (error);
706 }
707 /*
708 * Check for illegal behavior
709 */
710 if (uap->behav < 0 || uap->behav > MADV_CORE)
711 return (EINVAL);
712 /*
713 * Check for illegal addresses. Watch out for address wrap... Note
714 * that VM_*_ADDRESS are not constants due to casts (argh).
715 */
716 map = &td->td_proc->p_vmspace->vm_map;
717 if ((vm_offset_t)uap->addr < vm_map_min(map) ||
718 (vm_offset_t)uap->addr + uap->len > vm_map_max(map))
719 return (EINVAL);
720 if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr)
721 return (EINVAL);
722
723 /*
724 * Since this routine is only advisory, we default to conservative
725 * behavior.
726 */
727 start = trunc_page((vm_offset_t) uap->addr);
728 end = round_page((vm_offset_t) uap->addr + uap->len);
729
730 if (vm_map_madvise(map, start, end, uap->behav))
731 return (EINVAL);
732 return (0);
733 }
734
735 #ifndef _SYS_SYSPROTO_H_
736 struct mincore_args {
737 const void *addr;
738 size_t len;
739 char *vec;
740 };
741 #endif
742
743 /*
744 * MPSAFE
745 */
746 /* ARGSUSED */
747 int
748 mincore(td, uap)
749 struct thread *td;
750 struct mincore_args *uap;
751 {
752 vm_offset_t addr, first_addr;
753 vm_offset_t end, cend;
754 pmap_t pmap;
755 vm_map_t map;
756 char *vec;
757 int error = 0;
758 int vecindex, lastvecindex;
759 vm_map_entry_t current;
760 vm_map_entry_t entry;
761 int mincoreinfo;
762 unsigned int timestamp;
763
764 /*
765 * Make sure that the addresses presented are valid for user
766 * mode.
767 */
768 first_addr = addr = trunc_page((vm_offset_t) uap->addr);
769 end = addr + (vm_size_t)round_page(uap->len);
770 map = &td->td_proc->p_vmspace->vm_map;
771 if (end > vm_map_max(map) || end < addr)
772 return (ENOMEM);
773
774 /*
775 * Address of byte vector
776 */
777 vec = uap->vec;
778
779 pmap = vmspace_pmap(td->td_proc->p_vmspace);
780
781 vm_map_lock_read(map);
782 RestartScan:
783 timestamp = map->timestamp;
784
785 if (!vm_map_lookup_entry(map, addr, &entry)) {
786 vm_map_unlock_read(map);
787 return (ENOMEM);
788 }
789
790 /*
791 * Do this on a map entry basis so that if the pages are not
792 * in the current processes address space, we can easily look
793 * up the pages elsewhere.
794 */
795 lastvecindex = -1;
796 for (current = entry;
797 (current != &map->header) && (current->start < end);
798 current = current->next) {
799
800 /*
801 * check for contiguity
802 */
803 if (current->end < end &&
804 (entry->next == &map->header ||
805 current->next->start > current->end)) {
806 vm_map_unlock_read(map);
807 return (ENOMEM);
808 }
809
810 /*
811 * ignore submaps (for now) or null objects
812 */
813 if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) ||
814 current->object.vm_object == NULL)
815 continue;
816
817 /*
818 * limit this scan to the current map entry and the
819 * limits for the mincore call
820 */
821 if (addr < current->start)
822 addr = current->start;
823 cend = current->end;
824 if (cend > end)
825 cend = end;
826
827 /*
828 * scan this entry one page at a time
829 */
830 while (addr < cend) {
831 /*
832 * Check pmap first, it is likely faster, also
833 * it can provide info as to whether we are the
834 * one referencing or modifying the page.
835 */
836 mincoreinfo = pmap_mincore(pmap, addr);
837 if (!mincoreinfo) {
838 vm_pindex_t pindex;
839 vm_ooffset_t offset;
840 vm_page_t m;
841 /*
842 * calculate the page index into the object
843 */
844 offset = current->offset + (addr - current->start);
845 pindex = OFF_TO_IDX(offset);
846 VM_OBJECT_LOCK(current->object.vm_object);
847 m = vm_page_lookup(current->object.vm_object,
848 pindex);
849 /*
850 * if the page is resident, then gather information about
851 * it.
852 */
853 if (m != NULL && m->valid != 0) {
854 mincoreinfo = MINCORE_INCORE;
855 vm_page_lock_queues();
856 if (m->dirty ||
857 pmap_is_modified(m))
858 mincoreinfo |= MINCORE_MODIFIED_OTHER;
859 if ((m->flags & PG_REFERENCED) ||
860 pmap_ts_referenced(m)) {
861 vm_page_flag_set(m, PG_REFERENCED);
862 mincoreinfo |= MINCORE_REFERENCED_OTHER;
863 }
864 vm_page_unlock_queues();
865 }
866 VM_OBJECT_UNLOCK(current->object.vm_object);
867 }
868
869 /*
870 * subyte may page fault. In case it needs to modify
871 * the map, we release the lock.
872 */
873 vm_map_unlock_read(map);
874
875 /*
876 * calculate index into user supplied byte vector
877 */
878 vecindex = OFF_TO_IDX(addr - first_addr);
879
880 /*
881 * If we have skipped map entries, we need to make sure that
882 * the byte vector is zeroed for those skipped entries.
883 */
884 while ((lastvecindex + 1) < vecindex) {
885 error = subyte(vec + lastvecindex, 0);
886 if (error) {
887 error = EFAULT;
888 goto done2;
889 }
890 ++lastvecindex;
891 }
892
893 /*
894 * Pass the page information to the user
895 */
896 error = subyte(vec + vecindex, mincoreinfo);
897 if (error) {
898 error = EFAULT;
899 goto done2;
900 }
901
902 /*
903 * If the map has changed, due to the subyte, the previous
904 * output may be invalid.
905 */
906 vm_map_lock_read(map);
907 if (timestamp != map->timestamp)
908 goto RestartScan;
909
910 lastvecindex = vecindex;
911 addr += PAGE_SIZE;
912 }
913 }
914
915 /*
916 * subyte may page fault. In case it needs to modify
917 * the map, we release the lock.
918 */
919 vm_map_unlock_read(map);
920
921 /*
922 * Zero the last entries in the byte vector.
923 */
924 vecindex = OFF_TO_IDX(end - first_addr);
925 while ((lastvecindex + 1) < vecindex) {
926 error = subyte(vec + lastvecindex, 0);
927 if (error) {
928 error = EFAULT;
929 goto done2;
930 }
931 ++lastvecindex;
932 }
933
934 /*
935 * If the map has changed, due to the subyte, the previous
936 * output may be invalid.
937 */
938 vm_map_lock_read(map);
939 if (timestamp != map->timestamp)
940 goto RestartScan;
941 vm_map_unlock_read(map);
942 done2:
943 return (error);
944 }
945
946 #ifndef _SYS_SYSPROTO_H_
947 struct mlock_args {
948 const void *addr;
949 size_t len;
950 };
951 #endif
952 /*
953 * MPSAFE
954 */
955 int
956 mlock(td, uap)
957 struct thread *td;
958 struct mlock_args *uap;
959 {
960 struct proc *proc;
961 vm_offset_t addr, end, last, start;
962 vm_size_t npages, size;
963 int error;
964
965 error = priv_check(td, PRIV_VM_MLOCK);
966 if (error)
967 return (error);
968 addr = (vm_offset_t)uap->addr;
969 size = uap->len;
970 last = addr + size;
971 start = trunc_page(addr);
972 end = round_page(last);
973 if (last < addr || end < addr)
974 return (EINVAL);
975 npages = atop(end - start);
976 if (npages > vm_page_max_wired)
977 return (ENOMEM);
978 proc = td->td_proc;
979 PROC_LOCK(proc);
980 if (ptoa(npages +
981 pmap_wired_count(vm_map_pmap(&proc->p_vmspace->vm_map))) >
982 lim_cur(proc, RLIMIT_MEMLOCK)) {
983 PROC_UNLOCK(proc);
984 return (ENOMEM);
985 }
986 PROC_UNLOCK(proc);
987 if (npages + cnt.v_wire_count > vm_page_max_wired)
988 return (EAGAIN);
989 error = vm_map_wire(&proc->p_vmspace->vm_map, start, end,
990 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
991 return (error == KERN_SUCCESS ? 0 : ENOMEM);
992 }
993
994 #ifndef _SYS_SYSPROTO_H_
995 struct mlockall_args {
996 int how;
997 };
998 #endif
999
1000 /*
1001 * MPSAFE
1002 */
1003 int
1004 mlockall(td, uap)
1005 struct thread *td;
1006 struct mlockall_args *uap;
1007 {
1008 vm_map_t map;
1009 int error;
1010
1011 map = &td->td_proc->p_vmspace->vm_map;
1012 error = 0;
1013
1014 if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0))
1015 return (EINVAL);
1016
1017 #if 0
1018 /*
1019 * If wiring all pages in the process would cause it to exceed
1020 * a hard resource limit, return ENOMEM.
1021 */
1022 PROC_LOCK(td->td_proc);
1023 if (map->size - ptoa(pmap_wired_count(vm_map_pmap(map)) >
1024 lim_cur(td->td_proc, RLIMIT_MEMLOCK))) {
1025 PROC_UNLOCK(td->td_proc);
1026 return (ENOMEM);
1027 }
1028 PROC_UNLOCK(td->td_proc);
1029 #else
1030 error = priv_check(td, PRIV_VM_MLOCK);
1031 if (error)
1032 return (error);
1033 #endif
1034
1035 if (uap->how & MCL_FUTURE) {
1036 vm_map_lock(map);
1037 vm_map_modflags(map, MAP_WIREFUTURE, 0);
1038 vm_map_unlock(map);
1039 error = 0;
1040 }
1041
1042 if (uap->how & MCL_CURRENT) {
1043 /*
1044 * P1003.1-2001 mandates that all currently mapped pages
1045 * will be memory resident and locked (wired) upon return
1046 * from mlockall(). vm_map_wire() will wire pages, by
1047 * calling vm_fault_wire() for each page in the region.
1048 */
1049 error = vm_map_wire(map, vm_map_min(map), vm_map_max(map),
1050 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1051 error = (error == KERN_SUCCESS ? 0 : EAGAIN);
1052 }
1053
1054 return (error);
1055 }
1056
1057 #ifndef _SYS_SYSPROTO_H_
1058 struct munlockall_args {
1059 register_t dummy;
1060 };
1061 #endif
1062
1063 /*
1064 * MPSAFE
1065 */
1066 int
1067 munlockall(td, uap)
1068 struct thread *td;
1069 struct munlockall_args *uap;
1070 {
1071 vm_map_t map;
1072 int error;
1073
1074 map = &td->td_proc->p_vmspace->vm_map;
1075 error = priv_check(td, PRIV_VM_MUNLOCK);
1076 if (error)
1077 return (error);
1078
1079 /* Clear the MAP_WIREFUTURE flag from this vm_map. */
1080 vm_map_lock(map);
1081 vm_map_modflags(map, 0, MAP_WIREFUTURE);
1082 vm_map_unlock(map);
1083
1084 /* Forcibly unwire all pages. */
1085 error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map),
1086 VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
1087
1088 return (error);
1089 }
1090
1091 #ifndef _SYS_SYSPROTO_H_
1092 struct munlock_args {
1093 const void *addr;
1094 size_t len;
1095 };
1096 #endif
1097 /*
1098 * MPSAFE
1099 */
1100 int
1101 munlock(td, uap)
1102 struct thread *td;
1103 struct munlock_args *uap;
1104 {
1105 vm_offset_t addr, end, last, start;
1106 vm_size_t size;
1107 int error;
1108
1109 error = priv_check(td, PRIV_VM_MUNLOCK);
1110 if (error)
1111 return (error);
1112 addr = (vm_offset_t)uap->addr;
1113 size = uap->len;
1114 last = addr + size;
1115 start = trunc_page(addr);
1116 end = round_page(last);
1117 if (last < addr || end < addr)
1118 return (EINVAL);
1119 error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end,
1120 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
1121 return (error == KERN_SUCCESS ? 0 : ENOMEM);
1122 }
1123
1124 /*
1125 * vm_mmap_vnode()
1126 *
1127 * MPSAFE
1128 *
1129 * Helper function for vm_mmap. Perform sanity check specific for mmap
1130 * operations on vnodes.
1131 */
1132 int
1133 vm_mmap_vnode(struct thread *td, vm_size_t objsize,
1134 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
1135 struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp)
1136 {
1137 struct vattr va;
1138 vm_object_t obj;
1139 vm_offset_t foff;
1140 struct mount *mp;
1141 int error, flags;
1142 int vfslocked;
1143
1144 mp = vp->v_mount;
1145 vfslocked = VFS_LOCK_GIANT(mp);
1146 if ((error = vget(vp, LK_EXCLUSIVE, td)) != 0) {
1147 VFS_UNLOCK_GIANT(vfslocked);
1148 return (error);
1149 }
1150 foff = *foffp;
1151 flags = *flagsp;
1152 obj = vp->v_object;
1153 if (vp->v_type == VREG) {
1154 /*
1155 * Get the proper underlying object
1156 */
1157 if (obj == NULL) {
1158 error = EINVAL;
1159 goto done;
1160 }
1161 if (obj->handle != vp) {
1162 vput(vp);
1163 vp = (struct vnode*)obj->handle;
1164 vget(vp, LK_EXCLUSIVE, td);
1165 }
1166 } else if (vp->v_type == VCHR) {
1167 error = vm_mmap_cdev(td, objsize, prot, maxprotp, flagsp,
1168 vp->v_rdev, foffp, objp);
1169 if (error == 0)
1170 goto mark_atime;
1171 goto done;
1172 } else {
1173 error = EINVAL;
1174 goto done;
1175 }
1176 if ((error = VOP_GETATTR(vp, &va, td->td_ucred, td))) {
1177 goto done;
1178 }
1179 #ifdef MAC
1180 error = mac_check_vnode_mmap(td->td_ucred, vp, prot, flags);
1181 if (error != 0)
1182 goto done;
1183 #endif
1184 if ((flags & MAP_SHARED) != 0) {
1185 if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) {
1186 if (prot & PROT_WRITE) {
1187 error = EPERM;
1188 goto done;
1189 }
1190 *maxprotp &= ~VM_PROT_WRITE;
1191 }
1192 }
1193 /*
1194 * If it is a regular file without any references
1195 * we do not need to sync it.
1196 * Adjust object size to be the size of actual file.
1197 */
1198 objsize = round_page(va.va_size);
1199 if (va.va_nlink == 0)
1200 flags |= MAP_NOSYNC;
1201 obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff);
1202 if (obj == NULL) {
1203 error = ENOMEM;
1204 goto done;
1205 }
1206 *objp = obj;
1207 *flagsp = flags;
1208
1209 mark_atime:
1210 vfs_mark_atime(vp, td);
1211
1212 done:
1213 vput(vp);
1214 VFS_UNLOCK_GIANT(vfslocked);
1215 return (error);
1216 }
1217
1218 /*
1219 * vm_mmap_cdev()
1220 *
1221 * MPSAFE
1222 *
1223 * Helper function for vm_mmap. Perform sanity check specific for mmap
1224 * operations on cdevs.
1225 */
1226 int
1227 vm_mmap_cdev(struct thread *td, vm_size_t objsize,
1228 vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
1229 struct cdev *cdev, vm_ooffset_t *foff, vm_object_t *objp)
1230 {
1231 vm_object_t obj;
1232 struct cdevsw *dsw;
1233 int error, flags;
1234
1235 flags = *flagsp;
1236
1237 dsw = dev_refthread(cdev);
1238 if (dsw == NULL)
1239 return (ENXIO);
1240 if (dsw->d_flags & D_MMAP_ANON) {
1241 dev_relthread(cdev);
1242 *maxprotp = VM_PROT_ALL;
1243 *flagsp |= MAP_ANON;
1244 return (0);
1245 }
1246 /*
1247 * cdevs do not provide private mappings of any kind.
1248 */
1249 if ((*maxprotp & VM_PROT_WRITE) == 0 &&
1250 (prot & PROT_WRITE) != 0) {
1251 dev_relthread(cdev);
1252 return (EACCES);
1253 }
1254 if (flags & (MAP_PRIVATE|MAP_COPY)) {
1255 dev_relthread(cdev);
1256 return (EINVAL);
1257 }
1258 /*
1259 * Force device mappings to be shared.
1260 */
1261 flags |= MAP_SHARED;
1262 #ifdef MAC_XXX
1263 error = mac_cdev_check_mmap(td->td_ucred, cdev, prot);
1264 if (error != 0) {
1265 dev_relthread(cdev);
1266 return (error);
1267 }
1268 #endif
1269 /*
1270 * First, try d_mmap_single(). If that is not implemented
1271 * (returns ENODEV), fall back to using the device pager.
1272 * Note that d_mmap_single() must return a reference to the
1273 * object (it needs to bump the reference count of the object
1274 * it returns somehow).
1275 *
1276 * XXX assumes VM_PROT_* == PROT_*
1277 */
1278 error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot);
1279 dev_relthread(cdev);
1280 if (error != ENODEV)
1281 return (error);
1282 obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff);
1283 if (obj == NULL)
1284 return (EINVAL);
1285 *objp = obj;
1286 *flagsp = flags;
1287 return (0);
1288 }
1289
1290 /*
1291 * vm_mmap()
1292 *
1293 * MPSAFE
1294 *
1295 * Internal version of mmap. Currently used by mmap, exec, and sys5
1296 * shared memory. Handle is either a vnode pointer or NULL for MAP_ANON.
1297 */
1298 int
1299 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
1300 vm_prot_t maxprot, int flags,
1301 objtype_t handle_type, void *handle,
1302 vm_ooffset_t foff)
1303 {
1304 boolean_t fitit;
1305 vm_object_t object = NULL;
1306 int rv = KERN_SUCCESS;
1307 int docow, error;
1308 struct thread *td = curthread;
1309
1310 if (size == 0)
1311 return (0);
1312
1313 size = round_page(size);
1314
1315 PROC_LOCK(td->td_proc);
1316 if (td->td_proc->p_vmspace->vm_map.size + size >
1317 lim_cur(td->td_proc, RLIMIT_VMEM)) {
1318 PROC_UNLOCK(td->td_proc);
1319 return(ENOMEM);
1320 }
1321 PROC_UNLOCK(td->td_proc);
1322
1323 /*
1324 * We currently can only deal with page aligned file offsets.
1325 * The check is here rather than in the syscall because the
1326 * kernel calls this function internally for other mmaping
1327 * operations (such as in exec) and non-aligned offsets will
1328 * cause pmap inconsistencies...so we want to be sure to
1329 * disallow this in all cases.
1330 */
1331 if (foff & PAGE_MASK)
1332 return (EINVAL);
1333
1334 if ((flags & MAP_FIXED) == 0) {
1335 fitit = TRUE;
1336 *addr = round_page(*addr);
1337 } else {
1338 if (*addr != trunc_page(*addr))
1339 return (EINVAL);
1340 fitit = FALSE;
1341 }
1342 /*
1343 * Lookup/allocate object.
1344 */
1345 switch (handle_type) {
1346 case OBJT_DEVICE:
1347 error = vm_mmap_cdev(td, size, prot, &maxprot, &flags,
1348 handle, &foff, &object);
1349 break;
1350 case OBJT_VNODE:
1351 error = vm_mmap_vnode(td, size, prot, &maxprot, &flags,
1352 handle, &foff, &object);
1353 break;
1354 case OBJT_DEFAULT:
1355 if (handle == NULL) {
1356 error = 0;
1357 break;
1358 }
1359 /* FALLTHROUGH */
1360 default:
1361 error = EINVAL;
1362 break;
1363 }
1364 if (error)
1365 return (error);
1366 if (flags & MAP_ANON) {
1367 object = NULL;
1368 docow = 0;
1369 /*
1370 * Unnamed anonymous regions always start at 0.
1371 */
1372 if (handle == 0)
1373 foff = 0;
1374 } else {
1375 docow = MAP_PREFAULT_PARTIAL;
1376 }
1377
1378 if ((flags & (MAP_ANON|MAP_SHARED)) == 0)
1379 docow |= MAP_COPY_ON_WRITE;
1380 if (flags & MAP_NOSYNC)
1381 docow |= MAP_DISABLE_SYNCER;
1382 if (flags & MAP_NOCORE)
1383 docow |= MAP_DISABLE_COREDUMP;
1384
1385 #if defined(VM_PROT_READ_IS_EXEC)
1386 if (prot & VM_PROT_READ)
1387 prot |= VM_PROT_EXECUTE;
1388
1389 if (maxprot & VM_PROT_READ)
1390 maxprot |= VM_PROT_EXECUTE;
1391 #endif
1392
1393 if (flags & MAP_STACK)
1394 rv = vm_map_stack(map, *addr, size, prot, maxprot,
1395 docow | MAP_STACK_GROWS_DOWN);
1396 else if (fitit)
1397 rv = vm_map_find(map, object, foff, addr, size,
1398 object != NULL && object->type == OBJT_DEVICE ?
1399 VMFS_ALIGNED_SPACE : VMFS_ANY_SPACE, prot, maxprot, docow);
1400 else
1401 rv = vm_map_fixed(map, object, foff, *addr, size,
1402 prot, maxprot, docow);
1403
1404 if (rv != KERN_SUCCESS) {
1405 /*
1406 * Lose the object reference. Will destroy the
1407 * object if it's an unnamed anonymous mapping
1408 * or named anonymous without other references.
1409 */
1410 vm_object_deallocate(object);
1411 } else if (flags & MAP_SHARED) {
1412 /*
1413 * Shared memory is also shared with children.
1414 */
1415 rv = vm_map_inherit(map, *addr, *addr + size, VM_INHERIT_SHARE);
1416 if (rv != KERN_SUCCESS)
1417 (void) vm_map_remove(map, *addr, *addr + size);
1418 }
1419
1420 /*
1421 * If the process has requested that all future mappings
1422 * be wired, then heed this.
1423 */
1424 if ((rv == KERN_SUCCESS) && (map->flags & MAP_WIREFUTURE))
1425 vm_map_wire(map, *addr, *addr + size,
1426 VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
1427
1428 switch (rv) {
1429 case KERN_SUCCESS:
1430 return (0);
1431 case KERN_INVALID_ADDRESS:
1432 case KERN_NO_SPACE:
1433 return (ENOMEM);
1434 case KERN_PROTECTION_FAILURE:
1435 return (EACCES);
1436 default:
1437 return (EINVAL);
1438 }
1439 }
Cache object: 77319fa6c2f4fcf73b1cecf0d4a7463a
|