The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_mmap.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1988 University of Utah.
    3  * Copyright (c) 1991, 1993
    4  *      The Regents of the University of California.  All rights reserved.
    5  *
    6  * This code is derived from software contributed to Berkeley by
    7  * the Systems Programming Group of the University of Utah Computer
    8  * Science Department.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  * 4. Neither the name of the University nor the names of its contributors
   19  *    may be used to endorse or promote products derived from this software
   20  *    without specific prior written permission.
   21  *
   22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   32  * SUCH DAMAGE.
   33  *
   34  * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$
   35  *
   36  *      @(#)vm_mmap.c   8.4 (Berkeley) 1/12/94
   37  */
   38 
   39 /*
   40  * Mapped file (mmap) interface to VM
   41  */
   42 
   43 #include <sys/cdefs.h>
   44 __FBSDID("$FreeBSD: stable/10/sys/vm/vm_mmap.c 321717 2017-07-30 10:36:20Z kib $");
   45 
   46 #include "opt_compat.h"
   47 #include "opt_hwpmc_hooks.h"
   48 
   49 #include <sys/param.h>
   50 #include <sys/systm.h>
   51 #include <sys/capsicum.h>
   52 #include <sys/kernel.h>
   53 #include <sys/lock.h>
   54 #include <sys/mutex.h>
   55 #include <sys/sysproto.h>
   56 #include <sys/filedesc.h>
   57 #include <sys/priv.h>
   58 #include <sys/proc.h>
   59 #include <sys/procctl.h>
   60 #include <sys/racct.h>
   61 #include <sys/resource.h>
   62 #include <sys/resourcevar.h>
   63 #include <sys/rwlock.h>
   64 #include <sys/sysctl.h>
   65 #include <sys/vnode.h>
   66 #include <sys/fcntl.h>
   67 #include <sys/file.h>
   68 #include <sys/mman.h>
   69 #include <sys/mount.h>
   70 #include <sys/conf.h>
   71 #include <sys/stat.h>
   72 #include <sys/syscallsubr.h>
   73 #include <sys/sysent.h>
   74 #include <sys/vmmeter.h>
   75 
   76 #include <security/mac/mac_framework.h>
   77 
   78 #include <vm/vm.h>
   79 #include <vm/vm_param.h>
   80 #include <vm/pmap.h>
   81 #include <vm/vm_map.h>
   82 #include <vm/vm_object.h>
   83 #include <vm/vm_page.h>
   84 #include <vm/vm_pager.h>
   85 #include <vm/vm_pageout.h>
   86 #include <vm/vm_extern.h>
   87 #include <vm/vm_page.h>
   88 #include <vm/vnode_pager.h>
   89 
   90 #ifdef HWPMC_HOOKS
   91 #include <sys/pmckern.h>
   92 #endif
   93 
   94 int old_mlock = 0;
   95 SYSCTL_INT(_vm, OID_AUTO, old_mlock, CTLFLAG_RW | CTLFLAG_TUN, &old_mlock, 0,
   96     "Do not apply RLIMIT_MEMLOCK on mlockall");
   97 TUNABLE_INT("vm.old_mlock", &old_mlock);
   98 
   99 #ifdef MAP_32BIT
  100 #define MAP_32BIT_MAX_ADDR      ((vm_offset_t)1 << 31)
  101 #endif
  102 
  103 static int vm_mmap_vnode(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
  104     int *, struct vnode *, vm_ooffset_t *, vm_object_t *, boolean_t *);
  105 static int vm_mmap_cdev(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
  106     int *, struct cdev *, vm_ooffset_t *, vm_object_t *);
  107 static int vm_mmap_shm(struct thread *, vm_size_t, vm_prot_t, vm_prot_t *,
  108     int *, struct shmfd *, vm_ooffset_t, vm_object_t *);
  109 
  110 #ifndef _SYS_SYSPROTO_H_
  111 struct sbrk_args {
  112         int incr;
  113 };
  114 #endif
  115 
  116 int
  117 sys_sbrk(struct thread *td, struct sbrk_args *uap)
  118 {
  119         /* Not yet implemented */
  120         return (EOPNOTSUPP);
  121 }
  122 
  123 #ifndef _SYS_SYSPROTO_H_
  124 struct sstk_args {
  125         int incr;
  126 };
  127 #endif
  128 
  129 int
  130 sys_sstk(struct thread *td, struct sstk_args *uap)
  131 {
  132         /* Not yet implemented */
  133         return (EOPNOTSUPP);
  134 }
  135 
  136 #if defined(COMPAT_43)
  137 #ifndef _SYS_SYSPROTO_H_
  138 struct getpagesize_args {
  139         int dummy;
  140 };
  141 #endif
  142 
  143 int
  144 ogetpagesize(struct thread *td, struct getpagesize_args *uap)
  145 {
  146 
  147         td->td_retval[0] = PAGE_SIZE;
  148         return (0);
  149 }
  150 #endif                          /* COMPAT_43 */
  151 
  152 
  153 /*
  154  * Memory Map (mmap) system call.  Note that the file offset
  155  * and address are allowed to be NOT page aligned, though if
  156  * the MAP_FIXED flag it set, both must have the same remainder
  157  * modulo the PAGE_SIZE (POSIX 1003.1b).  If the address is not
  158  * page-aligned, the actual mapping starts at trunc_page(addr)
  159  * and the return value is adjusted up by the page offset.
  160  *
  161  * Generally speaking, only character devices which are themselves
  162  * memory-based, such as a video framebuffer, can be mmap'd.  Otherwise
  163  * there would be no cache coherency between a descriptor and a VM mapping
  164  * both to the same character device.
  165  */
  166 #ifndef _SYS_SYSPROTO_H_
  167 struct mmap_args {
  168         void *addr;
  169         size_t len;
  170         int prot;
  171         int flags;
  172         int fd;
  173         long pad;
  174         off_t pos;
  175 };
  176 #endif
  177 
  178 int
  179 sys_mmap(td, uap)
  180         struct thread *td;
  181         struct mmap_args *uap;
  182 {
  183 #ifdef HWPMC_HOOKS
  184         struct pmckern_map_in pkm;
  185 #endif
  186         struct file *fp;
  187         struct vnode *vp;
  188         vm_offset_t addr;
  189         vm_size_t size, pageoff;
  190         vm_prot_t cap_maxprot, prot, maxprot;
  191         void *handle;
  192         objtype_t handle_type;
  193         int align, error, flags;
  194         off_t pos;
  195         struct vmspace *vms = td->td_proc->p_vmspace;
  196         cap_rights_t rights;
  197 
  198         addr = (vm_offset_t) uap->addr;
  199         size = uap->len;
  200         prot = uap->prot & VM_PROT_ALL;
  201         flags = uap->flags;
  202         pos = uap->pos;
  203 
  204         fp = NULL;
  205 
  206         /*
  207          * Enforce the constraints.
  208          * Mapping of length 0 is only allowed for old binaries.
  209          * Anonymous mapping shall specify -1 as filedescriptor and
  210          * zero position for new code. Be nice to ancient a.out
  211          * binaries and correct pos for anonymous mapping, since old
  212          * ld.so sometimes issues anonymous map requests with non-zero
  213          * pos.
  214          */
  215         if (!SV_CURPROC_FLAG(SV_AOUT)) {
  216                 if ((uap->len == 0 && curproc->p_osrel >= P_OSREL_MAP_ANON) ||
  217                     ((flags & MAP_ANON) != 0 && (uap->fd != -1 || pos != 0)))
  218                         return (EINVAL);
  219         } else {
  220                 if ((flags & MAP_ANON) != 0)
  221                         pos = 0;
  222         }
  223 
  224         if (flags & MAP_STACK) {
  225                 if ((uap->fd != -1) ||
  226                     ((prot & (PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE)))
  227                         return (EINVAL);
  228                 flags |= MAP_ANON;
  229                 pos = 0;
  230         }
  231         if ((flags & (MAP_EXCL | MAP_FIXED)) == MAP_EXCL)
  232                 return (EINVAL);
  233         if ((flags & MAP_GUARD) != 0 && (prot != PROT_NONE || uap->fd != -1 ||
  234             pos != 0 || (flags & (MAP_SHARED | MAP_PRIVATE | MAP_PREFAULT |
  235             MAP_PREFAULT_READ | MAP_ANON | MAP_STACK)) != 0))
  236                 return (EINVAL);
  237 
  238         /*
  239          * Align the file position to a page boundary,
  240          * and save its page offset component.
  241          */
  242         pageoff = (pos & PAGE_MASK);
  243         pos -= pageoff;
  244 
  245         /* Adjust size for rounding (on both ends). */
  246         size += pageoff;                        /* low end... */
  247         size = (vm_size_t) round_page(size);    /* hi end */
  248 
  249         /* Ensure alignment is at least a page and fits in a pointer. */
  250         align = flags & MAP_ALIGNMENT_MASK;
  251         if (align != 0 && align != MAP_ALIGNED_SUPER &&
  252             (align >> MAP_ALIGNMENT_SHIFT >= sizeof(void *) * NBBY ||
  253             align >> MAP_ALIGNMENT_SHIFT < PAGE_SHIFT))
  254                 return (EINVAL);
  255 
  256         /*
  257          * Check for illegal addresses.  Watch out for address wrap... Note
  258          * that VM_*_ADDRESS are not constants due to casts (argh).
  259          */
  260         if (flags & MAP_FIXED) {
  261                 /*
  262                  * The specified address must have the same remainder
  263                  * as the file offset taken modulo PAGE_SIZE, so it
  264                  * should be aligned after adjustment by pageoff.
  265                  */
  266                 addr -= pageoff;
  267                 if (addr & PAGE_MASK)
  268                         return (EINVAL);
  269 
  270                 /* Address range must be all in user VM space. */
  271                 if (addr < vm_map_min(&vms->vm_map) ||
  272                     addr + size > vm_map_max(&vms->vm_map))
  273                         return (EINVAL);
  274                 if (addr + size < addr)
  275                         return (EINVAL);
  276 #ifdef MAP_32BIT
  277                 if (flags & MAP_32BIT && addr + size > MAP_32BIT_MAX_ADDR)
  278                         return (EINVAL);
  279         } else if (flags & MAP_32BIT) {
  280                 /*
  281                  * For MAP_32BIT, override the hint if it is too high and
  282                  * do not bother moving the mapping past the heap (since
  283                  * the heap is usually above 2GB).
  284                  */
  285                 if (addr + size > MAP_32BIT_MAX_ADDR)
  286                         addr = 0;
  287 #endif
  288         } else {
  289                 /*
  290                  * XXX for non-fixed mappings where no hint is provided or
  291                  * the hint would fall in the potential heap space,
  292                  * place it after the end of the largest possible heap.
  293                  *
  294                  * There should really be a pmap call to determine a reasonable
  295                  * location.
  296                  */
  297                 PROC_LOCK(td->td_proc);
  298                 if (addr == 0 ||
  299                     (addr >= round_page((vm_offset_t)vms->vm_taddr) &&
  300                     addr < round_page((vm_offset_t)vms->vm_daddr +
  301                     lim_max(td->td_proc, RLIMIT_DATA))))
  302                         addr = round_page((vm_offset_t)vms->vm_daddr +
  303                             lim_max(td->td_proc, RLIMIT_DATA));
  304                 PROC_UNLOCK(td->td_proc);
  305         }
  306         if ((flags & MAP_GUARD) != 0) {
  307                 handle = NULL;
  308                 handle_type = OBJT_DEFAULT;
  309                 maxprot = VM_PROT_NONE;
  310                 cap_maxprot = VM_PROT_NONE;
  311         } else if ((flags & MAP_ANON) != 0) {
  312                 /*
  313                  * Mapping blank space is trivial.
  314                  */
  315                 handle = NULL;
  316                 handle_type = OBJT_DEFAULT;
  317                 maxprot = VM_PROT_ALL;
  318                 cap_maxprot = VM_PROT_ALL;
  319         } else {
  320                 /*
  321                  * Mapping file, get fp for validation and don't let the
  322                  * descriptor disappear on us if we block. Check capability
  323                  * rights, but also return the maximum rights to be combined
  324                  * with maxprot later.
  325                  */
  326                 cap_rights_init(&rights, CAP_MMAP);
  327                 if (prot & PROT_READ)
  328                         cap_rights_set(&rights, CAP_MMAP_R);
  329                 if ((flags & MAP_SHARED) != 0) {
  330                         if (prot & PROT_WRITE)
  331                                 cap_rights_set(&rights, CAP_MMAP_W);
  332                 }
  333                 if (prot & PROT_EXEC)
  334                         cap_rights_set(&rights, CAP_MMAP_X);
  335                 error = fget_mmap(td, uap->fd, &rights, &cap_maxprot, &fp);
  336                 if (error != 0)
  337                         goto done;
  338                 if (fp->f_type == DTYPE_SHM) {
  339                         handle = fp->f_data;
  340                         handle_type = OBJT_SWAP;
  341                         maxprot = VM_PROT_NONE;
  342 
  343                         /* FREAD should always be set. */
  344                         if (fp->f_flag & FREAD)
  345                                 maxprot |= VM_PROT_EXECUTE | VM_PROT_READ;
  346                         if (fp->f_flag & FWRITE)
  347                                 maxprot |= VM_PROT_WRITE;
  348                         goto map;
  349                 }
  350                 if (fp->f_type != DTYPE_VNODE) {
  351                         error = ENODEV;
  352                         goto done;
  353                 }
  354 #if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \
  355     defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4)
  356                 /*
  357                  * POSIX shared-memory objects are defined to have
  358                  * kernel persistence, and are not defined to support
  359                  * read(2)/write(2) -- or even open(2).  Thus, we can
  360                  * use MAP_ASYNC to trade on-disk coherence for speed.
  361                  * The shm_open(3) library routine turns on the FPOSIXSHM
  362                  * flag to request this behavior.
  363                  */
  364                 if (fp->f_flag & FPOSIXSHM)
  365                         flags |= MAP_NOSYNC;
  366 #endif
  367                 vp = fp->f_vnode;
  368                 /*
  369                  * Ensure that file and memory protections are
  370                  * compatible.  Note that we only worry about
  371                  * writability if mapping is shared; in this case,
  372                  * current and max prot are dictated by the open file.
  373                  * XXX use the vnode instead?  Problem is: what
  374                  * credentials do we use for determination? What if
  375                  * proc does a setuid?
  376                  */
  377                 if (vp->v_mount != NULL && vp->v_mount->mnt_flag & MNT_NOEXEC)
  378                         maxprot = VM_PROT_NONE;
  379                 else
  380                         maxprot = VM_PROT_EXECUTE;
  381                 if (fp->f_flag & FREAD) {
  382                         maxprot |= VM_PROT_READ;
  383                 } else if (prot & PROT_READ) {
  384                         error = EACCES;
  385                         goto done;
  386                 }
  387                 /*
  388                  * If we are sharing potential changes (either via
  389                  * MAP_SHARED or via the implicit sharing of character
  390                  * device mappings), and we are trying to get write
  391                  * permission although we opened it without asking
  392                  * for it, bail out.
  393                  */
  394                 if ((flags & MAP_SHARED) != 0) {
  395                         if ((fp->f_flag & FWRITE) != 0) {
  396                                 maxprot |= VM_PROT_WRITE;
  397                         } else if ((prot & PROT_WRITE) != 0) {
  398                                 error = EACCES;
  399                                 goto done;
  400                         }
  401                 } else if (vp->v_type != VCHR || (fp->f_flag & FWRITE) != 0) {
  402                         maxprot |= VM_PROT_WRITE;
  403                         cap_maxprot |= VM_PROT_WRITE;
  404                 }
  405                 handle = (void *)vp;
  406                 handle_type = OBJT_VNODE;
  407         }
  408 map:
  409         td->td_fpop = fp;
  410         maxprot &= cap_maxprot;
  411         error = vm_mmap(&vms->vm_map, &addr, size, prot, maxprot,
  412             flags, handle_type, handle, pos);
  413         td->td_fpop = NULL;
  414 #ifdef HWPMC_HOOKS
  415         /* inform hwpmc(4) if an executable is being mapped */
  416         if (error == 0 && handle_type == OBJT_VNODE &&
  417             (prot & PROT_EXEC)) {
  418                 pkm.pm_file = handle;
  419                 pkm.pm_address = (uintptr_t) addr;
  420                 PMC_CALL_HOOK(td, PMC_FN_MMAP, (void *) &pkm);
  421         }
  422 #endif
  423         if (error == 0)
  424                 td->td_retval[0] = (register_t) (addr + pageoff);
  425 done:
  426         if (fp)
  427                 fdrop(fp, td);
  428 
  429         return (error);
  430 }
  431 
  432 int
  433 freebsd6_mmap(struct thread *td, struct freebsd6_mmap_args *uap)
  434 {
  435         struct mmap_args oargs;
  436 
  437         oargs.addr = uap->addr;
  438         oargs.len = uap->len;
  439         oargs.prot = uap->prot;
  440         oargs.flags = uap->flags;
  441         oargs.fd = uap->fd;
  442         oargs.pos = uap->pos;
  443         return (sys_mmap(td, &oargs));
  444 }
  445 
  446 #ifdef COMPAT_43
  447 #ifndef _SYS_SYSPROTO_H_
  448 struct ommap_args {
  449         caddr_t addr;
  450         int len;
  451         int prot;
  452         int flags;
  453         int fd;
  454         long pos;
  455 };
  456 #endif
  457 int
  458 ommap(td, uap)
  459         struct thread *td;
  460         struct ommap_args *uap;
  461 {
  462         struct mmap_args nargs;
  463         static const char cvtbsdprot[8] = {
  464                 0,
  465                 PROT_EXEC,
  466                 PROT_WRITE,
  467                 PROT_EXEC | PROT_WRITE,
  468                 PROT_READ,
  469                 PROT_EXEC | PROT_READ,
  470                 PROT_WRITE | PROT_READ,
  471                 PROT_EXEC | PROT_WRITE | PROT_READ,
  472         };
  473 
  474 #define OMAP_ANON       0x0002
  475 #define OMAP_COPY       0x0020
  476 #define OMAP_SHARED     0x0010
  477 #define OMAP_FIXED      0x0100
  478 
  479         nargs.addr = uap->addr;
  480         nargs.len = uap->len;
  481         nargs.prot = cvtbsdprot[uap->prot & 0x7];
  482 #ifdef COMPAT_FREEBSD32
  483 #if defined(__amd64__) || defined(__ia64__)
  484         if (i386_read_exec && SV_PROC_FLAG(td->td_proc, SV_ILP32) &&
  485             nargs.prot != 0)
  486                 nargs.prot |= PROT_EXEC;
  487 #endif
  488 #endif
  489         nargs.flags = 0;
  490         if (uap->flags & OMAP_ANON)
  491                 nargs.flags |= MAP_ANON;
  492         if (uap->flags & OMAP_COPY)
  493                 nargs.flags |= MAP_COPY;
  494         if (uap->flags & OMAP_SHARED)
  495                 nargs.flags |= MAP_SHARED;
  496         else
  497                 nargs.flags |= MAP_PRIVATE;
  498         if (uap->flags & OMAP_FIXED)
  499                 nargs.flags |= MAP_FIXED;
  500         nargs.fd = uap->fd;
  501         nargs.pos = uap->pos;
  502         return (sys_mmap(td, &nargs));
  503 }
  504 #endif                          /* COMPAT_43 */
  505 
  506 
  507 #ifndef _SYS_SYSPROTO_H_
  508 struct msync_args {
  509         void *addr;
  510         size_t len;
  511         int flags;
  512 };
  513 #endif
  514 int
  515 sys_msync(td, uap)
  516         struct thread *td;
  517         struct msync_args *uap;
  518 {
  519         vm_offset_t addr;
  520         vm_size_t size, pageoff;
  521         int flags;
  522         vm_map_t map;
  523         int rv;
  524 
  525         addr = (vm_offset_t) uap->addr;
  526         size = uap->len;
  527         flags = uap->flags;
  528 
  529         pageoff = (addr & PAGE_MASK);
  530         addr -= pageoff;
  531         size += pageoff;
  532         size = (vm_size_t) round_page(size);
  533         if (addr + size < addr)
  534                 return (EINVAL);
  535 
  536         if ((flags & (MS_ASYNC|MS_INVALIDATE)) == (MS_ASYNC|MS_INVALIDATE))
  537                 return (EINVAL);
  538 
  539         map = &td->td_proc->p_vmspace->vm_map;
  540 
  541         /*
  542          * Clean the pages and interpret the return value.
  543          */
  544         rv = vm_map_sync(map, addr, addr + size, (flags & MS_ASYNC) == 0,
  545             (flags & MS_INVALIDATE) != 0);
  546         switch (rv) {
  547         case KERN_SUCCESS:
  548                 return (0);
  549         case KERN_INVALID_ADDRESS:
  550                 return (ENOMEM);
  551         case KERN_INVALID_ARGUMENT:
  552                 return (EBUSY);
  553         case KERN_FAILURE:
  554                 return (EIO);
  555         default:
  556                 return (EINVAL);
  557         }
  558 }
  559 
  560 #ifndef _SYS_SYSPROTO_H_
  561 struct munmap_args {
  562         void *addr;
  563         size_t len;
  564 };
  565 #endif
  566 int
  567 sys_munmap(td, uap)
  568         struct thread *td;
  569         struct munmap_args *uap;
  570 {
  571 #ifdef HWPMC_HOOKS
  572         struct pmckern_map_out pkm;
  573         vm_map_entry_t entry;
  574 #endif
  575         vm_offset_t addr;
  576         vm_size_t size, pageoff;
  577         vm_map_t map;
  578 
  579         addr = (vm_offset_t) uap->addr;
  580         size = uap->len;
  581         if (size == 0)
  582                 return (EINVAL);
  583 
  584         pageoff = (addr & PAGE_MASK);
  585         addr -= pageoff;
  586         size += pageoff;
  587         size = (vm_size_t) round_page(size);
  588         if (addr + size < addr)
  589                 return (EINVAL);
  590 
  591         /*
  592          * Check for illegal addresses.  Watch out for address wrap...
  593          */
  594         map = &td->td_proc->p_vmspace->vm_map;
  595         if (addr < vm_map_min(map) || addr + size > vm_map_max(map))
  596                 return (EINVAL);
  597         vm_map_lock(map);
  598 #ifdef HWPMC_HOOKS
  599         /*
  600          * Inform hwpmc if the address range being unmapped contains
  601          * an executable region.
  602          */
  603         pkm.pm_address = (uintptr_t) NULL;
  604         if (vm_map_lookup_entry(map, addr, &entry)) {
  605                 for (;
  606                      entry != &map->header && entry->start < addr + size;
  607                      entry = entry->next) {
  608                         if (vm_map_check_protection(map, entry->start,
  609                                 entry->end, VM_PROT_EXECUTE) == TRUE) {
  610                                 pkm.pm_address = (uintptr_t) addr;
  611                                 pkm.pm_size = (size_t) size;
  612                                 break;
  613                         }
  614                 }
  615         }
  616 #endif
  617         vm_map_delete(map, addr, addr + size);
  618 
  619 #ifdef HWPMC_HOOKS
  620         /* downgrade the lock to prevent a LOR with the pmc-sx lock */
  621         vm_map_lock_downgrade(map);
  622         if (pkm.pm_address != (uintptr_t) NULL)
  623                 PMC_CALL_HOOK(td, PMC_FN_MUNMAP, (void *) &pkm);
  624         vm_map_unlock_read(map);
  625 #else
  626         vm_map_unlock(map);
  627 #endif
  628         /* vm_map_delete returns nothing but KERN_SUCCESS anyway */
  629         return (0);
  630 }
  631 
  632 #ifndef _SYS_SYSPROTO_H_
  633 struct mprotect_args {
  634         const void *addr;
  635         size_t len;
  636         int prot;
  637 };
  638 #endif
  639 int
  640 sys_mprotect(td, uap)
  641         struct thread *td;
  642         struct mprotect_args *uap;
  643 {
  644         vm_offset_t addr;
  645         vm_size_t size, pageoff;
  646         vm_prot_t prot;
  647 
  648         addr = (vm_offset_t) uap->addr;
  649         size = uap->len;
  650         prot = uap->prot & VM_PROT_ALL;
  651 
  652         pageoff = (addr & PAGE_MASK);
  653         addr -= pageoff;
  654         size += pageoff;
  655         size = (vm_size_t) round_page(size);
  656         if (addr + size < addr)
  657                 return (EINVAL);
  658 
  659         switch (vm_map_protect(&td->td_proc->p_vmspace->vm_map, addr,
  660             addr + size, prot, FALSE)) {
  661         case KERN_SUCCESS:
  662                 return (0);
  663         case KERN_PROTECTION_FAILURE:
  664                 return (EACCES);
  665         case KERN_RESOURCE_SHORTAGE:
  666                 return (ENOMEM);
  667         }
  668         return (EINVAL);
  669 }
  670 
  671 #ifndef _SYS_SYSPROTO_H_
  672 struct minherit_args {
  673         void *addr;
  674         size_t len;
  675         int inherit;
  676 };
  677 #endif
  678 int
  679 sys_minherit(struct thread *td, struct minherit_args *uap)
  680 {
  681         vm_offset_t addr;
  682         vm_size_t size, pageoff;
  683         vm_inherit_t inherit;
  684 
  685         addr = (vm_offset_t)uap->addr;
  686         size = uap->len;
  687         inherit = uap->inherit;
  688 
  689         pageoff = (addr & PAGE_MASK);
  690         addr -= pageoff;
  691         size += pageoff;
  692         size = (vm_size_t) round_page(size);
  693         if (addr + size < addr)
  694                 return (EINVAL);
  695 
  696         switch (vm_map_inherit(&td->td_proc->p_vmspace->vm_map, addr,
  697             addr + size, inherit)) {
  698         case KERN_SUCCESS:
  699                 return (0);
  700         case KERN_PROTECTION_FAILURE:
  701                 return (EACCES);
  702         }
  703         return (EINVAL);
  704 }
  705 
  706 #ifndef _SYS_SYSPROTO_H_
  707 struct madvise_args {
  708         void *addr;
  709         size_t len;
  710         int behav;
  711 };
  712 #endif
  713 
  714 int
  715 sys_madvise(struct thread *td, struct madvise_args *uap)
  716 {
  717         vm_offset_t start, end;
  718         vm_map_t map;
  719         int flags;
  720 
  721         /*
  722          * Check for our special case, advising the swap pager we are
  723          * "immortal."
  724          */
  725         if (uap->behav == MADV_PROTECT) {
  726                 flags = PPROT_SET;
  727                 return (kern_procctl(td, P_PID, td->td_proc->p_pid,
  728                     PROC_SPROTECT, &flags));
  729         }
  730 
  731         /*
  732          * Check for illegal behavior
  733          */
  734         if (uap->behav < 0 || uap->behav > MADV_CORE)
  735                 return (EINVAL);
  736         /*
  737          * Check for illegal addresses.  Watch out for address wrap... Note
  738          * that VM_*_ADDRESS are not constants due to casts (argh).
  739          */
  740         map = &td->td_proc->p_vmspace->vm_map;
  741         if ((vm_offset_t)uap->addr < vm_map_min(map) ||
  742             (vm_offset_t)uap->addr + uap->len > vm_map_max(map))
  743                 return (EINVAL);
  744         if (((vm_offset_t) uap->addr + uap->len) < (vm_offset_t) uap->addr)
  745                 return (EINVAL);
  746 
  747         /*
  748          * Since this routine is only advisory, we default to conservative
  749          * behavior.
  750          */
  751         start = trunc_page((vm_offset_t) uap->addr);
  752         end = round_page((vm_offset_t) uap->addr + uap->len);
  753 
  754         if (vm_map_madvise(map, start, end, uap->behav))
  755                 return (EINVAL);
  756         return (0);
  757 }
  758 
  759 #ifndef _SYS_SYSPROTO_H_
  760 struct mincore_args {
  761         const void *addr;
  762         size_t len;
  763         char *vec;
  764 };
  765 #endif
  766 
  767 int
  768 sys_mincore(struct thread *td, struct mincore_args *uap)
  769 {
  770         vm_offset_t addr, first_addr;
  771         vm_offset_t end, cend;
  772         pmap_t pmap;
  773         vm_map_t map;
  774         char *vec;
  775         int error = 0;
  776         int vecindex, lastvecindex;
  777         vm_map_entry_t current;
  778         vm_map_entry_t entry;
  779         vm_object_t object;
  780         vm_paddr_t locked_pa;
  781         vm_page_t m;
  782         vm_pindex_t pindex;
  783         int mincoreinfo;
  784         unsigned int timestamp;
  785         boolean_t locked;
  786 
  787         /*
  788          * Make sure that the addresses presented are valid for user
  789          * mode.
  790          */
  791         first_addr = addr = trunc_page((vm_offset_t) uap->addr);
  792         end = addr + (vm_size_t)round_page(uap->len);
  793         map = &td->td_proc->p_vmspace->vm_map;
  794         if (end > vm_map_max(map) || end < addr)
  795                 return (ENOMEM);
  796 
  797         /*
  798          * Address of byte vector
  799          */
  800         vec = uap->vec;
  801 
  802         pmap = vmspace_pmap(td->td_proc->p_vmspace);
  803 
  804         vm_map_lock_read(map);
  805 RestartScan:
  806         timestamp = map->timestamp;
  807 
  808         if (!vm_map_lookup_entry(map, addr, &entry)) {
  809                 vm_map_unlock_read(map);
  810                 return (ENOMEM);
  811         }
  812 
  813         /*
  814          * Do this on a map entry basis so that if the pages are not
  815          * in the current processes address space, we can easily look
  816          * up the pages elsewhere.
  817          */
  818         lastvecindex = -1;
  819         for (current = entry;
  820             (current != &map->header) && (current->start < end);
  821             current = current->next) {
  822 
  823                 /*
  824                  * check for contiguity
  825                  */
  826                 if (current->end < end &&
  827                     (entry->next == &map->header ||
  828                      current->next->start > current->end)) {
  829                         vm_map_unlock_read(map);
  830                         return (ENOMEM);
  831                 }
  832 
  833                 /*
  834                  * ignore submaps (for now) or null objects
  835                  */
  836                 if ((current->eflags & MAP_ENTRY_IS_SUB_MAP) ||
  837                         current->object.vm_object == NULL)
  838                         continue;
  839 
  840                 /*
  841                  * limit this scan to the current map entry and the
  842                  * limits for the mincore call
  843                  */
  844                 if (addr < current->start)
  845                         addr = current->start;
  846                 cend = current->end;
  847                 if (cend > end)
  848                         cend = end;
  849 
  850                 /*
  851                  * scan this entry one page at a time
  852                  */
  853                 while (addr < cend) {
  854                         /*
  855                          * Check pmap first, it is likely faster, also
  856                          * it can provide info as to whether we are the
  857                          * one referencing or modifying the page.
  858                          */
  859                         object = NULL;
  860                         locked_pa = 0;
  861                 retry:
  862                         m = NULL;
  863                         mincoreinfo = pmap_mincore(pmap, addr, &locked_pa);
  864                         if (locked_pa != 0) {
  865                                 /*
  866                                  * The page is mapped by this process but not
  867                                  * both accessed and modified.  It is also
  868                                  * managed.  Acquire the object lock so that
  869                                  * other mappings might be examined.
  870                                  */
  871                                 m = PHYS_TO_VM_PAGE(locked_pa);
  872                                 if (m->object != object) {
  873                                         if (object != NULL)
  874                                                 VM_OBJECT_WUNLOCK(object);
  875                                         object = m->object;
  876                                         locked = VM_OBJECT_TRYWLOCK(object);
  877                                         vm_page_unlock(m);
  878                                         if (!locked) {
  879                                                 VM_OBJECT_WLOCK(object);
  880                                                 vm_page_lock(m);
  881                                                 goto retry;
  882                                         }
  883                                 } else
  884                                         vm_page_unlock(m);
  885                                 KASSERT(m->valid == VM_PAGE_BITS_ALL,
  886                                     ("mincore: page %p is mapped but invalid",
  887                                     m));
  888                         } else if (mincoreinfo == 0) {
  889                                 /*
  890                                  * The page is not mapped by this process.  If
  891                                  * the object implements managed pages, then
  892                                  * determine if the page is resident so that
  893                                  * the mappings might be examined.
  894                                  */
  895                                 if (current->object.vm_object != object) {
  896                                         if (object != NULL)
  897                                                 VM_OBJECT_WUNLOCK(object);
  898                                         object = current->object.vm_object;
  899                                         VM_OBJECT_WLOCK(object);
  900                                 }
  901                                 if (object->type == OBJT_DEFAULT ||
  902                                     object->type == OBJT_SWAP ||
  903                                     object->type == OBJT_VNODE) {
  904                                         pindex = OFF_TO_IDX(current->offset +
  905                                             (addr - current->start));
  906                                         m = vm_page_lookup(object, pindex);
  907                                         if (m == NULL &&
  908                                             vm_page_is_cached(object, pindex))
  909                                                 mincoreinfo = MINCORE_INCORE;
  910                                         if (m != NULL && m->valid == 0)
  911                                                 m = NULL;
  912                                         if (m != NULL)
  913                                                 mincoreinfo = MINCORE_INCORE;
  914                                 }
  915                         }
  916                         if (m != NULL) {
  917                                 /* Examine other mappings to the page. */
  918                                 if (m->dirty == 0 && pmap_is_modified(m))
  919                                         vm_page_dirty(m);
  920                                 if (m->dirty != 0)
  921                                         mincoreinfo |= MINCORE_MODIFIED_OTHER;
  922                                 /*
  923                                  * The first test for PGA_REFERENCED is an
  924                                  * optimization.  The second test is
  925                                  * required because a concurrent pmap
  926                                  * operation could clear the last reference
  927                                  * and set PGA_REFERENCED before the call to
  928                                  * pmap_is_referenced(). 
  929                                  */
  930                                 if ((m->aflags & PGA_REFERENCED) != 0 ||
  931                                     pmap_is_referenced(m) ||
  932                                     (m->aflags & PGA_REFERENCED) != 0)
  933                                         mincoreinfo |= MINCORE_REFERENCED_OTHER;
  934                         }
  935                         if (object != NULL)
  936                                 VM_OBJECT_WUNLOCK(object);
  937 
  938                         /*
  939                          * subyte may page fault.  In case it needs to modify
  940                          * the map, we release the lock.
  941                          */
  942                         vm_map_unlock_read(map);
  943 
  944                         /*
  945                          * calculate index into user supplied byte vector
  946                          */
  947                         vecindex = OFF_TO_IDX(addr - first_addr);
  948 
  949                         /*
  950                          * If we have skipped map entries, we need to make sure that
  951                          * the byte vector is zeroed for those skipped entries.
  952                          */
  953                         while ((lastvecindex + 1) < vecindex) {
  954                                 ++lastvecindex;
  955                                 error = subyte(vec + lastvecindex, 0);
  956                                 if (error) {
  957                                         error = EFAULT;
  958                                         goto done2;
  959                                 }
  960                         }
  961 
  962                         /*
  963                          * Pass the page information to the user
  964                          */
  965                         error = subyte(vec + vecindex, mincoreinfo);
  966                         if (error) {
  967                                 error = EFAULT;
  968                                 goto done2;
  969                         }
  970 
  971                         /*
  972                          * If the map has changed, due to the subyte, the previous
  973                          * output may be invalid.
  974                          */
  975                         vm_map_lock_read(map);
  976                         if (timestamp != map->timestamp)
  977                                 goto RestartScan;
  978 
  979                         lastvecindex = vecindex;
  980                         addr += PAGE_SIZE;
  981                 }
  982         }
  983 
  984         /*
  985          * subyte may page fault.  In case it needs to modify
  986          * the map, we release the lock.
  987          */
  988         vm_map_unlock_read(map);
  989 
  990         /*
  991          * Zero the last entries in the byte vector.
  992          */
  993         vecindex = OFF_TO_IDX(end - first_addr);
  994         while ((lastvecindex + 1) < vecindex) {
  995                 ++lastvecindex;
  996                 error = subyte(vec + lastvecindex, 0);
  997                 if (error) {
  998                         error = EFAULT;
  999                         goto done2;
 1000                 }
 1001         }
 1002 
 1003         /*
 1004          * If the map has changed, due to the subyte, the previous
 1005          * output may be invalid.
 1006          */
 1007         vm_map_lock_read(map);
 1008         if (timestamp != map->timestamp)
 1009                 goto RestartScan;
 1010         vm_map_unlock_read(map);
 1011 done2:
 1012         return (error);
 1013 }
 1014 
 1015 #ifndef _SYS_SYSPROTO_H_
 1016 struct mlock_args {
 1017         const void *addr;
 1018         size_t len;
 1019 };
 1020 #endif
 1021 int
 1022 sys_mlock(struct thread *td, struct mlock_args *uap)
 1023 {
 1024 
 1025         return (vm_mlock(td->td_proc, td->td_ucred, uap->addr, uap->len));
 1026 }
 1027 
 1028 int
 1029 vm_mlock(struct proc *proc, struct ucred *cred, const void *addr0, size_t len)
 1030 {
 1031         vm_offset_t addr, end, last, start;
 1032         vm_size_t npages, size;
 1033         vm_map_t map;
 1034         unsigned long nsize;
 1035         int error;
 1036 
 1037         error = priv_check_cred(cred, PRIV_VM_MLOCK, 0);
 1038         if (error)
 1039                 return (error);
 1040         addr = (vm_offset_t)addr0;
 1041         size = len;
 1042         last = addr + size;
 1043         start = trunc_page(addr);
 1044         end = round_page(last);
 1045         if (last < addr || end < addr)
 1046                 return (EINVAL);
 1047         npages = atop(end - start);
 1048         if (npages > vm_page_max_wired)
 1049                 return (ENOMEM);
 1050         map = &proc->p_vmspace->vm_map;
 1051         PROC_LOCK(proc);
 1052         nsize = ptoa(npages + pmap_wired_count(map->pmap));
 1053         if (nsize > lim_cur(proc, RLIMIT_MEMLOCK)) {
 1054                 PROC_UNLOCK(proc);
 1055                 return (ENOMEM);
 1056         }
 1057         PROC_UNLOCK(proc);
 1058         if (npages + cnt.v_wire_count > vm_page_max_wired)
 1059                 return (EAGAIN);
 1060 #ifdef RACCT
 1061         if (racct_enable) {
 1062                 PROC_LOCK(proc);
 1063                 error = racct_set(proc, RACCT_MEMLOCK, nsize);
 1064                 PROC_UNLOCK(proc);
 1065                 if (error != 0)
 1066                         return (ENOMEM);
 1067         }
 1068 #endif
 1069         error = vm_map_wire(map, start, end,
 1070             VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
 1071 #ifdef RACCT
 1072         if (racct_enable && error != KERN_SUCCESS) {
 1073                 PROC_LOCK(proc);
 1074                 racct_set(proc, RACCT_MEMLOCK,
 1075                     ptoa(pmap_wired_count(map->pmap)));
 1076                 PROC_UNLOCK(proc);
 1077         }
 1078 #endif
 1079         return (error == KERN_SUCCESS ? 0 : ENOMEM);
 1080 }
 1081 
 1082 #ifndef _SYS_SYSPROTO_H_
 1083 struct mlockall_args {
 1084         int     how;
 1085 };
 1086 #endif
 1087 
 1088 int
 1089 sys_mlockall(struct thread *td, struct mlockall_args *uap)
 1090 {
 1091         vm_map_t map;
 1092         int error;
 1093 
 1094         map = &td->td_proc->p_vmspace->vm_map;
 1095         error = priv_check(td, PRIV_VM_MLOCK);
 1096         if (error)
 1097                 return (error);
 1098 
 1099         if ((uap->how == 0) || ((uap->how & ~(MCL_CURRENT|MCL_FUTURE)) != 0))
 1100                 return (EINVAL);
 1101 
 1102         /*
 1103          * If wiring all pages in the process would cause it to exceed
 1104          * a hard resource limit, return ENOMEM.
 1105          */
 1106         if (!old_mlock && uap->how & MCL_CURRENT) {
 1107                 PROC_LOCK(td->td_proc);
 1108                 if (map->size > lim_cur(td->td_proc, RLIMIT_MEMLOCK)) {
 1109                         PROC_UNLOCK(td->td_proc);
 1110                         return (ENOMEM);
 1111                 }
 1112                 PROC_UNLOCK(td->td_proc);
 1113         }
 1114 #ifdef RACCT
 1115         if (racct_enable) {
 1116                 PROC_LOCK(td->td_proc);
 1117                 error = racct_set(td->td_proc, RACCT_MEMLOCK, map->size);
 1118                 PROC_UNLOCK(td->td_proc);
 1119                 if (error != 0)
 1120                         return (ENOMEM);
 1121         }
 1122 #endif
 1123 
 1124         if (uap->how & MCL_FUTURE) {
 1125                 vm_map_lock(map);
 1126                 vm_map_modflags(map, MAP_WIREFUTURE, 0);
 1127                 vm_map_unlock(map);
 1128                 error = 0;
 1129         }
 1130 
 1131         if (uap->how & MCL_CURRENT) {
 1132                 /*
 1133                  * P1003.1-2001 mandates that all currently mapped pages
 1134                  * will be memory resident and locked (wired) upon return
 1135                  * from mlockall(). vm_map_wire() will wire pages, by
 1136                  * calling vm_fault_wire() for each page in the region.
 1137                  */
 1138                 error = vm_map_wire(map, vm_map_min(map), vm_map_max(map),
 1139                     VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
 1140                 error = (error == KERN_SUCCESS ? 0 : EAGAIN);
 1141         }
 1142 #ifdef RACCT
 1143         if (racct_enable && error != KERN_SUCCESS) {
 1144                 PROC_LOCK(td->td_proc);
 1145                 racct_set(td->td_proc, RACCT_MEMLOCK,
 1146                     ptoa(pmap_wired_count(map->pmap)));
 1147                 PROC_UNLOCK(td->td_proc);
 1148         }
 1149 #endif
 1150 
 1151         return (error);
 1152 }
 1153 
 1154 #ifndef _SYS_SYSPROTO_H_
 1155 struct munlockall_args {
 1156         register_t dummy;
 1157 };
 1158 #endif
 1159 
 1160 int
 1161 sys_munlockall(struct thread *td, struct munlockall_args *uap)
 1162 {
 1163         vm_map_t map;
 1164         int error;
 1165 
 1166         map = &td->td_proc->p_vmspace->vm_map;
 1167         error = priv_check(td, PRIV_VM_MUNLOCK);
 1168         if (error)
 1169                 return (error);
 1170 
 1171         /* Clear the MAP_WIREFUTURE flag from this vm_map. */
 1172         vm_map_lock(map);
 1173         vm_map_modflags(map, 0, MAP_WIREFUTURE);
 1174         vm_map_unlock(map);
 1175 
 1176         /* Forcibly unwire all pages. */
 1177         error = vm_map_unwire(map, vm_map_min(map), vm_map_max(map),
 1178             VM_MAP_WIRE_USER|VM_MAP_WIRE_HOLESOK);
 1179 #ifdef RACCT
 1180         if (racct_enable && error == KERN_SUCCESS) {
 1181                 PROC_LOCK(td->td_proc);
 1182                 racct_set(td->td_proc, RACCT_MEMLOCK, 0);
 1183                 PROC_UNLOCK(td->td_proc);
 1184         }
 1185 #endif
 1186 
 1187         return (error);
 1188 }
 1189 
 1190 #ifndef _SYS_SYSPROTO_H_
 1191 struct munlock_args {
 1192         const void *addr;
 1193         size_t len;
 1194 };
 1195 #endif
 1196 int
 1197 sys_munlock(td, uap)
 1198         struct thread *td;
 1199         struct munlock_args *uap;
 1200 {
 1201         vm_offset_t addr, end, last, start;
 1202         vm_size_t size;
 1203 #ifdef RACCT
 1204         vm_map_t map;
 1205 #endif
 1206         int error;
 1207 
 1208         error = priv_check(td, PRIV_VM_MUNLOCK);
 1209         if (error)
 1210                 return (error);
 1211         addr = (vm_offset_t)uap->addr;
 1212         size = uap->len;
 1213         last = addr + size;
 1214         start = trunc_page(addr);
 1215         end = round_page(last);
 1216         if (last < addr || end < addr)
 1217                 return (EINVAL);
 1218         error = vm_map_unwire(&td->td_proc->p_vmspace->vm_map, start, end,
 1219             VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
 1220 #ifdef RACCT
 1221         if (racct_enable && error == KERN_SUCCESS) {
 1222                 PROC_LOCK(td->td_proc);
 1223                 map = &td->td_proc->p_vmspace->vm_map;
 1224                 racct_set(td->td_proc, RACCT_MEMLOCK,
 1225                     ptoa(pmap_wired_count(map->pmap)));
 1226                 PROC_UNLOCK(td->td_proc);
 1227         }
 1228 #endif
 1229         return (error == KERN_SUCCESS ? 0 : ENOMEM);
 1230 }
 1231 
 1232 /*
 1233  * vm_mmap_vnode()
 1234  *
 1235  * Helper function for vm_mmap.  Perform sanity check specific for mmap
 1236  * operations on vnodes.
 1237  *
 1238  * For VCHR vnodes, the vnode lock is held over the call to
 1239  * vm_mmap_cdev() to keep vp->v_rdev valid.
 1240  */
 1241 int
 1242 vm_mmap_vnode(struct thread *td, vm_size_t objsize,
 1243     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
 1244     struct vnode *vp, vm_ooffset_t *foffp, vm_object_t *objp,
 1245     boolean_t *writecounted)
 1246 {
 1247         struct vattr va;
 1248         vm_object_t obj;
 1249         vm_offset_t foff;
 1250         struct ucred *cred;
 1251         int error, flags, locktype;
 1252 
 1253         cred = td->td_ucred;
 1254         if ((*maxprotp & VM_PROT_WRITE) && (*flagsp & MAP_SHARED))
 1255                 locktype = LK_EXCLUSIVE;
 1256         else
 1257                 locktype = LK_SHARED;
 1258         if ((error = vget(vp, locktype, td)) != 0)
 1259                 return (error);
 1260         foff = *foffp;
 1261         flags = *flagsp;
 1262         obj = vp->v_object;
 1263         if (vp->v_type == VREG) {
 1264                 /*
 1265                  * Get the proper underlying object
 1266                  */
 1267                 if (obj == NULL) {
 1268                         error = EINVAL;
 1269                         goto done;
 1270                 }
 1271                 if (obj->type == OBJT_VNODE && obj->handle != vp) {
 1272                         vput(vp);
 1273                         vp = (struct vnode *)obj->handle;
 1274                         /*
 1275                          * Bypass filesystems obey the mpsafety of the
 1276                          * underlying fs.  Tmpfs never bypasses.
 1277                          */
 1278                         error = vget(vp, locktype, td);
 1279                         if (error != 0)
 1280                                 return (error);
 1281                 }
 1282                 if (locktype == LK_EXCLUSIVE) {
 1283                         *writecounted = TRUE;
 1284                         vnode_pager_update_writecount(obj, 0, objsize);
 1285                 }
 1286         } else if (vp->v_type == VCHR) {
 1287                 error = vm_mmap_cdev(td, objsize, prot, maxprotp, flagsp,
 1288                     vp->v_rdev, foffp, objp);
 1289                 if (error == 0)
 1290                         goto mark_atime;
 1291                 goto done;
 1292         } else {
 1293                 error = EINVAL;
 1294                 goto done;
 1295         }
 1296         if ((error = VOP_GETATTR(vp, &va, cred)))
 1297                 goto done;
 1298 #ifdef MAC
 1299         error = mac_vnode_check_mmap(cred, vp, prot, flags);
 1300         if (error != 0)
 1301                 goto done;
 1302 #endif
 1303         if ((flags & MAP_SHARED) != 0) {
 1304                 if ((va.va_flags & (SF_SNAPSHOT|IMMUTABLE|APPEND)) != 0) {
 1305                         if (prot & PROT_WRITE) {
 1306                                 error = EPERM;
 1307                                 goto done;
 1308                         }
 1309                         *maxprotp &= ~VM_PROT_WRITE;
 1310                 }
 1311         }
 1312         /*
 1313          * If it is a regular file without any references
 1314          * we do not need to sync it.
 1315          * Adjust object size to be the size of actual file.
 1316          */
 1317         objsize = round_page(va.va_size);
 1318         if (va.va_nlink == 0)
 1319                 flags |= MAP_NOSYNC;
 1320         if (obj->type == OBJT_VNODE)
 1321                 obj = vm_pager_allocate(OBJT_VNODE, vp, objsize, prot, foff,
 1322                     cred);
 1323         else {
 1324                 KASSERT(obj->type == OBJT_DEFAULT || obj->type == OBJT_SWAP,
 1325                     ("wrong object type"));
 1326                 vm_object_reference(obj);
 1327         }
 1328         if (obj == NULL) {
 1329                 error = ENOMEM;
 1330                 goto done;
 1331         }
 1332         *objp = obj;
 1333         *flagsp = flags;
 1334 
 1335 mark_atime:
 1336         vfs_mark_atime(vp, cred);
 1337 
 1338 done:
 1339         if (error != 0 && *writecounted) {
 1340                 *writecounted = FALSE;
 1341                 vnode_pager_update_writecount(obj, objsize, 0);
 1342         }
 1343         vput(vp);
 1344         return (error);
 1345 }
 1346 
 1347 /*
 1348  * vm_mmap_cdev()
 1349  *
 1350  * Helper function for vm_mmap.  Perform sanity check specific for mmap
 1351  * operations on cdevs.
 1352  */
 1353 int
 1354 vm_mmap_cdev(struct thread *td, vm_size_t objsize,
 1355     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
 1356     struct cdev *cdev, vm_ooffset_t *foff, vm_object_t *objp)
 1357 {
 1358         vm_object_t obj;
 1359         struct cdevsw *dsw;
 1360         int error, flags, ref;
 1361 
 1362         flags = *flagsp;
 1363 
 1364         dsw = dev_refthread(cdev, &ref);
 1365         if (dsw == NULL)
 1366                 return (ENXIO);
 1367         if (dsw->d_flags & D_MMAP_ANON) {
 1368                 dev_relthread(cdev, ref);
 1369                 *maxprotp = VM_PROT_ALL;
 1370                 *flagsp |= MAP_ANON;
 1371                 return (0);
 1372         }
 1373         /*
 1374          * cdevs do not provide private mappings of any kind.
 1375          */
 1376         if ((*maxprotp & VM_PROT_WRITE) == 0 &&
 1377             (prot & PROT_WRITE) != 0) {
 1378                 dev_relthread(cdev, ref);
 1379                 return (EACCES);
 1380         }
 1381         if (flags & (MAP_PRIVATE|MAP_COPY)) {
 1382                 dev_relthread(cdev, ref);
 1383                 return (EINVAL);
 1384         }
 1385         /*
 1386          * Force device mappings to be shared.
 1387          */
 1388         flags |= MAP_SHARED;
 1389 #ifdef MAC_XXX
 1390         error = mac_cdev_check_mmap(td->td_ucred, cdev, prot);
 1391         if (error != 0) {
 1392                 dev_relthread(cdev, ref);
 1393                 return (error);
 1394         }
 1395 #endif
 1396         /*
 1397          * First, try d_mmap_single().  If that is not implemented
 1398          * (returns ENODEV), fall back to using the device pager.
 1399          * Note that d_mmap_single() must return a reference to the
 1400          * object (it needs to bump the reference count of the object
 1401          * it returns somehow).
 1402          *
 1403          * XXX assumes VM_PROT_* == PROT_*
 1404          */
 1405         error = dsw->d_mmap_single(cdev, foff, objsize, objp, (int)prot);
 1406         dev_relthread(cdev, ref);
 1407         if (error != ENODEV)
 1408                 return (error);
 1409         obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff,
 1410             td->td_ucred);
 1411         if (obj == NULL)
 1412                 return (EINVAL);
 1413         *objp = obj;
 1414         *flagsp = flags;
 1415         return (0);
 1416 }
 1417 
 1418 /*
 1419  * vm_mmap_shm()
 1420  *
 1421  * MPSAFE
 1422  *
 1423  * Helper function for vm_mmap.  Perform sanity check specific for mmap
 1424  * operations on shm file descriptors.
 1425  */
 1426 int
 1427 vm_mmap_shm(struct thread *td, vm_size_t objsize,
 1428     vm_prot_t prot, vm_prot_t *maxprotp, int *flagsp,
 1429     struct shmfd *shmfd, vm_ooffset_t foff, vm_object_t *objp)
 1430 {
 1431         int error;
 1432 
 1433         if ((*flagsp & MAP_SHARED) != 0 &&
 1434             (*maxprotp & VM_PROT_WRITE) == 0 &&
 1435             (prot & PROT_WRITE) != 0)
 1436                 return (EACCES);
 1437 #ifdef MAC
 1438         error = mac_posixshm_check_mmap(td->td_ucred, shmfd, prot, *flagsp);
 1439         if (error != 0)
 1440                 return (error);
 1441 #endif
 1442         error = shm_mmap(shmfd, objsize, foff, objp);
 1443         if (error)
 1444                 return (error);
 1445         return (0);
 1446 }
 1447 
 1448 /*
 1449  * vm_mmap()
 1450  *
 1451  * MPSAFE
 1452  *
 1453  * Internal version of mmap.  Currently used by mmap, exec, and sys5
 1454  * shared memory.  Handle is either a vnode pointer or NULL for MAP_ANON.
 1455  */
 1456 int
 1457 vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
 1458         vm_prot_t maxprot, int flags,
 1459         objtype_t handle_type, void *handle,
 1460         vm_ooffset_t foff)
 1461 {
 1462         boolean_t curmap, fitit;
 1463         vm_offset_t max_addr;
 1464         vm_object_t object = NULL;
 1465         struct thread *td = curthread;
 1466         int docow, error, findspace, rv;
 1467         boolean_t writecounted;
 1468 
 1469         if (size == 0)
 1470                 return (0);
 1471 
 1472         size = round_page(size);
 1473 
 1474         curmap = map == &td->td_proc->p_vmspace->vm_map;
 1475         if (curmap) {
 1476                 PROC_LOCK(td->td_proc);
 1477                 if (map->size + size > lim_cur(td->td_proc, RLIMIT_VMEM)) {
 1478                         PROC_UNLOCK(td->td_proc);
 1479                         return (ENOMEM);
 1480                 }
 1481                 if (racct_set(td->td_proc, RACCT_VMEM, map->size + size)) {
 1482                         PROC_UNLOCK(td->td_proc);
 1483                         return (ENOMEM);
 1484                 }
 1485                 if (!old_mlock && map->flags & MAP_WIREFUTURE) {
 1486                         if (ptoa(pmap_wired_count(map->pmap)) + size >
 1487                             lim_cur(td->td_proc, RLIMIT_MEMLOCK)) {
 1488                                 racct_set_force(td->td_proc, RACCT_VMEM,
 1489                                     map->size);
 1490                                 PROC_UNLOCK(td->td_proc);
 1491                                 return (ENOMEM);
 1492                         }
 1493                         error = racct_set(td->td_proc, RACCT_MEMLOCK,
 1494                             ptoa(pmap_wired_count(map->pmap)) + size);
 1495                         if (error != 0) {
 1496                                 racct_set_force(td->td_proc, RACCT_VMEM,
 1497                                     map->size);
 1498                                 PROC_UNLOCK(td->td_proc);
 1499                                 return (error);
 1500                         }
 1501                 }
 1502                 PROC_UNLOCK(td->td_proc);
 1503         }
 1504 
 1505         /*
 1506          * We currently can only deal with page aligned file offsets.
 1507          * The check is here rather than in the syscall because the
 1508          * kernel calls this function internally for other mmaping
 1509          * operations (such as in exec) and non-aligned offsets will
 1510          * cause pmap inconsistencies...so we want to be sure to
 1511          * disallow this in all cases.
 1512          */
 1513         if (foff & PAGE_MASK)
 1514                 return (EINVAL);
 1515 
 1516         if ((flags & MAP_FIXED) == 0) {
 1517                 fitit = TRUE;
 1518                 *addr = round_page(*addr);
 1519         } else {
 1520                 if (*addr != trunc_page(*addr))
 1521                         return (EINVAL);
 1522                 fitit = FALSE;
 1523         }
 1524         writecounted = FALSE;
 1525 
 1526         /*
 1527          * Lookup/allocate object.
 1528          */
 1529         switch (handle_type) {
 1530         case OBJT_DEVICE:
 1531                 error = vm_mmap_cdev(td, size, prot, &maxprot, &flags,
 1532                     handle, &foff, &object);
 1533                 break;
 1534         case OBJT_VNODE:
 1535                 error = vm_mmap_vnode(td, size, prot, &maxprot, &flags,
 1536                     handle, &foff, &object, &writecounted);
 1537                 break;
 1538         case OBJT_SWAP:
 1539                 error = vm_mmap_shm(td, size, prot, &maxprot, &flags,
 1540                     handle, foff, &object);
 1541                 break;
 1542         case OBJT_DEFAULT:
 1543                 if (handle == NULL) {
 1544                         error = 0;
 1545                         break;
 1546                 }
 1547                 /* FALLTHROUGH */
 1548         default:
 1549                 error = EINVAL;
 1550                 break;
 1551         }
 1552         if (error)
 1553                 return (error);
 1554         if (flags & MAP_ANON) {
 1555                 object = NULL;
 1556                 docow = 0;
 1557                 /*
 1558                  * Unnamed anonymous regions always start at 0.
 1559                  */
 1560                 if (handle == 0)
 1561                         foff = 0;
 1562         } else if (flags & MAP_PREFAULT_READ)
 1563                 docow = MAP_PREFAULT;
 1564         else
 1565                 docow = MAP_PREFAULT_PARTIAL;
 1566 
 1567         if ((flags & (MAP_ANON|MAP_SHARED)) == 0)
 1568                 docow |= MAP_COPY_ON_WRITE;
 1569         if (flags & MAP_NOSYNC)
 1570                 docow |= MAP_DISABLE_SYNCER;
 1571         if (flags & MAP_NOCORE)
 1572                 docow |= MAP_DISABLE_COREDUMP;
 1573         /* Shared memory is also shared with children. */
 1574         if (flags & MAP_SHARED)
 1575                 docow |= MAP_INHERIT_SHARE;
 1576         if (writecounted)
 1577                 docow |= MAP_VN_WRITECOUNT;
 1578         if (flags & MAP_STACK) {
 1579                 if (object != NULL)
 1580                         return (EINVAL);
 1581                 docow |= MAP_STACK_GROWS_DOWN;
 1582         }
 1583         if ((flags & MAP_EXCL) != 0)
 1584                 docow |= MAP_CHECK_EXCL;
 1585         if ((flags & MAP_GUARD) != 0)
 1586                 docow |= MAP_CREATE_GUARD;
 1587 
 1588         if (fitit) {
 1589                 if ((flags & MAP_ALIGNMENT_MASK) == MAP_ALIGNED_SUPER)
 1590                         findspace = VMFS_SUPER_SPACE;
 1591                 else if ((flags & MAP_ALIGNMENT_MASK) != 0)
 1592                         findspace = VMFS_ALIGNED_SPACE(flags >>
 1593                             MAP_ALIGNMENT_SHIFT);
 1594                 else
 1595                         findspace = VMFS_OPTIMAL_SPACE;
 1596                 max_addr = 0;
 1597 #ifdef MAP_32BIT
 1598                 if ((flags & MAP_32BIT) != 0)
 1599                         max_addr = MAP_32BIT_MAX_ADDR;
 1600 #endif
 1601                 if (curmap) {
 1602                         vm_offset_t min_addr;
 1603 
 1604                         PROC_LOCK(td->td_proc);
 1605                         min_addr = round_page((vm_offset_t)td->td_proc->
 1606                             p_vmspace->vm_daddr + lim_max(td->td_proc,
 1607                             RLIMIT_DATA));
 1608                         PROC_UNLOCK(td->td_proc);
 1609                         rv = vm_map_find_min(map, object, foff, addr, size,
 1610                             min_addr, max_addr,
 1611                             findspace, prot, maxprot, docow);
 1612                 } else {
 1613                         rv = vm_map_find(map, object, foff, addr, size,
 1614                             max_addr, findspace, prot, maxprot, docow);
 1615                 }
 1616         } else {
 1617                 rv = vm_map_fixed(map, object, foff, *addr, size,
 1618                     prot, maxprot, docow);
 1619         }
 1620 
 1621         if (rv == KERN_SUCCESS) {
 1622                 /*
 1623                  * If the process has requested that all future mappings
 1624                  * be wired, then heed this.
 1625                  */
 1626                 if (map->flags & MAP_WIREFUTURE) {
 1627                         vm_map_wire(map, *addr, *addr + size,
 1628                             VM_MAP_WIRE_USER | ((flags & MAP_STACK) ?
 1629                             VM_MAP_WIRE_HOLESOK : VM_MAP_WIRE_NOHOLES));
 1630                 }
 1631         } else {
 1632                 /*
 1633                  * If this mapping was accounted for in the vnode's
 1634                  * writecount, then undo that now.
 1635                  */
 1636                 if (writecounted)
 1637                         vnode_pager_release_writecount(object, 0, size);
 1638                 /*
 1639                  * Lose the object reference.  Will destroy the
 1640                  * object if it's an unnamed anonymous mapping
 1641                  * or named anonymous without other references.
 1642                  */
 1643                 vm_object_deallocate(object);
 1644         }
 1645         return (vm_mmap_to_errno(rv));
 1646 }
 1647 
 1648 /*
 1649  * Translate a Mach VM return code to zero on success or the appropriate errno
 1650  * on failure.
 1651  */
 1652 int
 1653 vm_mmap_to_errno(int rv)
 1654 {
 1655 
 1656         switch (rv) {
 1657         case KERN_SUCCESS:
 1658                 return (0);
 1659         case KERN_INVALID_ADDRESS:
 1660         case KERN_NO_SPACE:
 1661                 return (ENOMEM);
 1662         case KERN_PROTECTION_FAILURE:
 1663                 return (EACCES);
 1664         default:
 1665                 return (EINVAL);
 1666         }
 1667 }

Cache object: 6b62d88dcd97a6b26805bee661090080


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.