The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_map.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * The Mach Operating System project at Carnegie-Mellon University.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 4. Neither the name of the University nor the names of its contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      from: @(#)vm_map.c      8.3 (Berkeley) 1/12/94
   33  *
   34  *
   35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   36  * All rights reserved.
   37  *
   38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   39  *
   40  * Permission to use, copy, modify and distribute this software and
   41  * its documentation is hereby granted, provided that both the copyright
   42  * notice and this permission notice appear in all copies of the
   43  * software, derivative works or modified versions, and any portions
   44  * thereof, and that both notices appear in supporting documentation.
   45  *
   46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   49  *
   50  * Carnegie Mellon requests users of this software to return to
   51  *
   52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   53  *  School of Computer Science
   54  *  Carnegie Mellon University
   55  *  Pittsburgh PA 15213-3890
   56  *
   57  * any improvements or extensions that they make and grant Carnegie the
   58  * rights to redistribute these changes.
   59  */
   60 
   61 /*
   62  *      Virtual memory mapping module.
   63  */
   64 
   65 #include <sys/cdefs.h>
   66 __FBSDID("$FreeBSD: releng/10.0/sys/vm/vm_map.c 267017 2014-06-03 19:02:52Z delphij $");
   67 
   68 #include <sys/param.h>
   69 #include <sys/systm.h>
   70 #include <sys/kernel.h>
   71 #include <sys/ktr.h>
   72 #include <sys/lock.h>
   73 #include <sys/mutex.h>
   74 #include <sys/proc.h>
   75 #include <sys/vmmeter.h>
   76 #include <sys/mman.h>
   77 #include <sys/vnode.h>
   78 #include <sys/racct.h>
   79 #include <sys/resourcevar.h>
   80 #include <sys/rwlock.h>
   81 #include <sys/file.h>
   82 #include <sys/sysctl.h>
   83 #include <sys/sysent.h>
   84 #include <sys/shm.h>
   85 
   86 #include <vm/vm.h>
   87 #include <vm/vm_param.h>
   88 #include <vm/pmap.h>
   89 #include <vm/vm_map.h>
   90 #include <vm/vm_page.h>
   91 #include <vm/vm_object.h>
   92 #include <vm/vm_pager.h>
   93 #include <vm/vm_kern.h>
   94 #include <vm/vm_extern.h>
   95 #include <vm/vnode_pager.h>
   96 #include <vm/swap_pager.h>
   97 #include <vm/uma.h>
   98 
   99 /*
  100  *      Virtual memory maps provide for the mapping, protection,
  101  *      and sharing of virtual memory objects.  In addition,
  102  *      this module provides for an efficient virtual copy of
  103  *      memory from one map to another.
  104  *
  105  *      Synchronization is required prior to most operations.
  106  *
  107  *      Maps consist of an ordered doubly-linked list of simple
  108  *      entries; a self-adjusting binary search tree of these
  109  *      entries is used to speed up lookups.
  110  *
  111  *      Since portions of maps are specified by start/end addresses,
  112  *      which may not align with existing map entries, all
  113  *      routines merely "clip" entries to these start/end values.
  114  *      [That is, an entry is split into two, bordering at a
  115  *      start or end value.]  Note that these clippings may not
  116  *      always be necessary (as the two resulting entries are then
  117  *      not changed); however, the clipping is done for convenience.
  118  *
  119  *      As mentioned above, virtual copy operations are performed
  120  *      by copying VM object references from one map to
  121  *      another, and then marking both regions as copy-on-write.
  122  */
  123 
  124 static struct mtx map_sleep_mtx;
  125 static uma_zone_t mapentzone;
  126 static uma_zone_t kmapentzone;
  127 static uma_zone_t mapzone;
  128 static uma_zone_t vmspace_zone;
  129 static int vmspace_zinit(void *mem, int size, int flags);
  130 static int vm_map_zinit(void *mem, int ize, int flags);
  131 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
  132     vm_offset_t max);
  133 static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map);
  134 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
  135 #ifdef INVARIANTS
  136 static void vm_map_zdtor(void *mem, int size, void *arg);
  137 static void vmspace_zdtor(void *mem, int size, void *arg);
  138 #endif
  139 
  140 #define ENTRY_CHARGED(e) ((e)->cred != NULL || \
  141     ((e)->object.vm_object != NULL && (e)->object.vm_object->cred != NULL && \
  142      !((e)->eflags & MAP_ENTRY_NEEDS_COPY)))
  143 
  144 /* 
  145  * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type
  146  * stable.
  147  */
  148 #define PROC_VMSPACE_LOCK(p) do { } while (0)
  149 #define PROC_VMSPACE_UNLOCK(p) do { } while (0)
  150 
  151 /*
  152  *      VM_MAP_RANGE_CHECK:     [ internal use only ]
  153  *
  154  *      Asserts that the starting and ending region
  155  *      addresses fall within the valid range of the map.
  156  */
  157 #define VM_MAP_RANGE_CHECK(map, start, end)             \
  158                 {                                       \
  159                 if (start < vm_map_min(map))            \
  160                         start = vm_map_min(map);        \
  161                 if (end > vm_map_max(map))              \
  162                         end = vm_map_max(map);          \
  163                 if (start > end)                        \
  164                         start = end;                    \
  165                 }
  166 
  167 /*
  168  *      vm_map_startup:
  169  *
  170  *      Initialize the vm_map module.  Must be called before
  171  *      any other vm_map routines.
  172  *
  173  *      Map and entry structures are allocated from the general
  174  *      purpose memory pool with some exceptions:
  175  *
  176  *      - The kernel map and kmem submap are allocated statically.
  177  *      - Kernel map entries are allocated out of a static pool.
  178  *
  179  *      These restrictions are necessary since malloc() uses the
  180  *      maps and requires map entries.
  181  */
  182 
  183 void
  184 vm_map_startup(void)
  185 {
  186         mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF);
  187         mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL,
  188 #ifdef INVARIANTS
  189             vm_map_zdtor,
  190 #else
  191             NULL,
  192 #endif
  193             vm_map_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  194         uma_prealloc(mapzone, MAX_KMAP);
  195         kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry),
  196             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
  197             UMA_ZONE_MTXCLASS | UMA_ZONE_VM);
  198         mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry),
  199             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
  200         vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL,
  201 #ifdef INVARIANTS
  202             vmspace_zdtor,
  203 #else
  204             NULL,
  205 #endif
  206             vmspace_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  207 }
  208 
  209 static int
  210 vmspace_zinit(void *mem, int size, int flags)
  211 {
  212         struct vmspace *vm;
  213 
  214         vm = (struct vmspace *)mem;
  215 
  216         vm->vm_map.pmap = NULL;
  217         (void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags);
  218         PMAP_LOCK_INIT(vmspace_pmap(vm));
  219         return (0);
  220 }
  221 
  222 static int
  223 vm_map_zinit(void *mem, int size, int flags)
  224 {
  225         vm_map_t map;
  226 
  227         map = (vm_map_t)mem;
  228         memset(map, 0, sizeof(*map));
  229         mtx_init(&map->system_mtx, "vm map (system)", NULL, MTX_DEF | MTX_DUPOK);
  230         sx_init(&map->lock, "vm map (user)");
  231         return (0);
  232 }
  233 
  234 #ifdef INVARIANTS
  235 static void
  236 vmspace_zdtor(void *mem, int size, void *arg)
  237 {
  238         struct vmspace *vm;
  239 
  240         vm = (struct vmspace *)mem;
  241 
  242         vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg);
  243 }
  244 static void
  245 vm_map_zdtor(void *mem, int size, void *arg)
  246 {
  247         vm_map_t map;
  248 
  249         map = (vm_map_t)mem;
  250         KASSERT(map->nentries == 0,
  251             ("map %p nentries == %d on free.",
  252             map, map->nentries));
  253         KASSERT(map->size == 0,
  254             ("map %p size == %lu on free.",
  255             map, (unsigned long)map->size));
  256 }
  257 #endif  /* INVARIANTS */
  258 
  259 /*
  260  * Allocate a vmspace structure, including a vm_map and pmap,
  261  * and initialize those structures.  The refcnt is set to 1.
  262  *
  263  * If 'pinit' is NULL then the embedded pmap is initialized via pmap_pinit().
  264  */
  265 struct vmspace *
  266 vmspace_alloc(vm_offset_t min, vm_offset_t max, pmap_pinit_t pinit)
  267 {
  268         struct vmspace *vm;
  269 
  270         vm = uma_zalloc(vmspace_zone, M_WAITOK);
  271 
  272         KASSERT(vm->vm_map.pmap == NULL, ("vm_map.pmap must be NULL"));
  273 
  274         if (pinit == NULL)
  275                 pinit = &pmap_pinit;
  276 
  277         if (!pinit(vmspace_pmap(vm))) {
  278                 uma_zfree(vmspace_zone, vm);
  279                 return (NULL);
  280         }
  281         CTR1(KTR_VM, "vmspace_alloc: %p", vm);
  282         _vm_map_init(&vm->vm_map, vmspace_pmap(vm), min, max);
  283         vm->vm_refcnt = 1;
  284         vm->vm_shm = NULL;
  285         vm->vm_swrss = 0;
  286         vm->vm_tsize = 0;
  287         vm->vm_dsize = 0;
  288         vm->vm_ssize = 0;
  289         vm->vm_taddr = 0;
  290         vm->vm_daddr = 0;
  291         vm->vm_maxsaddr = 0;
  292         return (vm);
  293 }
  294 
  295 static void
  296 vmspace_container_reset(struct proc *p)
  297 {
  298 
  299 #ifdef RACCT
  300         PROC_LOCK(p);
  301         racct_set(p, RACCT_DATA, 0);
  302         racct_set(p, RACCT_STACK, 0);
  303         racct_set(p, RACCT_RSS, 0);
  304         racct_set(p, RACCT_MEMLOCK, 0);
  305         racct_set(p, RACCT_VMEM, 0);
  306         PROC_UNLOCK(p);
  307 #endif
  308 }
  309 
  310 static inline void
  311 vmspace_dofree(struct vmspace *vm)
  312 {
  313 
  314         CTR1(KTR_VM, "vmspace_free: %p", vm);
  315 
  316         /*
  317          * Make sure any SysV shm is freed, it might not have been in
  318          * exit1().
  319          */
  320         shmexit(vm);
  321 
  322         /*
  323          * Lock the map, to wait out all other references to it.
  324          * Delete all of the mappings and pages they hold, then call
  325          * the pmap module to reclaim anything left.
  326          */
  327         (void)vm_map_remove(&vm->vm_map, vm->vm_map.min_offset,
  328             vm->vm_map.max_offset);
  329 
  330         pmap_release(vmspace_pmap(vm));
  331         vm->vm_map.pmap = NULL;
  332         uma_zfree(vmspace_zone, vm);
  333 }
  334 
  335 void
  336 vmspace_free(struct vmspace *vm)
  337 {
  338 
  339         if (vm->vm_refcnt == 0)
  340                 panic("vmspace_free: attempt to free already freed vmspace");
  341 
  342         if (atomic_fetchadd_int(&vm->vm_refcnt, -1) == 1)
  343                 vmspace_dofree(vm);
  344 }
  345 
  346 void
  347 vmspace_exitfree(struct proc *p)
  348 {
  349         struct vmspace *vm;
  350 
  351         PROC_VMSPACE_LOCK(p);
  352         vm = p->p_vmspace;
  353         p->p_vmspace = NULL;
  354         PROC_VMSPACE_UNLOCK(p);
  355         KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace"));
  356         vmspace_free(vm);
  357 }
  358 
  359 void
  360 vmspace_exit(struct thread *td)
  361 {
  362         int refcnt;
  363         struct vmspace *vm;
  364         struct proc *p;
  365 
  366         /*
  367          * Release user portion of address space.
  368          * This releases references to vnodes,
  369          * which could cause I/O if the file has been unlinked.
  370          * Need to do this early enough that we can still sleep.
  371          *
  372          * The last exiting process to reach this point releases as
  373          * much of the environment as it can. vmspace_dofree() is the
  374          * slower fallback in case another process had a temporary
  375          * reference to the vmspace.
  376          */
  377 
  378         p = td->td_proc;
  379         vm = p->p_vmspace;
  380         atomic_add_int(&vmspace0.vm_refcnt, 1);
  381         do {
  382                 refcnt = vm->vm_refcnt;
  383                 if (refcnt > 1 && p->p_vmspace != &vmspace0) {
  384                         /* Switch now since other proc might free vmspace */
  385                         PROC_VMSPACE_LOCK(p);
  386                         p->p_vmspace = &vmspace0;
  387                         PROC_VMSPACE_UNLOCK(p);
  388                         pmap_activate(td);
  389                 }
  390         } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
  391         if (refcnt == 1) {
  392                 if (p->p_vmspace != vm) {
  393                         /* vmspace not yet freed, switch back */
  394                         PROC_VMSPACE_LOCK(p);
  395                         p->p_vmspace = vm;
  396                         PROC_VMSPACE_UNLOCK(p);
  397                         pmap_activate(td);
  398                 }
  399                 pmap_remove_pages(vmspace_pmap(vm));
  400                 /* Switch now since this proc will free vmspace */
  401                 PROC_VMSPACE_LOCK(p);
  402                 p->p_vmspace = &vmspace0;
  403                 PROC_VMSPACE_UNLOCK(p);
  404                 pmap_activate(td);
  405                 vmspace_dofree(vm);
  406         }
  407         vmspace_container_reset(p);
  408 }
  409 
  410 /* Acquire reference to vmspace owned by another process. */
  411 
  412 struct vmspace *
  413 vmspace_acquire_ref(struct proc *p)
  414 {
  415         struct vmspace *vm;
  416         int refcnt;
  417 
  418         PROC_VMSPACE_LOCK(p);
  419         vm = p->p_vmspace;
  420         if (vm == NULL) {
  421                 PROC_VMSPACE_UNLOCK(p);
  422                 return (NULL);
  423         }
  424         do {
  425                 refcnt = vm->vm_refcnt;
  426                 if (refcnt <= 0) {      /* Avoid 0->1 transition */
  427                         PROC_VMSPACE_UNLOCK(p);
  428                         return (NULL);
  429                 }
  430         } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt + 1));
  431         if (vm != p->p_vmspace) {
  432                 PROC_VMSPACE_UNLOCK(p);
  433                 vmspace_free(vm);
  434                 return (NULL);
  435         }
  436         PROC_VMSPACE_UNLOCK(p);
  437         return (vm);
  438 }
  439 
  440 void
  441 _vm_map_lock(vm_map_t map, const char *file, int line)
  442 {
  443 
  444         if (map->system_map)
  445                 mtx_lock_flags_(&map->system_mtx, 0, file, line);
  446         else
  447                 sx_xlock_(&map->lock, file, line);
  448         map->timestamp++;
  449 }
  450 
  451 static void
  452 vm_map_process_deferred(void)
  453 {
  454         struct thread *td;
  455         vm_map_entry_t entry, next;
  456         vm_object_t object;
  457 
  458         td = curthread;
  459         entry = td->td_map_def_user;
  460         td->td_map_def_user = NULL;
  461         while (entry != NULL) {
  462                 next = entry->next;
  463                 if ((entry->eflags & MAP_ENTRY_VN_WRITECNT) != 0) {
  464                         /*
  465                          * Decrement the object's writemappings and
  466                          * possibly the vnode's v_writecount.
  467                          */
  468                         KASSERT((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0,
  469                             ("Submap with writecount"));
  470                         object = entry->object.vm_object;
  471                         KASSERT(object != NULL, ("No object for writecount"));
  472                         vnode_pager_release_writecount(object, entry->start,
  473                             entry->end);
  474                 }
  475                 vm_map_entry_deallocate(entry, FALSE);
  476                 entry = next;
  477         }
  478 }
  479 
  480 void
  481 _vm_map_unlock(vm_map_t map, const char *file, int line)
  482 {
  483 
  484         if (map->system_map)
  485                 mtx_unlock_flags_(&map->system_mtx, 0, file, line);
  486         else {
  487                 sx_xunlock_(&map->lock, file, line);
  488                 vm_map_process_deferred();
  489         }
  490 }
  491 
  492 void
  493 _vm_map_lock_read(vm_map_t map, const char *file, int line)
  494 {
  495 
  496         if (map->system_map)
  497                 mtx_lock_flags_(&map->system_mtx, 0, file, line);
  498         else
  499                 sx_slock_(&map->lock, file, line);
  500 }
  501 
  502 void
  503 _vm_map_unlock_read(vm_map_t map, const char *file, int line)
  504 {
  505 
  506         if (map->system_map)
  507                 mtx_unlock_flags_(&map->system_mtx, 0, file, line);
  508         else {
  509                 sx_sunlock_(&map->lock, file, line);
  510                 vm_map_process_deferred();
  511         }
  512 }
  513 
  514 int
  515 _vm_map_trylock(vm_map_t map, const char *file, int line)
  516 {
  517         int error;
  518 
  519         error = map->system_map ?
  520             !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
  521             !sx_try_xlock_(&map->lock, file, line);
  522         if (error == 0)
  523                 map->timestamp++;
  524         return (error == 0);
  525 }
  526 
  527 int
  528 _vm_map_trylock_read(vm_map_t map, const char *file, int line)
  529 {
  530         int error;
  531 
  532         error = map->system_map ?
  533             !mtx_trylock_flags_(&map->system_mtx, 0, file, line) :
  534             !sx_try_slock_(&map->lock, file, line);
  535         return (error == 0);
  536 }
  537 
  538 /*
  539  *      _vm_map_lock_upgrade:   [ internal use only ]
  540  *
  541  *      Tries to upgrade a read (shared) lock on the specified map to a write
  542  *      (exclusive) lock.  Returns the value "" if the upgrade succeeds and a
  543  *      non-zero value if the upgrade fails.  If the upgrade fails, the map is
  544  *      returned without a read or write lock held.
  545  *
  546  *      Requires that the map be read locked.
  547  */
  548 int
  549 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line)
  550 {
  551         unsigned int last_timestamp;
  552 
  553         if (map->system_map) {
  554                 mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
  555         } else {
  556                 if (!sx_try_upgrade_(&map->lock, file, line)) {
  557                         last_timestamp = map->timestamp;
  558                         sx_sunlock_(&map->lock, file, line);
  559                         vm_map_process_deferred();
  560                         /*
  561                          * If the map's timestamp does not change while the
  562                          * map is unlocked, then the upgrade succeeds.
  563                          */
  564                         sx_xlock_(&map->lock, file, line);
  565                         if (last_timestamp != map->timestamp) {
  566                                 sx_xunlock_(&map->lock, file, line);
  567                                 return (1);
  568                         }
  569                 }
  570         }
  571         map->timestamp++;
  572         return (0);
  573 }
  574 
  575 void
  576 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
  577 {
  578 
  579         if (map->system_map) {
  580                 mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
  581         } else
  582                 sx_downgrade_(&map->lock, file, line);
  583 }
  584 
  585 /*
  586  *      vm_map_locked:
  587  *
  588  *      Returns a non-zero value if the caller holds a write (exclusive) lock
  589  *      on the specified map and the value "" otherwise.
  590  */
  591 int
  592 vm_map_locked(vm_map_t map)
  593 {
  594 
  595         if (map->system_map)
  596                 return (mtx_owned(&map->system_mtx));
  597         else
  598                 return (sx_xlocked(&map->lock));
  599 }
  600 
  601 #ifdef INVARIANTS
  602 static void
  603 _vm_map_assert_locked(vm_map_t map, const char *file, int line)
  604 {
  605 
  606         if (map->system_map)
  607                 mtx_assert_(&map->system_mtx, MA_OWNED, file, line);
  608         else
  609                 sx_assert_(&map->lock, SA_XLOCKED, file, line);
  610 }
  611 
  612 #define VM_MAP_ASSERT_LOCKED(map) \
  613     _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
  614 #else
  615 #define VM_MAP_ASSERT_LOCKED(map)
  616 #endif
  617 
  618 /*
  619  *      _vm_map_unlock_and_wait:
  620  *
  621  *      Atomically releases the lock on the specified map and puts the calling
  622  *      thread to sleep.  The calling thread will remain asleep until either
  623  *      vm_map_wakeup() is performed on the map or the specified timeout is
  624  *      exceeded.
  625  *
  626  *      WARNING!  This function does not perform deferred deallocations of
  627  *      objects and map entries.  Therefore, the calling thread is expected to
  628  *      reacquire the map lock after reawakening and later perform an ordinary
  629  *      unlock operation, such as vm_map_unlock(), before completing its
  630  *      operation on the map.
  631  */
  632 int
  633 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line)
  634 {
  635 
  636         mtx_lock(&map_sleep_mtx);
  637         if (map->system_map)
  638                 mtx_unlock_flags_(&map->system_mtx, 0, file, line);
  639         else
  640                 sx_xunlock_(&map->lock, file, line);
  641         return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps",
  642             timo));
  643 }
  644 
  645 /*
  646  *      vm_map_wakeup:
  647  *
  648  *      Awaken any threads that have slept on the map using
  649  *      vm_map_unlock_and_wait().
  650  */
  651 void
  652 vm_map_wakeup(vm_map_t map)
  653 {
  654 
  655         /*
  656          * Acquire and release map_sleep_mtx to prevent a wakeup()
  657          * from being performed (and lost) between the map unlock
  658          * and the msleep() in _vm_map_unlock_and_wait().
  659          */
  660         mtx_lock(&map_sleep_mtx);
  661         mtx_unlock(&map_sleep_mtx);
  662         wakeup(&map->root);
  663 }
  664 
  665 void
  666 vm_map_busy(vm_map_t map)
  667 {
  668 
  669         VM_MAP_ASSERT_LOCKED(map);
  670         map->busy++;
  671 }
  672 
  673 void
  674 vm_map_unbusy(vm_map_t map)
  675 {
  676 
  677         VM_MAP_ASSERT_LOCKED(map);
  678         KASSERT(map->busy, ("vm_map_unbusy: not busy"));
  679         if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) {
  680                 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP);
  681                 wakeup(&map->busy);
  682         }
  683 }
  684 
  685 void 
  686 vm_map_wait_busy(vm_map_t map)
  687 {
  688 
  689         VM_MAP_ASSERT_LOCKED(map);
  690         while (map->busy) {
  691                 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0);
  692                 if (map->system_map)
  693                         msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0);
  694                 else
  695                         sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0);
  696         }
  697         map->timestamp++;
  698 }
  699 
  700 long
  701 vmspace_resident_count(struct vmspace *vmspace)
  702 {
  703         return pmap_resident_count(vmspace_pmap(vmspace));
  704 }
  705 
  706 /*
  707  *      vm_map_create:
  708  *
  709  *      Creates and returns a new empty VM map with
  710  *      the given physical map structure, and having
  711  *      the given lower and upper address bounds.
  712  */
  713 vm_map_t
  714 vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max)
  715 {
  716         vm_map_t result;
  717 
  718         result = uma_zalloc(mapzone, M_WAITOK);
  719         CTR1(KTR_VM, "vm_map_create: %p", result);
  720         _vm_map_init(result, pmap, min, max);
  721         return (result);
  722 }
  723 
  724 /*
  725  * Initialize an existing vm_map structure
  726  * such as that in the vmspace structure.
  727  */
  728 static void
  729 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)
  730 {
  731 
  732         map->header.next = map->header.prev = &map->header;
  733         map->needs_wakeup = FALSE;
  734         map->system_map = 0;
  735         map->pmap = pmap;
  736         map->min_offset = min;
  737         map->max_offset = max;
  738         map->flags = 0;
  739         map->root = NULL;
  740         map->timestamp = 0;
  741         map->busy = 0;
  742 }
  743 
  744 void
  745 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max)
  746 {
  747 
  748         _vm_map_init(map, pmap, min, max);
  749         mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
  750         sx_init(&map->lock, "user map");
  751 }
  752 
  753 /*
  754  *      vm_map_entry_dispose:   [ internal use only ]
  755  *
  756  *      Inverse of vm_map_entry_create.
  757  */
  758 static void
  759 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry)
  760 {
  761         uma_zfree(map->system_map ? kmapentzone : mapentzone, entry);
  762 }
  763 
  764 /*
  765  *      vm_map_entry_create:    [ internal use only ]
  766  *
  767  *      Allocates a VM map entry for insertion.
  768  *      No entry fields are filled in.
  769  */
  770 static vm_map_entry_t
  771 vm_map_entry_create(vm_map_t map)
  772 {
  773         vm_map_entry_t new_entry;
  774 
  775         if (map->system_map)
  776                 new_entry = uma_zalloc(kmapentzone, M_NOWAIT);
  777         else
  778                 new_entry = uma_zalloc(mapentzone, M_WAITOK);
  779         if (new_entry == NULL)
  780                 panic("vm_map_entry_create: kernel resources exhausted");
  781         return (new_entry);
  782 }
  783 
  784 /*
  785  *      vm_map_entry_set_behavior:
  786  *
  787  *      Set the expected access behavior, either normal, random, or
  788  *      sequential.
  789  */
  790 static inline void
  791 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior)
  792 {
  793         entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) |
  794             (behavior & MAP_ENTRY_BEHAV_MASK);
  795 }
  796 
  797 /*
  798  *      vm_map_entry_set_max_free:
  799  *
  800  *      Set the max_free field in a vm_map_entry.
  801  */
  802 static inline void
  803 vm_map_entry_set_max_free(vm_map_entry_t entry)
  804 {
  805 
  806         entry->max_free = entry->adj_free;
  807         if (entry->left != NULL && entry->left->max_free > entry->max_free)
  808                 entry->max_free = entry->left->max_free;
  809         if (entry->right != NULL && entry->right->max_free > entry->max_free)
  810                 entry->max_free = entry->right->max_free;
  811 }
  812 
  813 /*
  814  *      vm_map_entry_splay:
  815  *
  816  *      The Sleator and Tarjan top-down splay algorithm with the
  817  *      following variation.  Max_free must be computed bottom-up, so
  818  *      on the downward pass, maintain the left and right spines in
  819  *      reverse order.  Then, make a second pass up each side to fix
  820  *      the pointers and compute max_free.  The time bound is O(log n)
  821  *      amortized.
  822  *
  823  *      The new root is the vm_map_entry containing "addr", or else an
  824  *      adjacent entry (lower or higher) if addr is not in the tree.
  825  *
  826  *      The map must be locked, and leaves it so.
  827  *
  828  *      Returns: the new root.
  829  */
  830 static vm_map_entry_t
  831 vm_map_entry_splay(vm_offset_t addr, vm_map_entry_t root)
  832 {
  833         vm_map_entry_t llist, rlist;
  834         vm_map_entry_t ltree, rtree;
  835         vm_map_entry_t y;
  836 
  837         /* Special case of empty tree. */
  838         if (root == NULL)
  839                 return (root);
  840 
  841         /*
  842          * Pass One: Splay down the tree until we find addr or a NULL
  843          * pointer where addr would go.  llist and rlist are the two
  844          * sides in reverse order (bottom-up), with llist linked by
  845          * the right pointer and rlist linked by the left pointer in
  846          * the vm_map_entry.  Wait until Pass Two to set max_free on
  847          * the two spines.
  848          */
  849         llist = NULL;
  850         rlist = NULL;
  851         for (;;) {
  852                 /* root is never NULL in here. */
  853                 if (addr < root->start) {
  854                         y = root->left;
  855                         if (y == NULL)
  856                                 break;
  857                         if (addr < y->start && y->left != NULL) {
  858                                 /* Rotate right and put y on rlist. */
  859                                 root->left = y->right;
  860                                 y->right = root;
  861                                 vm_map_entry_set_max_free(root);
  862                                 root = y->left;
  863                                 y->left = rlist;
  864                                 rlist = y;
  865                         } else {
  866                                 /* Put root on rlist. */
  867                                 root->left = rlist;
  868                                 rlist = root;
  869                                 root = y;
  870                         }
  871                 } else if (addr >= root->end) {
  872                         y = root->right;
  873                         if (y == NULL)
  874                                 break;
  875                         if (addr >= y->end && y->right != NULL) {
  876                                 /* Rotate left and put y on llist. */
  877                                 root->right = y->left;
  878                                 y->left = root;
  879                                 vm_map_entry_set_max_free(root);
  880                                 root = y->right;
  881                                 y->right = llist;
  882                                 llist = y;
  883                         } else {
  884                                 /* Put root on llist. */
  885                                 root->right = llist;
  886                                 llist = root;
  887                                 root = y;
  888                         }
  889                 } else
  890                         break;
  891         }
  892 
  893         /*
  894          * Pass Two: Walk back up the two spines, flip the pointers
  895          * and set max_free.  The subtrees of the root go at the
  896          * bottom of llist and rlist.
  897          */
  898         ltree = root->left;
  899         while (llist != NULL) {
  900                 y = llist->right;
  901                 llist->right = ltree;
  902                 vm_map_entry_set_max_free(llist);
  903                 ltree = llist;
  904                 llist = y;
  905         }
  906         rtree = root->right;
  907         while (rlist != NULL) {
  908                 y = rlist->left;
  909                 rlist->left = rtree;
  910                 vm_map_entry_set_max_free(rlist);
  911                 rtree = rlist;
  912                 rlist = y;
  913         }
  914 
  915         /*
  916          * Final assembly: add ltree and rtree as subtrees of root.
  917          */
  918         root->left = ltree;
  919         root->right = rtree;
  920         vm_map_entry_set_max_free(root);
  921 
  922         return (root);
  923 }
  924 
  925 /*
  926  *      vm_map_entry_{un,}link:
  927  *
  928  *      Insert/remove entries from maps.
  929  */
  930 static void
  931 vm_map_entry_link(vm_map_t map,
  932                   vm_map_entry_t after_where,
  933                   vm_map_entry_t entry)
  934 {
  935 
  936         CTR4(KTR_VM,
  937             "vm_map_entry_link: map %p, nentries %d, entry %p, after %p", map,
  938             map->nentries, entry, after_where);
  939         VM_MAP_ASSERT_LOCKED(map);
  940         map->nentries++;
  941         entry->prev = after_where;
  942         entry->next = after_where->next;
  943         entry->next->prev = entry;
  944         after_where->next = entry;
  945 
  946         if (after_where != &map->header) {
  947                 if (after_where != map->root)
  948                         vm_map_entry_splay(after_where->start, map->root);
  949                 entry->right = after_where->right;
  950                 entry->left = after_where;
  951                 after_where->right = NULL;
  952                 after_where->adj_free = entry->start - after_where->end;
  953                 vm_map_entry_set_max_free(after_where);
  954         } else {
  955                 entry->right = map->root;
  956                 entry->left = NULL;
  957         }
  958         entry->adj_free = (entry->next == &map->header ? map->max_offset :
  959             entry->next->start) - entry->end;
  960         vm_map_entry_set_max_free(entry);
  961         map->root = entry;
  962 }
  963 
  964 static void
  965 vm_map_entry_unlink(vm_map_t map,
  966                     vm_map_entry_t entry)
  967 {
  968         vm_map_entry_t next, prev, root;
  969 
  970         VM_MAP_ASSERT_LOCKED(map);
  971         if (entry != map->root)
  972                 vm_map_entry_splay(entry->start, map->root);
  973         if (entry->left == NULL)
  974                 root = entry->right;
  975         else {
  976                 root = vm_map_entry_splay(entry->start, entry->left);
  977                 root->right = entry->right;
  978                 root->adj_free = (entry->next == &map->header ? map->max_offset :
  979                     entry->next->start) - root->end;
  980                 vm_map_entry_set_max_free(root);
  981         }
  982         map->root = root;
  983 
  984         prev = entry->prev;
  985         next = entry->next;
  986         next->prev = prev;
  987         prev->next = next;
  988         map->nentries--;
  989         CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map,
  990             map->nentries, entry);
  991 }
  992 
  993 /*
  994  *      vm_map_entry_resize_free:
  995  *
  996  *      Recompute the amount of free space following a vm_map_entry
  997  *      and propagate that value up the tree.  Call this function after
  998  *      resizing a map entry in-place, that is, without a call to
  999  *      vm_map_entry_link() or _unlink().
 1000  *
 1001  *      The map must be locked, and leaves it so.
 1002  */
 1003 static void
 1004 vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t entry)
 1005 {
 1006 
 1007         /*
 1008          * Using splay trees without parent pointers, propagating
 1009          * max_free up the tree is done by moving the entry to the
 1010          * root and making the change there.
 1011          */
 1012         if (entry != map->root)
 1013                 map->root = vm_map_entry_splay(entry->start, map->root);
 1014 
 1015         entry->adj_free = (entry->next == &map->header ? map->max_offset :
 1016             entry->next->start) - entry->end;
 1017         vm_map_entry_set_max_free(entry);
 1018 }
 1019 
 1020 /*
 1021  *      vm_map_lookup_entry:    [ internal use only ]
 1022  *
 1023  *      Finds the map entry containing (or
 1024  *      immediately preceding) the specified address
 1025  *      in the given map; the entry is returned
 1026  *      in the "entry" parameter.  The boolean
 1027  *      result indicates whether the address is
 1028  *      actually contained in the map.
 1029  */
 1030 boolean_t
 1031 vm_map_lookup_entry(
 1032         vm_map_t map,
 1033         vm_offset_t address,
 1034         vm_map_entry_t *entry)  /* OUT */
 1035 {
 1036         vm_map_entry_t cur;
 1037         boolean_t locked;
 1038 
 1039         /*
 1040          * If the map is empty, then the map entry immediately preceding
 1041          * "address" is the map's header.
 1042          */
 1043         cur = map->root;
 1044         if (cur == NULL)
 1045                 *entry = &map->header;
 1046         else if (address >= cur->start && cur->end > address) {
 1047                 *entry = cur;
 1048                 return (TRUE);
 1049         } else if ((locked = vm_map_locked(map)) ||
 1050             sx_try_upgrade(&map->lock)) {
 1051                 /*
 1052                  * Splay requires a write lock on the map.  However, it only
 1053                  * restructures the binary search tree; it does not otherwise
 1054                  * change the map.  Thus, the map's timestamp need not change
 1055                  * on a temporary upgrade.
 1056                  */
 1057                 map->root = cur = vm_map_entry_splay(address, cur);
 1058                 if (!locked)
 1059                         sx_downgrade(&map->lock);
 1060 
 1061                 /*
 1062                  * If "address" is contained within a map entry, the new root
 1063                  * is that map entry.  Otherwise, the new root is a map entry
 1064                  * immediately before or after "address".
 1065                  */
 1066                 if (address >= cur->start) {
 1067                         *entry = cur;
 1068                         if (cur->end > address)
 1069                                 return (TRUE);
 1070                 } else
 1071                         *entry = cur->prev;
 1072         } else
 1073                 /*
 1074                  * Since the map is only locked for read access, perform a
 1075                  * standard binary search tree lookup for "address".
 1076                  */
 1077                 for (;;) {
 1078                         if (address < cur->start) {
 1079                                 if (cur->left == NULL) {
 1080                                         *entry = cur->prev;
 1081                                         break;
 1082                                 }
 1083                                 cur = cur->left;
 1084                         } else if (cur->end > address) {
 1085                                 *entry = cur;
 1086                                 return (TRUE);
 1087                         } else {
 1088                                 if (cur->right == NULL) {
 1089                                         *entry = cur;
 1090                                         break;
 1091                                 }
 1092                                 cur = cur->right;
 1093                         }
 1094                 }
 1095         return (FALSE);
 1096 }
 1097 
 1098 /*
 1099  *      vm_map_insert:
 1100  *
 1101  *      Inserts the given whole VM object into the target
 1102  *      map at the specified address range.  The object's
 1103  *      size should match that of the address range.
 1104  *
 1105  *      Requires that the map be locked, and leaves it so.
 1106  *
 1107  *      If object is non-NULL, ref count must be bumped by caller
 1108  *      prior to making call to account for the new entry.
 1109  */
 1110 int
 1111 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 1112               vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max,
 1113               int cow)
 1114 {
 1115         vm_map_entry_t new_entry;
 1116         vm_map_entry_t prev_entry;
 1117         vm_map_entry_t temp_entry;
 1118         vm_eflags_t protoeflags;
 1119         struct ucred *cred;
 1120         vm_inherit_t inheritance;
 1121         boolean_t charge_prev_obj;
 1122 
 1123         VM_MAP_ASSERT_LOCKED(map);
 1124 
 1125         /*
 1126          * Check that the start and end points are not bogus.
 1127          */
 1128         if ((start < map->min_offset) || (end > map->max_offset) ||
 1129             (start >= end))
 1130                 return (KERN_INVALID_ADDRESS);
 1131 
 1132         /*
 1133          * Find the entry prior to the proposed starting address; if it's part
 1134          * of an existing entry, this range is bogus.
 1135          */
 1136         if (vm_map_lookup_entry(map, start, &temp_entry))
 1137                 return (KERN_NO_SPACE);
 1138 
 1139         prev_entry = temp_entry;
 1140 
 1141         /*
 1142          * Assert that the next entry doesn't overlap the end point.
 1143          */
 1144         if ((prev_entry->next != &map->header) &&
 1145             (prev_entry->next->start < end))
 1146                 return (KERN_NO_SPACE);
 1147 
 1148         protoeflags = 0;
 1149         charge_prev_obj = FALSE;
 1150 
 1151         if (cow & MAP_COPY_ON_WRITE)
 1152                 protoeflags |= MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY;
 1153 
 1154         if (cow & MAP_NOFAULT) {
 1155                 protoeflags |= MAP_ENTRY_NOFAULT;
 1156 
 1157                 KASSERT(object == NULL,
 1158                         ("vm_map_insert: paradoxical MAP_NOFAULT request"));
 1159         }
 1160         if (cow & MAP_DISABLE_SYNCER)
 1161                 protoeflags |= MAP_ENTRY_NOSYNC;
 1162         if (cow & MAP_DISABLE_COREDUMP)
 1163                 protoeflags |= MAP_ENTRY_NOCOREDUMP;
 1164         if (cow & MAP_VN_WRITECOUNT)
 1165                 protoeflags |= MAP_ENTRY_VN_WRITECNT;
 1166         if (cow & MAP_INHERIT_SHARE)
 1167                 inheritance = VM_INHERIT_SHARE;
 1168         else
 1169                 inheritance = VM_INHERIT_DEFAULT;
 1170 
 1171         cred = NULL;
 1172         KASSERT((object != kmem_object && object != kernel_object) ||
 1173             ((object == kmem_object || object == kernel_object) &&
 1174                 !(protoeflags & MAP_ENTRY_NEEDS_COPY)),
 1175             ("kmem or kernel object and cow"));
 1176         if (cow & (MAP_ACC_NO_CHARGE | MAP_NOFAULT))
 1177                 goto charged;
 1178         if ((cow & MAP_ACC_CHARGED) || ((prot & VM_PROT_WRITE) &&
 1179             ((protoeflags & MAP_ENTRY_NEEDS_COPY) || object == NULL))) {
 1180                 if (!(cow & MAP_ACC_CHARGED) && !swap_reserve(end - start))
 1181                         return (KERN_RESOURCE_SHORTAGE);
 1182                 KASSERT(object == NULL || (protoeflags & MAP_ENTRY_NEEDS_COPY) ||
 1183                     object->cred == NULL,
 1184                     ("OVERCOMMIT: vm_map_insert o %p", object));
 1185                 cred = curthread->td_ucred;
 1186                 crhold(cred);
 1187                 if (object == NULL && !(protoeflags & MAP_ENTRY_NEEDS_COPY))
 1188                         charge_prev_obj = TRUE;
 1189         }
 1190 
 1191 charged:
 1192         /* Expand the kernel pmap, if necessary. */
 1193         if (map == kernel_map && end > kernel_vm_end)
 1194                 pmap_growkernel(end);
 1195         if (object != NULL) {
 1196                 /*
 1197                  * OBJ_ONEMAPPING must be cleared unless this mapping
 1198                  * is trivially proven to be the only mapping for any
 1199                  * of the object's pages.  (Object granularity
 1200                  * reference counting is insufficient to recognize
 1201                  * aliases with precision.)
 1202                  */
 1203                 VM_OBJECT_WLOCK(object);
 1204                 if (object->ref_count > 1 || object->shadow_count != 0)
 1205                         vm_object_clear_flag(object, OBJ_ONEMAPPING);
 1206                 VM_OBJECT_WUNLOCK(object);
 1207         }
 1208         else if ((prev_entry != &map->header) &&
 1209                  (prev_entry->eflags == protoeflags) &&
 1210                  (cow & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) == 0 &&
 1211                  (prev_entry->end == start) &&
 1212                  (prev_entry->wired_count == 0) &&
 1213                  (prev_entry->cred == cred ||
 1214                   (prev_entry->object.vm_object != NULL &&
 1215                    (prev_entry->object.vm_object->cred == cred))) &&
 1216                    vm_object_coalesce(prev_entry->object.vm_object,
 1217                        prev_entry->offset,
 1218                        (vm_size_t)(prev_entry->end - prev_entry->start),
 1219                        (vm_size_t)(end - prev_entry->end), charge_prev_obj)) {
 1220                 /*
 1221                  * We were able to extend the object.  Determine if we
 1222                  * can extend the previous map entry to include the
 1223                  * new range as well.
 1224                  */
 1225                 if ((prev_entry->inheritance == inheritance) &&
 1226                     (prev_entry->protection == prot) &&
 1227                     (prev_entry->max_protection == max)) {
 1228                         map->size += (end - prev_entry->end);
 1229                         prev_entry->end = end;
 1230                         vm_map_entry_resize_free(map, prev_entry);
 1231                         vm_map_simplify_entry(map, prev_entry);
 1232                         if (cred != NULL)
 1233                                 crfree(cred);
 1234                         return (KERN_SUCCESS);
 1235                 }
 1236 
 1237                 /*
 1238                  * If we can extend the object but cannot extend the
 1239                  * map entry, we have to create a new map entry.  We
 1240                  * must bump the ref count on the extended object to
 1241                  * account for it.  object may be NULL.
 1242                  */
 1243                 object = prev_entry->object.vm_object;
 1244                 offset = prev_entry->offset +
 1245                         (prev_entry->end - prev_entry->start);
 1246                 vm_object_reference(object);
 1247                 if (cred != NULL && object != NULL && object->cred != NULL &&
 1248                     !(prev_entry->eflags & MAP_ENTRY_NEEDS_COPY)) {
 1249                         /* Object already accounts for this uid. */
 1250                         crfree(cred);
 1251                         cred = NULL;
 1252                 }
 1253         }
 1254 
 1255         /*
 1256          * NOTE: if conditionals fail, object can be NULL here.  This occurs
 1257          * in things like the buffer map where we manage kva but do not manage
 1258          * backing objects.
 1259          */
 1260 
 1261         /*
 1262          * Create a new entry
 1263          */
 1264         new_entry = vm_map_entry_create(map);
 1265         new_entry->start = start;
 1266         new_entry->end = end;
 1267         new_entry->cred = NULL;
 1268 
 1269         new_entry->eflags = protoeflags;
 1270         new_entry->object.vm_object = object;
 1271         new_entry->offset = offset;
 1272         new_entry->avail_ssize = 0;
 1273 
 1274         new_entry->inheritance = inheritance;
 1275         new_entry->protection = prot;
 1276         new_entry->max_protection = max;
 1277         new_entry->wired_count = 0;
 1278         new_entry->read_ahead = VM_FAULT_READ_AHEAD_INIT;
 1279         new_entry->next_read = OFF_TO_IDX(offset);
 1280 
 1281         KASSERT(cred == NULL || !ENTRY_CHARGED(new_entry),
 1282             ("OVERCOMMIT: vm_map_insert leaks vm_map %p", new_entry));
 1283         new_entry->cred = cred;
 1284 
 1285         /*
 1286          * Insert the new entry into the list
 1287          */
 1288         vm_map_entry_link(map, prev_entry, new_entry);
 1289         map->size += new_entry->end - new_entry->start;
 1290 
 1291         /*
 1292          * It may be possible to merge the new entry with the next and/or
 1293          * previous entries.  However, due to MAP_STACK_* being a hack, a
 1294          * panic can result from merging such entries.
 1295          */
 1296         if ((cow & (MAP_STACK_GROWS_DOWN | MAP_STACK_GROWS_UP)) == 0)
 1297                 vm_map_simplify_entry(map, new_entry);
 1298 
 1299         if (cow & (MAP_PREFAULT|MAP_PREFAULT_PARTIAL)) {
 1300                 vm_map_pmap_enter(map, start, prot,
 1301                                     object, OFF_TO_IDX(offset), end - start,
 1302                                     cow & MAP_PREFAULT_PARTIAL);
 1303         }
 1304 
 1305         return (KERN_SUCCESS);
 1306 }
 1307 
 1308 /*
 1309  *      vm_map_findspace:
 1310  *
 1311  *      Find the first fit (lowest VM address) for "length" free bytes
 1312  *      beginning at address >= start in the given map.
 1313  *
 1314  *      In a vm_map_entry, "adj_free" is the amount of free space
 1315  *      adjacent (higher address) to this entry, and "max_free" is the
 1316  *      maximum amount of contiguous free space in its subtree.  This
 1317  *      allows finding a free region in one path down the tree, so
 1318  *      O(log n) amortized with splay trees.
 1319  *
 1320  *      The map must be locked, and leaves it so.
 1321  *
 1322  *      Returns: 0 on success, and starting address in *addr,
 1323  *               1 if insufficient space.
 1324  */
 1325 int
 1326 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length,
 1327     vm_offset_t *addr)  /* OUT */
 1328 {
 1329         vm_map_entry_t entry;
 1330         vm_offset_t st;
 1331 
 1332         /*
 1333          * Request must fit within min/max VM address and must avoid
 1334          * address wrap.
 1335          */
 1336         if (start < map->min_offset)
 1337                 start = map->min_offset;
 1338         if (start + length > map->max_offset || start + length < start)
 1339                 return (1);
 1340 
 1341         /* Empty tree means wide open address space. */
 1342         if (map->root == NULL) {
 1343                 *addr = start;
 1344                 return (0);
 1345         }
 1346 
 1347         /*
 1348          * After splay, if start comes before root node, then there
 1349          * must be a gap from start to the root.
 1350          */
 1351         map->root = vm_map_entry_splay(start, map->root);
 1352         if (start + length <= map->root->start) {
 1353                 *addr = start;
 1354                 return (0);
 1355         }
 1356 
 1357         /*
 1358          * Root is the last node that might begin its gap before
 1359          * start, and this is the last comparison where address
 1360          * wrap might be a problem.
 1361          */
 1362         st = (start > map->root->end) ? start : map->root->end;
 1363         if (length <= map->root->end + map->root->adj_free - st) {
 1364                 *addr = st;
 1365                 return (0);
 1366         }
 1367 
 1368         /* With max_free, can immediately tell if no solution. */
 1369         entry = map->root->right;
 1370         if (entry == NULL || length > entry->max_free)
 1371                 return (1);
 1372 
 1373         /*
 1374          * Search the right subtree in the order: left subtree, root,
 1375          * right subtree (first fit).  The previous splay implies that
 1376          * all regions in the right subtree have addresses > start.
 1377          */
 1378         while (entry != NULL) {
 1379                 if (entry->left != NULL && entry->left->max_free >= length)
 1380                         entry = entry->left;
 1381                 else if (entry->adj_free >= length) {
 1382                         *addr = entry->end;
 1383                         return (0);
 1384                 } else
 1385                         entry = entry->right;
 1386         }
 1387 
 1388         /* Can't get here, so panic if we do. */
 1389         panic("vm_map_findspace: max_free corrupt");
 1390 }
 1391 
 1392 int
 1393 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 1394     vm_offset_t start, vm_size_t length, vm_prot_t prot,
 1395     vm_prot_t max, int cow)
 1396 {
 1397         vm_offset_t end;
 1398         int result;
 1399 
 1400         end = start + length;
 1401         vm_map_lock(map);
 1402         VM_MAP_RANGE_CHECK(map, start, end);
 1403         (void) vm_map_delete(map, start, end);
 1404         result = vm_map_insert(map, object, offset, start, end, prot,
 1405             max, cow);
 1406         vm_map_unlock(map);
 1407         return (result);
 1408 }
 1409 
 1410 /*
 1411  *      vm_map_find finds an unallocated region in the target address
 1412  *      map with the given length.  The search is defined to be
 1413  *      first-fit from the specified address; the region found is
 1414  *      returned in the same parameter.
 1415  *
 1416  *      If object is non-NULL, ref count must be bumped by caller
 1417  *      prior to making call to account for the new entry.
 1418  */
 1419 int
 1420 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 1421             vm_offset_t *addr,  /* IN/OUT */
 1422             vm_size_t length, vm_offset_t max_addr, int find_space,
 1423             vm_prot_t prot, vm_prot_t max, int cow)
 1424 {
 1425         vm_offset_t alignment, initial_addr, start;
 1426         int result;
 1427 
 1428         if (find_space == VMFS_OPTIMAL_SPACE && (object == NULL ||
 1429             (object->flags & OBJ_COLORED) == 0))
 1430                 find_space = VMFS_ANY_SPACE;
 1431         if (find_space >> 8 != 0) {
 1432                 KASSERT((find_space & 0xff) == 0, ("bad VMFS flags"));
 1433                 alignment = (vm_offset_t)1 << (find_space >> 8);
 1434         } else
 1435                 alignment = 0;
 1436         initial_addr = *addr;
 1437 again:
 1438         start = initial_addr;
 1439         vm_map_lock(map);
 1440         do {
 1441                 if (find_space != VMFS_NO_SPACE) {
 1442                         if (vm_map_findspace(map, start, length, addr) ||
 1443                             (max_addr != 0 && *addr + length > max_addr)) {
 1444                                 vm_map_unlock(map);
 1445                                 if (find_space == VMFS_OPTIMAL_SPACE) {
 1446                                         find_space = VMFS_ANY_SPACE;
 1447                                         goto again;
 1448                                 }
 1449                                 return (KERN_NO_SPACE);
 1450                         }
 1451                         switch (find_space) {
 1452                         case VMFS_SUPER_SPACE:
 1453                         case VMFS_OPTIMAL_SPACE:
 1454                                 pmap_align_superpage(object, offset, addr,
 1455                                     length);
 1456                                 break;
 1457                         case VMFS_ANY_SPACE:
 1458                                 break;
 1459                         default:
 1460                                 if ((*addr & (alignment - 1)) != 0) {
 1461                                         *addr &= ~(alignment - 1);
 1462                                         *addr += alignment;
 1463                                 }
 1464                                 break;
 1465                         }
 1466 
 1467                         start = *addr;
 1468                 }
 1469                 result = vm_map_insert(map, object, offset, start, start +
 1470                     length, prot, max, cow);
 1471         } while (result == KERN_NO_SPACE && find_space != VMFS_NO_SPACE &&
 1472             find_space != VMFS_ANY_SPACE);
 1473         vm_map_unlock(map);
 1474         return (result);
 1475 }
 1476 
 1477 /*
 1478  *      vm_map_simplify_entry:
 1479  *
 1480  *      Simplify the given map entry by merging with either neighbor.  This
 1481  *      routine also has the ability to merge with both neighbors.
 1482  *
 1483  *      The map must be locked.
 1484  *
 1485  *      This routine guarentees that the passed entry remains valid (though
 1486  *      possibly extended).  When merging, this routine may delete one or
 1487  *      both neighbors.
 1488  */
 1489 void
 1490 vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry)
 1491 {
 1492         vm_map_entry_t next, prev;
 1493         vm_size_t prevsize, esize;
 1494 
 1495         if (entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP))
 1496                 return;
 1497 
 1498         prev = entry->prev;
 1499         if (prev != &map->header) {
 1500                 prevsize = prev->end - prev->start;
 1501                 if ( (prev->end == entry->start) &&
 1502                      (prev->object.vm_object == entry->object.vm_object) &&
 1503                      (!prev->object.vm_object ||
 1504                         (prev->offset + prevsize == entry->offset)) &&
 1505                      (prev->eflags == entry->eflags) &&
 1506                      (prev->protection == entry->protection) &&
 1507                      (prev->max_protection == entry->max_protection) &&
 1508                      (prev->inheritance == entry->inheritance) &&
 1509                      (prev->wired_count == entry->wired_count) &&
 1510                      (prev->cred == entry->cred)) {
 1511                         vm_map_entry_unlink(map, prev);
 1512                         entry->start = prev->start;
 1513                         entry->offset = prev->offset;
 1514                         if (entry->prev != &map->header)
 1515                                 vm_map_entry_resize_free(map, entry->prev);
 1516 
 1517                         /*
 1518                          * If the backing object is a vnode object,
 1519                          * vm_object_deallocate() calls vrele().
 1520                          * However, vrele() does not lock the vnode
 1521                          * because the vnode has additional
 1522                          * references.  Thus, the map lock can be kept
 1523                          * without causing a lock-order reversal with
 1524                          * the vnode lock.
 1525                          *
 1526                          * Since we count the number of virtual page
 1527                          * mappings in object->un_pager.vnp.writemappings,
 1528                          * the writemappings value should not be adjusted
 1529                          * when the entry is disposed of.
 1530                          */
 1531                         if (prev->object.vm_object)
 1532                                 vm_object_deallocate(prev->object.vm_object);
 1533                         if (prev->cred != NULL)
 1534                                 crfree(prev->cred);
 1535                         vm_map_entry_dispose(map, prev);
 1536                 }
 1537         }
 1538 
 1539         next = entry->next;
 1540         if (next != &map->header) {
 1541                 esize = entry->end - entry->start;
 1542                 if ((entry->end == next->start) &&
 1543                     (next->object.vm_object == entry->object.vm_object) &&
 1544                      (!entry->object.vm_object ||
 1545                         (entry->offset + esize == next->offset)) &&
 1546                     (next->eflags == entry->eflags) &&
 1547                     (next->protection == entry->protection) &&
 1548                     (next->max_protection == entry->max_protection) &&
 1549                     (next->inheritance == entry->inheritance) &&
 1550                     (next->wired_count == entry->wired_count) &&
 1551                     (next->cred == entry->cred)) {
 1552                         vm_map_entry_unlink(map, next);
 1553                         entry->end = next->end;
 1554                         vm_map_entry_resize_free(map, entry);
 1555 
 1556                         /*
 1557                          * See comment above.
 1558                          */
 1559                         if (next->object.vm_object)
 1560                                 vm_object_deallocate(next->object.vm_object);
 1561                         if (next->cred != NULL)
 1562                                 crfree(next->cred);
 1563                         vm_map_entry_dispose(map, next);
 1564                 }
 1565         }
 1566 }
 1567 /*
 1568  *      vm_map_clip_start:      [ internal use only ]
 1569  *
 1570  *      Asserts that the given entry begins at or after
 1571  *      the specified address; if necessary,
 1572  *      it splits the entry into two.
 1573  */
 1574 #define vm_map_clip_start(map, entry, startaddr) \
 1575 { \
 1576         if (startaddr > entry->start) \
 1577                 _vm_map_clip_start(map, entry, startaddr); \
 1578 }
 1579 
 1580 /*
 1581  *      This routine is called only when it is known that
 1582  *      the entry must be split.
 1583  */
 1584 static void
 1585 _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start)
 1586 {
 1587         vm_map_entry_t new_entry;
 1588 
 1589         VM_MAP_ASSERT_LOCKED(map);
 1590 
 1591         /*
 1592          * Split off the front portion -- note that we must insert the new
 1593          * entry BEFORE this one, so that this entry has the specified
 1594          * starting address.
 1595          */
 1596         vm_map_simplify_entry(map, entry);
 1597 
 1598         /*
 1599          * If there is no object backing this entry, we might as well create
 1600          * one now.  If we defer it, an object can get created after the map
 1601          * is clipped, and individual objects will be created for the split-up
 1602          * map.  This is a bit of a hack, but is also about the best place to
 1603          * put this improvement.
 1604          */
 1605         if (entry->object.vm_object == NULL && !map->system_map) {
 1606                 vm_object_t object;
 1607                 object = vm_object_allocate(OBJT_DEFAULT,
 1608                                 atop(entry->end - entry->start));
 1609                 entry->object.vm_object = object;
 1610                 entry->offset = 0;
 1611                 if (entry->cred != NULL) {
 1612                         object->cred = entry->cred;
 1613                         object->charge = entry->end - entry->start;
 1614                         entry->cred = NULL;
 1615                 }
 1616         } else if (entry->object.vm_object != NULL &&
 1617                    ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
 1618                    entry->cred != NULL) {
 1619                 VM_OBJECT_WLOCK(entry->object.vm_object);
 1620                 KASSERT(entry->object.vm_object->cred == NULL,
 1621                     ("OVERCOMMIT: vm_entry_clip_start: both cred e %p", entry));
 1622                 entry->object.vm_object->cred = entry->cred;
 1623                 entry->object.vm_object->charge = entry->end - entry->start;
 1624                 VM_OBJECT_WUNLOCK(entry->object.vm_object);
 1625                 entry->cred = NULL;
 1626         }
 1627 
 1628         new_entry = vm_map_entry_create(map);
 1629         *new_entry = *entry;
 1630 
 1631         new_entry->end = start;
 1632         entry->offset += (start - entry->start);
 1633         entry->start = start;
 1634         if (new_entry->cred != NULL)
 1635                 crhold(entry->cred);
 1636 
 1637         vm_map_entry_link(map, entry->prev, new_entry);
 1638 
 1639         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 1640                 vm_object_reference(new_entry->object.vm_object);
 1641                 /*
 1642                  * The object->un_pager.vnp.writemappings for the
 1643                  * object of MAP_ENTRY_VN_WRITECNT type entry shall be
 1644                  * kept as is here.  The virtual pages are
 1645                  * re-distributed among the clipped entries, so the sum is
 1646                  * left the same.
 1647                  */
 1648         }
 1649 }
 1650 
 1651 /*
 1652  *      vm_map_clip_end:        [ internal use only ]
 1653  *
 1654  *      Asserts that the given entry ends at or before
 1655  *      the specified address; if necessary,
 1656  *      it splits the entry into two.
 1657  */
 1658 #define vm_map_clip_end(map, entry, endaddr) \
 1659 { \
 1660         if ((endaddr) < (entry->end)) \
 1661                 _vm_map_clip_end((map), (entry), (endaddr)); \
 1662 }
 1663 
 1664 /*
 1665  *      This routine is called only when it is known that
 1666  *      the entry must be split.
 1667  */
 1668 static void
 1669 _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end)
 1670 {
 1671         vm_map_entry_t new_entry;
 1672 
 1673         VM_MAP_ASSERT_LOCKED(map);
 1674 
 1675         /*
 1676          * If there is no object backing this entry, we might as well create
 1677          * one now.  If we defer it, an object can get created after the map
 1678          * is clipped, and individual objects will be created for the split-up
 1679          * map.  This is a bit of a hack, but is also about the best place to
 1680          * put this improvement.
 1681          */
 1682         if (entry->object.vm_object == NULL && !map->system_map) {
 1683                 vm_object_t object;
 1684                 object = vm_object_allocate(OBJT_DEFAULT,
 1685                                 atop(entry->end - entry->start));
 1686                 entry->object.vm_object = object;
 1687                 entry->offset = 0;
 1688                 if (entry->cred != NULL) {
 1689                         object->cred = entry->cred;
 1690                         object->charge = entry->end - entry->start;
 1691                         entry->cred = NULL;
 1692                 }
 1693         } else if (entry->object.vm_object != NULL &&
 1694                    ((entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) &&
 1695                    entry->cred != NULL) {
 1696                 VM_OBJECT_WLOCK(entry->object.vm_object);
 1697                 KASSERT(entry->object.vm_object->cred == NULL,
 1698                     ("OVERCOMMIT: vm_entry_clip_end: both cred e %p", entry));
 1699                 entry->object.vm_object->cred = entry->cred;
 1700                 entry->object.vm_object->charge = entry->end - entry->start;
 1701                 VM_OBJECT_WUNLOCK(entry->object.vm_object);
 1702                 entry->cred = NULL;
 1703         }
 1704 
 1705         /*
 1706          * Create a new entry and insert it AFTER the specified entry
 1707          */
 1708         new_entry = vm_map_entry_create(map);
 1709         *new_entry = *entry;
 1710 
 1711         new_entry->start = entry->end = end;
 1712         new_entry->offset += (end - entry->start);
 1713         if (new_entry->cred != NULL)
 1714                 crhold(entry->cred);
 1715 
 1716         vm_map_entry_link(map, entry, new_entry);
 1717 
 1718         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 1719                 vm_object_reference(new_entry->object.vm_object);
 1720         }
 1721 }
 1722 
 1723 /*
 1724  *      vm_map_submap:          [ kernel use only ]
 1725  *
 1726  *      Mark the given range as handled by a subordinate map.
 1727  *
 1728  *      This range must have been created with vm_map_find,
 1729  *      and no other operations may have been performed on this
 1730  *      range prior to calling vm_map_submap.
 1731  *
 1732  *      Only a limited number of operations can be performed
 1733  *      within this rage after calling vm_map_submap:
 1734  *              vm_fault
 1735  *      [Don't try vm_map_copy!]
 1736  *
 1737  *      To remove a submapping, one must first remove the
 1738  *      range from the superior map, and then destroy the
 1739  *      submap (if desired).  [Better yet, don't try it.]
 1740  */
 1741 int
 1742 vm_map_submap(
 1743         vm_map_t map,
 1744         vm_offset_t start,
 1745         vm_offset_t end,
 1746         vm_map_t submap)
 1747 {
 1748         vm_map_entry_t entry;
 1749         int result = KERN_INVALID_ARGUMENT;
 1750 
 1751         vm_map_lock(map);
 1752 
 1753         VM_MAP_RANGE_CHECK(map, start, end);
 1754 
 1755         if (vm_map_lookup_entry(map, start, &entry)) {
 1756                 vm_map_clip_start(map, entry, start);
 1757         } else
 1758                 entry = entry->next;
 1759 
 1760         vm_map_clip_end(map, entry, end);
 1761 
 1762         if ((entry->start == start) && (entry->end == end) &&
 1763             ((entry->eflags & MAP_ENTRY_COW) == 0) &&
 1764             (entry->object.vm_object == NULL)) {
 1765                 entry->object.sub_map = submap;
 1766                 entry->eflags |= MAP_ENTRY_IS_SUB_MAP;
 1767                 result = KERN_SUCCESS;
 1768         }
 1769         vm_map_unlock(map);
 1770 
 1771         return (result);
 1772 }
 1773 
 1774 /*
 1775  * The maximum number of pages to map
 1776  */
 1777 #define MAX_INIT_PT     96
 1778 
 1779 /*
 1780  *      vm_map_pmap_enter:
 1781  *
 1782  *      Preload read-only mappings for the specified object's resident pages
 1783  *      into the target map.  If "flags" is MAP_PREFAULT_PARTIAL, then only
 1784  *      the resident pages within the address range [addr, addr + ulmin(size,
 1785  *      ptoa(MAX_INIT_PT))) are mapped.  Otherwise, all resident pages within
 1786  *      the specified address range are mapped.  This eliminates many soft
 1787  *      faults on process startup and immediately after an mmap(2).  Because
 1788  *      these are speculative mappings, cached pages are not reactivated and
 1789  *      mapped.
 1790  */
 1791 void
 1792 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
 1793     vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags)
 1794 {
 1795         vm_offset_t start;
 1796         vm_page_t p, p_start;
 1797         vm_pindex_t psize, tmpidx;
 1798 
 1799         if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
 1800                 return;
 1801         VM_OBJECT_RLOCK(object);
 1802         if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
 1803                 VM_OBJECT_RUNLOCK(object);
 1804                 VM_OBJECT_WLOCK(object);
 1805                 if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
 1806                         pmap_object_init_pt(map->pmap, addr, object, pindex,
 1807                             size);
 1808                         VM_OBJECT_WUNLOCK(object);
 1809                         return;
 1810                 }
 1811                 VM_OBJECT_LOCK_DOWNGRADE(object);
 1812         }
 1813 
 1814         psize = atop(size);
 1815         if (psize > MAX_INIT_PT && (flags & MAP_PREFAULT_PARTIAL) != 0)
 1816                 psize = MAX_INIT_PT;
 1817         if (psize + pindex > object->size) {
 1818                 if (object->size < pindex) {
 1819                         VM_OBJECT_RUNLOCK(object);
 1820                         return;
 1821                 }
 1822                 psize = object->size - pindex;
 1823         }
 1824 
 1825         start = 0;
 1826         p_start = NULL;
 1827 
 1828         p = vm_page_find_least(object, pindex);
 1829         /*
 1830          * Assert: the variable p is either (1) the page with the
 1831          * least pindex greater than or equal to the parameter pindex
 1832          * or (2) NULL.
 1833          */
 1834         for (;
 1835              p != NULL && (tmpidx = p->pindex - pindex) < psize;
 1836              p = TAILQ_NEXT(p, listq)) {
 1837                 /*
 1838                  * don't allow an madvise to blow away our really
 1839                  * free pages allocating pv entries.
 1840                  */
 1841                 if ((flags & MAP_PREFAULT_MADVISE) &&
 1842                     cnt.v_free_count < cnt.v_free_reserved) {
 1843                         psize = tmpidx;
 1844                         break;
 1845                 }
 1846                 if (p->valid == VM_PAGE_BITS_ALL) {
 1847                         if (p_start == NULL) {
 1848                                 start = addr + ptoa(tmpidx);
 1849                                 p_start = p;
 1850                         }
 1851                 } else if (p_start != NULL) {
 1852                         pmap_enter_object(map->pmap, start, addr +
 1853                             ptoa(tmpidx), p_start, prot);
 1854                         p_start = NULL;
 1855                 }
 1856         }
 1857         if (p_start != NULL)
 1858                 pmap_enter_object(map->pmap, start, addr + ptoa(psize),
 1859                     p_start, prot);
 1860         VM_OBJECT_RUNLOCK(object);
 1861 }
 1862 
 1863 /*
 1864  *      vm_map_protect:
 1865  *
 1866  *      Sets the protection of the specified address
 1867  *      region in the target map.  If "set_max" is
 1868  *      specified, the maximum protection is to be set;
 1869  *      otherwise, only the current protection is affected.
 1870  */
 1871 int
 1872 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1873                vm_prot_t new_prot, boolean_t set_max)
 1874 {
 1875         vm_map_entry_t current, entry;
 1876         vm_object_t obj;
 1877         struct ucred *cred;
 1878         vm_prot_t old_prot;
 1879 
 1880         vm_map_lock(map);
 1881 
 1882         VM_MAP_RANGE_CHECK(map, start, end);
 1883 
 1884         if (vm_map_lookup_entry(map, start, &entry)) {
 1885                 vm_map_clip_start(map, entry, start);
 1886         } else {
 1887                 entry = entry->next;
 1888         }
 1889 
 1890         /*
 1891          * Make a first pass to check for protection violations.
 1892          */
 1893         current = entry;
 1894         while ((current != &map->header) && (current->start < end)) {
 1895                 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
 1896                         vm_map_unlock(map);
 1897                         return (KERN_INVALID_ARGUMENT);
 1898                 }
 1899                 if ((new_prot & current->max_protection) != new_prot) {
 1900                         vm_map_unlock(map);
 1901                         return (KERN_PROTECTION_FAILURE);
 1902                 }
 1903                 current = current->next;
 1904         }
 1905 
 1906 
 1907         /*
 1908          * Do an accounting pass for private read-only mappings that
 1909          * now will do cow due to allowed write (e.g. debugger sets
 1910          * breakpoint on text segment)
 1911          */
 1912         for (current = entry; (current != &map->header) &&
 1913              (current->start < end); current = current->next) {
 1914 
 1915                 vm_map_clip_end(map, current, end);
 1916 
 1917                 if (set_max ||
 1918                     ((new_prot & ~(current->protection)) & VM_PROT_WRITE) == 0 ||
 1919                     ENTRY_CHARGED(current)) {
 1920                         continue;
 1921                 }
 1922 
 1923                 cred = curthread->td_ucred;
 1924                 obj = current->object.vm_object;
 1925 
 1926                 if (obj == NULL || (current->eflags & MAP_ENTRY_NEEDS_COPY)) {
 1927                         if (!swap_reserve(current->end - current->start)) {
 1928                                 vm_map_unlock(map);
 1929                                 return (KERN_RESOURCE_SHORTAGE);
 1930                         }
 1931                         crhold(cred);
 1932                         current->cred = cred;
 1933                         continue;
 1934                 }
 1935 
 1936                 VM_OBJECT_WLOCK(obj);
 1937                 if (obj->type != OBJT_DEFAULT && obj->type != OBJT_SWAP) {
 1938                         VM_OBJECT_WUNLOCK(obj);
 1939                         continue;
 1940                 }
 1941 
 1942                 /*
 1943                  * Charge for the whole object allocation now, since
 1944                  * we cannot distinguish between non-charged and
 1945                  * charged clipped mapping of the same object later.
 1946                  */
 1947                 KASSERT(obj->charge == 0,
 1948                     ("vm_map_protect: object %p overcharged\n", obj));
 1949                 if (!swap_reserve(ptoa(obj->size))) {
 1950                         VM_OBJECT_WUNLOCK(obj);
 1951                         vm_map_unlock(map);
 1952                         return (KERN_RESOURCE_SHORTAGE);
 1953                 }
 1954 
 1955                 crhold(cred);
 1956                 obj->cred = cred;
 1957                 obj->charge = ptoa(obj->size);
 1958                 VM_OBJECT_WUNLOCK(obj);
 1959         }
 1960 
 1961         /*
 1962          * Go back and fix up protections. [Note that clipping is not
 1963          * necessary the second time.]
 1964          */
 1965         current = entry;
 1966         while ((current != &map->header) && (current->start < end)) {
 1967                 old_prot = current->protection;
 1968 
 1969                 if (set_max)
 1970                         current->protection =
 1971                             (current->max_protection = new_prot) &
 1972                             old_prot;
 1973                 else
 1974                         current->protection = new_prot;
 1975 
 1976                 if ((current->eflags & (MAP_ENTRY_COW | MAP_ENTRY_USER_WIRED))
 1977                      == (MAP_ENTRY_COW | MAP_ENTRY_USER_WIRED) &&
 1978                     (current->protection & VM_PROT_WRITE) != 0 &&
 1979                     (old_prot & VM_PROT_WRITE) == 0) {
 1980                         vm_fault_copy_entry(map, map, current, current, NULL);
 1981                 }
 1982 
 1983                 /*
 1984                  * When restricting access, update the physical map.  Worry
 1985                  * about copy-on-write here.
 1986                  */
 1987                 if ((old_prot & ~current->protection) != 0) {
 1988 #define MASK(entry)     (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \
 1989                                                         VM_PROT_ALL)
 1990                         pmap_protect(map->pmap, current->start,
 1991                             current->end,
 1992                             current->protection & MASK(current));
 1993 #undef  MASK
 1994                 }
 1995                 vm_map_simplify_entry(map, current);
 1996                 current = current->next;
 1997         }
 1998         vm_map_unlock(map);
 1999         return (KERN_SUCCESS);
 2000 }
 2001 
 2002 /*
 2003  *      vm_map_madvise:
 2004  *
 2005  *      This routine traverses a processes map handling the madvise
 2006  *      system call.  Advisories are classified as either those effecting
 2007  *      the vm_map_entry structure, or those effecting the underlying
 2008  *      objects.
 2009  */
 2010 int
 2011 vm_map_madvise(
 2012         vm_map_t map,
 2013         vm_offset_t start,
 2014         vm_offset_t end,
 2015         int behav)
 2016 {
 2017         vm_map_entry_t current, entry;
 2018         int modify_map = 0;
 2019 
 2020         /*
 2021          * Some madvise calls directly modify the vm_map_entry, in which case
 2022          * we need to use an exclusive lock on the map and we need to perform
 2023          * various clipping operations.  Otherwise we only need a read-lock
 2024          * on the map.
 2025          */
 2026         switch(behav) {
 2027         case MADV_NORMAL:
 2028         case MADV_SEQUENTIAL:
 2029         case MADV_RANDOM:
 2030         case MADV_NOSYNC:
 2031         case MADV_AUTOSYNC:
 2032         case MADV_NOCORE:
 2033         case MADV_CORE:
 2034                 modify_map = 1;
 2035                 vm_map_lock(map);
 2036                 break;
 2037         case MADV_WILLNEED:
 2038         case MADV_DONTNEED:
 2039         case MADV_FREE:
 2040                 vm_map_lock_read(map);
 2041                 break;
 2042         default:
 2043                 return (KERN_INVALID_ARGUMENT);
 2044         }
 2045 
 2046         /*
 2047          * Locate starting entry and clip if necessary.
 2048          */
 2049         VM_MAP_RANGE_CHECK(map, start, end);
 2050 
 2051         if (vm_map_lookup_entry(map, start, &entry)) {
 2052                 if (modify_map)
 2053                         vm_map_clip_start(map, entry, start);
 2054         } else {
 2055                 entry = entry->next;
 2056         }
 2057 
 2058         if (modify_map) {
 2059                 /*
 2060                  * madvise behaviors that are implemented in the vm_map_entry.
 2061                  *
 2062                  * We clip the vm_map_entry so that behavioral changes are
 2063                  * limited to the specified address range.
 2064                  */
 2065                 for (current = entry;
 2066                      (current != &map->header) && (current->start < end);
 2067                      current = current->next
 2068                 ) {
 2069                         if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
 2070                                 continue;
 2071 
 2072                         vm_map_clip_end(map, current, end);
 2073 
 2074                         switch (behav) {
 2075                         case MADV_NORMAL:
 2076                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_NORMAL);
 2077                                 break;
 2078                         case MADV_SEQUENTIAL:
 2079                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_SEQUENTIAL);
 2080                                 break;
 2081                         case MADV_RANDOM:
 2082                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_RANDOM);
 2083                                 break;
 2084                         case MADV_NOSYNC:
 2085                                 current->eflags |= MAP_ENTRY_NOSYNC;
 2086                                 break;
 2087                         case MADV_AUTOSYNC:
 2088                                 current->eflags &= ~MAP_ENTRY_NOSYNC;
 2089                                 break;
 2090                         case MADV_NOCORE:
 2091                                 current->eflags |= MAP_ENTRY_NOCOREDUMP;
 2092                                 break;
 2093                         case MADV_CORE:
 2094                                 current->eflags &= ~MAP_ENTRY_NOCOREDUMP;
 2095                                 break;
 2096                         default:
 2097                                 break;
 2098                         }
 2099                         vm_map_simplify_entry(map, current);
 2100                 }
 2101                 vm_map_unlock(map);
 2102         } else {
 2103                 vm_pindex_t pstart, pend;
 2104 
 2105                 /*
 2106                  * madvise behaviors that are implemented in the underlying
 2107                  * vm_object.
 2108                  *
 2109                  * Since we don't clip the vm_map_entry, we have to clip
 2110                  * the vm_object pindex and count.
 2111                  */
 2112                 for (current = entry;
 2113                      (current != &map->header) && (current->start < end);
 2114                      current = current->next
 2115                 ) {
 2116                         vm_offset_t useEnd, useStart;
 2117 
 2118                         if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
 2119                                 continue;
 2120 
 2121                         pstart = OFF_TO_IDX(current->offset);
 2122                         pend = pstart + atop(current->end - current->start);
 2123                         useStart = current->start;
 2124                         useEnd = current->end;
 2125 
 2126                         if (current->start < start) {
 2127                                 pstart += atop(start - current->start);
 2128                                 useStart = start;
 2129                         }
 2130                         if (current->end > end) {
 2131                                 pend -= atop(current->end - end);
 2132                                 useEnd = end;
 2133                         }
 2134 
 2135                         if (pstart >= pend)
 2136                                 continue;
 2137 
 2138                         /*
 2139                          * Perform the pmap_advise() before clearing
 2140                          * PGA_REFERENCED in vm_page_advise().  Otherwise, a
 2141                          * concurrent pmap operation, such as pmap_remove(),
 2142                          * could clear a reference in the pmap and set
 2143                          * PGA_REFERENCED on the page before the pmap_advise()
 2144                          * had completed.  Consequently, the page would appear
 2145                          * referenced based upon an old reference that
 2146                          * occurred before this pmap_advise() ran.
 2147                          */
 2148                         if (behav == MADV_DONTNEED || behav == MADV_FREE)
 2149                                 pmap_advise(map->pmap, useStart, useEnd,
 2150                                     behav);
 2151 
 2152                         vm_object_madvise(current->object.vm_object, pstart,
 2153                             pend, behav);
 2154                         if (behav == MADV_WILLNEED) {
 2155                                 vm_map_pmap_enter(map,
 2156                                     useStart,
 2157                                     current->protection,
 2158                                     current->object.vm_object,
 2159                                     pstart,
 2160                                     ptoa(pend - pstart),
 2161                                     MAP_PREFAULT_MADVISE
 2162                                 );
 2163                         }
 2164                 }
 2165                 vm_map_unlock_read(map);
 2166         }
 2167         return (0);
 2168 }
 2169 
 2170 
 2171 /*
 2172  *      vm_map_inherit:
 2173  *
 2174  *      Sets the inheritance of the specified address
 2175  *      range in the target map.  Inheritance
 2176  *      affects how the map will be shared with
 2177  *      child maps at the time of vmspace_fork.
 2178  */
 2179 int
 2180 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end,
 2181                vm_inherit_t new_inheritance)
 2182 {
 2183         vm_map_entry_t entry;
 2184         vm_map_entry_t temp_entry;
 2185 
 2186         switch (new_inheritance) {
 2187         case VM_INHERIT_NONE:
 2188         case VM_INHERIT_COPY:
 2189         case VM_INHERIT_SHARE:
 2190                 break;
 2191         default:
 2192                 return (KERN_INVALID_ARGUMENT);
 2193         }
 2194         vm_map_lock(map);
 2195         VM_MAP_RANGE_CHECK(map, start, end);
 2196         if (vm_map_lookup_entry(map, start, &temp_entry)) {
 2197                 entry = temp_entry;
 2198                 vm_map_clip_start(map, entry, start);
 2199         } else
 2200                 entry = temp_entry->next;
 2201         while ((entry != &map->header) && (entry->start < end)) {
 2202                 vm_map_clip_end(map, entry, end);
 2203                 entry->inheritance = new_inheritance;
 2204                 vm_map_simplify_entry(map, entry);
 2205                 entry = entry->next;
 2206         }
 2207         vm_map_unlock(map);
 2208         return (KERN_SUCCESS);
 2209 }
 2210 
 2211 /*
 2212  *      vm_map_unwire:
 2213  *
 2214  *      Implements both kernel and user unwiring.
 2215  */
 2216 int
 2217 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 2218     int flags)
 2219 {
 2220         vm_map_entry_t entry, first_entry, tmp_entry;
 2221         vm_offset_t saved_start;
 2222         unsigned int last_timestamp;
 2223         int rv;
 2224         boolean_t need_wakeup, result, user_unwire;
 2225 
 2226         user_unwire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
 2227         vm_map_lock(map);
 2228         VM_MAP_RANGE_CHECK(map, start, end);
 2229         if (!vm_map_lookup_entry(map, start, &first_entry)) {
 2230                 if (flags & VM_MAP_WIRE_HOLESOK)
 2231                         first_entry = first_entry->next;
 2232                 else {
 2233                         vm_map_unlock(map);
 2234                         return (KERN_INVALID_ADDRESS);
 2235                 }
 2236         }
 2237         last_timestamp = map->timestamp;
 2238         entry = first_entry;
 2239         while (entry != &map->header && entry->start < end) {
 2240                 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
 2241                         /*
 2242                          * We have not yet clipped the entry.
 2243                          */
 2244                         saved_start = (start >= entry->start) ? start :
 2245                             entry->start;
 2246                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 2247                         if (vm_map_unlock_and_wait(map, 0)) {
 2248                                 /*
 2249                                  * Allow interruption of user unwiring?
 2250                                  */
 2251                         }
 2252                         vm_map_lock(map);
 2253                         if (last_timestamp+1 != map->timestamp) {
 2254                                 /*
 2255                                  * Look again for the entry because the map was
 2256                                  * modified while it was unlocked.
 2257                                  * Specifically, the entry may have been
 2258                                  * clipped, merged, or deleted.
 2259                                  */
 2260                                 if (!vm_map_lookup_entry(map, saved_start,
 2261                                     &tmp_entry)) {
 2262                                         if (flags & VM_MAP_WIRE_HOLESOK)
 2263                                                 tmp_entry = tmp_entry->next;
 2264                                         else {
 2265                                                 if (saved_start == start) {
 2266                                                         /*
 2267                                                          * First_entry has been deleted.
 2268                                                          */
 2269                                                         vm_map_unlock(map);
 2270                                                         return (KERN_INVALID_ADDRESS);
 2271                                                 }
 2272                                                 end = saved_start;
 2273                                                 rv = KERN_INVALID_ADDRESS;
 2274                                                 goto done;
 2275                                         }
 2276                                 }
 2277                                 if (entry == first_entry)
 2278                                         first_entry = tmp_entry;
 2279                                 else
 2280                                         first_entry = NULL;
 2281                                 entry = tmp_entry;
 2282                         }
 2283                         last_timestamp = map->timestamp;
 2284                         continue;
 2285                 }
 2286                 vm_map_clip_start(map, entry, start);
 2287                 vm_map_clip_end(map, entry, end);
 2288                 /*
 2289                  * Mark the entry in case the map lock is released.  (See
 2290                  * above.)
 2291                  */
 2292                 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
 2293                 entry->wiring_thread = curthread;
 2294                 /*
 2295                  * Check the map for holes in the specified region.
 2296                  * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
 2297                  */
 2298                 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
 2299                     (entry->end < end && (entry->next == &map->header ||
 2300                     entry->next->start > entry->end))) {
 2301                         end = entry->end;
 2302                         rv = KERN_INVALID_ADDRESS;
 2303                         goto done;
 2304                 }
 2305                 /*
 2306                  * If system unwiring, require that the entry is system wired.
 2307                  */
 2308                 if (!user_unwire &&
 2309                     vm_map_entry_system_wired_count(entry) == 0) {
 2310                         end = entry->end;
 2311                         rv = KERN_INVALID_ARGUMENT;
 2312                         goto done;
 2313                 }
 2314                 entry = entry->next;
 2315         }
 2316         rv = KERN_SUCCESS;
 2317 done:
 2318         need_wakeup = FALSE;
 2319         if (first_entry == NULL) {
 2320                 result = vm_map_lookup_entry(map, start, &first_entry);
 2321                 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
 2322                         first_entry = first_entry->next;
 2323                 else
 2324                         KASSERT(result, ("vm_map_unwire: lookup failed"));
 2325         }
 2326         for (entry = first_entry; entry != &map->header && entry->start < end;
 2327             entry = entry->next) {
 2328                 /*
 2329                  * If VM_MAP_WIRE_HOLESOK was specified, an empty
 2330                  * space in the unwired region could have been mapped
 2331                  * while the map lock was dropped for draining
 2332                  * MAP_ENTRY_IN_TRANSITION.  Moreover, another thread
 2333                  * could be simultaneously wiring this new mapping
 2334                  * entry.  Detect these cases and skip any entries
 2335                  * marked as in transition by us.
 2336                  */
 2337                 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 ||
 2338                     entry->wiring_thread != curthread) {
 2339                         KASSERT((flags & VM_MAP_WIRE_HOLESOK) != 0,
 2340                             ("vm_map_unwire: !HOLESOK and new/changed entry"));
 2341                         continue;
 2342                 }
 2343 
 2344                 if (rv == KERN_SUCCESS && (!user_unwire ||
 2345                     (entry->eflags & MAP_ENTRY_USER_WIRED))) {
 2346                         if (user_unwire)
 2347                                 entry->eflags &= ~MAP_ENTRY_USER_WIRED;
 2348                         entry->wired_count--;
 2349                         if (entry->wired_count == 0) {
 2350                                 /*
 2351                                  * Retain the map lock.
 2352                                  */
 2353                                 vm_fault_unwire(map, entry->start, entry->end,
 2354                                     entry->object.vm_object != NULL &&
 2355                                     (entry->object.vm_object->flags &
 2356                                     OBJ_FICTITIOUS) != 0);
 2357                         }
 2358                 }
 2359                 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0,
 2360                     ("vm_map_unwire: in-transition flag missing"));
 2361                 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
 2362                 entry->wiring_thread = NULL;
 2363                 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
 2364                         entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
 2365                         need_wakeup = TRUE;
 2366                 }
 2367                 vm_map_simplify_entry(map, entry);
 2368         }
 2369         vm_map_unlock(map);
 2370         if (need_wakeup)
 2371                 vm_map_wakeup(map);
 2372         return (rv);
 2373 }
 2374 
 2375 /*
 2376  *      vm_map_wire:
 2377  *
 2378  *      Implements both kernel and user wiring.
 2379  */
 2380 int
 2381 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 2382     int flags)
 2383 {
 2384         vm_map_entry_t entry, first_entry, tmp_entry;
 2385         vm_offset_t saved_end, saved_start;
 2386         unsigned int last_timestamp;
 2387         int rv;
 2388         boolean_t fictitious, need_wakeup, result, user_wire;
 2389         vm_prot_t prot;
 2390 
 2391         prot = 0;
 2392         if (flags & VM_MAP_WIRE_WRITE)
 2393                 prot |= VM_PROT_WRITE;
 2394         user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
 2395         vm_map_lock(map);
 2396         VM_MAP_RANGE_CHECK(map, start, end);
 2397         if (!vm_map_lookup_entry(map, start, &first_entry)) {
 2398                 if (flags & VM_MAP_WIRE_HOLESOK)
 2399                         first_entry = first_entry->next;
 2400                 else {
 2401                         vm_map_unlock(map);
 2402                         return (KERN_INVALID_ADDRESS);
 2403                 }
 2404         }
 2405         last_timestamp = map->timestamp;
 2406         entry = first_entry;
 2407         while (entry != &map->header && entry->start < end) {
 2408                 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
 2409                         /*
 2410                          * We have not yet clipped the entry.
 2411                          */
 2412                         saved_start = (start >= entry->start) ? start :
 2413                             entry->start;
 2414                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 2415                         if (vm_map_unlock_and_wait(map, 0)) {
 2416                                 /*
 2417                                  * Allow interruption of user wiring?
 2418                                  */
 2419                         }
 2420                         vm_map_lock(map);
 2421                         if (last_timestamp + 1 != map->timestamp) {
 2422                                 /*
 2423                                  * Look again for the entry because the map was
 2424                                  * modified while it was unlocked.
 2425                                  * Specifically, the entry may have been
 2426                                  * clipped, merged, or deleted.
 2427                                  */
 2428                                 if (!vm_map_lookup_entry(map, saved_start,
 2429                                     &tmp_entry)) {
 2430                                         if (flags & VM_MAP_WIRE_HOLESOK)
 2431                                                 tmp_entry = tmp_entry->next;
 2432                                         else {
 2433                                                 if (saved_start == start) {
 2434                                                         /*
 2435                                                          * first_entry has been deleted.
 2436                                                          */
 2437                                                         vm_map_unlock(map);
 2438                                                         return (KERN_INVALID_ADDRESS);
 2439                                                 }
 2440                                                 end = saved_start;
 2441                                                 rv = KERN_INVALID_ADDRESS;
 2442                                                 goto done;
 2443                                         }
 2444                                 }
 2445                                 if (entry == first_entry)
 2446                                         first_entry = tmp_entry;
 2447                                 else
 2448                                         first_entry = NULL;
 2449                                 entry = tmp_entry;
 2450                         }
 2451                         last_timestamp = map->timestamp;
 2452                         continue;
 2453                 }
 2454                 vm_map_clip_start(map, entry, start);
 2455                 vm_map_clip_end(map, entry, end);
 2456                 /*
 2457                  * Mark the entry in case the map lock is released.  (See
 2458                  * above.)
 2459                  */
 2460                 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
 2461                 entry->wiring_thread = curthread;
 2462                 if ((entry->protection & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0
 2463                     || (entry->protection & prot) != prot) {
 2464                         entry->eflags |= MAP_ENTRY_WIRE_SKIPPED;
 2465                         if ((flags & VM_MAP_WIRE_HOLESOK) == 0) {
 2466                                 end = entry->end;
 2467                                 rv = KERN_INVALID_ADDRESS;
 2468                                 goto done;
 2469                         }
 2470                         goto next_entry;
 2471                 }
 2472                 if (entry->wired_count == 0) {
 2473                         entry->wired_count++;
 2474                         saved_start = entry->start;
 2475                         saved_end = entry->end;
 2476                         fictitious = entry->object.vm_object != NULL &&
 2477                             (entry->object.vm_object->flags &
 2478                             OBJ_FICTITIOUS) != 0;
 2479                         /*
 2480                          * Release the map lock, relying on the in-transition
 2481                          * mark.  Mark the map busy for fork.
 2482                          */
 2483                         vm_map_busy(map);
 2484                         vm_map_unlock(map);
 2485                         rv = vm_fault_wire(map, saved_start, saved_end,
 2486                             fictitious);
 2487                         vm_map_lock(map);
 2488                         vm_map_unbusy(map);
 2489                         if (last_timestamp + 1 != map->timestamp) {
 2490                                 /*
 2491                                  * Look again for the entry because the map was
 2492                                  * modified while it was unlocked.  The entry
 2493                                  * may have been clipped, but NOT merged or
 2494                                  * deleted.
 2495                                  */
 2496                                 result = vm_map_lookup_entry(map, saved_start,
 2497                                     &tmp_entry);
 2498                                 KASSERT(result, ("vm_map_wire: lookup failed"));
 2499                                 if (entry == first_entry)
 2500                                         first_entry = tmp_entry;
 2501                                 else
 2502                                         first_entry = NULL;
 2503                                 entry = tmp_entry;
 2504                                 while (entry->end < saved_end) {
 2505                                         if (rv != KERN_SUCCESS) {
 2506                                                 KASSERT(entry->wired_count == 1,
 2507                                                     ("vm_map_wire: bad count"));
 2508                                                 entry->wired_count = -1;
 2509                                         }
 2510                                         entry = entry->next;
 2511                                 }
 2512                         }
 2513                         last_timestamp = map->timestamp;
 2514                         if (rv != KERN_SUCCESS) {
 2515                                 KASSERT(entry->wired_count == 1,
 2516                                     ("vm_map_wire: bad count"));
 2517                                 /*
 2518                                  * Assign an out-of-range value to represent
 2519                                  * the failure to wire this entry.
 2520                                  */
 2521                                 entry->wired_count = -1;
 2522                                 end = entry->end;
 2523                                 goto done;
 2524                         }
 2525                 } else if (!user_wire ||
 2526                            (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
 2527                         entry->wired_count++;
 2528                 }
 2529                 /*
 2530                  * Check the map for holes in the specified region.
 2531                  * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
 2532                  */
 2533         next_entry:
 2534                 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
 2535                     (entry->end < end && (entry->next == &map->header ||
 2536                     entry->next->start > entry->end))) {
 2537                         end = entry->end;
 2538                         rv = KERN_INVALID_ADDRESS;
 2539                         goto done;
 2540                 }
 2541                 entry = entry->next;
 2542         }
 2543         rv = KERN_SUCCESS;
 2544 done:
 2545         need_wakeup = FALSE;
 2546         if (first_entry == NULL) {
 2547                 result = vm_map_lookup_entry(map, start, &first_entry);
 2548                 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
 2549                         first_entry = first_entry->next;
 2550                 else
 2551                         KASSERT(result, ("vm_map_wire: lookup failed"));
 2552         }
 2553         for (entry = first_entry; entry != &map->header && entry->start < end;
 2554             entry = entry->next) {
 2555                 if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0)
 2556                         goto next_entry_done;
 2557 
 2558                 /*
 2559                  * If VM_MAP_WIRE_HOLESOK was specified, an empty
 2560                  * space in the unwired region could have been mapped
 2561                  * while the map lock was dropped for faulting in the
 2562                  * pages or draining MAP_ENTRY_IN_TRANSITION.
 2563                  * Moreover, another thread could be simultaneously
 2564                  * wiring this new mapping entry.  Detect these cases
 2565                  * and skip any entries marked as in transition by us.
 2566                  */
 2567                 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) == 0 ||
 2568                     entry->wiring_thread != curthread) {
 2569                         KASSERT((flags & VM_MAP_WIRE_HOLESOK) != 0,
 2570                             ("vm_map_wire: !HOLESOK and new/changed entry"));
 2571                         continue;
 2572                 }
 2573 
 2574                 if (rv == KERN_SUCCESS) {
 2575                         if (user_wire)
 2576                                 entry->eflags |= MAP_ENTRY_USER_WIRED;
 2577                 } else if (entry->wired_count == -1) {
 2578                         /*
 2579                          * Wiring failed on this entry.  Thus, unwiring is
 2580                          * unnecessary.
 2581                          */
 2582                         entry->wired_count = 0;
 2583                 } else {
 2584                         if (!user_wire ||
 2585                             (entry->eflags & MAP_ENTRY_USER_WIRED) == 0)
 2586                                 entry->wired_count--;
 2587                         if (entry->wired_count == 0) {
 2588                                 /*
 2589                                  * Retain the map lock.
 2590                                  */
 2591                                 vm_fault_unwire(map, entry->start, entry->end,
 2592                                     entry->object.vm_object != NULL &&
 2593                                     (entry->object.vm_object->flags &
 2594                                     OBJ_FICTITIOUS) != 0);
 2595                         }
 2596                 }
 2597         next_entry_done:
 2598                 KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0,
 2599                     ("vm_map_wire: in-transition flag missing %p", entry));
 2600                 KASSERT(entry->wiring_thread == curthread,
 2601                     ("vm_map_wire: alien wire %p", entry));
 2602                 entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION |
 2603                     MAP_ENTRY_WIRE_SKIPPED);
 2604                 entry->wiring_thread = NULL;
 2605                 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
 2606                         entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
 2607                         need_wakeup = TRUE;
 2608                 }
 2609                 vm_map_simplify_entry(map, entry);
 2610         }
 2611         vm_map_unlock(map);
 2612         if (need_wakeup)
 2613                 vm_map_wakeup(map);
 2614         return (rv);
 2615 }
 2616 
 2617 /*
 2618  * vm_map_sync
 2619  *
 2620  * Push any dirty cached pages in the address range to their pager.
 2621  * If syncio is TRUE, dirty pages are written synchronously.
 2622  * If invalidate is TRUE, any cached pages are freed as well.
 2623  *
 2624  * If the size of the region from start to end is zero, we are
 2625  * supposed to flush all modified pages within the region containing
 2626  * start.  Unfortunately, a region can be split or coalesced with
 2627  * neighboring regions, making it difficult to determine what the
 2628  * original region was.  Therefore, we approximate this requirement by
 2629  * flushing the current region containing start.
 2630  *
 2631  * Returns an error if any part of the specified range is not mapped.
 2632  */
 2633 int
 2634 vm_map_sync(
 2635         vm_map_t map,
 2636         vm_offset_t start,
 2637         vm_offset_t end,
 2638         boolean_t syncio,
 2639         boolean_t invalidate)
 2640 {
 2641         vm_map_entry_t current;
 2642         vm_map_entry_t entry;
 2643         vm_size_t size;
 2644         vm_object_t object;
 2645         vm_ooffset_t offset;
 2646         unsigned int last_timestamp;
 2647         boolean_t failed;
 2648 
 2649         vm_map_lock_read(map);
 2650         VM_MAP_RANGE_CHECK(map, start, end);
 2651         if (!vm_map_lookup_entry(map, start, &entry)) {
 2652                 vm_map_unlock_read(map);
 2653                 return (KERN_INVALID_ADDRESS);
 2654         } else if (start == end) {
 2655                 start = entry->start;
 2656                 end = entry->end;
 2657         }
 2658         /*
 2659          * Make a first pass to check for user-wired memory and holes.
 2660          */
 2661         for (current = entry; current != &map->header && current->start < end;
 2662             current = current->next) {
 2663                 if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) {
 2664                         vm_map_unlock_read(map);
 2665                         return (KERN_INVALID_ARGUMENT);
 2666                 }
 2667                 if (end > current->end &&
 2668                     (current->next == &map->header ||
 2669                         current->end != current->next->start)) {
 2670                         vm_map_unlock_read(map);
 2671                         return (KERN_INVALID_ADDRESS);
 2672                 }
 2673         }
 2674 
 2675         if (invalidate)
 2676                 pmap_remove(map->pmap, start, end);
 2677         failed = FALSE;
 2678 
 2679         /*
 2680          * Make a second pass, cleaning/uncaching pages from the indicated
 2681          * objects as we go.
 2682          */
 2683         for (current = entry; current != &map->header && current->start < end;) {
 2684                 offset = current->offset + (start - current->start);
 2685                 size = (end <= current->end ? end : current->end) - start;
 2686                 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
 2687                         vm_map_t smap;
 2688                         vm_map_entry_t tentry;
 2689                         vm_size_t tsize;
 2690 
 2691                         smap = current->object.sub_map;
 2692                         vm_map_lock_read(smap);
 2693                         (void) vm_map_lookup_entry(smap, offset, &tentry);
 2694                         tsize = tentry->end - offset;
 2695                         if (tsize < size)
 2696                                 size = tsize;
 2697                         object = tentry->object.vm_object;
 2698                         offset = tentry->offset + (offset - tentry->start);
 2699                         vm_map_unlock_read(smap);
 2700                 } else {
 2701                         object = current->object.vm_object;
 2702                 }
 2703                 vm_object_reference(object);
 2704                 last_timestamp = map->timestamp;
 2705                 vm_map_unlock_read(map);
 2706                 if (!vm_object_sync(object, offset, size, syncio, invalidate))
 2707                         failed = TRUE;
 2708                 start += size;
 2709                 vm_object_deallocate(object);
 2710                 vm_map_lock_read(map);
 2711                 if (last_timestamp == map->timestamp ||
 2712                     !vm_map_lookup_entry(map, start, &current))
 2713                         current = current->next;
 2714         }
 2715 
 2716         vm_map_unlock_read(map);
 2717         return (failed ? KERN_FAILURE : KERN_SUCCESS);
 2718 }
 2719 
 2720 /*
 2721  *      vm_map_entry_unwire:    [ internal use only ]
 2722  *
 2723  *      Make the region specified by this entry pageable.
 2724  *
 2725  *      The map in question should be locked.
 2726  *      [This is the reason for this routine's existence.]
 2727  */
 2728 static void
 2729 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry)
 2730 {
 2731         vm_fault_unwire(map, entry->start, entry->end,
 2732             entry->object.vm_object != NULL &&
 2733             (entry->object.vm_object->flags & OBJ_FICTITIOUS) != 0);
 2734         entry->wired_count = 0;
 2735 }
 2736 
 2737 static void
 2738 vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map)
 2739 {
 2740 
 2741         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0)
 2742                 vm_object_deallocate(entry->object.vm_object);
 2743         uma_zfree(system_map ? kmapentzone : mapentzone, entry);
 2744 }
 2745 
 2746 /*
 2747  *      vm_map_entry_delete:    [ internal use only ]
 2748  *
 2749  *      Deallocate the given entry from the target map.
 2750  */
 2751 static void
 2752 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry)
 2753 {
 2754         vm_object_t object;
 2755         vm_pindex_t offidxstart, offidxend, count, size1;
 2756         vm_ooffset_t size;
 2757 
 2758         vm_map_entry_unlink(map, entry);
 2759         object = entry->object.vm_object;
 2760         size = entry->end - entry->start;
 2761         map->size -= size;
 2762 
 2763         if (entry->cred != NULL) {
 2764                 swap_release_by_cred(size, entry->cred);
 2765                 crfree(entry->cred);
 2766         }
 2767 
 2768         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2769             (object != NULL)) {
 2770                 KASSERT(entry->cred == NULL || object->cred == NULL ||
 2771                     (entry->eflags & MAP_ENTRY_NEEDS_COPY),
 2772                     ("OVERCOMMIT vm_map_entry_delete: both cred %p", entry));
 2773                 count = OFF_TO_IDX(size);
 2774                 offidxstart = OFF_TO_IDX(entry->offset);
 2775                 offidxend = offidxstart + count;
 2776                 VM_OBJECT_WLOCK(object);
 2777                 if (object->ref_count != 1 &&
 2778                     ((object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING ||
 2779                     object == kernel_object || object == kmem_object)) {
 2780                         vm_object_collapse(object);
 2781 
 2782                         /*
 2783                          * The option OBJPR_NOTMAPPED can be passed here
 2784                          * because vm_map_delete() already performed
 2785                          * pmap_remove() on the only mapping to this range
 2786                          * of pages. 
 2787                          */
 2788                         vm_object_page_remove(object, offidxstart, offidxend,
 2789                             OBJPR_NOTMAPPED);
 2790                         if (object->type == OBJT_SWAP)
 2791                                 swap_pager_freespace(object, offidxstart, count);
 2792                         if (offidxend >= object->size &&
 2793                             offidxstart < object->size) {
 2794                                 size1 = object->size;
 2795                                 object->size = offidxstart;
 2796                                 if (object->cred != NULL) {
 2797                                         size1 -= object->size;
 2798                                         KASSERT(object->charge >= ptoa(size1),
 2799                                             ("vm_map_entry_delete: object->charge < 0"));
 2800                                         swap_release_by_cred(ptoa(size1), object->cred);
 2801                                         object->charge -= ptoa(size1);
 2802                                 }
 2803                         }
 2804                 }
 2805                 VM_OBJECT_WUNLOCK(object);
 2806         } else
 2807                 entry->object.vm_object = NULL;
 2808         if (map->system_map)
 2809                 vm_map_entry_deallocate(entry, TRUE);
 2810         else {
 2811                 entry->next = curthread->td_map_def_user;
 2812                 curthread->td_map_def_user = entry;
 2813         }
 2814 }
 2815 
 2816 /*
 2817  *      vm_map_delete:  [ internal use only ]
 2818  *
 2819  *      Deallocates the given address range from the target
 2820  *      map.
 2821  */
 2822 int
 2823 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
 2824 {
 2825         vm_map_entry_t entry;
 2826         vm_map_entry_t first_entry;
 2827 
 2828         VM_MAP_ASSERT_LOCKED(map);
 2829 
 2830         /*
 2831          * Find the start of the region, and clip it
 2832          */
 2833         if (!vm_map_lookup_entry(map, start, &first_entry))
 2834                 entry = first_entry->next;
 2835         else {
 2836                 entry = first_entry;
 2837                 vm_map_clip_start(map, entry, start);
 2838         }
 2839 
 2840         /*
 2841          * Step through all entries in this region
 2842          */
 2843         while ((entry != &map->header) && (entry->start < end)) {
 2844                 vm_map_entry_t next;
 2845 
 2846                 /*
 2847                  * Wait for wiring or unwiring of an entry to complete.
 2848                  * Also wait for any system wirings to disappear on
 2849                  * user maps.
 2850                  */
 2851                 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 ||
 2852                     (vm_map_pmap(map) != kernel_pmap &&
 2853                     vm_map_entry_system_wired_count(entry) != 0)) {
 2854                         unsigned int last_timestamp;
 2855                         vm_offset_t saved_start;
 2856                         vm_map_entry_t tmp_entry;
 2857 
 2858                         saved_start = entry->start;
 2859                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 2860                         last_timestamp = map->timestamp;
 2861                         (void) vm_map_unlock_and_wait(map, 0);
 2862                         vm_map_lock(map);
 2863                         if (last_timestamp + 1 != map->timestamp) {
 2864                                 /*
 2865                                  * Look again for the entry because the map was
 2866                                  * modified while it was unlocked.
 2867                                  * Specifically, the entry may have been
 2868                                  * clipped, merged, or deleted.
 2869                                  */
 2870                                 if (!vm_map_lookup_entry(map, saved_start,
 2871                                                          &tmp_entry))
 2872                                         entry = tmp_entry->next;
 2873                                 else {
 2874                                         entry = tmp_entry;
 2875                                         vm_map_clip_start(map, entry,
 2876                                                           saved_start);
 2877                                 }
 2878                         }
 2879                         continue;
 2880                 }
 2881                 vm_map_clip_end(map, entry, end);
 2882 
 2883                 next = entry->next;
 2884 
 2885                 /*
 2886                  * Unwire before removing addresses from the pmap; otherwise,
 2887                  * unwiring will put the entries back in the pmap.
 2888                  */
 2889                 if (entry->wired_count != 0) {
 2890                         vm_map_entry_unwire(map, entry);
 2891                 }
 2892 
 2893                 pmap_remove(map->pmap, entry->start, entry->end);
 2894 
 2895                 /*
 2896                  * Delete the entry only after removing all pmap
 2897                  * entries pointing to its pages.  (Otherwise, its
 2898                  * page frames may be reallocated, and any modify bits
 2899                  * will be set in the wrong object!)
 2900                  */
 2901                 vm_map_entry_delete(map, entry);
 2902                 entry = next;
 2903         }
 2904         return (KERN_SUCCESS);
 2905 }
 2906 
 2907 /*
 2908  *      vm_map_remove:
 2909  *
 2910  *      Remove the given address range from the target map.
 2911  *      This is the exported form of vm_map_delete.
 2912  */
 2913 int
 2914 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end)
 2915 {
 2916         int result;
 2917 
 2918         vm_map_lock(map);
 2919         VM_MAP_RANGE_CHECK(map, start, end);
 2920         result = vm_map_delete(map, start, end);
 2921         vm_map_unlock(map);
 2922         return (result);
 2923 }
 2924 
 2925 /*
 2926  *      vm_map_check_protection:
 2927  *
 2928  *      Assert that the target map allows the specified privilege on the
 2929  *      entire address region given.  The entire region must be allocated.
 2930  *
 2931  *      WARNING!  This code does not and should not check whether the
 2932  *      contents of the region is accessible.  For example a smaller file
 2933  *      might be mapped into a larger address space.
 2934  *
 2935  *      NOTE!  This code is also called by munmap().
 2936  *
 2937  *      The map must be locked.  A read lock is sufficient.
 2938  */
 2939 boolean_t
 2940 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end,
 2941                         vm_prot_t protection)
 2942 {
 2943         vm_map_entry_t entry;
 2944         vm_map_entry_t tmp_entry;
 2945 
 2946         if (!vm_map_lookup_entry(map, start, &tmp_entry))
 2947                 return (FALSE);
 2948         entry = tmp_entry;
 2949 
 2950         while (start < end) {
 2951                 if (entry == &map->header)
 2952                         return (FALSE);
 2953                 /*
 2954                  * No holes allowed!
 2955                  */
 2956                 if (start < entry->start)
 2957                         return (FALSE);
 2958                 /*
 2959                  * Check protection associated with entry.
 2960                  */
 2961                 if ((entry->protection & protection) != protection)
 2962                         return (FALSE);
 2963                 /* go to next entry */
 2964                 start = entry->end;
 2965                 entry = entry->next;
 2966         }
 2967         return (TRUE);
 2968 }
 2969 
 2970 /*
 2971  *      vm_map_copy_entry:
 2972  *
 2973  *      Copies the contents of the source entry to the destination
 2974  *      entry.  The entries *must* be aligned properly.
 2975  */
 2976 static void
 2977 vm_map_copy_entry(
 2978         vm_map_t src_map,
 2979         vm_map_t dst_map,
 2980         vm_map_entry_t src_entry,
 2981         vm_map_entry_t dst_entry,
 2982         vm_ooffset_t *fork_charge)
 2983 {
 2984         vm_object_t src_object;
 2985         vm_map_entry_t fake_entry;
 2986         vm_offset_t size;
 2987         struct ucred *cred;
 2988         int charged;
 2989 
 2990         VM_MAP_ASSERT_LOCKED(dst_map);
 2991 
 2992         if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP)
 2993                 return;
 2994 
 2995         if (src_entry->wired_count == 0) {
 2996 
 2997                 /*
 2998                  * If the source entry is marked needs_copy, it is already
 2999                  * write-protected.
 3000                  */
 3001                 if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
 3002                         pmap_protect(src_map->pmap,
 3003                             src_entry->start,
 3004                             src_entry->end,
 3005                             src_entry->protection & ~VM_PROT_WRITE);
 3006                 }
 3007 
 3008                 /*
 3009                  * Make a copy of the object.
 3010                  */
 3011                 size = src_entry->end - src_entry->start;
 3012                 if ((src_object = src_entry->object.vm_object) != NULL) {
 3013                         VM_OBJECT_WLOCK(src_object);
 3014                         charged = ENTRY_CHARGED(src_entry);
 3015                         if ((src_object->handle == NULL) &&
 3016                                 (src_object->type == OBJT_DEFAULT ||
 3017                                  src_object->type == OBJT_SWAP)) {
 3018                                 vm_object_collapse(src_object);
 3019                                 if ((src_object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) {
 3020                                         vm_object_split(src_entry);
 3021                                         src_object = src_entry->object.vm_object;
 3022                                 }
 3023                         }
 3024                         vm_object_reference_locked(src_object);
 3025                         vm_object_clear_flag(src_object, OBJ_ONEMAPPING);
 3026                         if (src_entry->cred != NULL &&
 3027                             !(src_entry->eflags & MAP_ENTRY_NEEDS_COPY)) {
 3028                                 KASSERT(src_object->cred == NULL,
 3029                                     ("OVERCOMMIT: vm_map_copy_entry: cred %p",
 3030                                      src_object));
 3031                                 src_object->cred = src_entry->cred;
 3032                                 src_object->charge = size;
 3033                         }
 3034                         VM_OBJECT_WUNLOCK(src_object);
 3035                         dst_entry->object.vm_object = src_object;
 3036                         if (charged) {
 3037                                 cred = curthread->td_ucred;
 3038                                 crhold(cred);
 3039                                 dst_entry->cred = cred;
 3040                                 *fork_charge += size;
 3041                                 if (!(src_entry->eflags &
 3042                                       MAP_ENTRY_NEEDS_COPY)) {
 3043                                         crhold(cred);
 3044                                         src_entry->cred = cred;
 3045                                         *fork_charge += size;
 3046                                 }
 3047                         }
 3048                         src_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
 3049                         dst_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
 3050                         dst_entry->offset = src_entry->offset;
 3051                         if (src_entry->eflags & MAP_ENTRY_VN_WRITECNT) {
 3052                                 /*
 3053                                  * MAP_ENTRY_VN_WRITECNT cannot
 3054                                  * indicate write reference from
 3055                                  * src_entry, since the entry is
 3056                                  * marked as needs copy.  Allocate a
 3057                                  * fake entry that is used to
 3058                                  * decrement object->un_pager.vnp.writecount
 3059                                  * at the appropriate time.  Attach
 3060                                  * fake_entry to the deferred list.
 3061                                  */
 3062                                 fake_entry = vm_map_entry_create(dst_map);
 3063                                 fake_entry->eflags = MAP_ENTRY_VN_WRITECNT;
 3064                                 src_entry->eflags &= ~MAP_ENTRY_VN_WRITECNT;
 3065                                 vm_object_reference(src_object);
 3066                                 fake_entry->object.vm_object = src_object;
 3067                                 fake_entry->start = src_entry->start;
 3068                                 fake_entry->end = src_entry->end;
 3069                                 fake_entry->next = curthread->td_map_def_user;
 3070                                 curthread->td_map_def_user = fake_entry;
 3071                         }
 3072                 } else {
 3073                         dst_entry->object.vm_object = NULL;
 3074                         dst_entry->offset = 0;
 3075                         if (src_entry->cred != NULL) {
 3076                                 dst_entry->cred = curthread->td_ucred;
 3077                                 crhold(dst_entry->cred);
 3078                                 *fork_charge += size;
 3079                         }
 3080                 }
 3081 
 3082                 pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start,
 3083                     dst_entry->end - dst_entry->start, src_entry->start);
 3084         } else {
 3085                 /*
 3086                  * Of course, wired down pages can't be set copy-on-write.
 3087                  * Cause wired pages to be copied into the new map by
 3088                  * simulating faults (the new pages are pageable)
 3089                  */
 3090                 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry,
 3091                     fork_charge);
 3092         }
 3093 }
 3094 
 3095 /*
 3096  * vmspace_map_entry_forked:
 3097  * Update the newly-forked vmspace each time a map entry is inherited
 3098  * or copied.  The values for vm_dsize and vm_tsize are approximate
 3099  * (and mostly-obsolete ideas in the face of mmap(2) et al.)
 3100  */
 3101 static void
 3102 vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2,
 3103     vm_map_entry_t entry)
 3104 {
 3105         vm_size_t entrysize;
 3106         vm_offset_t newend;
 3107 
 3108         entrysize = entry->end - entry->start;
 3109         vm2->vm_map.size += entrysize;
 3110         if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) {
 3111                 vm2->vm_ssize += btoc(entrysize);
 3112         } else if (entry->start >= (vm_offset_t)vm1->vm_daddr &&
 3113             entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) {
 3114                 newend = MIN(entry->end,
 3115                     (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize));
 3116                 vm2->vm_dsize += btoc(newend - entry->start);
 3117         } else if (entry->start >= (vm_offset_t)vm1->vm_taddr &&
 3118             entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) {
 3119                 newend = MIN(entry->end,
 3120                     (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize));
 3121                 vm2->vm_tsize += btoc(newend - entry->start);
 3122         }
 3123 }
 3124 
 3125 /*
 3126  * vmspace_fork:
 3127  * Create a new process vmspace structure and vm_map
 3128  * based on those of an existing process.  The new map
 3129  * is based on the old map, according to the inheritance
 3130  * values on the regions in that map.
 3131  *
 3132  * XXX It might be worth coalescing the entries added to the new vmspace.
 3133  *
 3134  * The source map must not be locked.
 3135  */
 3136 struct vmspace *
 3137 vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge)
 3138 {
 3139         struct vmspace *vm2;
 3140         vm_map_t new_map, old_map;
 3141         vm_map_entry_t new_entry, old_entry;
 3142         vm_object_t object;
 3143         int locked;
 3144 
 3145         old_map = &vm1->vm_map;
 3146         /* Copy immutable fields of vm1 to vm2. */
 3147         vm2 = vmspace_alloc(old_map->min_offset, old_map->max_offset, NULL);
 3148         if (vm2 == NULL)
 3149                 return (NULL);
 3150         vm2->vm_taddr = vm1->vm_taddr;
 3151         vm2->vm_daddr = vm1->vm_daddr;
 3152         vm2->vm_maxsaddr = vm1->vm_maxsaddr;
 3153         vm_map_lock(old_map);
 3154         if (old_map->busy)
 3155                 vm_map_wait_busy(old_map);
 3156         new_map = &vm2->vm_map;
 3157         locked = vm_map_trylock(new_map); /* trylock to silence WITNESS */
 3158         KASSERT(locked, ("vmspace_fork: lock failed"));
 3159 
 3160         old_entry = old_map->header.next;
 3161 
 3162         while (old_entry != &old_map->header) {
 3163                 if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP)
 3164                         panic("vm_map_fork: encountered a submap");
 3165 
 3166                 switch (old_entry->inheritance) {
 3167                 case VM_INHERIT_NONE:
 3168                         break;
 3169 
 3170                 case VM_INHERIT_SHARE:
 3171                         /*
 3172                          * Clone the entry, creating the shared object if necessary.
 3173                          */
 3174                         object = old_entry->object.vm_object;
 3175                         if (object == NULL) {
 3176                                 object = vm_object_allocate(OBJT_DEFAULT,
 3177                                         atop(old_entry->end - old_entry->start));
 3178                                 old_entry->object.vm_object = object;
 3179                                 old_entry->offset = 0;
 3180                                 if (old_entry->cred != NULL) {
 3181                                         object->cred = old_entry->cred;
 3182                                         object->charge = old_entry->end -
 3183                                             old_entry->start;
 3184                                         old_entry->cred = NULL;
 3185                                 }
 3186                         }
 3187 
 3188                         /*
 3189                          * Add the reference before calling vm_object_shadow
 3190                          * to insure that a shadow object is created.
 3191                          */
 3192                         vm_object_reference(object);
 3193                         if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 3194                                 vm_object_shadow(&old_entry->object.vm_object,
 3195                                     &old_entry->offset,
 3196                                     old_entry->end - old_entry->start);
 3197                                 old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
 3198                                 /* Transfer the second reference too. */
 3199                                 vm_object_reference(
 3200                                     old_entry->object.vm_object);
 3201 
 3202                                 /*
 3203                                  * As in vm_map_simplify_entry(), the
 3204                                  * vnode lock will not be acquired in
 3205                                  * this call to vm_object_deallocate().
 3206                                  */
 3207                                 vm_object_deallocate(object);
 3208                                 object = old_entry->object.vm_object;
 3209                         }
 3210                         VM_OBJECT_WLOCK(object);
 3211                         vm_object_clear_flag(object, OBJ_ONEMAPPING);
 3212                         if (old_entry->cred != NULL) {
 3213                                 KASSERT(object->cred == NULL, ("vmspace_fork both cred"));
 3214                                 object->cred = old_entry->cred;
 3215                                 object->charge = old_entry->end - old_entry->start;
 3216                                 old_entry->cred = NULL;
 3217                         }
 3218 
 3219                         /*
 3220                          * Assert the correct state of the vnode
 3221                          * v_writecount while the object is locked, to
 3222                          * not relock it later for the assertion
 3223                          * correctness.
 3224                          */
 3225                         if (old_entry->eflags & MAP_ENTRY_VN_WRITECNT &&
 3226                             object->type == OBJT_VNODE) {
 3227                                 KASSERT(((struct vnode *)object->handle)->
 3228                                     v_writecount > 0,
 3229                                     ("vmspace_fork: v_writecount %p", object));
 3230                                 KASSERT(object->un_pager.vnp.writemappings > 0,
 3231                                     ("vmspace_fork: vnp.writecount %p",
 3232                                     object));
 3233                         }
 3234                         VM_OBJECT_WUNLOCK(object);
 3235 
 3236                         /*
 3237                          * Clone the entry, referencing the shared object.
 3238                          */
 3239                         new_entry = vm_map_entry_create(new_map);
 3240                         *new_entry = *old_entry;
 3241                         new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
 3242                             MAP_ENTRY_IN_TRANSITION);
 3243                         new_entry->wiring_thread = NULL;
 3244                         new_entry->wired_count = 0;
 3245                         if (new_entry->eflags & MAP_ENTRY_VN_WRITECNT) {
 3246                                 vnode_pager_update_writecount(object,
 3247                                     new_entry->start, new_entry->end);
 3248                         }
 3249 
 3250                         /*
 3251                          * Insert the entry into the new map -- we know we're
 3252                          * inserting at the end of the new map.
 3253                          */
 3254                         vm_map_entry_link(new_map, new_map->header.prev,
 3255                             new_entry);
 3256                         vmspace_map_entry_forked(vm1, vm2, new_entry);
 3257 
 3258                         /*
 3259                          * Update the physical map
 3260                          */
 3261                         pmap_copy(new_map->pmap, old_map->pmap,
 3262                             new_entry->start,
 3263                             (old_entry->end - old_entry->start),
 3264                             old_entry->start);
 3265                         break;
 3266 
 3267                 case VM_INHERIT_COPY:
 3268                         /*
 3269                          * Clone the entry and link into the map.
 3270                          */
 3271                         new_entry = vm_map_entry_create(new_map);
 3272                         *new_entry = *old_entry;
 3273                         /*
 3274                          * Copied entry is COW over the old object.
 3275                          */
 3276                         new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
 3277                             MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_VN_WRITECNT);
 3278                         new_entry->wiring_thread = NULL;
 3279                         new_entry->wired_count = 0;
 3280                         new_entry->object.vm_object = NULL;
 3281                         new_entry->cred = NULL;
 3282                         vm_map_entry_link(new_map, new_map->header.prev,
 3283                             new_entry);
 3284                         vmspace_map_entry_forked(vm1, vm2, new_entry);
 3285                         vm_map_copy_entry(old_map, new_map, old_entry,
 3286                             new_entry, fork_charge);
 3287                         break;
 3288                 }
 3289                 old_entry = old_entry->next;
 3290         }
 3291         /*
 3292          * Use inlined vm_map_unlock() to postpone handling the deferred
 3293          * map entries, which cannot be done until both old_map and
 3294          * new_map locks are released.
 3295          */
 3296         sx_xunlock(&old_map->lock);
 3297         sx_xunlock(&new_map->lock);
 3298         vm_map_process_deferred();
 3299 
 3300         return (vm2);
 3301 }
 3302 
 3303 int
 3304 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
 3305     vm_prot_t prot, vm_prot_t max, int cow)
 3306 {
 3307         vm_map_entry_t new_entry, prev_entry;
 3308         vm_offset_t bot, top;
 3309         vm_size_t growsize, init_ssize;
 3310         int orient, rv;
 3311         rlim_t lmemlim, vmemlim;
 3312 
 3313         /*
 3314          * The stack orientation is piggybacked with the cow argument.
 3315          * Extract it into orient and mask the cow argument so that we
 3316          * don't pass it around further.
 3317          * NOTE: We explicitly allow bi-directional stacks.
 3318          */
 3319         orient = cow & (MAP_STACK_GROWS_DOWN|MAP_STACK_GROWS_UP);
 3320         KASSERT(orient != 0, ("No stack grow direction"));
 3321 
 3322         if (addrbos < vm_map_min(map) ||
 3323             addrbos > vm_map_max(map) ||
 3324             addrbos + max_ssize < addrbos)
 3325                 return (KERN_NO_SPACE);
 3326 
 3327         growsize = sgrowsiz;
 3328         init_ssize = (max_ssize < growsize) ? max_ssize : growsize;
 3329 
 3330         PROC_LOCK(curproc);
 3331         lmemlim = lim_cur(curproc, RLIMIT_MEMLOCK);
 3332         vmemlim = lim_cur(curproc, RLIMIT_VMEM);
 3333         PROC_UNLOCK(curproc);
 3334 
 3335         vm_map_lock(map);
 3336 
 3337         /* If addr is already mapped, no go */
 3338         if (vm_map_lookup_entry(map, addrbos, &prev_entry)) {
 3339                 vm_map_unlock(map);
 3340                 return (KERN_NO_SPACE);
 3341         }
 3342 
 3343         if (!old_mlock && map->flags & MAP_WIREFUTURE) {
 3344                 if (ptoa(pmap_wired_count(map->pmap)) + init_ssize > lmemlim) {
 3345                         vm_map_unlock(map);
 3346                         return (KERN_NO_SPACE);
 3347                 }
 3348         }
 3349 
 3350         /* If we would blow our VMEM resource limit, no go */
 3351         if (map->size + init_ssize > vmemlim) {
 3352                 vm_map_unlock(map);
 3353                 return (KERN_NO_SPACE);
 3354         }
 3355 
 3356         /*
 3357          * If we can't accomodate max_ssize in the current mapping, no go.
 3358          * However, we need to be aware that subsequent user mappings might
 3359          * map into the space we have reserved for stack, and currently this
 3360          * space is not protected.
 3361          *
 3362          * Hopefully we will at least detect this condition when we try to
 3363          * grow the stack.
 3364          */
 3365         if ((prev_entry->next != &map->header) &&
 3366             (prev_entry->next->start < addrbos + max_ssize)) {
 3367                 vm_map_unlock(map);
 3368                 return (KERN_NO_SPACE);
 3369         }
 3370 
 3371         /*
 3372          * We initially map a stack of only init_ssize.  We will grow as
 3373          * needed later.  Depending on the orientation of the stack (i.e.
 3374          * the grow direction) we either map at the top of the range, the
 3375          * bottom of the range or in the middle.
 3376          *
 3377          * Note: we would normally expect prot and max to be VM_PROT_ALL,
 3378          * and cow to be 0.  Possibly we should eliminate these as input
 3379          * parameters, and just pass these values here in the insert call.
 3380          */
 3381         if (orient == MAP_STACK_GROWS_DOWN)
 3382                 bot = addrbos + max_ssize - init_ssize;
 3383         else if (orient == MAP_STACK_GROWS_UP)
 3384                 bot = addrbos;
 3385         else
 3386                 bot = round_page(addrbos + max_ssize/2 - init_ssize/2);
 3387         top = bot + init_ssize;
 3388         rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow);
 3389 
 3390         /* Now set the avail_ssize amount. */
 3391         if (rv == KERN_SUCCESS) {
 3392                 if (prev_entry != &map->header)
 3393                         vm_map_clip_end(map, prev_entry, bot);
 3394                 new_entry = prev_entry->next;
 3395                 if (new_entry->end != top || new_entry->start != bot)
 3396                         panic("Bad entry start/end for new stack entry");
 3397 
 3398                 new_entry->avail_ssize = max_ssize - init_ssize;
 3399                 if (orient & MAP_STACK_GROWS_DOWN)
 3400                         new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
 3401                 if (orient & MAP_STACK_GROWS_UP)
 3402                         new_entry->eflags |= MAP_ENTRY_GROWS_UP;
 3403         }
 3404 
 3405         vm_map_unlock(map);
 3406         return (rv);
 3407 }
 3408 
 3409 static int stack_guard_page = 0;
 3410 TUNABLE_INT("security.bsd.stack_guard_page", &stack_guard_page);
 3411 SYSCTL_INT(_security_bsd, OID_AUTO, stack_guard_page, CTLFLAG_RW,
 3412     &stack_guard_page, 0,
 3413     "Insert stack guard page ahead of the growable segments.");
 3414 
 3415 /* Attempts to grow a vm stack entry.  Returns KERN_SUCCESS if the
 3416  * desired address is already mapped, or if we successfully grow
 3417  * the stack.  Also returns KERN_SUCCESS if addr is outside the
 3418  * stack range (this is strange, but preserves compatibility with
 3419  * the grow function in vm_machdep.c).
 3420  */
 3421 int
 3422 vm_map_growstack(struct proc *p, vm_offset_t addr)
 3423 {
 3424         vm_map_entry_t next_entry, prev_entry;
 3425         vm_map_entry_t new_entry, stack_entry;
 3426         struct vmspace *vm = p->p_vmspace;
 3427         vm_map_t map = &vm->vm_map;
 3428         vm_offset_t end;
 3429         vm_size_t growsize;
 3430         size_t grow_amount, max_grow;
 3431         rlim_t lmemlim, stacklim, vmemlim;
 3432         int is_procstack, rv;
 3433         struct ucred *cred;
 3434 #ifdef notyet
 3435         uint64_t limit;
 3436 #endif
 3437 #ifdef RACCT
 3438         int error;
 3439 #endif
 3440 
 3441 Retry:
 3442         PROC_LOCK(p);
 3443         lmemlim = lim_cur(p, RLIMIT_MEMLOCK);
 3444         stacklim = lim_cur(p, RLIMIT_STACK);
 3445         vmemlim = lim_cur(p, RLIMIT_VMEM);
 3446         PROC_UNLOCK(p);
 3447 
 3448         vm_map_lock_read(map);
 3449 
 3450         /* If addr is already in the entry range, no need to grow.*/
 3451         if (vm_map_lookup_entry(map, addr, &prev_entry)) {
 3452                 vm_map_unlock_read(map);
 3453                 return (KERN_SUCCESS);
 3454         }
 3455 
 3456         next_entry = prev_entry->next;
 3457         if (!(prev_entry->eflags & MAP_ENTRY_GROWS_UP)) {
 3458                 /*
 3459                  * This entry does not grow upwards. Since the address lies
 3460                  * beyond this entry, the next entry (if one exists) has to
 3461                  * be a downward growable entry. The entry list header is
 3462                  * never a growable entry, so it suffices to check the flags.
 3463                  */
 3464                 if (!(next_entry->eflags & MAP_ENTRY_GROWS_DOWN)) {
 3465                         vm_map_unlock_read(map);
 3466                         return (KERN_SUCCESS);
 3467                 }
 3468                 stack_entry = next_entry;
 3469         } else {
 3470                 /*
 3471                  * This entry grows upward. If the next entry does not at
 3472                  * least grow downwards, this is the entry we need to grow.
 3473                  * otherwise we have two possible choices and we have to
 3474                  * select one.
 3475                  */
 3476                 if (next_entry->eflags & MAP_ENTRY_GROWS_DOWN) {
 3477                         /*
 3478                          * We have two choices; grow the entry closest to
 3479                          * the address to minimize the amount of growth.
 3480                          */
 3481                         if (addr - prev_entry->end <= next_entry->start - addr)
 3482                                 stack_entry = prev_entry;
 3483                         else
 3484                                 stack_entry = next_entry;
 3485                 } else
 3486                         stack_entry = prev_entry;
 3487         }
 3488 
 3489         if (stack_entry == next_entry) {
 3490                 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_DOWN, ("foo"));
 3491                 KASSERT(addr < stack_entry->start, ("foo"));
 3492                 end = (prev_entry != &map->header) ? prev_entry->end :
 3493                     stack_entry->start - stack_entry->avail_ssize;
 3494                 grow_amount = roundup(stack_entry->start - addr, PAGE_SIZE);
 3495                 max_grow = stack_entry->start - end;
 3496         } else {
 3497                 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_UP, ("foo"));
 3498                 KASSERT(addr >= stack_entry->end, ("foo"));
 3499                 end = (next_entry != &map->header) ? next_entry->start :
 3500                     stack_entry->end + stack_entry->avail_ssize;
 3501                 grow_amount = roundup(addr + 1 - stack_entry->end, PAGE_SIZE);
 3502                 max_grow = end - stack_entry->end;
 3503         }
 3504 
 3505         if (grow_amount > stack_entry->avail_ssize) {
 3506                 vm_map_unlock_read(map);
 3507                 return (KERN_NO_SPACE);
 3508         }
 3509 
 3510         /*
 3511          * If there is no longer enough space between the entries nogo, and
 3512          * adjust the available space.  Note: this  should only happen if the
 3513          * user has mapped into the stack area after the stack was created,
 3514          * and is probably an error.
 3515          *
 3516          * This also effectively destroys any guard page the user might have
 3517          * intended by limiting the stack size.
 3518          */
 3519         if (grow_amount + (stack_guard_page ? PAGE_SIZE : 0) > max_grow) {
 3520                 if (vm_map_lock_upgrade(map))
 3521                         goto Retry;
 3522 
 3523                 stack_entry->avail_ssize = max_grow;
 3524 
 3525                 vm_map_unlock(map);
 3526                 return (KERN_NO_SPACE);
 3527         }
 3528 
 3529         is_procstack = (addr >= (vm_offset_t)vm->vm_maxsaddr) ? 1 : 0;
 3530 
 3531         /*
 3532          * If this is the main process stack, see if we're over the stack
 3533          * limit.
 3534          */
 3535         if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
 3536                 vm_map_unlock_read(map);
 3537                 return (KERN_NO_SPACE);
 3538         }
 3539 #ifdef RACCT
 3540         PROC_LOCK(p);
 3541         if (is_procstack &&
 3542             racct_set(p, RACCT_STACK, ctob(vm->vm_ssize) + grow_amount)) {
 3543                 PROC_UNLOCK(p);
 3544                 vm_map_unlock_read(map);
 3545                 return (KERN_NO_SPACE);
 3546         }
 3547         PROC_UNLOCK(p);
 3548 #endif
 3549 
 3550         /* Round up the grow amount modulo sgrowsiz */
 3551         growsize = sgrowsiz;
 3552         grow_amount = roundup(grow_amount, growsize);
 3553         if (grow_amount > stack_entry->avail_ssize)
 3554                 grow_amount = stack_entry->avail_ssize;
 3555         if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
 3556                 grow_amount = trunc_page((vm_size_t)stacklim) -
 3557                     ctob(vm->vm_ssize);
 3558         }
 3559 #ifdef notyet
 3560         PROC_LOCK(p);
 3561         limit = racct_get_available(p, RACCT_STACK);
 3562         PROC_UNLOCK(p);
 3563         if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > limit))
 3564                 grow_amount = limit - ctob(vm->vm_ssize);
 3565 #endif
 3566         if (!old_mlock && map->flags & MAP_WIREFUTURE) {
 3567                 if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) {
 3568                         vm_map_unlock_read(map);
 3569                         rv = KERN_NO_SPACE;
 3570                         goto out;
 3571                 }
 3572 #ifdef RACCT
 3573                 PROC_LOCK(p);
 3574                 if (racct_set(p, RACCT_MEMLOCK,
 3575                     ptoa(pmap_wired_count(map->pmap)) + grow_amount)) {
 3576                         PROC_UNLOCK(p);
 3577                         vm_map_unlock_read(map);
 3578                         rv = KERN_NO_SPACE;
 3579                         goto out;
 3580                 }
 3581                 PROC_UNLOCK(p);
 3582 #endif
 3583         }
 3584         /* If we would blow our VMEM resource limit, no go */
 3585         if (map->size + grow_amount > vmemlim) {
 3586                 vm_map_unlock_read(map);
 3587                 rv = KERN_NO_SPACE;
 3588                 goto out;
 3589         }
 3590 #ifdef RACCT
 3591         PROC_LOCK(p);
 3592         if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) {
 3593                 PROC_UNLOCK(p);
 3594                 vm_map_unlock_read(map);
 3595                 rv = KERN_NO_SPACE;
 3596                 goto out;
 3597         }
 3598         PROC_UNLOCK(p);
 3599 #endif
 3600 
 3601         if (vm_map_lock_upgrade(map))
 3602                 goto Retry;
 3603 
 3604         if (stack_entry == next_entry) {
 3605                 /*
 3606                  * Growing downward.
 3607                  */
 3608                 /* Get the preliminary new entry start value */
 3609                 addr = stack_entry->start - grow_amount;
 3610 
 3611                 /*
 3612                  * If this puts us into the previous entry, cut back our
 3613                  * growth to the available space. Also, see the note above.
 3614                  */
 3615                 if (addr < end) {
 3616                         stack_entry->avail_ssize = max_grow;
 3617                         addr = end;
 3618                         if (stack_guard_page)
 3619                                 addr += PAGE_SIZE;
 3620                 }
 3621 
 3622                 rv = vm_map_insert(map, NULL, 0, addr, stack_entry->start,
 3623                     next_entry->protection, next_entry->max_protection, 0);
 3624 
 3625                 /* Adjust the available stack space by the amount we grew. */
 3626                 if (rv == KERN_SUCCESS) {
 3627                         if (prev_entry != &map->header)
 3628                                 vm_map_clip_end(map, prev_entry, addr);
 3629                         new_entry = prev_entry->next;
 3630                         KASSERT(new_entry == stack_entry->prev, ("foo"));
 3631                         KASSERT(new_entry->end == stack_entry->start, ("foo"));
 3632                         KASSERT(new_entry->start == addr, ("foo"));
 3633                         grow_amount = new_entry->end - new_entry->start;
 3634                         new_entry->avail_ssize = stack_entry->avail_ssize -
 3635                             grow_amount;
 3636                         stack_entry->eflags &= ~MAP_ENTRY_GROWS_DOWN;
 3637                         new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
 3638                 }
 3639         } else {
 3640                 /*
 3641                  * Growing upward.
 3642                  */
 3643                 addr = stack_entry->end + grow_amount;
 3644 
 3645                 /*
 3646                  * If this puts us into the next entry, cut back our growth
 3647                  * to the available space. Also, see the note above.
 3648                  */
 3649                 if (addr > end) {
 3650                         stack_entry->avail_ssize = end - stack_entry->end;
 3651                         addr = end;
 3652                         if (stack_guard_page)
 3653                                 addr -= PAGE_SIZE;
 3654                 }
 3655 
 3656                 grow_amount = addr - stack_entry->end;
 3657                 cred = stack_entry->cred;
 3658                 if (cred == NULL && stack_entry->object.vm_object != NULL)
 3659                         cred = stack_entry->object.vm_object->cred;
 3660                 if (cred != NULL && !swap_reserve_by_cred(grow_amount, cred))
 3661                         rv = KERN_NO_SPACE;
 3662                 /* Grow the underlying object if applicable. */
 3663                 else if (stack_entry->object.vm_object == NULL ||
 3664                          vm_object_coalesce(stack_entry->object.vm_object,
 3665                          stack_entry->offset,
 3666                          (vm_size_t)(stack_entry->end - stack_entry->start),
 3667                          (vm_size_t)grow_amount, cred != NULL)) {
 3668                         map->size += (addr - stack_entry->end);
 3669                         /* Update the current entry. */
 3670                         stack_entry->end = addr;
 3671                         stack_entry->avail_ssize -= grow_amount;
 3672                         vm_map_entry_resize_free(map, stack_entry);
 3673                         rv = KERN_SUCCESS;
 3674 
 3675                         if (next_entry != &map->header)
 3676                                 vm_map_clip_start(map, next_entry, addr);
 3677                 } else
 3678                         rv = KERN_FAILURE;
 3679         }
 3680 
 3681         if (rv == KERN_SUCCESS && is_procstack)
 3682                 vm->vm_ssize += btoc(grow_amount);
 3683 
 3684         vm_map_unlock(map);
 3685 
 3686         /*
 3687          * Heed the MAP_WIREFUTURE flag if it was set for this process.
 3688          */
 3689         if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE)) {
 3690                 vm_map_wire(map,
 3691                     (stack_entry == next_entry) ? addr : addr - grow_amount,
 3692                     (stack_entry == next_entry) ? stack_entry->start : addr,
 3693                     (p->p_flag & P_SYSTEM)
 3694                     ? VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES
 3695                     : VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
 3696         }
 3697 
 3698 out:
 3699 #ifdef RACCT
 3700         if (rv != KERN_SUCCESS) {
 3701                 PROC_LOCK(p);
 3702                 error = racct_set(p, RACCT_VMEM, map->size);
 3703                 KASSERT(error == 0, ("decreasing RACCT_VMEM failed"));
 3704                 if (!old_mlock) {
 3705                         error = racct_set(p, RACCT_MEMLOCK,
 3706                             ptoa(pmap_wired_count(map->pmap)));
 3707                         KASSERT(error == 0, ("decreasing RACCT_MEMLOCK failed"));
 3708                 }
 3709                 error = racct_set(p, RACCT_STACK, ctob(vm->vm_ssize));
 3710                 KASSERT(error == 0, ("decreasing RACCT_STACK failed"));
 3711                 PROC_UNLOCK(p);
 3712         }
 3713 #endif
 3714 
 3715         return (rv);
 3716 }
 3717 
 3718 /*
 3719  * Unshare the specified VM space for exec.  If other processes are
 3720  * mapped to it, then create a new one.  The new vmspace is null.
 3721  */
 3722 int
 3723 vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser)
 3724 {
 3725         struct vmspace *oldvmspace = p->p_vmspace;
 3726         struct vmspace *newvmspace;
 3727 
 3728         KASSERT((curthread->td_pflags & TDP_EXECVMSPC) == 0,
 3729             ("vmspace_exec recursed"));
 3730         newvmspace = vmspace_alloc(minuser, maxuser, NULL);
 3731         if (newvmspace == NULL)
 3732                 return (ENOMEM);
 3733         newvmspace->vm_swrss = oldvmspace->vm_swrss;
 3734         /*
 3735          * This code is written like this for prototype purposes.  The
 3736          * goal is to avoid running down the vmspace here, but let the
 3737          * other process's that are still using the vmspace to finally
 3738          * run it down.  Even though there is little or no chance of blocking
 3739          * here, it is a good idea to keep this form for future mods.
 3740          */
 3741         PROC_VMSPACE_LOCK(p);
 3742         p->p_vmspace = newvmspace;
 3743         PROC_VMSPACE_UNLOCK(p);
 3744         if (p == curthread->td_proc)
 3745                 pmap_activate(curthread);
 3746         curthread->td_pflags |= TDP_EXECVMSPC;
 3747         return (0);
 3748 }
 3749 
 3750 /*
 3751  * Unshare the specified VM space for forcing COW.  This
 3752  * is called by rfork, for the (RFMEM|RFPROC) == 0 case.
 3753  */
 3754 int
 3755 vmspace_unshare(struct proc *p)
 3756 {
 3757         struct vmspace *oldvmspace = p->p_vmspace;
 3758         struct vmspace *newvmspace;
 3759         vm_ooffset_t fork_charge;
 3760 
 3761         if (oldvmspace->vm_refcnt == 1)
 3762                 return (0);
 3763         fork_charge = 0;
 3764         newvmspace = vmspace_fork(oldvmspace, &fork_charge);
 3765         if (newvmspace == NULL)
 3766                 return (ENOMEM);
 3767         if (!swap_reserve_by_cred(fork_charge, p->p_ucred)) {
 3768                 vmspace_free(newvmspace);
 3769                 return (ENOMEM);
 3770         }
 3771         PROC_VMSPACE_LOCK(p);
 3772         p->p_vmspace = newvmspace;
 3773         PROC_VMSPACE_UNLOCK(p);
 3774         if (p == curthread->td_proc)
 3775                 pmap_activate(curthread);
 3776         vmspace_free(oldvmspace);
 3777         return (0);
 3778 }
 3779 
 3780 /*
 3781  *      vm_map_lookup:
 3782  *
 3783  *      Finds the VM object, offset, and
 3784  *      protection for a given virtual address in the
 3785  *      specified map, assuming a page fault of the
 3786  *      type specified.
 3787  *
 3788  *      Leaves the map in question locked for read; return
 3789  *      values are guaranteed until a vm_map_lookup_done
 3790  *      call is performed.  Note that the map argument
 3791  *      is in/out; the returned map must be used in
 3792  *      the call to vm_map_lookup_done.
 3793  *
 3794  *      A handle (out_entry) is returned for use in
 3795  *      vm_map_lookup_done, to make that fast.
 3796  *
 3797  *      If a lookup is requested with "write protection"
 3798  *      specified, the map may be changed to perform virtual
 3799  *      copying operations, although the data referenced will
 3800  *      remain the same.
 3801  */
 3802 int
 3803 vm_map_lookup(vm_map_t *var_map,                /* IN/OUT */
 3804               vm_offset_t vaddr,
 3805               vm_prot_t fault_typea,
 3806               vm_map_entry_t *out_entry,        /* OUT */
 3807               vm_object_t *object,              /* OUT */
 3808               vm_pindex_t *pindex,              /* OUT */
 3809               vm_prot_t *out_prot,              /* OUT */
 3810               boolean_t *wired)                 /* OUT */
 3811 {
 3812         vm_map_entry_t entry;
 3813         vm_map_t map = *var_map;
 3814         vm_prot_t prot;
 3815         vm_prot_t fault_type = fault_typea;
 3816         vm_object_t eobject;
 3817         vm_size_t size;
 3818         struct ucred *cred;
 3819 
 3820 RetryLookup:;
 3821 
 3822         vm_map_lock_read(map);
 3823 
 3824         /*
 3825          * Lookup the faulting address.
 3826          */
 3827         if (!vm_map_lookup_entry(map, vaddr, out_entry)) {
 3828                 vm_map_unlock_read(map);
 3829                 return (KERN_INVALID_ADDRESS);
 3830         }
 3831 
 3832         entry = *out_entry;
 3833 
 3834         /*
 3835          * Handle submaps.
 3836          */
 3837         if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 3838                 vm_map_t old_map = map;
 3839 
 3840                 *var_map = map = entry->object.sub_map;
 3841                 vm_map_unlock_read(old_map);
 3842                 goto RetryLookup;
 3843         }
 3844 
 3845         /*
 3846          * Check whether this task is allowed to have this page.
 3847          */
 3848         prot = entry->protection;
 3849         fault_type &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
 3850         if ((fault_type & prot) != fault_type || prot == VM_PROT_NONE) {
 3851                 vm_map_unlock_read(map);
 3852                 return (KERN_PROTECTION_FAILURE);
 3853         }
 3854         if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
 3855             (entry->eflags & MAP_ENTRY_COW) &&
 3856             (fault_type & VM_PROT_WRITE)) {
 3857                 vm_map_unlock_read(map);
 3858                 return (KERN_PROTECTION_FAILURE);
 3859         }
 3860         if ((fault_typea & VM_PROT_COPY) != 0 &&
 3861             (entry->max_protection & VM_PROT_WRITE) == 0 &&
 3862             (entry->eflags & MAP_ENTRY_COW) == 0) {
 3863                 vm_map_unlock_read(map);
 3864                 return (KERN_PROTECTION_FAILURE);
 3865         }
 3866 
 3867         /*
 3868          * If this page is not pageable, we have to get it for all possible
 3869          * accesses.
 3870          */
 3871         *wired = (entry->wired_count != 0);
 3872         if (*wired)
 3873                 fault_type = entry->protection;
 3874         size = entry->end - entry->start;
 3875         /*
 3876          * If the entry was copy-on-write, we either ...
 3877          */
 3878         if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 3879                 /*
 3880                  * If we want to write the page, we may as well handle that
 3881                  * now since we've got the map locked.
 3882                  *
 3883                  * If we don't need to write the page, we just demote the
 3884                  * permissions allowed.
 3885                  */
 3886                 if ((fault_type & VM_PROT_WRITE) != 0 ||
 3887                     (fault_typea & VM_PROT_COPY) != 0) {
 3888                         /*
 3889                          * Make a new object, and place it in the object
 3890                          * chain.  Note that no new references have appeared
 3891                          * -- one just moved from the map to the new
 3892                          * object.
 3893                          */
 3894                         if (vm_map_lock_upgrade(map))
 3895                                 goto RetryLookup;
 3896 
 3897                         if (entry->cred == NULL) {
 3898                                 /*
 3899                                  * The debugger owner is charged for
 3900                                  * the memory.
 3901                                  */
 3902                                 cred = curthread->td_ucred;
 3903                                 crhold(cred);
 3904                                 if (!swap_reserve_by_cred(size, cred)) {
 3905                                         crfree(cred);
 3906                                         vm_map_unlock(map);
 3907                                         return (KERN_RESOURCE_SHORTAGE);
 3908                                 }
 3909                                 entry->cred = cred;
 3910                         }
 3911                         vm_object_shadow(&entry->object.vm_object,
 3912                             &entry->offset, size);
 3913                         entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
 3914                         eobject = entry->object.vm_object;
 3915                         if (eobject->cred != NULL) {
 3916                                 /*
 3917                                  * The object was not shadowed.
 3918                                  */
 3919                                 swap_release_by_cred(size, entry->cred);
 3920                                 crfree(entry->cred);
 3921                                 entry->cred = NULL;
 3922                         } else if (entry->cred != NULL) {
 3923                                 VM_OBJECT_WLOCK(eobject);
 3924                                 eobject->cred = entry->cred;
 3925                                 eobject->charge = size;
 3926                                 VM_OBJECT_WUNLOCK(eobject);
 3927                                 entry->cred = NULL;
 3928                         }
 3929 
 3930                         vm_map_lock_downgrade(map);
 3931                 } else {
 3932                         /*
 3933                          * We're attempting to read a copy-on-write page --
 3934                          * don't allow writes.
 3935                          */
 3936                         prot &= ~VM_PROT_WRITE;
 3937                 }
 3938         }
 3939 
 3940         /*
 3941          * Create an object if necessary.
 3942          */
 3943         if (entry->object.vm_object == NULL &&
 3944             !map->system_map) {
 3945                 if (vm_map_lock_upgrade(map))
 3946                         goto RetryLookup;
 3947                 entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT,
 3948                     atop(size));
 3949                 entry->offset = 0;
 3950                 if (entry->cred != NULL) {
 3951                         VM_OBJECT_WLOCK(entry->object.vm_object);
 3952                         entry->object.vm_object->cred = entry->cred;
 3953                         entry->object.vm_object->charge = size;
 3954                         VM_OBJECT_WUNLOCK(entry->object.vm_object);
 3955                         entry->cred = NULL;
 3956                 }
 3957                 vm_map_lock_downgrade(map);
 3958         }
 3959 
 3960         /*
 3961          * Return the object/offset from this entry.  If the entry was
 3962          * copy-on-write or empty, it has been fixed up.
 3963          */
 3964         *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
 3965         *object = entry->object.vm_object;
 3966 
 3967         *out_prot = prot;
 3968         return (KERN_SUCCESS);
 3969 }
 3970 
 3971 /*
 3972  *      vm_map_lookup_locked:
 3973  *
 3974  *      Lookup the faulting address.  A version of vm_map_lookup that returns 
 3975  *      KERN_FAILURE instead of blocking on map lock or memory allocation.
 3976  */
 3977 int
 3978 vm_map_lookup_locked(vm_map_t *var_map,         /* IN/OUT */
 3979                      vm_offset_t vaddr,
 3980                      vm_prot_t fault_typea,
 3981                      vm_map_entry_t *out_entry, /* OUT */
 3982                      vm_object_t *object,       /* OUT */
 3983                      vm_pindex_t *pindex,       /* OUT */
 3984                      vm_prot_t *out_prot,       /* OUT */
 3985                      boolean_t *wired)          /* OUT */
 3986 {
 3987         vm_map_entry_t entry;
 3988         vm_map_t map = *var_map;
 3989         vm_prot_t prot;
 3990         vm_prot_t fault_type = fault_typea;
 3991 
 3992         /*
 3993          * Lookup the faulting address.
 3994          */
 3995         if (!vm_map_lookup_entry(map, vaddr, out_entry))
 3996                 return (KERN_INVALID_ADDRESS);
 3997 
 3998         entry = *out_entry;
 3999 
 4000         /*
 4001          * Fail if the entry refers to a submap.
 4002          */
 4003         if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
 4004                 return (KERN_FAILURE);
 4005 
 4006         /*
 4007          * Check whether this task is allowed to have this page.
 4008          */
 4009         prot = entry->protection;
 4010         fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
 4011         if ((fault_type & prot) != fault_type)
 4012                 return (KERN_PROTECTION_FAILURE);
 4013         if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
 4014             (entry->eflags & MAP_ENTRY_COW) &&
 4015             (fault_type & VM_PROT_WRITE))
 4016                 return (KERN_PROTECTION_FAILURE);
 4017 
 4018         /*
 4019          * If this page is not pageable, we have to get it for all possible
 4020          * accesses.
 4021          */
 4022         *wired = (entry->wired_count != 0);
 4023         if (*wired)
 4024                 fault_type = entry->protection;
 4025 
 4026         if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 4027                 /*
 4028                  * Fail if the entry was copy-on-write for a write fault.
 4029                  */
 4030                 if (fault_type & VM_PROT_WRITE)
 4031                         return (KERN_FAILURE);
 4032                 /*
 4033                  * We're attempting to read a copy-on-write page --
 4034                  * don't allow writes.
 4035                  */
 4036                 prot &= ~VM_PROT_WRITE;
 4037         }
 4038 
 4039         /*
 4040          * Fail if an object should be created.
 4041          */
 4042         if (entry->object.vm_object == NULL && !map->system_map)
 4043                 return (KERN_FAILURE);
 4044 
 4045         /*
 4046          * Return the object/offset from this entry.  If the entry was
 4047          * copy-on-write or empty, it has been fixed up.
 4048          */
 4049         *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
 4050         *object = entry->object.vm_object;
 4051 
 4052         *out_prot = prot;
 4053         return (KERN_SUCCESS);
 4054 }
 4055 
 4056 /*
 4057  *      vm_map_lookup_done:
 4058  *
 4059  *      Releases locks acquired by a vm_map_lookup
 4060  *      (according to the handle returned by that lookup).
 4061  */
 4062 void
 4063 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry)
 4064 {
 4065         /*
 4066          * Unlock the main-level map
 4067          */
 4068         vm_map_unlock_read(map);
 4069 }
 4070 
 4071 #include "opt_ddb.h"
 4072 #ifdef DDB
 4073 #include <sys/kernel.h>
 4074 
 4075 #include <ddb/ddb.h>
 4076 
 4077 static void
 4078 vm_map_print(vm_map_t map)
 4079 {
 4080         vm_map_entry_t entry;
 4081 
 4082         db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n",
 4083             (void *)map,
 4084             (void *)map->pmap, map->nentries, map->timestamp);
 4085 
 4086         db_indent += 2;
 4087         for (entry = map->header.next; entry != &map->header;
 4088             entry = entry->next) {
 4089                 db_iprintf("map entry %p: start=%p, end=%p\n",
 4090                     (void *)entry, (void *)entry->start, (void *)entry->end);
 4091                 {
 4092                         static char *inheritance_name[4] =
 4093                         {"share", "copy", "none", "donate_copy"};
 4094 
 4095                         db_iprintf(" prot=%x/%x/%s",
 4096                             entry->protection,
 4097                             entry->max_protection,
 4098                             inheritance_name[(int)(unsigned char)entry->inheritance]);
 4099                         if (entry->wired_count != 0)
 4100                                 db_printf(", wired");
 4101                 }
 4102                 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 4103                         db_printf(", share=%p, offset=0x%jx\n",
 4104                             (void *)entry->object.sub_map,
 4105                             (uintmax_t)entry->offset);
 4106                         if ((entry->prev == &map->header) ||
 4107                             (entry->prev->object.sub_map !=
 4108                                 entry->object.sub_map)) {
 4109                                 db_indent += 2;
 4110                                 vm_map_print((vm_map_t)entry->object.sub_map);
 4111                                 db_indent -= 2;
 4112                         }
 4113                 } else {
 4114                         if (entry->cred != NULL)
 4115                                 db_printf(", ruid %d", entry->cred->cr_ruid);
 4116                         db_printf(", object=%p, offset=0x%jx",
 4117                             (void *)entry->object.vm_object,
 4118                             (uintmax_t)entry->offset);
 4119                         if (entry->object.vm_object && entry->object.vm_object->cred)
 4120                                 db_printf(", obj ruid %d charge %jx",
 4121                                     entry->object.vm_object->cred->cr_ruid,
 4122                                     (uintmax_t)entry->object.vm_object->charge);
 4123                         if (entry->eflags & MAP_ENTRY_COW)
 4124                                 db_printf(", copy (%s)",
 4125                                     (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done");
 4126                         db_printf("\n");
 4127 
 4128                         if ((entry->prev == &map->header) ||
 4129                             (entry->prev->object.vm_object !=
 4130                                 entry->object.vm_object)) {
 4131                                 db_indent += 2;
 4132                                 vm_object_print((db_expr_t)(intptr_t)
 4133                                                 entry->object.vm_object,
 4134                                                 1, 0, (char *)0);
 4135                                 db_indent -= 2;
 4136                         }
 4137                 }
 4138         }
 4139         db_indent -= 2;
 4140 }
 4141 
 4142 DB_SHOW_COMMAND(map, map)
 4143 {
 4144 
 4145         if (!have_addr) {
 4146                 db_printf("usage: show map <addr>\n");
 4147                 return;
 4148         }
 4149         vm_map_print((vm_map_t)addr);
 4150 }
 4151 
 4152 DB_SHOW_COMMAND(procvm, procvm)
 4153 {
 4154         struct proc *p;
 4155 
 4156         if (have_addr) {
 4157                 p = (struct proc *) addr;
 4158         } else {
 4159                 p = curproc;
 4160         }
 4161 
 4162         db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n",
 4163             (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map,
 4164             (void *)vmspace_pmap(p->p_vmspace));
 4165 
 4166         vm_map_print((vm_map_t)&p->p_vmspace->vm_map);
 4167 }
 4168 
 4169 #endif /* DDB */

Cache object: 0b0ca894fbbd06b6962c3debbaaed3cd


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.