The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_map.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * The Mach Operating System project at Carnegie-Mellon University.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 4. Neither the name of the University nor the names of its contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      from: @(#)vm_map.c      8.3 (Berkeley) 1/12/94
   33  *
   34  *
   35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   36  * All rights reserved.
   37  *
   38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   39  *
   40  * Permission to use, copy, modify and distribute this software and
   41  * its documentation is hereby granted, provided that both the copyright
   42  * notice and this permission notice appear in all copies of the
   43  * software, derivative works or modified versions, and any portions
   44  * thereof, and that both notices appear in supporting documentation.
   45  *
   46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   49  *
   50  * Carnegie Mellon requests users of this software to return to
   51  *
   52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   53  *  School of Computer Science
   54  *  Carnegie Mellon University
   55  *  Pittsburgh PA 15213-3890
   56  *
   57  * any improvements or extensions that they make and grant Carnegie the
   58  * rights to redistribute these changes.
   59  */
   60 
   61 /*
   62  *      Virtual memory mapping module.
   63  */
   64 
   65 #include <sys/cdefs.h>
   66 __FBSDID("$FreeBSD: releng/7.4/sys/vm/vm_map.c 216554 2010-12-19 06:09:02Z mlaier $");
   67 
   68 #include <sys/param.h>
   69 #include <sys/systm.h>
   70 #include <sys/ktr.h>
   71 #include <sys/lock.h>
   72 #include <sys/mutex.h>
   73 #include <sys/proc.h>
   74 #include <sys/vmmeter.h>
   75 #include <sys/mman.h>
   76 #include <sys/vnode.h>
   77 #include <sys/resourcevar.h>
   78 #include <sys/file.h>
   79 #include <sys/sysent.h>
   80 #include <sys/shm.h>
   81 
   82 #include <vm/vm.h>
   83 #include <vm/vm_param.h>
   84 #include <vm/pmap.h>
   85 #include <vm/vm_map.h>
   86 #include <vm/vm_page.h>
   87 #include <vm/vm_object.h>
   88 #include <vm/vm_pager.h>
   89 #include <vm/vm_kern.h>
   90 #include <vm/vm_extern.h>
   91 #include <vm/swap_pager.h>
   92 #include <vm/uma.h>
   93 
   94 /*
   95  *      Virtual memory maps provide for the mapping, protection,
   96  *      and sharing of virtual memory objects.  In addition,
   97  *      this module provides for an efficient virtual copy of
   98  *      memory from one map to another.
   99  *
  100  *      Synchronization is required prior to most operations.
  101  *
  102  *      Maps consist of an ordered doubly-linked list of simple
  103  *      entries; a single hint is used to speed up lookups.
  104  *
  105  *      Since portions of maps are specified by start/end addresses,
  106  *      which may not align with existing map entries, all
  107  *      routines merely "clip" entries to these start/end values.
  108  *      [That is, an entry is split into two, bordering at a
  109  *      start or end value.]  Note that these clippings may not
  110  *      always be necessary (as the two resulting entries are then
  111  *      not changed); however, the clipping is done for convenience.
  112  *
  113  *      As mentioned above, virtual copy operations are performed
  114  *      by copying VM object references from one map to
  115  *      another, and then marking both regions as copy-on-write.
  116  */
  117 
  118 static struct mtx map_sleep_mtx;
  119 static uma_zone_t mapentzone;
  120 static uma_zone_t kmapentzone;
  121 static uma_zone_t mapzone;
  122 static uma_zone_t vmspace_zone;
  123 static struct vm_object kmapentobj;
  124 static int vmspace_zinit(void *mem, int size, int flags);
  125 static void vmspace_zfini(void *mem, int size);
  126 static int vm_map_zinit(void *mem, int ize, int flags);
  127 static void vm_map_zfini(void *mem, int size);
  128 static void _vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max);
  129 
  130 #ifdef INVARIANTS
  131 static void vm_map_zdtor(void *mem, int size, void *arg);
  132 static void vmspace_zdtor(void *mem, int size, void *arg);
  133 #endif
  134 
  135 /* 
  136  * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type
  137  * stable.
  138  */
  139 #define PROC_VMSPACE_LOCK(p) do { } while (0)
  140 #define PROC_VMSPACE_UNLOCK(p) do { } while (0)
  141 
  142 /*
  143  *      VM_MAP_RANGE_CHECK:     [ internal use only ]
  144  *
  145  *      Asserts that the starting and ending region
  146  *      addresses fall within the valid range of the map.
  147  */
  148 #define VM_MAP_RANGE_CHECK(map, start, end)             \
  149                 {                                       \
  150                 if (start < vm_map_min(map))            \
  151                         start = vm_map_min(map);        \
  152                 if (end > vm_map_max(map))              \
  153                         end = vm_map_max(map);          \
  154                 if (start > end)                        \
  155                         start = end;                    \
  156                 }
  157 
  158 /*
  159  *      vm_map_startup:
  160  *
  161  *      Initialize the vm_map module.  Must be called before
  162  *      any other vm_map routines.
  163  *
  164  *      Map and entry structures are allocated from the general
  165  *      purpose memory pool with some exceptions:
  166  *
  167  *      - The kernel map and kmem submap are allocated statically.
  168  *      - Kernel map entries are allocated out of a static pool.
  169  *
  170  *      These restrictions are necessary since malloc() uses the
  171  *      maps and requires map entries.
  172  */
  173 
  174 void
  175 vm_map_startup(void)
  176 {
  177         mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF);
  178         mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL,
  179 #ifdef INVARIANTS
  180             vm_map_zdtor,
  181 #else
  182             NULL,
  183 #endif
  184             vm_map_zinit, vm_map_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  185         uma_prealloc(mapzone, MAX_KMAP);
  186         kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry),
  187             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
  188             UMA_ZONE_MTXCLASS | UMA_ZONE_VM);
  189         uma_prealloc(kmapentzone, MAX_KMAPENT);
  190         mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry),
  191             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
  192 }
  193 
  194 static void
  195 vmspace_zfini(void *mem, int size)
  196 {
  197         struct vmspace *vm;
  198 
  199         vm = (struct vmspace *)mem;
  200         vm_map_zfini(&vm->vm_map, sizeof(vm->vm_map));
  201 }
  202 
  203 static int
  204 vmspace_zinit(void *mem, int size, int flags)
  205 {
  206         struct vmspace *vm;
  207 
  208         vm = (struct vmspace *)mem;
  209 
  210         vm->vm_map.pmap = NULL;
  211         (void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags);
  212         return (0);
  213 }
  214 
  215 static void
  216 vm_map_zfini(void *mem, int size)
  217 {
  218         vm_map_t map;
  219 
  220         map = (vm_map_t)mem;
  221         mtx_destroy(&map->system_mtx);
  222         sx_destroy(&map->lock);
  223 }
  224 
  225 static int
  226 vm_map_zinit(void *mem, int size, int flags)
  227 {
  228         vm_map_t map;
  229 
  230         map = (vm_map_t)mem;
  231         map->nentries = 0;
  232         map->size = 0;
  233         mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
  234         sx_init(&map->lock, "user map");
  235         return (0);
  236 }
  237 
  238 #ifdef INVARIANTS
  239 static void
  240 vmspace_zdtor(void *mem, int size, void *arg)
  241 {
  242         struct vmspace *vm;
  243 
  244         vm = (struct vmspace *)mem;
  245 
  246         vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg);
  247 }
  248 static void
  249 vm_map_zdtor(void *mem, int size, void *arg)
  250 {
  251         vm_map_t map;
  252 
  253         map = (vm_map_t)mem;
  254         KASSERT(map->nentries == 0,
  255             ("map %p nentries == %d on free.",
  256             map, map->nentries));
  257         KASSERT(map->size == 0,
  258             ("map %p size == %lu on free.",
  259             map, (unsigned long)map->size));
  260 }
  261 #endif  /* INVARIANTS */
  262 
  263 /*
  264  * Allocate a vmspace structure, including a vm_map and pmap,
  265  * and initialize those structures.  The refcnt is set to 1.
  266  */
  267 struct vmspace *
  268 vmspace_alloc(min, max)
  269         vm_offset_t min, max;
  270 {
  271         struct vmspace *vm;
  272 
  273         vm = uma_zalloc(vmspace_zone, M_WAITOK);
  274         if (vm->vm_map.pmap == NULL && !pmap_pinit(vmspace_pmap(vm))) {
  275                 uma_zfree(vmspace_zone, vm);
  276                 return (NULL);
  277         }
  278         CTR1(KTR_VM, "vmspace_alloc: %p", vm);
  279         _vm_map_init(&vm->vm_map, min, max);
  280         vm->vm_map.pmap = vmspace_pmap(vm);             /* XXX */
  281         vm->vm_refcnt = 1;
  282         vm->vm_shm = NULL;
  283         vm->vm_swrss = 0;
  284         vm->vm_tsize = 0;
  285         vm->vm_dsize = 0;
  286         vm->vm_ssize = 0;
  287         vm->vm_taddr = 0;
  288         vm->vm_daddr = 0;
  289         vm->vm_maxsaddr = 0;
  290         return (vm);
  291 }
  292 
  293 void
  294 vm_init2(void)
  295 {
  296         uma_zone_set_obj(kmapentzone, &kmapentobj, lmin(cnt.v_page_count,
  297             (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE) / 8 +
  298              maxproc * 2 + maxfiles);
  299         vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL,
  300 #ifdef INVARIANTS
  301             vmspace_zdtor,
  302 #else
  303             NULL,
  304 #endif
  305             vmspace_zinit, vmspace_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  306 }
  307 
  308 static inline void
  309 vmspace_dofree(struct vmspace *vm)
  310 {
  311         CTR1(KTR_VM, "vmspace_free: %p", vm);
  312 
  313         /*
  314          * Make sure any SysV shm is freed, it might not have been in
  315          * exit1().
  316          */
  317         shmexit(vm);
  318 
  319         /*
  320          * Lock the map, to wait out all other references to it.
  321          * Delete all of the mappings and pages they hold, then call
  322          * the pmap module to reclaim anything left.
  323          */
  324         (void)vm_map_remove(&vm->vm_map, vm->vm_map.min_offset,
  325             vm->vm_map.max_offset);
  326 
  327         /*
  328          * XXX Comment out the pmap_release call for now. The
  329          * vmspace_zone is marked as UMA_ZONE_NOFREE, and bugs cause
  330          * pmap.resident_count to be != 0 on exit sometimes.
  331          */
  332 /*      pmap_release(vmspace_pmap(vm)); */
  333         uma_zfree(vmspace_zone, vm);
  334 }
  335 
  336 void
  337 vmspace_free(struct vmspace *vm)
  338 {
  339         int refcnt;
  340 
  341         if (vm->vm_refcnt == 0)
  342                 panic("vmspace_free: attempt to free already freed vmspace");
  343 
  344         do
  345                 refcnt = vm->vm_refcnt;
  346         while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
  347         if (refcnt == 1)
  348                 vmspace_dofree(vm);
  349 }
  350 
  351 void
  352 vmspace_exitfree(struct proc *p)
  353 {
  354         struct vmspace *vm;
  355 
  356         PROC_VMSPACE_LOCK(p);
  357         vm = p->p_vmspace;
  358         p->p_vmspace = NULL;
  359         PROC_VMSPACE_UNLOCK(p);
  360         KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace"));
  361         vmspace_free(vm);
  362 }
  363 
  364 void
  365 vmspace_exit(struct thread *td)
  366 {
  367         int refcnt;
  368         struct vmspace *vm;
  369         struct proc *p;
  370 
  371         /*
  372          * Release user portion of address space.
  373          * This releases references to vnodes,
  374          * which could cause I/O if the file has been unlinked.
  375          * Need to do this early enough that we can still sleep.
  376          *
  377          * The last exiting process to reach this point releases as
  378          * much of the environment as it can. vmspace_dofree() is the
  379          * slower fallback in case another process had a temporary
  380          * reference to the vmspace.
  381          */
  382 
  383         p = td->td_proc;
  384         vm = p->p_vmspace;
  385         atomic_add_int(&vmspace0.vm_refcnt, 1);
  386         do {
  387                 refcnt = vm->vm_refcnt;
  388                 if (refcnt > 1 && p->p_vmspace != &vmspace0) {
  389                         /* Switch now since other proc might free vmspace */
  390                         PROC_VMSPACE_LOCK(p);
  391                         p->p_vmspace = &vmspace0;
  392                         PROC_VMSPACE_UNLOCK(p);
  393                         pmap_activate(td);
  394                 }
  395         } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
  396         if (refcnt == 1) {
  397                 if (p->p_vmspace != vm) {
  398                         /* vmspace not yet freed, switch back */
  399                         PROC_VMSPACE_LOCK(p);
  400                         p->p_vmspace = vm;
  401                         PROC_VMSPACE_UNLOCK(p);
  402                         pmap_activate(td);
  403                 }
  404                 pmap_remove_pages(vmspace_pmap(vm));
  405                 /* Switch now since this proc will free vmspace */
  406                 PROC_VMSPACE_LOCK(p);
  407                 p->p_vmspace = &vmspace0;
  408                 PROC_VMSPACE_UNLOCK(p);
  409                 pmap_activate(td);
  410                 vmspace_dofree(vm);
  411         }
  412 }
  413 
  414 /* Acquire reference to vmspace owned by another process. */
  415 
  416 struct vmspace *
  417 vmspace_acquire_ref(struct proc *p)
  418 {
  419         struct vmspace *vm;
  420         int refcnt;
  421 
  422         PROC_VMSPACE_LOCK(p);
  423         vm = p->p_vmspace;
  424         if (vm == NULL) {
  425                 PROC_VMSPACE_UNLOCK(p);
  426                 return (NULL);
  427         }
  428         do {
  429                 refcnt = vm->vm_refcnt;
  430                 if (refcnt <= 0) {      /* Avoid 0->1 transition */
  431                         PROC_VMSPACE_UNLOCK(p);
  432                         return (NULL);
  433                 }
  434         } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt + 1));
  435         if (vm != p->p_vmspace) {
  436                 PROC_VMSPACE_UNLOCK(p);
  437                 vmspace_free(vm);
  438                 return (NULL);
  439         }
  440         PROC_VMSPACE_UNLOCK(p);
  441         return (vm);
  442 }
  443 
  444 void
  445 _vm_map_lock(vm_map_t map, const char *file, int line)
  446 {
  447 
  448         if (map->system_map)
  449                 _mtx_lock_flags(&map->system_mtx, 0, file, line);
  450         else
  451                 (void)_sx_xlock(&map->lock, 0, file, line);
  452         map->timestamp++;
  453 }
  454 
  455 void
  456 _vm_map_unlock(vm_map_t map, const char *file, int line)
  457 {
  458 
  459         if (map->system_map)
  460                 _mtx_unlock_flags(&map->system_mtx, 0, file, line);
  461         else
  462                 _sx_xunlock(&map->lock, file, line);
  463 }
  464 
  465 void
  466 _vm_map_lock_read(vm_map_t map, const char *file, int line)
  467 {
  468 
  469         if (map->system_map)
  470                 _mtx_lock_flags(&map->system_mtx, 0, file, line);
  471         else
  472                 (void)_sx_xlock(&map->lock, 0, file, line);
  473 }
  474 
  475 void
  476 _vm_map_unlock_read(vm_map_t map, const char *file, int line)
  477 {
  478 
  479         if (map->system_map)
  480                 _mtx_unlock_flags(&map->system_mtx, 0, file, line);
  481         else
  482                 _sx_xunlock(&map->lock, file, line);
  483 }
  484 
  485 int
  486 _vm_map_trylock(vm_map_t map, const char *file, int line)
  487 {
  488         int error;
  489 
  490         error = map->system_map ?
  491             !_mtx_trylock(&map->system_mtx, 0, file, line) :
  492             !_sx_try_xlock(&map->lock, file, line);
  493         if (error == 0)
  494                 map->timestamp++;
  495         return (error == 0);
  496 }
  497 
  498 int
  499 _vm_map_trylock_read(vm_map_t map, const char *file, int line)
  500 {
  501         int error;
  502 
  503         error = map->system_map ?
  504             !_mtx_trylock(&map->system_mtx, 0, file, line) :
  505             !_sx_try_xlock(&map->lock, file, line);
  506         return (error == 0);
  507 }
  508 
  509 int
  510 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line)
  511 {
  512 
  513 #ifdef INVARIANTS
  514         if (map->system_map) {
  515                 _mtx_assert(&map->system_mtx, MA_OWNED, file, line);
  516         } else
  517                 _sx_assert(&map->lock, SX_XLOCKED, file, line);
  518 #endif
  519         map->timestamp++;
  520         return (0);
  521 }
  522 
  523 void
  524 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
  525 {
  526 
  527 #ifdef INVARIANTS
  528         if (map->system_map) {
  529                 _mtx_assert(&map->system_mtx, MA_OWNED, file, line);
  530         } else
  531                 _sx_assert(&map->lock, SX_XLOCKED, file, line);
  532 #endif
  533 }
  534 
  535 /*
  536  *      vm_map_unlock_and_wait:
  537  */
  538 int
  539 vm_map_unlock_and_wait(vm_map_t map, boolean_t user_wait)
  540 {
  541 
  542         mtx_lock(&map_sleep_mtx);
  543         vm_map_unlock(map);
  544         return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", 0));
  545 }
  546 
  547 /*
  548  *      vm_map_wakeup:
  549  */
  550 void
  551 vm_map_wakeup(vm_map_t map)
  552 {
  553 
  554         /*
  555          * Acquire and release map_sleep_mtx to prevent a wakeup()
  556          * from being performed (and lost) between the vm_map_unlock()
  557          * and the msleep() in vm_map_unlock_and_wait().
  558          */
  559         mtx_lock(&map_sleep_mtx);
  560         mtx_unlock(&map_sleep_mtx);
  561         wakeup(&map->root);
  562 }
  563 
  564 void
  565 vm_map_busy(vm_map_t map)
  566 {
  567 
  568 #ifdef INVARIANTS
  569         if (map->system_map) {
  570                 mtx_assert(&map->system_mtx, MA_OWNED);
  571         } else
  572                 sx_assert(&map->lock, SX_XLOCKED);
  573 #endif
  574         map->busy++;
  575 }
  576 
  577 void
  578 vm_map_unbusy(vm_map_t map)
  579 {
  580 
  581 #ifdef INVARIANTS
  582         if (map->system_map) {
  583                 mtx_assert(&map->system_mtx, MA_OWNED);
  584         } else
  585                 sx_assert(&map->lock, SX_XLOCKED);
  586 #endif
  587         KASSERT(map->busy, ("vm_map_unbusy: not busy"));
  588         if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) {
  589                 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP);
  590                 wakeup(&map->busy);
  591         }
  592 }
  593 
  594 void 
  595 vm_map_wait_busy(vm_map_t map)
  596 {
  597 
  598 #ifdef INVARIANTS
  599         if (map->system_map) {
  600                 mtx_assert(&map->system_mtx, MA_OWNED);
  601         } else
  602                 sx_assert(&map->lock, SX_XLOCKED);
  603 #endif
  604         while (map->busy) {
  605                 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0);
  606                 if (map->system_map)
  607                         msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0);
  608                 else
  609                         sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0);
  610         }
  611         map->timestamp++;
  612 }
  613 
  614 long
  615 vmspace_resident_count(struct vmspace *vmspace)
  616 {
  617         return pmap_resident_count(vmspace_pmap(vmspace));
  618 }
  619 
  620 long
  621 vmspace_wired_count(struct vmspace *vmspace)
  622 {
  623         return pmap_wired_count(vmspace_pmap(vmspace));
  624 }
  625 
  626 /*
  627  *      vm_map_create:
  628  *
  629  *      Creates and returns a new empty VM map with
  630  *      the given physical map structure, and having
  631  *      the given lower and upper address bounds.
  632  */
  633 vm_map_t
  634 vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max)
  635 {
  636         vm_map_t result;
  637 
  638         result = uma_zalloc(mapzone, M_WAITOK);
  639         CTR1(KTR_VM, "vm_map_create: %p", result);
  640         _vm_map_init(result, min, max);
  641         result->pmap = pmap;
  642         return (result);
  643 }
  644 
  645 /*
  646  * Initialize an existing vm_map structure
  647  * such as that in the vmspace structure.
  648  * The pmap is set elsewhere.
  649  */
  650 static void
  651 _vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max)
  652 {
  653 
  654         map->header.next = map->header.prev = &map->header;
  655         map->needs_wakeup = FALSE;
  656         map->system_map = 0;
  657         map->min_offset = min;
  658         map->max_offset = max;
  659         map->flags = 0;
  660         map->root = NULL;
  661         map->timestamp = 0;
  662         map->busy = 0;
  663 }
  664 
  665 void
  666 vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max)
  667 {
  668         _vm_map_init(map, min, max);
  669         mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
  670         sx_init(&map->lock, "user map");
  671 }
  672 
  673 /*
  674  *      vm_map_entry_dispose:   [ internal use only ]
  675  *
  676  *      Inverse of vm_map_entry_create.
  677  */
  678 static void
  679 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry)
  680 {
  681         uma_zfree(map->system_map ? kmapentzone : mapentzone, entry);
  682 }
  683 
  684 /*
  685  *      vm_map_entry_create:    [ internal use only ]
  686  *
  687  *      Allocates a VM map entry for insertion.
  688  *      No entry fields are filled in.
  689  */
  690 static vm_map_entry_t
  691 vm_map_entry_create(vm_map_t map)
  692 {
  693         vm_map_entry_t new_entry;
  694 
  695         if (map->system_map)
  696                 new_entry = uma_zalloc(kmapentzone, M_NOWAIT);
  697         else
  698                 new_entry = uma_zalloc(mapentzone, M_WAITOK);
  699         if (new_entry == NULL)
  700                 panic("vm_map_entry_create: kernel resources exhausted");
  701         return (new_entry);
  702 }
  703 
  704 /*
  705  *      vm_map_entry_set_behavior:
  706  *
  707  *      Set the expected access behavior, either normal, random, or
  708  *      sequential.
  709  */
  710 static inline void
  711 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior)
  712 {
  713         entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) |
  714             (behavior & MAP_ENTRY_BEHAV_MASK);
  715 }
  716 
  717 /*
  718  *      vm_map_entry_set_max_free:
  719  *
  720  *      Set the max_free field in a vm_map_entry.
  721  */
  722 static inline void
  723 vm_map_entry_set_max_free(vm_map_entry_t entry)
  724 {
  725 
  726         entry->max_free = entry->adj_free;
  727         if (entry->left != NULL && entry->left->max_free > entry->max_free)
  728                 entry->max_free = entry->left->max_free;
  729         if (entry->right != NULL && entry->right->max_free > entry->max_free)
  730                 entry->max_free = entry->right->max_free;
  731 }
  732 
  733 /*
  734  *      vm_map_entry_splay:
  735  *
  736  *      The Sleator and Tarjan top-down splay algorithm with the
  737  *      following variation.  Max_free must be computed bottom-up, so
  738  *      on the downward pass, maintain the left and right spines in
  739  *      reverse order.  Then, make a second pass up each side to fix
  740  *      the pointers and compute max_free.  The time bound is O(log n)
  741  *      amortized.
  742  *
  743  *      The new root is the vm_map_entry containing "addr", or else an
  744  *      adjacent entry (lower or higher) if addr is not in the tree.
  745  *
  746  *      The map must be locked, and leaves it so.
  747  *
  748  *      Returns: the new root.
  749  */
  750 static vm_map_entry_t
  751 vm_map_entry_splay(vm_offset_t addr, vm_map_entry_t root)
  752 {
  753         vm_map_entry_t llist, rlist;
  754         vm_map_entry_t ltree, rtree;
  755         vm_map_entry_t y;
  756 
  757         /* Special case of empty tree. */
  758         if (root == NULL)
  759                 return (root);
  760 
  761         /*
  762          * Pass One: Splay down the tree until we find addr or a NULL
  763          * pointer where addr would go.  llist and rlist are the two
  764          * sides in reverse order (bottom-up), with llist linked by
  765          * the right pointer and rlist linked by the left pointer in
  766          * the vm_map_entry.  Wait until Pass Two to set max_free on
  767          * the two spines.
  768          */
  769         llist = NULL;
  770         rlist = NULL;
  771         for (;;) {
  772                 /* root is never NULL in here. */
  773                 if (addr < root->start) {
  774                         y = root->left;
  775                         if (y == NULL)
  776                                 break;
  777                         if (addr < y->start && y->left != NULL) {
  778                                 /* Rotate right and put y on rlist. */
  779                                 root->left = y->right;
  780                                 y->right = root;
  781                                 vm_map_entry_set_max_free(root);
  782                                 root = y->left;
  783                                 y->left = rlist;
  784                                 rlist = y;
  785                         } else {
  786                                 /* Put root on rlist. */
  787                                 root->left = rlist;
  788                                 rlist = root;
  789                                 root = y;
  790                         }
  791                 } else {
  792                         y = root->right;
  793                         if (addr < root->end || y == NULL)
  794                                 break;
  795                         if (addr >= y->end && y->right != NULL) {
  796                                 /* Rotate left and put y on llist. */
  797                                 root->right = y->left;
  798                                 y->left = root;
  799                                 vm_map_entry_set_max_free(root);
  800                                 root = y->right;
  801                                 y->right = llist;
  802                                 llist = y;
  803                         } else {
  804                                 /* Put root on llist. */
  805                                 root->right = llist;
  806                                 llist = root;
  807                                 root = y;
  808                         }
  809                 }
  810         }
  811 
  812         /*
  813          * Pass Two: Walk back up the two spines, flip the pointers
  814          * and set max_free.  The subtrees of the root go at the
  815          * bottom of llist and rlist.
  816          */
  817         ltree = root->left;
  818         while (llist != NULL) {
  819                 y = llist->right;
  820                 llist->right = ltree;
  821                 vm_map_entry_set_max_free(llist);
  822                 ltree = llist;
  823                 llist = y;
  824         }
  825         rtree = root->right;
  826         while (rlist != NULL) {
  827                 y = rlist->left;
  828                 rlist->left = rtree;
  829                 vm_map_entry_set_max_free(rlist);
  830                 rtree = rlist;
  831                 rlist = y;
  832         }
  833 
  834         /*
  835          * Final assembly: add ltree and rtree as subtrees of root.
  836          */
  837         root->left = ltree;
  838         root->right = rtree;
  839         vm_map_entry_set_max_free(root);
  840 
  841         return (root);
  842 }
  843 
  844 /*
  845  *      vm_map_entry_{un,}link:
  846  *
  847  *      Insert/remove entries from maps.
  848  */
  849 static void
  850 vm_map_entry_link(vm_map_t map,
  851                   vm_map_entry_t after_where,
  852                   vm_map_entry_t entry)
  853 {
  854 
  855         CTR4(KTR_VM,
  856             "vm_map_entry_link: map %p, nentries %d, entry %p, after %p", map,
  857             map->nentries, entry, after_where);
  858         map->nentries++;
  859         entry->prev = after_where;
  860         entry->next = after_where->next;
  861         entry->next->prev = entry;
  862         after_where->next = entry;
  863 
  864         if (after_where != &map->header) {
  865                 if (after_where != map->root)
  866                         vm_map_entry_splay(after_where->start, map->root);
  867                 entry->right = after_where->right;
  868                 entry->left = after_where;
  869                 after_where->right = NULL;
  870                 after_where->adj_free = entry->start - after_where->end;
  871                 vm_map_entry_set_max_free(after_where);
  872         } else {
  873                 entry->right = map->root;
  874                 entry->left = NULL;
  875         }
  876         entry->adj_free = (entry->next == &map->header ? map->max_offset :
  877             entry->next->start) - entry->end;
  878         vm_map_entry_set_max_free(entry);
  879         map->root = entry;
  880 }
  881 
  882 static void
  883 vm_map_entry_unlink(vm_map_t map,
  884                     vm_map_entry_t entry)
  885 {
  886         vm_map_entry_t next, prev, root;
  887 
  888         if (entry != map->root)
  889                 vm_map_entry_splay(entry->start, map->root);
  890         if (entry->left == NULL)
  891                 root = entry->right;
  892         else {
  893                 root = vm_map_entry_splay(entry->start, entry->left);
  894                 root->right = entry->right;
  895                 root->adj_free = (entry->next == &map->header ? map->max_offset :
  896                     entry->next->start) - root->end;
  897                 vm_map_entry_set_max_free(root);
  898         }
  899         map->root = root;
  900 
  901         prev = entry->prev;
  902         next = entry->next;
  903         next->prev = prev;
  904         prev->next = next;
  905         map->nentries--;
  906         CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map,
  907             map->nentries, entry);
  908 }
  909 
  910 /*
  911  *      vm_map_entry_resize_free:
  912  *
  913  *      Recompute the amount of free space following a vm_map_entry
  914  *      and propagate that value up the tree.  Call this function after
  915  *      resizing a map entry in-place, that is, without a call to
  916  *      vm_map_entry_link() or _unlink().
  917  *
  918  *      The map must be locked, and leaves it so.
  919  */
  920 static void
  921 vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t entry)
  922 {
  923 
  924         /*
  925          * Using splay trees without parent pointers, propagating
  926          * max_free up the tree is done by moving the entry to the
  927          * root and making the change there.
  928          */
  929         if (entry != map->root)
  930                 map->root = vm_map_entry_splay(entry->start, map->root);
  931 
  932         entry->adj_free = (entry->next == &map->header ? map->max_offset :
  933             entry->next->start) - entry->end;
  934         vm_map_entry_set_max_free(entry);
  935 }
  936 
  937 /*
  938  *      vm_map_lookup_entry:    [ internal use only ]
  939  *
  940  *      Finds the map entry containing (or
  941  *      immediately preceding) the specified address
  942  *      in the given map; the entry is returned
  943  *      in the "entry" parameter.  The boolean
  944  *      result indicates whether the address is
  945  *      actually contained in the map.
  946  */
  947 boolean_t
  948 vm_map_lookup_entry(
  949         vm_map_t map,
  950         vm_offset_t address,
  951         vm_map_entry_t *entry)  /* OUT */
  952 {
  953         vm_map_entry_t cur;
  954 
  955         cur = vm_map_entry_splay(address, map->root);
  956         if (cur == NULL)
  957                 *entry = &map->header;
  958         else {
  959                 map->root = cur;
  960 
  961                 if (address >= cur->start) {
  962                         *entry = cur;
  963                         if (cur->end > address)
  964                                 return (TRUE);
  965                 } else
  966                         *entry = cur->prev;
  967         }
  968         return (FALSE);
  969 }
  970 
  971 /*
  972  *      vm_map_insert:
  973  *
  974  *      Inserts the given whole VM object into the target
  975  *      map at the specified address range.  The object's
  976  *      size should match that of the address range.
  977  *
  978  *      Requires that the map be locked, and leaves it so.
  979  *
  980  *      If object is non-NULL, ref count must be bumped by caller
  981  *      prior to making call to account for the new entry.
  982  */
  983 int
  984 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
  985               vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max,
  986               int cow)
  987 {
  988         vm_map_entry_t new_entry;
  989         vm_map_entry_t prev_entry;
  990         vm_map_entry_t temp_entry;
  991         vm_eflags_t protoeflags;
  992 
  993         /*
  994          * Check that the start and end points are not bogus.
  995          */
  996         if ((start < map->min_offset) || (end > map->max_offset) ||
  997             (start >= end))
  998                 return (KERN_INVALID_ADDRESS);
  999 
 1000         /*
 1001          * Find the entry prior to the proposed starting address; if it's part
 1002          * of an existing entry, this range is bogus.
 1003          */
 1004         if (vm_map_lookup_entry(map, start, &temp_entry))
 1005                 return (KERN_NO_SPACE);
 1006 
 1007         prev_entry = temp_entry;
 1008 
 1009         /*
 1010          * Assert that the next entry doesn't overlap the end point.
 1011          */
 1012         if ((prev_entry->next != &map->header) &&
 1013             (prev_entry->next->start < end))
 1014                 return (KERN_NO_SPACE);
 1015 
 1016         protoeflags = 0;
 1017 
 1018         if (cow & MAP_COPY_ON_WRITE)
 1019                 protoeflags |= MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY;
 1020 
 1021         if (cow & MAP_NOFAULT) {
 1022                 protoeflags |= MAP_ENTRY_NOFAULT;
 1023 
 1024                 KASSERT(object == NULL,
 1025                         ("vm_map_insert: paradoxical MAP_NOFAULT request"));
 1026         }
 1027         if (cow & MAP_DISABLE_SYNCER)
 1028                 protoeflags |= MAP_ENTRY_NOSYNC;
 1029         if (cow & MAP_DISABLE_COREDUMP)
 1030                 protoeflags |= MAP_ENTRY_NOCOREDUMP;
 1031 
 1032         /* Expand the kernel pmap, if necessary. */
 1033         if (map == kernel_map && end > kernel_vm_end)
 1034                 pmap_growkernel(end);
 1035         if (object != NULL) {
 1036                 /*
 1037                  * OBJ_ONEMAPPING must be cleared unless this mapping
 1038                  * is trivially proven to be the only mapping for any
 1039                  * of the object's pages.  (Object granularity
 1040                  * reference counting is insufficient to recognize
 1041                  * aliases with precision.)
 1042                  */
 1043                 VM_OBJECT_LOCK(object);
 1044                 if (object->ref_count > 1 || object->shadow_count != 0)
 1045                         vm_object_clear_flag(object, OBJ_ONEMAPPING);
 1046                 VM_OBJECT_UNLOCK(object);
 1047         }
 1048         else if ((prev_entry != &map->header) &&
 1049                  (prev_entry->eflags == protoeflags) &&
 1050                  (prev_entry->end == start) &&
 1051                  (prev_entry->wired_count == 0) &&
 1052                  ((prev_entry->object.vm_object == NULL) ||
 1053                   vm_object_coalesce(prev_entry->object.vm_object,
 1054                                      prev_entry->offset,
 1055                                      (vm_size_t)(prev_entry->end - prev_entry->start),
 1056                                      (vm_size_t)(end - prev_entry->end)))) {
 1057                 /*
 1058                  * We were able to extend the object.  Determine if we
 1059                  * can extend the previous map entry to include the
 1060                  * new range as well.
 1061                  */
 1062                 if ((prev_entry->inheritance == VM_INHERIT_DEFAULT) &&
 1063                     (prev_entry->protection == prot) &&
 1064                     (prev_entry->max_protection == max)) {
 1065                         map->size += (end - prev_entry->end);
 1066                         prev_entry->end = end;
 1067                         vm_map_entry_resize_free(map, prev_entry);
 1068                         vm_map_simplify_entry(map, prev_entry);
 1069                         return (KERN_SUCCESS);
 1070                 }
 1071 
 1072                 /*
 1073                  * If we can extend the object but cannot extend the
 1074                  * map entry, we have to create a new map entry.  We
 1075                  * must bump the ref count on the extended object to
 1076                  * account for it.  object may be NULL.
 1077                  */
 1078                 object = prev_entry->object.vm_object;
 1079                 offset = prev_entry->offset +
 1080                         (prev_entry->end - prev_entry->start);
 1081                 vm_object_reference(object);
 1082         }
 1083 
 1084         /*
 1085          * NOTE: if conditionals fail, object can be NULL here.  This occurs
 1086          * in things like the buffer map where we manage kva but do not manage
 1087          * backing objects.
 1088          */
 1089 
 1090         /*
 1091          * Create a new entry
 1092          */
 1093         new_entry = vm_map_entry_create(map);
 1094         new_entry->start = start;
 1095         new_entry->end = end;
 1096 
 1097         new_entry->eflags = protoeflags;
 1098         new_entry->object.vm_object = object;
 1099         new_entry->offset = offset;
 1100         new_entry->avail_ssize = 0;
 1101 
 1102         new_entry->inheritance = VM_INHERIT_DEFAULT;
 1103         new_entry->protection = prot;
 1104         new_entry->max_protection = max;
 1105         new_entry->wired_count = 0;
 1106 
 1107         /*
 1108          * Insert the new entry into the list
 1109          */
 1110         vm_map_entry_link(map, prev_entry, new_entry);
 1111         map->size += new_entry->end - new_entry->start;
 1112 
 1113 #if 0
 1114         /*
 1115          * Temporarily removed to avoid MAP_STACK panic, due to
 1116          * MAP_STACK being a huge hack.  Will be added back in
 1117          * when MAP_STACK (and the user stack mapping) is fixed.
 1118          */
 1119         /*
 1120          * It may be possible to simplify the entry
 1121          */
 1122         vm_map_simplify_entry(map, new_entry);
 1123 #endif
 1124 
 1125         if (cow & (MAP_PREFAULT|MAP_PREFAULT_PARTIAL)) {
 1126                 vm_map_pmap_enter(map, start, prot,
 1127                                     object, OFF_TO_IDX(offset), end - start,
 1128                                     cow & MAP_PREFAULT_PARTIAL);
 1129         }
 1130 
 1131         return (KERN_SUCCESS);
 1132 }
 1133 
 1134 /*
 1135  *      vm_map_findspace:
 1136  *
 1137  *      Find the first fit (lowest VM address) for "length" free bytes
 1138  *      beginning at address >= start in the given map.
 1139  *
 1140  *      In a vm_map_entry, "adj_free" is the amount of free space
 1141  *      adjacent (higher address) to this entry, and "max_free" is the
 1142  *      maximum amount of contiguous free space in its subtree.  This
 1143  *      allows finding a free region in one path down the tree, so
 1144  *      O(log n) amortized with splay trees.
 1145  *
 1146  *      The map must be locked, and leaves it so.
 1147  *
 1148  *      Returns: 0 on success, and starting address in *addr,
 1149  *               1 if insufficient space.
 1150  */
 1151 int
 1152 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length,
 1153     vm_offset_t *addr)  /* OUT */
 1154 {
 1155         vm_map_entry_t entry;
 1156         vm_offset_t st;
 1157 
 1158         /*
 1159          * Request must fit within min/max VM address and must avoid
 1160          * address wrap.
 1161          */
 1162         if (start < map->min_offset)
 1163                 start = map->min_offset;
 1164         if (start + length > map->max_offset || start + length < start)
 1165                 return (1);
 1166 
 1167         /* Empty tree means wide open address space. */
 1168         if (map->root == NULL) {
 1169                 *addr = start;
 1170                 return (0);
 1171         }
 1172 
 1173         /*
 1174          * After splay, if start comes before root node, then there
 1175          * must be a gap from start to the root.
 1176          */
 1177         map->root = vm_map_entry_splay(start, map->root);
 1178         if (start + length <= map->root->start) {
 1179                 *addr = start;
 1180                 return (0);
 1181         }
 1182 
 1183         /*
 1184          * Root is the last node that might begin its gap before
 1185          * start, and this is the last comparison where address
 1186          * wrap might be a problem.
 1187          */
 1188         st = (start > map->root->end) ? start : map->root->end;
 1189         if (length <= map->root->end + map->root->adj_free - st) {
 1190                 *addr = st;
 1191                 return (0);
 1192         }
 1193 
 1194         /* With max_free, can immediately tell if no solution. */
 1195         entry = map->root->right;
 1196         if (entry == NULL || length > entry->max_free)
 1197                 return (1);
 1198 
 1199         /*
 1200          * Search the right subtree in the order: left subtree, root,
 1201          * right subtree (first fit).  The previous splay implies that
 1202          * all regions in the right subtree have addresses > start.
 1203          */
 1204         while (entry != NULL) {
 1205                 if (entry->left != NULL && entry->left->max_free >= length)
 1206                         entry = entry->left;
 1207                 else if (entry->adj_free >= length) {
 1208                         *addr = entry->end;
 1209                         return (0);
 1210                 } else
 1211                         entry = entry->right;
 1212         }
 1213 
 1214         /* Can't get here, so panic if we do. */
 1215         panic("vm_map_findspace: max_free corrupt");
 1216 }
 1217 
 1218 int
 1219 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 1220     vm_offset_t start, vm_size_t length, vm_prot_t prot,
 1221     vm_prot_t max, int cow)
 1222 {
 1223         vm_offset_t end;
 1224         int result;
 1225 
 1226         vm_map_lock(map);
 1227         end = start + length;
 1228         VM_MAP_RANGE_CHECK(map, start, end);
 1229         (void) vm_map_delete(map, start, end);
 1230         result = vm_map_insert(map, object, offset, start, end, prot,
 1231             max, cow);
 1232         vm_map_unlock(map);
 1233         return (result);
 1234 }
 1235 
 1236 /*
 1237  *      vm_map_find finds an unallocated region in the target address
 1238  *      map with the given length.  The search is defined to be
 1239  *      first-fit from the specified address; the region found is
 1240  *      returned in the same parameter.
 1241  *
 1242  *      If object is non-NULL, ref count must be bumped by caller
 1243  *      prior to making call to account for the new entry.
 1244  */
 1245 int
 1246 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 1247             vm_offset_t *addr,  /* IN/OUT */
 1248             vm_size_t length, int find_space, vm_prot_t prot,
 1249             vm_prot_t max, int cow)
 1250 {
 1251         vm_offset_t start;
 1252         int result;
 1253 
 1254         start = *addr;
 1255         vm_map_lock(map);
 1256         do {
 1257                 if (find_space != VMFS_NO_SPACE) {
 1258                         if (vm_map_findspace(map, start, length, addr)) {
 1259                                 vm_map_unlock(map);
 1260                                 return (KERN_NO_SPACE);
 1261                         }
 1262                         if (find_space == VMFS_ALIGNED_SPACE)
 1263                                 pmap_align_superpage(object, offset, addr,
 1264                                     length);
 1265                         start = *addr;
 1266                 }
 1267                 result = vm_map_insert(map, object, offset, start, start +
 1268                     length, prot, max, cow);
 1269         } while (result == KERN_NO_SPACE && find_space == VMFS_ALIGNED_SPACE);
 1270         vm_map_unlock(map);
 1271         return (result);
 1272 }
 1273 
 1274 /*
 1275  *      vm_map_simplify_entry:
 1276  *
 1277  *      Simplify the given map entry by merging with either neighbor.  This
 1278  *      routine also has the ability to merge with both neighbors.
 1279  *
 1280  *      The map must be locked.
 1281  *
 1282  *      This routine guarentees that the passed entry remains valid (though
 1283  *      possibly extended).  When merging, this routine may delete one or
 1284  *      both neighbors.
 1285  */
 1286 void
 1287 vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry)
 1288 {
 1289         vm_map_entry_t next, prev;
 1290         vm_size_t prevsize, esize;
 1291 
 1292         if (entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP))
 1293                 return;
 1294 
 1295         prev = entry->prev;
 1296         if (prev != &map->header) {
 1297                 prevsize = prev->end - prev->start;
 1298                 if ( (prev->end == entry->start) &&
 1299                      (prev->object.vm_object == entry->object.vm_object) &&
 1300                      (!prev->object.vm_object ||
 1301                         (prev->offset + prevsize == entry->offset)) &&
 1302                      (prev->eflags == entry->eflags) &&
 1303                      (prev->protection == entry->protection) &&
 1304                      (prev->max_protection == entry->max_protection) &&
 1305                      (prev->inheritance == entry->inheritance) &&
 1306                      (prev->wired_count == entry->wired_count)) {
 1307                         vm_map_entry_unlink(map, prev);
 1308                         entry->start = prev->start;
 1309                         entry->offset = prev->offset;
 1310                         if (entry->prev != &map->header)
 1311                                 vm_map_entry_resize_free(map, entry->prev);
 1312                         if (prev->object.vm_object)
 1313                                 vm_object_deallocate(prev->object.vm_object);
 1314                         vm_map_entry_dispose(map, prev);
 1315                 }
 1316         }
 1317 
 1318         next = entry->next;
 1319         if (next != &map->header) {
 1320                 esize = entry->end - entry->start;
 1321                 if ((entry->end == next->start) &&
 1322                     (next->object.vm_object == entry->object.vm_object) &&
 1323                      (!entry->object.vm_object ||
 1324                         (entry->offset + esize == next->offset)) &&
 1325                     (next->eflags == entry->eflags) &&
 1326                     (next->protection == entry->protection) &&
 1327                     (next->max_protection == entry->max_protection) &&
 1328                     (next->inheritance == entry->inheritance) &&
 1329                     (next->wired_count == entry->wired_count)) {
 1330                         vm_map_entry_unlink(map, next);
 1331                         entry->end = next->end;
 1332                         vm_map_entry_resize_free(map, entry);
 1333                         if (next->object.vm_object)
 1334                                 vm_object_deallocate(next->object.vm_object);
 1335                         vm_map_entry_dispose(map, next);
 1336                 }
 1337         }
 1338 }
 1339 /*
 1340  *      vm_map_clip_start:      [ internal use only ]
 1341  *
 1342  *      Asserts that the given entry begins at or after
 1343  *      the specified address; if necessary,
 1344  *      it splits the entry into two.
 1345  */
 1346 #define vm_map_clip_start(map, entry, startaddr) \
 1347 { \
 1348         if (startaddr > entry->start) \
 1349                 _vm_map_clip_start(map, entry, startaddr); \
 1350 }
 1351 
 1352 /*
 1353  *      This routine is called only when it is known that
 1354  *      the entry must be split.
 1355  */
 1356 static void
 1357 _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start)
 1358 {
 1359         vm_map_entry_t new_entry;
 1360 
 1361         /*
 1362          * Split off the front portion -- note that we must insert the new
 1363          * entry BEFORE this one, so that this entry has the specified
 1364          * starting address.
 1365          */
 1366         vm_map_simplify_entry(map, entry);
 1367 
 1368         /*
 1369          * If there is no object backing this entry, we might as well create
 1370          * one now.  If we defer it, an object can get created after the map
 1371          * is clipped, and individual objects will be created for the split-up
 1372          * map.  This is a bit of a hack, but is also about the best place to
 1373          * put this improvement.
 1374          */
 1375         if (entry->object.vm_object == NULL && !map->system_map) {
 1376                 vm_object_t object;
 1377                 object = vm_object_allocate(OBJT_DEFAULT,
 1378                                 atop(entry->end - entry->start));
 1379                 entry->object.vm_object = object;
 1380                 entry->offset = 0;
 1381         }
 1382 
 1383         new_entry = vm_map_entry_create(map);
 1384         *new_entry = *entry;
 1385 
 1386         new_entry->end = start;
 1387         entry->offset += (start - entry->start);
 1388         entry->start = start;
 1389 
 1390         vm_map_entry_link(map, entry->prev, new_entry);
 1391 
 1392         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 1393                 vm_object_reference(new_entry->object.vm_object);
 1394         }
 1395 }
 1396 
 1397 /*
 1398  *      vm_map_clip_end:        [ internal use only ]
 1399  *
 1400  *      Asserts that the given entry ends at or before
 1401  *      the specified address; if necessary,
 1402  *      it splits the entry into two.
 1403  */
 1404 #define vm_map_clip_end(map, entry, endaddr) \
 1405 { \
 1406         if ((endaddr) < (entry->end)) \
 1407                 _vm_map_clip_end((map), (entry), (endaddr)); \
 1408 }
 1409 
 1410 /*
 1411  *      This routine is called only when it is known that
 1412  *      the entry must be split.
 1413  */
 1414 static void
 1415 _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end)
 1416 {
 1417         vm_map_entry_t new_entry;
 1418 
 1419         /*
 1420          * If there is no object backing this entry, we might as well create
 1421          * one now.  If we defer it, an object can get created after the map
 1422          * is clipped, and individual objects will be created for the split-up
 1423          * map.  This is a bit of a hack, but is also about the best place to
 1424          * put this improvement.
 1425          */
 1426         if (entry->object.vm_object == NULL && !map->system_map) {
 1427                 vm_object_t object;
 1428                 object = vm_object_allocate(OBJT_DEFAULT,
 1429                                 atop(entry->end - entry->start));
 1430                 entry->object.vm_object = object;
 1431                 entry->offset = 0;
 1432         }
 1433 
 1434         /*
 1435          * Create a new entry and insert it AFTER the specified entry
 1436          */
 1437         new_entry = vm_map_entry_create(map);
 1438         *new_entry = *entry;
 1439 
 1440         new_entry->start = entry->end = end;
 1441         new_entry->offset += (end - entry->start);
 1442 
 1443         vm_map_entry_link(map, entry, new_entry);
 1444 
 1445         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 1446                 vm_object_reference(new_entry->object.vm_object);
 1447         }
 1448 }
 1449 
 1450 /*
 1451  *      vm_map_submap:          [ kernel use only ]
 1452  *
 1453  *      Mark the given range as handled by a subordinate map.
 1454  *
 1455  *      This range must have been created with vm_map_find,
 1456  *      and no other operations may have been performed on this
 1457  *      range prior to calling vm_map_submap.
 1458  *
 1459  *      Only a limited number of operations can be performed
 1460  *      within this rage after calling vm_map_submap:
 1461  *              vm_fault
 1462  *      [Don't try vm_map_copy!]
 1463  *
 1464  *      To remove a submapping, one must first remove the
 1465  *      range from the superior map, and then destroy the
 1466  *      submap (if desired).  [Better yet, don't try it.]
 1467  */
 1468 int
 1469 vm_map_submap(
 1470         vm_map_t map,
 1471         vm_offset_t start,
 1472         vm_offset_t end,
 1473         vm_map_t submap)
 1474 {
 1475         vm_map_entry_t entry;
 1476         int result = KERN_INVALID_ARGUMENT;
 1477 
 1478         vm_map_lock(map);
 1479 
 1480         VM_MAP_RANGE_CHECK(map, start, end);
 1481 
 1482         if (vm_map_lookup_entry(map, start, &entry)) {
 1483                 vm_map_clip_start(map, entry, start);
 1484         } else
 1485                 entry = entry->next;
 1486 
 1487         vm_map_clip_end(map, entry, end);
 1488 
 1489         if ((entry->start == start) && (entry->end == end) &&
 1490             ((entry->eflags & MAP_ENTRY_COW) == 0) &&
 1491             (entry->object.vm_object == NULL)) {
 1492                 entry->object.sub_map = submap;
 1493                 entry->eflags |= MAP_ENTRY_IS_SUB_MAP;
 1494                 result = KERN_SUCCESS;
 1495         }
 1496         vm_map_unlock(map);
 1497 
 1498         return (result);
 1499 }
 1500 
 1501 /*
 1502  * The maximum number of pages to map
 1503  */
 1504 #define MAX_INIT_PT     96
 1505 
 1506 /*
 1507  *      vm_map_pmap_enter:
 1508  *
 1509  *      Preload read-only mappings for the given object's resident pages into
 1510  *      the given map.  This eliminates the soft faults on process startup and
 1511  *      immediately after an mmap(2).  Unless the given flags include
 1512  *      MAP_PREFAULT_MADVISE, cached pages are not reactivated and mapped.
 1513  */
 1514 void
 1515 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
 1516     vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags)
 1517 {
 1518         vm_offset_t start;
 1519         vm_page_t p, p_start;
 1520         vm_pindex_t psize, tmpidx;
 1521         boolean_t are_queues_locked;
 1522 
 1523         if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
 1524                 return;
 1525         VM_OBJECT_LOCK(object);
 1526         if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
 1527                 pmap_object_init_pt(map->pmap, addr, object, pindex, size);
 1528                 goto unlock_return;
 1529         }
 1530 
 1531         psize = atop(size);
 1532 
 1533         if ((flags & MAP_PREFAULT_PARTIAL) && psize > MAX_INIT_PT &&
 1534             object->resident_page_count > MAX_INIT_PT)
 1535                 goto unlock_return;
 1536 
 1537         if (psize + pindex > object->size) {
 1538                 if (object->size < pindex)
 1539                         goto unlock_return;
 1540                 psize = object->size - pindex;
 1541         }
 1542 
 1543         are_queues_locked = FALSE;
 1544         start = 0;
 1545         p_start = NULL;
 1546 
 1547         if ((p = TAILQ_FIRST(&object->memq)) != NULL) {
 1548                 if (p->pindex < pindex) {
 1549                         p = vm_page_splay(pindex, object->root);
 1550                         if ((object->root = p)->pindex < pindex)
 1551                                 p = TAILQ_NEXT(p, listq);
 1552                 }
 1553         }
 1554         /*
 1555          * Assert: the variable p is either (1) the page with the
 1556          * least pindex greater than or equal to the parameter pindex
 1557          * or (2) NULL.
 1558          */
 1559         for (;
 1560              p != NULL && (tmpidx = p->pindex - pindex) < psize;
 1561              p = TAILQ_NEXT(p, listq)) {
 1562                 /*
 1563                  * don't allow an madvise to blow away our really
 1564                  * free pages allocating pv entries.
 1565                  */
 1566                 if ((flags & MAP_PREFAULT_MADVISE) &&
 1567                     cnt.v_free_count < cnt.v_free_reserved) {
 1568                         psize = tmpidx;
 1569                         break;
 1570                 }
 1571                 if ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL &&
 1572                     (p->busy == 0)) {
 1573                         if (p_start == NULL) {
 1574                                 start = addr + ptoa(tmpidx);
 1575                                 p_start = p;
 1576                         }
 1577                 } else if (p_start != NULL) {
 1578                         if (!are_queues_locked) {
 1579                                 are_queues_locked = TRUE;
 1580                                 vm_page_lock_queues();
 1581                         }
 1582                         pmap_enter_object(map->pmap, start, addr +
 1583                             ptoa(tmpidx), p_start, prot);
 1584                         p_start = NULL;
 1585                 }
 1586         }
 1587         if (p_start != NULL) {
 1588                 if (!are_queues_locked) {
 1589                         are_queues_locked = TRUE;
 1590                         vm_page_lock_queues();
 1591                 }
 1592                 pmap_enter_object(map->pmap, start, addr + ptoa(psize),
 1593                     p_start, prot);
 1594         }
 1595         if (are_queues_locked)
 1596                 vm_page_unlock_queues();
 1597 unlock_return:
 1598         VM_OBJECT_UNLOCK(object);
 1599 }
 1600 
 1601 /*
 1602  *      vm_map_protect:
 1603  *
 1604  *      Sets the protection of the specified address
 1605  *      region in the target map.  If "set_max" is
 1606  *      specified, the maximum protection is to be set;
 1607  *      otherwise, only the current protection is affected.
 1608  */
 1609 int
 1610 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1611                vm_prot_t new_prot, boolean_t set_max)
 1612 {
 1613         vm_map_entry_t current;
 1614         vm_map_entry_t entry;
 1615 
 1616         vm_map_lock(map);
 1617 
 1618         VM_MAP_RANGE_CHECK(map, start, end);
 1619 
 1620         if (vm_map_lookup_entry(map, start, &entry)) {
 1621                 vm_map_clip_start(map, entry, start);
 1622         } else {
 1623                 entry = entry->next;
 1624         }
 1625 
 1626         /*
 1627          * Make a first pass to check for protection violations.
 1628          */
 1629         current = entry;
 1630         while ((current != &map->header) && (current->start < end)) {
 1631                 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
 1632                         vm_map_unlock(map);
 1633                         return (KERN_INVALID_ARGUMENT);
 1634                 }
 1635                 if ((new_prot & current->max_protection) != new_prot) {
 1636                         vm_map_unlock(map);
 1637                         return (KERN_PROTECTION_FAILURE);
 1638                 }
 1639                 current = current->next;
 1640         }
 1641 
 1642         /*
 1643          * Go back and fix up protections. [Note that clipping is not
 1644          * necessary the second time.]
 1645          */
 1646         current = entry;
 1647         while ((current != &map->header) && (current->start < end)) {
 1648                 vm_prot_t old_prot;
 1649 
 1650                 vm_map_clip_end(map, current, end);
 1651 
 1652                 old_prot = current->protection;
 1653                 if (set_max)
 1654                         current->protection =
 1655                             (current->max_protection = new_prot) &
 1656                             old_prot;
 1657                 else
 1658                         current->protection = new_prot;
 1659 
 1660                 /*
 1661                  * Update physical map if necessary. Worry about copy-on-write
 1662                  * here -- CHECK THIS XXX
 1663                  */
 1664                 if (current->protection != old_prot) {
 1665 #define MASK(entry)     (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \
 1666                                                         VM_PROT_ALL)
 1667                         pmap_protect(map->pmap, current->start,
 1668                             current->end,
 1669                             current->protection & MASK(current));
 1670 #undef  MASK
 1671                 }
 1672                 vm_map_simplify_entry(map, current);
 1673                 current = current->next;
 1674         }
 1675         vm_map_unlock(map);
 1676         return (KERN_SUCCESS);
 1677 }
 1678 
 1679 /*
 1680  *      vm_map_madvise:
 1681  *
 1682  *      This routine traverses a processes map handling the madvise
 1683  *      system call.  Advisories are classified as either those effecting
 1684  *      the vm_map_entry structure, or those effecting the underlying
 1685  *      objects.
 1686  */
 1687 int
 1688 vm_map_madvise(
 1689         vm_map_t map,
 1690         vm_offset_t start,
 1691         vm_offset_t end,
 1692         int behav)
 1693 {
 1694         vm_map_entry_t current, entry;
 1695         int modify_map = 0;
 1696 
 1697         /*
 1698          * Some madvise calls directly modify the vm_map_entry, in which case
 1699          * we need to use an exclusive lock on the map and we need to perform
 1700          * various clipping operations.  Otherwise we only need a read-lock
 1701          * on the map.
 1702          */
 1703         switch(behav) {
 1704         case MADV_NORMAL:
 1705         case MADV_SEQUENTIAL:
 1706         case MADV_RANDOM:
 1707         case MADV_NOSYNC:
 1708         case MADV_AUTOSYNC:
 1709         case MADV_NOCORE:
 1710         case MADV_CORE:
 1711                 modify_map = 1;
 1712                 vm_map_lock(map);
 1713                 break;
 1714         case MADV_WILLNEED:
 1715         case MADV_DONTNEED:
 1716         case MADV_FREE:
 1717                 vm_map_lock_read(map);
 1718                 break;
 1719         default:
 1720                 return (KERN_INVALID_ARGUMENT);
 1721         }
 1722 
 1723         /*
 1724          * Locate starting entry and clip if necessary.
 1725          */
 1726         VM_MAP_RANGE_CHECK(map, start, end);
 1727 
 1728         if (vm_map_lookup_entry(map, start, &entry)) {
 1729                 if (modify_map)
 1730                         vm_map_clip_start(map, entry, start);
 1731         } else {
 1732                 entry = entry->next;
 1733         }
 1734 
 1735         if (modify_map) {
 1736                 /*
 1737                  * madvise behaviors that are implemented in the vm_map_entry.
 1738                  *
 1739                  * We clip the vm_map_entry so that behavioral changes are
 1740                  * limited to the specified address range.
 1741                  */
 1742                 for (current = entry;
 1743                      (current != &map->header) && (current->start < end);
 1744                      current = current->next
 1745                 ) {
 1746                         if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
 1747                                 continue;
 1748 
 1749                         vm_map_clip_end(map, current, end);
 1750 
 1751                         switch (behav) {
 1752                         case MADV_NORMAL:
 1753                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_NORMAL);
 1754                                 break;
 1755                         case MADV_SEQUENTIAL:
 1756                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_SEQUENTIAL);
 1757                                 break;
 1758                         case MADV_RANDOM:
 1759                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_RANDOM);
 1760                                 break;
 1761                         case MADV_NOSYNC:
 1762                                 current->eflags |= MAP_ENTRY_NOSYNC;
 1763                                 break;
 1764                         case MADV_AUTOSYNC:
 1765                                 current->eflags &= ~MAP_ENTRY_NOSYNC;
 1766                                 break;
 1767                         case MADV_NOCORE:
 1768                                 current->eflags |= MAP_ENTRY_NOCOREDUMP;
 1769                                 break;
 1770                         case MADV_CORE:
 1771                                 current->eflags &= ~MAP_ENTRY_NOCOREDUMP;
 1772                                 break;
 1773                         default:
 1774                                 break;
 1775                         }
 1776                         vm_map_simplify_entry(map, current);
 1777                 }
 1778                 vm_map_unlock(map);
 1779         } else {
 1780                 vm_pindex_t pindex;
 1781                 int count;
 1782 
 1783                 /*
 1784                  * madvise behaviors that are implemented in the underlying
 1785                  * vm_object.
 1786                  *
 1787                  * Since we don't clip the vm_map_entry, we have to clip
 1788                  * the vm_object pindex and count.
 1789                  */
 1790                 for (current = entry;
 1791                      (current != &map->header) && (current->start < end);
 1792                      current = current->next
 1793                 ) {
 1794                         vm_offset_t useStart;
 1795 
 1796                         if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
 1797                                 continue;
 1798 
 1799                         pindex = OFF_TO_IDX(current->offset);
 1800                         count = atop(current->end - current->start);
 1801                         useStart = current->start;
 1802 
 1803                         if (current->start < start) {
 1804                                 pindex += atop(start - current->start);
 1805                                 count -= atop(start - current->start);
 1806                                 useStart = start;
 1807                         }
 1808                         if (current->end > end)
 1809                                 count -= atop(current->end - end);
 1810 
 1811                         if (count <= 0)
 1812                                 continue;
 1813 
 1814                         vm_object_madvise(current->object.vm_object,
 1815                                           pindex, count, behav);
 1816                         if (behav == MADV_WILLNEED) {
 1817                                 vm_map_pmap_enter(map,
 1818                                     useStart,
 1819                                     current->protection,
 1820                                     current->object.vm_object,
 1821                                     pindex,
 1822                                     (count << PAGE_SHIFT),
 1823                                     MAP_PREFAULT_MADVISE
 1824                                 );
 1825                         }
 1826                 }
 1827                 vm_map_unlock_read(map);
 1828         }
 1829         return (0);
 1830 }
 1831 
 1832 
 1833 /*
 1834  *      vm_map_inherit:
 1835  *
 1836  *      Sets the inheritance of the specified address
 1837  *      range in the target map.  Inheritance
 1838  *      affects how the map will be shared with
 1839  *      child maps at the time of vm_map_fork.
 1840  */
 1841 int
 1842 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1843                vm_inherit_t new_inheritance)
 1844 {
 1845         vm_map_entry_t entry;
 1846         vm_map_entry_t temp_entry;
 1847 
 1848         switch (new_inheritance) {
 1849         case VM_INHERIT_NONE:
 1850         case VM_INHERIT_COPY:
 1851         case VM_INHERIT_SHARE:
 1852                 break;
 1853         default:
 1854                 return (KERN_INVALID_ARGUMENT);
 1855         }
 1856         vm_map_lock(map);
 1857         VM_MAP_RANGE_CHECK(map, start, end);
 1858         if (vm_map_lookup_entry(map, start, &temp_entry)) {
 1859                 entry = temp_entry;
 1860                 vm_map_clip_start(map, entry, start);
 1861         } else
 1862                 entry = temp_entry->next;
 1863         while ((entry != &map->header) && (entry->start < end)) {
 1864                 vm_map_clip_end(map, entry, end);
 1865                 entry->inheritance = new_inheritance;
 1866                 vm_map_simplify_entry(map, entry);
 1867                 entry = entry->next;
 1868         }
 1869         vm_map_unlock(map);
 1870         return (KERN_SUCCESS);
 1871 }
 1872 
 1873 /*
 1874  *      vm_map_unwire:
 1875  *
 1876  *      Implements both kernel and user unwiring.
 1877  */
 1878 int
 1879 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1880     int flags)
 1881 {
 1882         vm_map_entry_t entry, first_entry, tmp_entry;
 1883         vm_offset_t saved_start;
 1884         unsigned int last_timestamp;
 1885         int rv;
 1886         boolean_t need_wakeup, result, user_unwire;
 1887 
 1888         user_unwire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
 1889         vm_map_lock(map);
 1890         VM_MAP_RANGE_CHECK(map, start, end);
 1891         if (!vm_map_lookup_entry(map, start, &first_entry)) {
 1892                 if (flags & VM_MAP_WIRE_HOLESOK)
 1893                         first_entry = first_entry->next;
 1894                 else {
 1895                         vm_map_unlock(map);
 1896                         return (KERN_INVALID_ADDRESS);
 1897                 }
 1898         }
 1899         last_timestamp = map->timestamp;
 1900         entry = first_entry;
 1901         while (entry != &map->header && entry->start < end) {
 1902                 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
 1903                         /*
 1904                          * We have not yet clipped the entry.
 1905                          */
 1906                         saved_start = (start >= entry->start) ? start :
 1907                             entry->start;
 1908                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 1909                         if (vm_map_unlock_and_wait(map, user_unwire)) {
 1910                                 /*
 1911                                  * Allow interruption of user unwiring?
 1912                                  */
 1913                         }
 1914                         vm_map_lock(map);
 1915                         if (last_timestamp+1 != map->timestamp) {
 1916                                 /*
 1917                                  * Look again for the entry because the map was
 1918                                  * modified while it was unlocked.
 1919                                  * Specifically, the entry may have been
 1920                                  * clipped, merged, or deleted.
 1921                                  */
 1922                                 if (!vm_map_lookup_entry(map, saved_start,
 1923                                     &tmp_entry)) {
 1924                                         if (flags & VM_MAP_WIRE_HOLESOK)
 1925                                                 tmp_entry = tmp_entry->next;
 1926                                         else {
 1927                                                 if (saved_start == start) {
 1928                                                         /*
 1929                                                          * First_entry has been deleted.
 1930                                                          */
 1931                                                         vm_map_unlock(map);
 1932                                                         return (KERN_INVALID_ADDRESS);
 1933                                                 }
 1934                                                 end = saved_start;
 1935                                                 rv = KERN_INVALID_ADDRESS;
 1936                                                 goto done;
 1937                                         }
 1938                                 }
 1939                                 if (entry == first_entry)
 1940                                         first_entry = tmp_entry;
 1941                                 else
 1942                                         first_entry = NULL;
 1943                                 entry = tmp_entry;
 1944                         }
 1945                         last_timestamp = map->timestamp;
 1946                         continue;
 1947                 }
 1948                 vm_map_clip_start(map, entry, start);
 1949                 vm_map_clip_end(map, entry, end);
 1950                 /*
 1951                  * Mark the entry in case the map lock is released.  (See
 1952                  * above.)
 1953                  */
 1954                 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
 1955                 /*
 1956                  * Check the map for holes in the specified region.
 1957                  * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
 1958                  */
 1959                 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
 1960                     (entry->end < end && (entry->next == &map->header ||
 1961                     entry->next->start > entry->end))) {
 1962                         end = entry->end;
 1963                         rv = KERN_INVALID_ADDRESS;
 1964                         goto done;
 1965                 }
 1966                 /*
 1967                  * If system unwiring, require that the entry is system wired.
 1968                  */
 1969                 if (!user_unwire &&
 1970                     vm_map_entry_system_wired_count(entry) == 0) {
 1971                         end = entry->end;
 1972                         rv = KERN_INVALID_ARGUMENT;
 1973                         goto done;
 1974                 }
 1975                 entry = entry->next;
 1976         }
 1977         rv = KERN_SUCCESS;
 1978 done:
 1979         need_wakeup = FALSE;
 1980         if (first_entry == NULL) {
 1981                 result = vm_map_lookup_entry(map, start, &first_entry);
 1982                 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
 1983                         first_entry = first_entry->next;
 1984                 else
 1985                         KASSERT(result, ("vm_map_unwire: lookup failed"));
 1986         }
 1987         entry = first_entry;
 1988         while (entry != &map->header && entry->start < end) {
 1989                 if (rv == KERN_SUCCESS && (!user_unwire ||
 1990                     (entry->eflags & MAP_ENTRY_USER_WIRED))) {
 1991                         if (user_unwire)
 1992                                 entry->eflags &= ~MAP_ENTRY_USER_WIRED;
 1993                         entry->wired_count--;
 1994                         if (entry->wired_count == 0) {
 1995                                 /*
 1996                                  * Retain the map lock.
 1997                                  */
 1998                                 vm_fault_unwire(map, entry->start, entry->end,
 1999                                     entry->object.vm_object != NULL &&
 2000                                     (entry->object.vm_object->type == OBJT_DEVICE ||
 2001                                     entry->object.vm_object->type == OBJT_SG));
 2002                         }
 2003                 }
 2004                 KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
 2005                         ("vm_map_unwire: in-transition flag missing"));
 2006                 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
 2007                 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
 2008                         entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
 2009                         need_wakeup = TRUE;
 2010                 }
 2011                 vm_map_simplify_entry(map, entry);
 2012                 entry = entry->next;
 2013         }
 2014         vm_map_unlock(map);
 2015         if (need_wakeup)
 2016                 vm_map_wakeup(map);
 2017         return (rv);
 2018 }
 2019 
 2020 /*
 2021  *      vm_map_wire:
 2022  *
 2023  *      Implements both kernel and user wiring.
 2024  */
 2025 int
 2026 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 2027     int flags)
 2028 {
 2029         vm_map_entry_t entry, first_entry, tmp_entry;
 2030         vm_offset_t saved_end, saved_start;
 2031         unsigned int last_timestamp;
 2032         int rv;
 2033         boolean_t fictitious, need_wakeup, result, user_wire;
 2034 
 2035         user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
 2036         vm_map_lock(map);
 2037         VM_MAP_RANGE_CHECK(map, start, end);
 2038         if (!vm_map_lookup_entry(map, start, &first_entry)) {
 2039                 if (flags & VM_MAP_WIRE_HOLESOK)
 2040                         first_entry = first_entry->next;
 2041                 else {
 2042                         vm_map_unlock(map);
 2043                         return (KERN_INVALID_ADDRESS);
 2044                 }
 2045         }
 2046         last_timestamp = map->timestamp;
 2047         entry = first_entry;
 2048         while (entry != &map->header && entry->start < end) {
 2049                 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
 2050                         /*
 2051                          * We have not yet clipped the entry.
 2052                          */
 2053                         saved_start = (start >= entry->start) ? start :
 2054                             entry->start;
 2055                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 2056                         if (vm_map_unlock_and_wait(map, user_wire)) {
 2057                                 /*
 2058                                  * Allow interruption of user wiring?
 2059                                  */
 2060                         }
 2061                         vm_map_lock(map);
 2062                         if (last_timestamp + 1 != map->timestamp) {
 2063                                 /*
 2064                                  * Look again for the entry because the map was
 2065                                  * modified while it was unlocked.
 2066                                  * Specifically, the entry may have been
 2067                                  * clipped, merged, or deleted.
 2068                                  */
 2069                                 if (!vm_map_lookup_entry(map, saved_start,
 2070                                     &tmp_entry)) {
 2071                                         if (flags & VM_MAP_WIRE_HOLESOK)
 2072                                                 tmp_entry = tmp_entry->next;
 2073                                         else {
 2074                                                 if (saved_start == start) {
 2075                                                         /*
 2076                                                          * first_entry has been deleted.
 2077                                                          */
 2078                                                         vm_map_unlock(map);
 2079                                                         return (KERN_INVALID_ADDRESS);
 2080                                                 }
 2081                                                 end = saved_start;
 2082                                                 rv = KERN_INVALID_ADDRESS;
 2083                                                 goto done;
 2084                                         }
 2085                                 }
 2086                                 if (entry == first_entry)
 2087                                         first_entry = tmp_entry;
 2088                                 else
 2089                                         first_entry = NULL;
 2090                                 entry = tmp_entry;
 2091                         }
 2092                         last_timestamp = map->timestamp;
 2093                         continue;
 2094                 }
 2095                 vm_map_clip_start(map, entry, start);
 2096                 vm_map_clip_end(map, entry, end);
 2097                 /*
 2098                  * Mark the entry in case the map lock is released.  (See
 2099                  * above.)
 2100                  */
 2101                 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
 2102                 /*
 2103                  *
 2104                  */
 2105                 if (entry->wired_count == 0) {
 2106                         if ((entry->protection & (VM_PROT_READ|VM_PROT_EXECUTE))
 2107                             == 0) {
 2108                                 entry->eflags |= MAP_ENTRY_WIRE_SKIPPED;
 2109                                 if ((flags & VM_MAP_WIRE_HOLESOK) == 0) {
 2110                                         end = entry->end;
 2111                                         rv = KERN_INVALID_ADDRESS;
 2112                                         goto done;
 2113                                 }
 2114                                 goto next_entry;
 2115                         }
 2116                         entry->wired_count++;
 2117                         saved_start = entry->start;
 2118                         saved_end = entry->end;
 2119                         fictitious = entry->object.vm_object != NULL &&
 2120                             (entry->object.vm_object->type == OBJT_DEVICE ||
 2121                             entry->object.vm_object->type == OBJT_SG);
 2122                         /*
 2123                          * Release the map lock, relying on the in-transition
 2124                          * mark.  Mark the map busy for fork.
 2125                          */
 2126                         vm_map_busy(map);
 2127                         vm_map_unlock(map);
 2128                         rv = vm_fault_wire(map, saved_start, saved_end,
 2129                             user_wire, fictitious);
 2130                         vm_map_lock(map);
 2131                         vm_map_unbusy(map);
 2132                         if (last_timestamp + 1 != map->timestamp) {
 2133                                 /*
 2134                                  * Look again for the entry because the map was
 2135                                  * modified while it was unlocked.  The entry
 2136                                  * may have been clipped, but NOT merged or
 2137                                  * deleted.
 2138                                  */
 2139                                 result = vm_map_lookup_entry(map, saved_start,
 2140                                     &tmp_entry);
 2141                                 KASSERT(result, ("vm_map_wire: lookup failed"));
 2142                                 if (entry == first_entry)
 2143                                         first_entry = tmp_entry;
 2144                                 else
 2145                                         first_entry = NULL;
 2146                                 entry = tmp_entry;
 2147                                 while (entry->end < saved_end) {
 2148                                         if (rv != KERN_SUCCESS) {
 2149                                                 KASSERT(entry->wired_count == 1,
 2150                                                     ("vm_map_wire: bad count"));
 2151                                                 entry->wired_count = -1;
 2152                                         }
 2153                                         entry = entry->next;
 2154                                 }
 2155                         }
 2156                         last_timestamp = map->timestamp;
 2157                         if (rv != KERN_SUCCESS) {
 2158                                 KASSERT(entry->wired_count == 1,
 2159                                     ("vm_map_wire: bad count"));
 2160                                 /*
 2161                                  * Assign an out-of-range value to represent
 2162                                  * the failure to wire this entry.
 2163                                  */
 2164                                 entry->wired_count = -1;
 2165                                 end = entry->end;
 2166                                 goto done;
 2167                         }
 2168                 } else if (!user_wire ||
 2169                            (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
 2170                         entry->wired_count++;
 2171                 }
 2172                 /*
 2173                  * Check the map for holes in the specified region.
 2174                  * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
 2175                  */
 2176         next_entry:
 2177                 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
 2178                     (entry->end < end && (entry->next == &map->header ||
 2179                     entry->next->start > entry->end))) {
 2180                         end = entry->end;
 2181                         rv = KERN_INVALID_ADDRESS;
 2182                         goto done;
 2183                 }
 2184                 entry = entry->next;
 2185         }
 2186         rv = KERN_SUCCESS;
 2187 done:
 2188         need_wakeup = FALSE;
 2189         if (first_entry == NULL) {
 2190                 result = vm_map_lookup_entry(map, start, &first_entry);
 2191                 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
 2192                         first_entry = first_entry->next;
 2193                 else
 2194                         KASSERT(result, ("vm_map_wire: lookup failed"));
 2195         }
 2196         entry = first_entry;
 2197         while (entry != &map->header && entry->start < end) {
 2198                 if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0)
 2199                         goto next_entry_done;
 2200                 if (rv == KERN_SUCCESS) {
 2201                         if (user_wire)
 2202                                 entry->eflags |= MAP_ENTRY_USER_WIRED;
 2203                 } else if (entry->wired_count == -1) {
 2204                         /*
 2205                          * Wiring failed on this entry.  Thus, unwiring is
 2206                          * unnecessary.
 2207                          */
 2208                         entry->wired_count = 0;
 2209                 } else {
 2210                         if (!user_wire ||
 2211                             (entry->eflags & MAP_ENTRY_USER_WIRED) == 0)
 2212                                 entry->wired_count--;
 2213                         if (entry->wired_count == 0) {
 2214                                 /*
 2215                                  * Retain the map lock.
 2216                                  */
 2217                                 vm_fault_unwire(map, entry->start, entry->end,
 2218                                     entry->object.vm_object != NULL &&
 2219                                     (entry->object.vm_object->type == OBJT_DEVICE ||
 2220                                     entry->object.vm_object->type == OBJT_SG));
 2221                         }
 2222                 }
 2223         next_entry_done:
 2224                 KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
 2225                         ("vm_map_wire: in-transition flag missing"));
 2226                 entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION|MAP_ENTRY_WIRE_SKIPPED);
 2227                 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
 2228                         entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
 2229                         need_wakeup = TRUE;
 2230                 }
 2231                 vm_map_simplify_entry(map, entry);
 2232                 entry = entry->next;
 2233         }
 2234         vm_map_unlock(map);
 2235         if (need_wakeup)
 2236                 vm_map_wakeup(map);
 2237         return (rv);
 2238 }
 2239 
 2240 /*
 2241  * vm_map_sync
 2242  *
 2243  * Push any dirty cached pages in the address range to their pager.
 2244  * If syncio is TRUE, dirty pages are written synchronously.
 2245  * If invalidate is TRUE, any cached pages are freed as well.
 2246  *
 2247  * If the size of the region from start to end is zero, we are
 2248  * supposed to flush all modified pages within the region containing
 2249  * start.  Unfortunately, a region can be split or coalesced with
 2250  * neighboring regions, making it difficult to determine what the
 2251  * original region was.  Therefore, we approximate this requirement by
 2252  * flushing the current region containing start.
 2253  *
 2254  * Returns an error if any part of the specified range is not mapped.
 2255  */
 2256 int
 2257 vm_map_sync(
 2258         vm_map_t map,
 2259         vm_offset_t start,
 2260         vm_offset_t end,
 2261         boolean_t syncio,
 2262         boolean_t invalidate)
 2263 {
 2264         vm_map_entry_t current;
 2265         vm_map_entry_t entry;
 2266         vm_size_t size;
 2267         vm_object_t object;
 2268         vm_ooffset_t offset;
 2269 
 2270         vm_map_lock_read(map);
 2271         VM_MAP_RANGE_CHECK(map, start, end);
 2272         if (!vm_map_lookup_entry(map, start, &entry)) {
 2273                 vm_map_unlock_read(map);
 2274                 return (KERN_INVALID_ADDRESS);
 2275         } else if (start == end) {
 2276                 start = entry->start;
 2277                 end = entry->end;
 2278         }
 2279         /*
 2280          * Make a first pass to check for user-wired memory and holes.
 2281          */
 2282         for (current = entry; current != &map->header && current->start < end;
 2283             current = current->next) {
 2284                 if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) {
 2285                         vm_map_unlock_read(map);
 2286                         return (KERN_INVALID_ARGUMENT);
 2287                 }
 2288                 if (end > current->end &&
 2289                     (current->next == &map->header ||
 2290                         current->end != current->next->start)) {
 2291                         vm_map_unlock_read(map);
 2292                         return (KERN_INVALID_ADDRESS);
 2293                 }
 2294         }
 2295 
 2296         if (invalidate)
 2297                 pmap_remove(map->pmap, start, end);
 2298 
 2299         /*
 2300          * Make a second pass, cleaning/uncaching pages from the indicated
 2301          * objects as we go.
 2302          */
 2303         for (current = entry; current != &map->header && current->start < end;
 2304             current = current->next) {
 2305                 offset = current->offset + (start - current->start);
 2306                 size = (end <= current->end ? end : current->end) - start;
 2307                 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
 2308                         vm_map_t smap;
 2309                         vm_map_entry_t tentry;
 2310                         vm_size_t tsize;
 2311 
 2312                         smap = current->object.sub_map;
 2313                         vm_map_lock_read(smap);
 2314                         (void) vm_map_lookup_entry(smap, offset, &tentry);
 2315                         tsize = tentry->end - offset;
 2316                         if (tsize < size)
 2317                                 size = tsize;
 2318                         object = tentry->object.vm_object;
 2319                         offset = tentry->offset + (offset - tentry->start);
 2320                         vm_map_unlock_read(smap);
 2321                 } else {
 2322                         object = current->object.vm_object;
 2323                 }
 2324                 vm_object_sync(object, offset, size, syncio, invalidate);
 2325                 start += size;
 2326         }
 2327 
 2328         vm_map_unlock_read(map);
 2329         return (KERN_SUCCESS);
 2330 }
 2331 
 2332 /*
 2333  *      vm_map_entry_unwire:    [ internal use only ]
 2334  *
 2335  *      Make the region specified by this entry pageable.
 2336  *
 2337  *      The map in question should be locked.
 2338  *      [This is the reason for this routine's existence.]
 2339  */
 2340 static void
 2341 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry)
 2342 {
 2343         vm_fault_unwire(map, entry->start, entry->end,
 2344             entry->object.vm_object != NULL &&
 2345             (entry->object.vm_object->type == OBJT_DEVICE ||
 2346             entry->object.vm_object->type == OBJT_SG));
 2347         entry->wired_count = 0;
 2348 }
 2349 
 2350 /*
 2351  *      vm_map_entry_delete:    [ internal use only ]
 2352  *
 2353  *      Deallocate the given entry from the target map.
 2354  */
 2355 static void
 2356 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry)
 2357 {
 2358         vm_object_t object;
 2359         vm_pindex_t offidxstart, offidxend, count;
 2360 
 2361         vm_map_entry_unlink(map, entry);
 2362         map->size -= entry->end - entry->start;
 2363 
 2364         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2365             (object = entry->object.vm_object) != NULL) {
 2366                 count = OFF_TO_IDX(entry->end - entry->start);
 2367                 offidxstart = OFF_TO_IDX(entry->offset);
 2368                 offidxend = offidxstart + count;
 2369                 VM_OBJECT_LOCK(object);
 2370                 if (object->ref_count != 1 &&
 2371                     ((object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING ||
 2372                     object == kernel_object || object == kmem_object)) {
 2373                         vm_object_collapse(object);
 2374                         vm_object_page_remove(object, offidxstart, offidxend, FALSE);
 2375                         if (object->type == OBJT_SWAP)
 2376                                 swap_pager_freespace(object, offidxstart, count);
 2377                         if (offidxend >= object->size &&
 2378                             offidxstart < object->size)
 2379                                 object->size = offidxstart;
 2380                 }
 2381                 VM_OBJECT_UNLOCK(object);
 2382                 vm_object_deallocate(object);
 2383         }
 2384 
 2385         vm_map_entry_dispose(map, entry);
 2386 }
 2387 
 2388 /*
 2389  *      vm_map_delete:  [ internal use only ]
 2390  *
 2391  *      Deallocates the given address range from the target
 2392  *      map.
 2393  */
 2394 int
 2395 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
 2396 {
 2397         vm_map_entry_t entry;
 2398         vm_map_entry_t first_entry;
 2399 
 2400         /*
 2401          * Find the start of the region, and clip it
 2402          */
 2403         if (!vm_map_lookup_entry(map, start, &first_entry))
 2404                 entry = first_entry->next;
 2405         else {
 2406                 entry = first_entry;
 2407                 vm_map_clip_start(map, entry, start);
 2408         }
 2409 
 2410         /*
 2411          * Step through all entries in this region
 2412          */
 2413         while ((entry != &map->header) && (entry->start < end)) {
 2414                 vm_map_entry_t next;
 2415 
 2416                 /*
 2417                  * Wait for wiring or unwiring of an entry to complete.
 2418                  * Also wait for any system wirings to disappear on
 2419                  * user maps.
 2420                  */
 2421                 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 ||
 2422                     (vm_map_pmap(map) != kernel_pmap &&
 2423                     vm_map_entry_system_wired_count(entry) != 0)) {
 2424                         unsigned int last_timestamp;
 2425                         vm_offset_t saved_start;
 2426                         vm_map_entry_t tmp_entry;
 2427 
 2428                         saved_start = entry->start;
 2429                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 2430                         last_timestamp = map->timestamp;
 2431                         (void) vm_map_unlock_and_wait(map, FALSE);
 2432                         vm_map_lock(map);
 2433                         if (last_timestamp + 1 != map->timestamp) {
 2434                                 /*
 2435                                  * Look again for the entry because the map was
 2436                                  * modified while it was unlocked.
 2437                                  * Specifically, the entry may have been
 2438                                  * clipped, merged, or deleted.
 2439                                  */
 2440                                 if (!vm_map_lookup_entry(map, saved_start,
 2441                                                          &tmp_entry))
 2442                                         entry = tmp_entry->next;
 2443                                 else {
 2444                                         entry = tmp_entry;
 2445                                         vm_map_clip_start(map, entry,
 2446                                                           saved_start);
 2447                                 }
 2448                         }
 2449                         continue;
 2450                 }
 2451                 vm_map_clip_end(map, entry, end);
 2452 
 2453                 next = entry->next;
 2454 
 2455                 /*
 2456                  * Unwire before removing addresses from the pmap; otherwise,
 2457                  * unwiring will put the entries back in the pmap.
 2458                  */
 2459                 if (entry->wired_count != 0) {
 2460                         vm_map_entry_unwire(map, entry);
 2461                 }
 2462 
 2463                 pmap_remove(map->pmap, entry->start, entry->end);
 2464 
 2465                 /*
 2466                  * Delete the entry (which may delete the object) only after
 2467                  * removing all pmap entries pointing to its pages.
 2468                  * (Otherwise, its page frames may be reallocated, and any
 2469                  * modify bits will be set in the wrong object!)
 2470                  */
 2471                 vm_map_entry_delete(map, entry);
 2472                 entry = next;
 2473         }
 2474         return (KERN_SUCCESS);
 2475 }
 2476 
 2477 /*
 2478  *      vm_map_remove:
 2479  *
 2480  *      Remove the given address range from the target map.
 2481  *      This is the exported form of vm_map_delete.
 2482  */
 2483 int
 2484 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end)
 2485 {
 2486         int result;
 2487 
 2488         vm_map_lock(map);
 2489         VM_MAP_RANGE_CHECK(map, start, end);
 2490         result = vm_map_delete(map, start, end);
 2491         vm_map_unlock(map);
 2492         return (result);
 2493 }
 2494 
 2495 /*
 2496  *      vm_map_check_protection:
 2497  *
 2498  *      Assert that the target map allows the specified privilege on the
 2499  *      entire address region given.  The entire region must be allocated.
 2500  *
 2501  *      WARNING!  This code does not and should not check whether the
 2502  *      contents of the region is accessible.  For example a smaller file
 2503  *      might be mapped into a larger address space.
 2504  *
 2505  *      NOTE!  This code is also called by munmap().
 2506  *
 2507  *      The map must be locked.  A read lock is sufficient.
 2508  */
 2509 boolean_t
 2510 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end,
 2511                         vm_prot_t protection)
 2512 {
 2513         vm_map_entry_t entry;
 2514         vm_map_entry_t tmp_entry;
 2515 
 2516         if (!vm_map_lookup_entry(map, start, &tmp_entry))
 2517                 return (FALSE);
 2518         entry = tmp_entry;
 2519 
 2520         while (start < end) {
 2521                 if (entry == &map->header)
 2522                         return (FALSE);
 2523                 /*
 2524                  * No holes allowed!
 2525                  */
 2526                 if (start < entry->start)
 2527                         return (FALSE);
 2528                 /*
 2529                  * Check protection associated with entry.
 2530                  */
 2531                 if ((entry->protection & protection) != protection)
 2532                         return (FALSE);
 2533                 /* go to next entry */
 2534                 start = entry->end;
 2535                 entry = entry->next;
 2536         }
 2537         return (TRUE);
 2538 }
 2539 
 2540 /*
 2541  *      vm_map_copy_entry:
 2542  *
 2543  *      Copies the contents of the source entry to the destination
 2544  *      entry.  The entries *must* be aligned properly.
 2545  */
 2546 static void
 2547 vm_map_copy_entry(
 2548         vm_map_t src_map,
 2549         vm_map_t dst_map,
 2550         vm_map_entry_t src_entry,
 2551         vm_map_entry_t dst_entry)
 2552 {
 2553         vm_object_t src_object;
 2554 
 2555         if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP)
 2556                 return;
 2557 
 2558         if (src_entry->wired_count == 0) {
 2559 
 2560                 /*
 2561                  * If the source entry is marked needs_copy, it is already
 2562                  * write-protected.
 2563                  */
 2564                 if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
 2565                         pmap_protect(src_map->pmap,
 2566                             src_entry->start,
 2567                             src_entry->end,
 2568                             src_entry->protection & ~VM_PROT_WRITE);
 2569                 }
 2570 
 2571                 /*
 2572                  * Make a copy of the object.
 2573                  */
 2574                 if ((src_object = src_entry->object.vm_object) != NULL) {
 2575                         VM_OBJECT_LOCK(src_object);
 2576                         if ((src_object->handle == NULL) &&
 2577                                 (src_object->type == OBJT_DEFAULT ||
 2578                                  src_object->type == OBJT_SWAP)) {
 2579                                 vm_object_collapse(src_object);
 2580                                 if ((src_object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) {
 2581                                         vm_object_split(src_entry);
 2582                                         src_object = src_entry->object.vm_object;
 2583                                 }
 2584                         }
 2585                         vm_object_reference_locked(src_object);
 2586                         vm_object_clear_flag(src_object, OBJ_ONEMAPPING);
 2587                         VM_OBJECT_UNLOCK(src_object);
 2588                         dst_entry->object.vm_object = src_object;
 2589                         src_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
 2590                         dst_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
 2591                         dst_entry->offset = src_entry->offset;
 2592                 } else {
 2593                         dst_entry->object.vm_object = NULL;
 2594                         dst_entry->offset = 0;
 2595                 }
 2596 
 2597                 pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start,
 2598                     dst_entry->end - dst_entry->start, src_entry->start);
 2599         } else {
 2600                 /*
 2601                  * Of course, wired down pages can't be set copy-on-write.
 2602                  * Cause wired pages to be copied into the new map by
 2603                  * simulating faults (the new pages are pageable)
 2604                  */
 2605                 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry);
 2606         }
 2607 }
 2608 
 2609 /*
 2610  * vmspace_map_entry_forked:
 2611  * Update the newly-forked vmspace each time a map entry is inherited
 2612  * or copied.  The values for vm_dsize and vm_tsize are approximate
 2613  * (and mostly-obsolete ideas in the face of mmap(2) et al.)
 2614  */
 2615 static void
 2616 vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2,
 2617     vm_map_entry_t entry)
 2618 {
 2619         vm_size_t entrysize;
 2620         vm_offset_t newend;
 2621 
 2622         entrysize = entry->end - entry->start;
 2623         vm2->vm_map.size += entrysize;
 2624         if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) {
 2625                 vm2->vm_ssize += btoc(entrysize);
 2626         } else if (entry->start >= (vm_offset_t)vm1->vm_daddr &&
 2627             entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) {
 2628                 newend = MIN(entry->end,
 2629                     (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize));
 2630                 vm2->vm_dsize += btoc(newend - entry->start);
 2631         } else if (entry->start >= (vm_offset_t)vm1->vm_taddr &&
 2632             entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) {
 2633                 newend = MIN(entry->end,
 2634                     (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize));
 2635                 vm2->vm_tsize += btoc(newend - entry->start);
 2636         }
 2637 }
 2638 
 2639 /*
 2640  * vmspace_fork:
 2641  * Create a new process vmspace structure and vm_map
 2642  * based on those of an existing process.  The new map
 2643  * is based on the old map, according to the inheritance
 2644  * values on the regions in that map.
 2645  *
 2646  * XXX It might be worth coalescing the entries added to the new vmspace.
 2647  *
 2648  * The source map must not be locked.
 2649  */
 2650 struct vmspace *
 2651 vmspace_fork(struct vmspace *vm1)
 2652 {
 2653         struct vmspace *vm2;
 2654         vm_map_t old_map = &vm1->vm_map;
 2655         vm_map_t new_map;
 2656         vm_map_entry_t old_entry;
 2657         vm_map_entry_t new_entry;
 2658         vm_object_t object;
 2659 
 2660         vm_map_lock(old_map);
 2661         if (old_map->busy)
 2662                 vm_map_wait_busy(old_map);
 2663 
 2664         vm2 = vmspace_alloc(old_map->min_offset, old_map->max_offset);
 2665         if (vm2 == NULL)
 2666                 goto unlock_and_return;
 2667         vm2->vm_taddr = vm1->vm_taddr;
 2668         vm2->vm_daddr = vm1->vm_daddr;
 2669         vm2->vm_maxsaddr = vm1->vm_maxsaddr;
 2670         new_map = &vm2->vm_map; /* XXX */
 2671         new_map->timestamp = 1;
 2672 
 2673         old_entry = old_map->header.next;
 2674 
 2675         while (old_entry != &old_map->header) {
 2676                 if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP)
 2677                         panic("vm_map_fork: encountered a submap");
 2678 
 2679                 switch (old_entry->inheritance) {
 2680                 case VM_INHERIT_NONE:
 2681                         break;
 2682 
 2683                 case VM_INHERIT_SHARE:
 2684                         /*
 2685                          * Clone the entry, creating the shared object if necessary.
 2686                          */
 2687                         object = old_entry->object.vm_object;
 2688                         if (object == NULL) {
 2689                                 object = vm_object_allocate(OBJT_DEFAULT,
 2690                                         atop(old_entry->end - old_entry->start));
 2691                                 old_entry->object.vm_object = object;
 2692                                 old_entry->offset = 0;
 2693                         }
 2694 
 2695                         /*
 2696                          * Add the reference before calling vm_object_shadow
 2697                          * to insure that a shadow object is created.
 2698                          */
 2699                         vm_object_reference(object);
 2700                         if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 2701                                 vm_object_shadow(&old_entry->object.vm_object,
 2702                                         &old_entry->offset,
 2703                                         atop(old_entry->end - old_entry->start));
 2704                                 old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
 2705                                 /* Transfer the second reference too. */
 2706                                 vm_object_reference(
 2707                                     old_entry->object.vm_object);
 2708                                 vm_object_deallocate(object);
 2709                                 object = old_entry->object.vm_object;
 2710                         }
 2711                         VM_OBJECT_LOCK(object);
 2712                         vm_object_clear_flag(object, OBJ_ONEMAPPING);
 2713                         VM_OBJECT_UNLOCK(object);
 2714 
 2715                         /*
 2716                          * Clone the entry, referencing the shared object.
 2717                          */
 2718                         new_entry = vm_map_entry_create(new_map);
 2719                         *new_entry = *old_entry;
 2720                         new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
 2721                             MAP_ENTRY_IN_TRANSITION);
 2722                         new_entry->wired_count = 0;
 2723 
 2724                         /*
 2725                          * Insert the entry into the new map -- we know we're
 2726                          * inserting at the end of the new map.
 2727                          */
 2728                         vm_map_entry_link(new_map, new_map->header.prev,
 2729                             new_entry);
 2730                         vmspace_map_entry_forked(vm1, vm2, new_entry);
 2731 
 2732                         /*
 2733                          * Update the physical map
 2734                          */
 2735                         pmap_copy(new_map->pmap, old_map->pmap,
 2736                             new_entry->start,
 2737                             (old_entry->end - old_entry->start),
 2738                             old_entry->start);
 2739                         break;
 2740 
 2741                 case VM_INHERIT_COPY:
 2742                         /*
 2743                          * Clone the entry and link into the map.
 2744                          */
 2745                         new_entry = vm_map_entry_create(new_map);
 2746                         *new_entry = *old_entry;
 2747                         new_entry->eflags &= ~(MAP_ENTRY_USER_WIRED |
 2748                             MAP_ENTRY_IN_TRANSITION);
 2749                         new_entry->wired_count = 0;
 2750                         new_entry->object.vm_object = NULL;
 2751                         vm_map_entry_link(new_map, new_map->header.prev,
 2752                             new_entry);
 2753                         vmspace_map_entry_forked(vm1, vm2, new_entry);
 2754                         vm_map_copy_entry(old_map, new_map, old_entry,
 2755                             new_entry);
 2756                         break;
 2757                 }
 2758                 old_entry = old_entry->next;
 2759         }
 2760 unlock_and_return:
 2761         vm_map_unlock(old_map);
 2762 
 2763         return (vm2);
 2764 }
 2765 
 2766 int
 2767 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
 2768     vm_prot_t prot, vm_prot_t max, int cow)
 2769 {
 2770         vm_map_entry_t new_entry, prev_entry;
 2771         vm_offset_t bot, top;
 2772         vm_size_t init_ssize;
 2773         int orient, rv;
 2774         rlim_t vmemlim;
 2775 
 2776         /*
 2777          * The stack orientation is piggybacked with the cow argument.
 2778          * Extract it into orient and mask the cow argument so that we
 2779          * don't pass it around further.
 2780          * NOTE: We explicitly allow bi-directional stacks.
 2781          */
 2782         orient = cow & (MAP_STACK_GROWS_DOWN|MAP_STACK_GROWS_UP);
 2783         cow &= ~orient;
 2784         KASSERT(orient != 0, ("No stack grow direction"));
 2785 
 2786         if (addrbos < vm_map_min(map) ||
 2787             addrbos > vm_map_max(map) ||
 2788             addrbos + max_ssize < addrbos)
 2789                 return (KERN_NO_SPACE);
 2790 
 2791         init_ssize = (max_ssize < sgrowsiz) ? max_ssize : sgrowsiz;
 2792 
 2793         PROC_LOCK(curthread->td_proc);
 2794         vmemlim = lim_cur(curthread->td_proc, RLIMIT_VMEM);
 2795         PROC_UNLOCK(curthread->td_proc);
 2796 
 2797         vm_map_lock(map);
 2798 
 2799         /* If addr is already mapped, no go */
 2800         if (vm_map_lookup_entry(map, addrbos, &prev_entry)) {
 2801                 vm_map_unlock(map);
 2802                 return (KERN_NO_SPACE);
 2803         }
 2804 
 2805         /* If we would blow our VMEM resource limit, no go */
 2806         if (map->size + init_ssize > vmemlim) {
 2807                 vm_map_unlock(map);
 2808                 return (KERN_NO_SPACE);
 2809         }
 2810 
 2811         /*
 2812          * If we can't accomodate max_ssize in the current mapping, no go.
 2813          * However, we need to be aware that subsequent user mappings might
 2814          * map into the space we have reserved for stack, and currently this
 2815          * space is not protected.
 2816          *
 2817          * Hopefully we will at least detect this condition when we try to
 2818          * grow the stack.
 2819          */
 2820         if ((prev_entry->next != &map->header) &&
 2821             (prev_entry->next->start < addrbos + max_ssize)) {
 2822                 vm_map_unlock(map);
 2823                 return (KERN_NO_SPACE);
 2824         }
 2825 
 2826         /*
 2827          * We initially map a stack of only init_ssize.  We will grow as
 2828          * needed later.  Depending on the orientation of the stack (i.e.
 2829          * the grow direction) we either map at the top of the range, the
 2830          * bottom of the range or in the middle.
 2831          *
 2832          * Note: we would normally expect prot and max to be VM_PROT_ALL,
 2833          * and cow to be 0.  Possibly we should eliminate these as input
 2834          * parameters, and just pass these values here in the insert call.
 2835          */
 2836         if (orient == MAP_STACK_GROWS_DOWN)
 2837                 bot = addrbos + max_ssize - init_ssize;
 2838         else if (orient == MAP_STACK_GROWS_UP)
 2839                 bot = addrbos;
 2840         else
 2841                 bot = round_page(addrbos + max_ssize/2 - init_ssize/2);
 2842         top = bot + init_ssize;
 2843         rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow);
 2844 
 2845         /* Now set the avail_ssize amount. */
 2846         if (rv == KERN_SUCCESS) {
 2847                 if (prev_entry != &map->header)
 2848                         vm_map_clip_end(map, prev_entry, bot);
 2849                 new_entry = prev_entry->next;
 2850                 if (new_entry->end != top || new_entry->start != bot)
 2851                         panic("Bad entry start/end for new stack entry");
 2852 
 2853                 new_entry->avail_ssize = max_ssize - init_ssize;
 2854                 if (orient & MAP_STACK_GROWS_DOWN)
 2855                         new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
 2856                 if (orient & MAP_STACK_GROWS_UP)
 2857                         new_entry->eflags |= MAP_ENTRY_GROWS_UP;
 2858         }
 2859 
 2860         vm_map_unlock(map);
 2861         return (rv);
 2862 }
 2863 
 2864 /* Attempts to grow a vm stack entry.  Returns KERN_SUCCESS if the
 2865  * desired address is already mapped, or if we successfully grow
 2866  * the stack.  Also returns KERN_SUCCESS if addr is outside the
 2867  * stack range (this is strange, but preserves compatibility with
 2868  * the grow function in vm_machdep.c).
 2869  */
 2870 int
 2871 vm_map_growstack(struct proc *p, vm_offset_t addr)
 2872 {
 2873         vm_map_entry_t next_entry, prev_entry;
 2874         vm_map_entry_t new_entry, stack_entry;
 2875         struct vmspace *vm = p->p_vmspace;
 2876         vm_map_t map = &vm->vm_map;
 2877         vm_offset_t end;
 2878         size_t grow_amount, max_grow;
 2879         rlim_t stacklim, vmemlim;
 2880         int is_procstack, rv;
 2881 
 2882 Retry:
 2883         PROC_LOCK(p);
 2884         stacklim = lim_cur(p, RLIMIT_STACK);
 2885         vmemlim = lim_cur(p, RLIMIT_VMEM);
 2886         PROC_UNLOCK(p);
 2887 
 2888         vm_map_lock_read(map);
 2889 
 2890         /* If addr is already in the entry range, no need to grow.*/
 2891         if (vm_map_lookup_entry(map, addr, &prev_entry)) {
 2892                 vm_map_unlock_read(map);
 2893                 return (KERN_SUCCESS);
 2894         }
 2895 
 2896         next_entry = prev_entry->next;
 2897         if (!(prev_entry->eflags & MAP_ENTRY_GROWS_UP)) {
 2898                 /*
 2899                  * This entry does not grow upwards. Since the address lies
 2900                  * beyond this entry, the next entry (if one exists) has to
 2901                  * be a downward growable entry. The entry list header is
 2902                  * never a growable entry, so it suffices to check the flags.
 2903                  */
 2904                 if (!(next_entry->eflags & MAP_ENTRY_GROWS_DOWN)) {
 2905                         vm_map_unlock_read(map);
 2906                         return (KERN_SUCCESS);
 2907                 }
 2908                 stack_entry = next_entry;
 2909         } else {
 2910                 /*
 2911                  * This entry grows upward. If the next entry does not at
 2912                  * least grow downwards, this is the entry we need to grow.
 2913                  * otherwise we have two possible choices and we have to
 2914                  * select one.
 2915                  */
 2916                 if (next_entry->eflags & MAP_ENTRY_GROWS_DOWN) {
 2917                         /*
 2918                          * We have two choices; grow the entry closest to
 2919                          * the address to minimize the amount of growth.
 2920                          */
 2921                         if (addr - prev_entry->end <= next_entry->start - addr)
 2922                                 stack_entry = prev_entry;
 2923                         else
 2924                                 stack_entry = next_entry;
 2925                 } else
 2926                         stack_entry = prev_entry;
 2927         }
 2928 
 2929         if (stack_entry == next_entry) {
 2930                 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_DOWN, ("foo"));
 2931                 KASSERT(addr < stack_entry->start, ("foo"));
 2932                 end = (prev_entry != &map->header) ? prev_entry->end :
 2933                     stack_entry->start - stack_entry->avail_ssize;
 2934                 grow_amount = roundup(stack_entry->start - addr, PAGE_SIZE);
 2935                 max_grow = stack_entry->start - end;
 2936         } else {
 2937                 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_UP, ("foo"));
 2938                 KASSERT(addr >= stack_entry->end, ("foo"));
 2939                 end = (next_entry != &map->header) ? next_entry->start :
 2940                     stack_entry->end + stack_entry->avail_ssize;
 2941                 grow_amount = roundup(addr + 1 - stack_entry->end, PAGE_SIZE);
 2942                 max_grow = end - stack_entry->end;
 2943         }
 2944 
 2945         if (grow_amount > stack_entry->avail_ssize) {
 2946                 vm_map_unlock_read(map);
 2947                 return (KERN_NO_SPACE);
 2948         }
 2949 
 2950         /*
 2951          * If there is no longer enough space between the entries nogo, and
 2952          * adjust the available space.  Note: this  should only happen if the
 2953          * user has mapped into the stack area after the stack was created,
 2954          * and is probably an error.
 2955          *
 2956          * This also effectively destroys any guard page the user might have
 2957          * intended by limiting the stack size.
 2958          */
 2959         if (grow_amount > max_grow) {
 2960                 if (vm_map_lock_upgrade(map))
 2961                         goto Retry;
 2962 
 2963                 stack_entry->avail_ssize = max_grow;
 2964 
 2965                 vm_map_unlock(map);
 2966                 return (KERN_NO_SPACE);
 2967         }
 2968 
 2969         is_procstack = (addr >= (vm_offset_t)vm->vm_maxsaddr) ? 1 : 0;
 2970 
 2971         /*
 2972          * If this is the main process stack, see if we're over the stack
 2973          * limit.
 2974          */
 2975         if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
 2976                 vm_map_unlock_read(map);
 2977                 return (KERN_NO_SPACE);
 2978         }
 2979 
 2980         /* Round up the grow amount modulo SGROWSIZ */
 2981         grow_amount = roundup (grow_amount, sgrowsiz);
 2982         if (grow_amount > stack_entry->avail_ssize)
 2983                 grow_amount = stack_entry->avail_ssize;
 2984         if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
 2985                 grow_amount = trunc_page((vm_size_t)stacklim) -
 2986                     ctob(vm->vm_ssize);
 2987         }
 2988 
 2989         /* If we would blow our VMEM resource limit, no go */
 2990         if (map->size + grow_amount > vmemlim) {
 2991                 vm_map_unlock_read(map);
 2992                 return (KERN_NO_SPACE);
 2993         }
 2994 
 2995         if (vm_map_lock_upgrade(map))
 2996                 goto Retry;
 2997 
 2998         if (stack_entry == next_entry) {
 2999                 /*
 3000                  * Growing downward.
 3001                  */
 3002                 /* Get the preliminary new entry start value */
 3003                 addr = stack_entry->start - grow_amount;
 3004 
 3005                 /*
 3006                  * If this puts us into the previous entry, cut back our
 3007                  * growth to the available space. Also, see the note above.
 3008                  */
 3009                 if (addr < end) {
 3010                         stack_entry->avail_ssize = max_grow;
 3011                         addr = end;
 3012                 }
 3013 
 3014                 rv = vm_map_insert(map, NULL, 0, addr, stack_entry->start,
 3015                     p->p_sysent->sv_stackprot, VM_PROT_ALL, 0);
 3016 
 3017                 /* Adjust the available stack space by the amount we grew. */
 3018                 if (rv == KERN_SUCCESS) {
 3019                         if (prev_entry != &map->header)
 3020                                 vm_map_clip_end(map, prev_entry, addr);
 3021                         new_entry = prev_entry->next;
 3022                         KASSERT(new_entry == stack_entry->prev, ("foo"));
 3023                         KASSERT(new_entry->end == stack_entry->start, ("foo"));
 3024                         KASSERT(new_entry->start == addr, ("foo"));
 3025                         grow_amount = new_entry->end - new_entry->start;
 3026                         new_entry->avail_ssize = stack_entry->avail_ssize -
 3027                             grow_amount;
 3028                         stack_entry->eflags &= ~MAP_ENTRY_GROWS_DOWN;
 3029                         new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
 3030                 }
 3031         } else {
 3032                 /*
 3033                  * Growing upward.
 3034                  */
 3035                 addr = stack_entry->end + grow_amount;
 3036 
 3037                 /*
 3038                  * If this puts us into the next entry, cut back our growth
 3039                  * to the available space. Also, see the note above.
 3040                  */
 3041                 if (addr > end) {
 3042                         stack_entry->avail_ssize = end - stack_entry->end;
 3043                         addr = end;
 3044                 }
 3045 
 3046                 grow_amount = addr - stack_entry->end;
 3047 
 3048                 /* Grow the underlying object if applicable. */
 3049                 if (stack_entry->object.vm_object == NULL ||
 3050                     vm_object_coalesce(stack_entry->object.vm_object,
 3051                     stack_entry->offset,
 3052                     (vm_size_t)(stack_entry->end - stack_entry->start),
 3053                     (vm_size_t)grow_amount)) {
 3054                         map->size += (addr - stack_entry->end);
 3055                         /* Update the current entry. */
 3056                         stack_entry->end = addr;
 3057                         stack_entry->avail_ssize -= grow_amount;
 3058                         vm_map_entry_resize_free(map, stack_entry);
 3059                         rv = KERN_SUCCESS;
 3060 
 3061                         if (next_entry != &map->header)
 3062                                 vm_map_clip_start(map, next_entry, addr);
 3063                 } else
 3064                         rv = KERN_FAILURE;
 3065         }
 3066 
 3067         if (rv == KERN_SUCCESS && is_procstack)
 3068                 vm->vm_ssize += btoc(grow_amount);
 3069 
 3070         vm_map_unlock(map);
 3071 
 3072         /*
 3073          * Heed the MAP_WIREFUTURE flag if it was set for this process.
 3074          */
 3075         if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE)) {
 3076                 vm_map_wire(map,
 3077                     (stack_entry == next_entry) ? addr : addr - grow_amount,
 3078                     (stack_entry == next_entry) ? stack_entry->start : addr,
 3079                     (p->p_flag & P_SYSTEM)
 3080                     ? VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES
 3081                     : VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
 3082         }
 3083 
 3084         return (rv);
 3085 }
 3086 
 3087 /*
 3088  * Unshare the specified VM space for exec.  If other processes are
 3089  * mapped to it, then create a new one.  The new vmspace is null.
 3090  */
 3091 int
 3092 vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser)
 3093 {
 3094         struct vmspace *oldvmspace = p->p_vmspace;
 3095         struct vmspace *newvmspace;
 3096 
 3097         newvmspace = vmspace_alloc(minuser, maxuser);
 3098         if (newvmspace == NULL)
 3099                 return (ENOMEM);
 3100         newvmspace->vm_swrss = oldvmspace->vm_swrss;
 3101         /*
 3102          * This code is written like this for prototype purposes.  The
 3103          * goal is to avoid running down the vmspace here, but let the
 3104          * other process's that are still using the vmspace to finally
 3105          * run it down.  Even though there is little or no chance of blocking
 3106          * here, it is a good idea to keep this form for future mods.
 3107          */
 3108         PROC_VMSPACE_LOCK(p);
 3109         p->p_vmspace = newvmspace;
 3110         PROC_VMSPACE_UNLOCK(p);
 3111         if (p == curthread->td_proc)            /* XXXKSE ? */
 3112                 pmap_activate(curthread);
 3113         vmspace_free(oldvmspace);
 3114         return (0);
 3115 }
 3116 
 3117 /*
 3118  * Unshare the specified VM space for forcing COW.  This
 3119  * is called by rfork, for the (RFMEM|RFPROC) == 0 case.
 3120  */
 3121 int
 3122 vmspace_unshare(struct proc *p)
 3123 {
 3124         struct vmspace *oldvmspace = p->p_vmspace;
 3125         struct vmspace *newvmspace;
 3126 
 3127         if (oldvmspace->vm_refcnt == 1)
 3128                 return (0);
 3129         newvmspace = vmspace_fork(oldvmspace);
 3130         if (newvmspace == NULL)
 3131                 return (ENOMEM);
 3132         PROC_VMSPACE_LOCK(p);
 3133         p->p_vmspace = newvmspace;
 3134         PROC_VMSPACE_UNLOCK(p);
 3135         if (p == curthread->td_proc)            /* XXXKSE ? */
 3136                 pmap_activate(curthread);
 3137         vmspace_free(oldvmspace);
 3138         return (0);
 3139 }
 3140 
 3141 /*
 3142  *      vm_map_lookup:
 3143  *
 3144  *      Finds the VM object, offset, and
 3145  *      protection for a given virtual address in the
 3146  *      specified map, assuming a page fault of the
 3147  *      type specified.
 3148  *
 3149  *      Leaves the map in question locked for read; return
 3150  *      values are guaranteed until a vm_map_lookup_done
 3151  *      call is performed.  Note that the map argument
 3152  *      is in/out; the returned map must be used in
 3153  *      the call to vm_map_lookup_done.
 3154  *
 3155  *      A handle (out_entry) is returned for use in
 3156  *      vm_map_lookup_done, to make that fast.
 3157  *
 3158  *      If a lookup is requested with "write protection"
 3159  *      specified, the map may be changed to perform virtual
 3160  *      copying operations, although the data referenced will
 3161  *      remain the same.
 3162  */
 3163 int
 3164 vm_map_lookup(vm_map_t *var_map,                /* IN/OUT */
 3165               vm_offset_t vaddr,
 3166               vm_prot_t fault_typea,
 3167               vm_map_entry_t *out_entry,        /* OUT */
 3168               vm_object_t *object,              /* OUT */
 3169               vm_pindex_t *pindex,              /* OUT */
 3170               vm_prot_t *out_prot,              /* OUT */
 3171               boolean_t *wired)                 /* OUT */
 3172 {
 3173         vm_map_entry_t entry;
 3174         vm_map_t map = *var_map;
 3175         vm_prot_t prot;
 3176         vm_prot_t fault_type = fault_typea;
 3177 
 3178 RetryLookup:;
 3179         /*
 3180          * Lookup the faulting address.
 3181          */
 3182 
 3183         vm_map_lock_read(map);
 3184 #define RETURN(why) \
 3185                 { \
 3186                 vm_map_unlock_read(map); \
 3187                 return (why); \
 3188                 }
 3189 
 3190         /*
 3191          * If the map has an interesting hint, try it before calling full
 3192          * blown lookup routine.
 3193          */
 3194         entry = map->root;
 3195         *out_entry = entry;
 3196         if (entry == NULL ||
 3197             (vaddr < entry->start) || (vaddr >= entry->end)) {
 3198                 /*
 3199                  * Entry was either not a valid hint, or the vaddr was not
 3200                  * contained in the entry, so do a full lookup.
 3201                  */
 3202                 if (!vm_map_lookup_entry(map, vaddr, out_entry))
 3203                         RETURN(KERN_INVALID_ADDRESS);
 3204 
 3205                 entry = *out_entry;
 3206         }
 3207 
 3208         /*
 3209          * Handle submaps.
 3210          */
 3211         if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 3212                 vm_map_t old_map = map;
 3213 
 3214                 *var_map = map = entry->object.sub_map;
 3215                 vm_map_unlock_read(old_map);
 3216                 goto RetryLookup;
 3217         }
 3218 
 3219         /*
 3220          * Check whether this task is allowed to have this page.
 3221          * Note the special case for MAP_ENTRY_COW
 3222          * pages with an override.  This is to implement a forced
 3223          * COW for debuggers.
 3224          */
 3225         if (fault_type & VM_PROT_OVERRIDE_WRITE)
 3226                 prot = entry->max_protection;
 3227         else
 3228                 prot = entry->protection;
 3229         fault_type &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
 3230         if ((fault_type & prot) != fault_type) {
 3231                         RETURN(KERN_PROTECTION_FAILURE);
 3232         }
 3233         if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
 3234             (entry->eflags & MAP_ENTRY_COW) &&
 3235             (fault_type & VM_PROT_WRITE) &&
 3236             (fault_typea & VM_PROT_OVERRIDE_WRITE) == 0) {
 3237                 RETURN(KERN_PROTECTION_FAILURE);
 3238         }
 3239 
 3240         /*
 3241          * If this page is not pageable, we have to get it for all possible
 3242          * accesses.
 3243          */
 3244         *wired = (entry->wired_count != 0);
 3245         if (*wired)
 3246                 prot = fault_type = entry->protection;
 3247 
 3248         /*
 3249          * If the entry was copy-on-write, we either ...
 3250          */
 3251         if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 3252                 /*
 3253                  * If we want to write the page, we may as well handle that
 3254                  * now since we've got the map locked.
 3255                  *
 3256                  * If we don't need to write the page, we just demote the
 3257                  * permissions allowed.
 3258                  */
 3259                 if (fault_type & VM_PROT_WRITE) {
 3260                         /*
 3261                          * Make a new object, and place it in the object
 3262                          * chain.  Note that no new references have appeared
 3263                          * -- one just moved from the map to the new
 3264                          * object.
 3265                          */
 3266                         if (vm_map_lock_upgrade(map))
 3267                                 goto RetryLookup;
 3268 
 3269                         vm_object_shadow(
 3270                             &entry->object.vm_object,
 3271                             &entry->offset,
 3272                             atop(entry->end - entry->start));
 3273                         entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
 3274 
 3275                         vm_map_lock_downgrade(map);
 3276                 } else {
 3277                         /*
 3278                          * We're attempting to read a copy-on-write page --
 3279                          * don't allow writes.
 3280                          */
 3281                         prot &= ~VM_PROT_WRITE;
 3282                 }
 3283         }
 3284 
 3285         /*
 3286          * Create an object if necessary.
 3287          */
 3288         if (entry->object.vm_object == NULL &&
 3289             !map->system_map) {
 3290                 if (vm_map_lock_upgrade(map))
 3291                         goto RetryLookup;
 3292                 entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT,
 3293                     atop(entry->end - entry->start));
 3294                 entry->offset = 0;
 3295                 vm_map_lock_downgrade(map);
 3296         }
 3297 
 3298         /*
 3299          * Return the object/offset from this entry.  If the entry was
 3300          * copy-on-write or empty, it has been fixed up.
 3301          */
 3302         *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
 3303         *object = entry->object.vm_object;
 3304 
 3305         *out_prot = prot;
 3306         return (KERN_SUCCESS);
 3307 
 3308 #undef  RETURN
 3309 }
 3310 
 3311 /*
 3312  *      vm_map_lookup_locked:
 3313  *
 3314  *      Lookup the faulting address.  A version of vm_map_lookup that returns 
 3315  *      KERN_FAILURE instead of blocking on map lock or memory allocation.
 3316  */
 3317 int
 3318 vm_map_lookup_locked(vm_map_t *var_map,         /* IN/OUT */
 3319                      vm_offset_t vaddr,
 3320                      vm_prot_t fault_typea,
 3321                      vm_map_entry_t *out_entry, /* OUT */
 3322                      vm_object_t *object,       /* OUT */
 3323                      vm_pindex_t *pindex,       /* OUT */
 3324                      vm_prot_t *out_prot,       /* OUT */
 3325                      boolean_t *wired)          /* OUT */
 3326 {
 3327         vm_map_entry_t entry;
 3328         vm_map_t map = *var_map;
 3329         vm_prot_t prot;
 3330         vm_prot_t fault_type = fault_typea;
 3331 
 3332         /*
 3333          * If the map has an interesting hint, try it before calling full
 3334          * blown lookup routine.
 3335          */
 3336         entry = map->root;
 3337         *out_entry = entry;
 3338         if (entry == NULL ||
 3339             (vaddr < entry->start) || (vaddr >= entry->end)) {
 3340                 /*
 3341                  * Entry was either not a valid hint, or the vaddr was not
 3342                  * contained in the entry, so do a full lookup.
 3343                  */
 3344                 if (!vm_map_lookup_entry(map, vaddr, out_entry))
 3345                         return (KERN_INVALID_ADDRESS);
 3346 
 3347                 entry = *out_entry;
 3348         }
 3349 
 3350         /*
 3351          * Fail if the entry refers to a submap.
 3352          */
 3353         if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
 3354                 return (KERN_FAILURE);
 3355 
 3356         /*
 3357          * Check whether this task is allowed to have this page.
 3358          * Note the special case for MAP_ENTRY_COW
 3359          * pages with an override.  This is to implement a forced
 3360          * COW for debuggers.
 3361          */
 3362         if (fault_type & VM_PROT_OVERRIDE_WRITE)
 3363                 prot = entry->max_protection;
 3364         else
 3365                 prot = entry->protection;
 3366         fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
 3367         if ((fault_type & prot) != fault_type)
 3368                 return (KERN_PROTECTION_FAILURE);
 3369         if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
 3370             (entry->eflags & MAP_ENTRY_COW) &&
 3371             (fault_type & VM_PROT_WRITE) &&
 3372             (fault_typea & VM_PROT_OVERRIDE_WRITE) == 0)
 3373                 return (KERN_PROTECTION_FAILURE);
 3374 
 3375         /*
 3376          * If this page is not pageable, we have to get it for all possible
 3377          * accesses.
 3378          */
 3379         *wired = (entry->wired_count != 0);
 3380         if (*wired)
 3381                 prot = fault_type = entry->protection;
 3382 
 3383         if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 3384                 /*
 3385                  * Fail if the entry was copy-on-write for a write fault.
 3386                  */
 3387                 if (fault_type & VM_PROT_WRITE)
 3388                         return (KERN_FAILURE);
 3389                 /*
 3390                  * We're attempting to read a copy-on-write page --
 3391                  * don't allow writes.
 3392                  */
 3393                 prot &= ~VM_PROT_WRITE;
 3394         }
 3395 
 3396         /*
 3397          * Fail if an object should be created.
 3398          */
 3399         if (entry->object.vm_object == NULL && !map->system_map)
 3400                 return (KERN_FAILURE);
 3401 
 3402         /*
 3403          * Return the object/offset from this entry.  If the entry was
 3404          * copy-on-write or empty, it has been fixed up.
 3405          */
 3406         *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
 3407         *object = entry->object.vm_object;
 3408 
 3409         *out_prot = prot;
 3410         return (KERN_SUCCESS);
 3411 }
 3412 
 3413 /*
 3414  *      vm_map_lookup_done:
 3415  *
 3416  *      Releases locks acquired by a vm_map_lookup
 3417  *      (according to the handle returned by that lookup).
 3418  */
 3419 void
 3420 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry)
 3421 {
 3422         /*
 3423          * Unlock the main-level map
 3424          */
 3425         vm_map_unlock_read(map);
 3426 }
 3427 
 3428 #include "opt_ddb.h"
 3429 #ifdef DDB
 3430 #include <sys/kernel.h>
 3431 
 3432 #include <ddb/ddb.h>
 3433 
 3434 /*
 3435  *      vm_map_print:   [ debug ]
 3436  */
 3437 DB_SHOW_COMMAND(map, vm_map_print)
 3438 {
 3439         static int nlines;
 3440         /* XXX convert args. */
 3441         vm_map_t map = (vm_map_t)addr;
 3442         boolean_t full = have_addr;
 3443 
 3444         vm_map_entry_t entry;
 3445 
 3446         db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n",
 3447             (void *)map,
 3448             (void *)map->pmap, map->nentries, map->timestamp);
 3449         nlines++;
 3450 
 3451         if (!full && db_indent)
 3452                 return;
 3453 
 3454         db_indent += 2;
 3455         for (entry = map->header.next; entry != &map->header;
 3456             entry = entry->next) {
 3457                 db_iprintf("map entry %p: start=%p, end=%p\n",
 3458                     (void *)entry, (void *)entry->start, (void *)entry->end);
 3459                 nlines++;
 3460                 {
 3461                         static char *inheritance_name[4] =
 3462                         {"share", "copy", "none", "donate_copy"};
 3463 
 3464                         db_iprintf(" prot=%x/%x/%s",
 3465                             entry->protection,
 3466                             entry->max_protection,
 3467                             inheritance_name[(int)(unsigned char)entry->inheritance]);
 3468                         if (entry->wired_count != 0)
 3469                                 db_printf(", wired");
 3470                 }
 3471                 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 3472                         db_printf(", share=%p, offset=0x%jx\n",
 3473                             (void *)entry->object.sub_map,
 3474                             (uintmax_t)entry->offset);
 3475                         nlines++;
 3476                         if ((entry->prev == &map->header) ||
 3477                             (entry->prev->object.sub_map !=
 3478                                 entry->object.sub_map)) {
 3479                                 db_indent += 2;
 3480                                 vm_map_print((db_expr_t)(intptr_t)
 3481                                              entry->object.sub_map,
 3482                                              full, 0, (char *)0);
 3483                                 db_indent -= 2;
 3484                         }
 3485                 } else {
 3486                         db_printf(", object=%p, offset=0x%jx",
 3487                             (void *)entry->object.vm_object,
 3488                             (uintmax_t)entry->offset);
 3489                         if (entry->eflags & MAP_ENTRY_COW)
 3490                                 db_printf(", copy (%s)",
 3491                                     (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done");
 3492                         db_printf("\n");
 3493                         nlines++;
 3494 
 3495                         if ((entry->prev == &map->header) ||
 3496                             (entry->prev->object.vm_object !=
 3497                                 entry->object.vm_object)) {
 3498                                 db_indent += 2;
 3499                                 vm_object_print((db_expr_t)(intptr_t)
 3500                                                 entry->object.vm_object,
 3501                                                 full, 0, (char *)0);
 3502                                 nlines += 4;
 3503                                 db_indent -= 2;
 3504                         }
 3505                 }
 3506         }
 3507         db_indent -= 2;
 3508         if (db_indent == 0)
 3509                 nlines = 0;
 3510 }
 3511 
 3512 
 3513 DB_SHOW_COMMAND(procvm, procvm)
 3514 {
 3515         struct proc *p;
 3516 
 3517         if (have_addr) {
 3518                 p = (struct proc *) addr;
 3519         } else {
 3520                 p = curproc;
 3521         }
 3522 
 3523         db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n",
 3524             (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map,
 3525             (void *)vmspace_pmap(p->p_vmspace));
 3526 
 3527         vm_map_print((db_expr_t)(intptr_t)&p->p_vmspace->vm_map, 1, 0, NULL);
 3528 }
 3529 
 3530 #endif /* DDB */

Cache object: 468d723cb91190b4105b38a7a8eacd3d


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.