The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_map.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * The Mach Operating System project at Carnegie-Mellon University.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 4. Neither the name of the University nor the names of its contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      from: @(#)vm_map.c      8.3 (Berkeley) 1/12/94
   33  *
   34  *
   35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   36  * All rights reserved.
   37  *
   38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   39  *
   40  * Permission to use, copy, modify and distribute this software and
   41  * its documentation is hereby granted, provided that both the copyright
   42  * notice and this permission notice appear in all copies of the
   43  * software, derivative works or modified versions, and any portions
   44  * thereof, and that both notices appear in supporting documentation.
   45  *
   46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   49  *
   50  * Carnegie Mellon requests users of this software to return to
   51  *
   52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   53  *  School of Computer Science
   54  *  Carnegie Mellon University
   55  *  Pittsburgh PA 15213-3890
   56  *
   57  * any improvements or extensions that they make and grant Carnegie the
   58  * rights to redistribute these changes.
   59  */
   60 
   61 /*
   62  *      Virtual memory mapping module.
   63  */
   64 
   65 #include <sys/cdefs.h>
   66 __FBSDID("$FreeBSD: src/sys/vm/vm_map.c,v 1.366.2.3 2006/06/02 00:26:27 tegge Exp $");
   67 
   68 #include <sys/param.h>
   69 #include <sys/systm.h>
   70 #include <sys/ktr.h>
   71 #include <sys/lock.h>
   72 #include <sys/mutex.h>
   73 #include <sys/proc.h>
   74 #include <sys/vmmeter.h>
   75 #include <sys/mman.h>
   76 #include <sys/vnode.h>
   77 #include <sys/resourcevar.h>
   78 #include <sys/file.h>
   79 #include <sys/sysent.h>
   80 #include <sys/shm.h>
   81 
   82 #include <vm/vm.h>
   83 #include <vm/vm_param.h>
   84 #include <vm/pmap.h>
   85 #include <vm/vm_map.h>
   86 #include <vm/vm_page.h>
   87 #include <vm/vm_object.h>
   88 #include <vm/vm_pager.h>
   89 #include <vm/vm_kern.h>
   90 #include <vm/vm_extern.h>
   91 #include <vm/swap_pager.h>
   92 #include <vm/uma.h>
   93 
   94 /*
   95  *      Virtual memory maps provide for the mapping, protection,
   96  *      and sharing of virtual memory objects.  In addition,
   97  *      this module provides for an efficient virtual copy of
   98  *      memory from one map to another.
   99  *
  100  *      Synchronization is required prior to most operations.
  101  *
  102  *      Maps consist of an ordered doubly-linked list of simple
  103  *      entries; a single hint is used to speed up lookups.
  104  *
  105  *      Since portions of maps are specified by start/end addresses,
  106  *      which may not align with existing map entries, all
  107  *      routines merely "clip" entries to these start/end values.
  108  *      [That is, an entry is split into two, bordering at a
  109  *      start or end value.]  Note that these clippings may not
  110  *      always be necessary (as the two resulting entries are then
  111  *      not changed); however, the clipping is done for convenience.
  112  *
  113  *      As mentioned above, virtual copy operations are performed
  114  *      by copying VM object references from one map to
  115  *      another, and then marking both regions as copy-on-write.
  116  */
  117 
  118 /*
  119  *      vm_map_startup:
  120  *
  121  *      Initialize the vm_map module.  Must be called before
  122  *      any other vm_map routines.
  123  *
  124  *      Map and entry structures are allocated from the general
  125  *      purpose memory pool with some exceptions:
  126  *
  127  *      - The kernel map and kmem submap are allocated statically.
  128  *      - Kernel map entries are allocated out of a static pool.
  129  *
  130  *      These restrictions are necessary since malloc() uses the
  131  *      maps and requires map entries.
  132  */
  133 
  134 static struct mtx map_sleep_mtx;
  135 static uma_zone_t mapentzone;
  136 static uma_zone_t kmapentzone;
  137 static uma_zone_t mapzone;
  138 static uma_zone_t vmspace_zone;
  139 static struct vm_object kmapentobj;
  140 static int vmspace_zinit(void *mem, int size, int flags);
  141 static void vmspace_zfini(void *mem, int size);
  142 static int vm_map_zinit(void *mem, int ize, int flags);
  143 static void vm_map_zfini(void *mem, int size);
  144 static void _vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max);
  145 
  146 #ifdef INVARIANTS
  147 static void vm_map_zdtor(void *mem, int size, void *arg);
  148 static void vmspace_zdtor(void *mem, int size, void *arg);
  149 #endif
  150 
  151 /* 
  152  * PROC_VMSPACE_{UN,}LOCK() can be a noop as long as vmspaces are type
  153  * stable.
  154  */
  155 #define PROC_VMSPACE_LOCK(p) do { } while (0)
  156 #define PROC_VMSPACE_UNLOCK(p) do { } while (0)
  157 
  158 void
  159 vm_map_startup(void)
  160 {
  161         mtx_init(&map_sleep_mtx, "vm map sleep mutex", NULL, MTX_DEF);
  162         mapzone = uma_zcreate("MAP", sizeof(struct vm_map), NULL,
  163 #ifdef INVARIANTS
  164             vm_map_zdtor,
  165 #else
  166             NULL,
  167 #endif
  168             vm_map_zinit, vm_map_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  169         uma_prealloc(mapzone, MAX_KMAP);
  170         kmapentzone = uma_zcreate("KMAP ENTRY", sizeof(struct vm_map_entry),
  171             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
  172             UMA_ZONE_MTXCLASS | UMA_ZONE_VM);
  173         uma_prealloc(kmapentzone, MAX_KMAPENT);
  174         mapentzone = uma_zcreate("MAP ENTRY", sizeof(struct vm_map_entry),
  175             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
  176         uma_prealloc(mapentzone, MAX_MAPENT);
  177 }
  178 
  179 static void
  180 vmspace_zfini(void *mem, int size)
  181 {
  182         struct vmspace *vm;
  183 
  184         vm = (struct vmspace *)mem;
  185         pmap_release(vmspace_pmap(vm));
  186         vm_map_zfini(&vm->vm_map, sizeof(vm->vm_map));
  187 }
  188 
  189 static int
  190 vmspace_zinit(void *mem, int size, int flags)
  191 {
  192         struct vmspace *vm;
  193 
  194         vm = (struct vmspace *)mem;
  195 
  196         (void)vm_map_zinit(&vm->vm_map, sizeof(vm->vm_map), flags);
  197         pmap_pinit(vmspace_pmap(vm));
  198         return (0);
  199 }
  200 
  201 static void
  202 vm_map_zfini(void *mem, int size)
  203 {
  204         vm_map_t map;
  205 
  206         map = (vm_map_t)mem;
  207         mtx_destroy(&map->system_mtx);
  208         sx_destroy(&map->lock);
  209 }
  210 
  211 static int
  212 vm_map_zinit(void *mem, int size, int flags)
  213 {
  214         vm_map_t map;
  215 
  216         map = (vm_map_t)mem;
  217         map->nentries = 0;
  218         map->size = 0;
  219         mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
  220         sx_init(&map->lock, "user map");
  221         return (0);
  222 }
  223 
  224 #ifdef INVARIANTS
  225 static void
  226 vmspace_zdtor(void *mem, int size, void *arg)
  227 {
  228         struct vmspace *vm;
  229 
  230         vm = (struct vmspace *)mem;
  231 
  232         vm_map_zdtor(&vm->vm_map, sizeof(vm->vm_map), arg);
  233 }
  234 static void
  235 vm_map_zdtor(void *mem, int size, void *arg)
  236 {
  237         vm_map_t map;
  238 
  239         map = (vm_map_t)mem;
  240         KASSERT(map->nentries == 0,
  241             ("map %p nentries == %d on free.",
  242             map, map->nentries));
  243         KASSERT(map->size == 0,
  244             ("map %p size == %lu on free.",
  245             map, (unsigned long)map->size));
  246 }
  247 #endif  /* INVARIANTS */
  248 
  249 /*
  250  * Allocate a vmspace structure, including a vm_map and pmap,
  251  * and initialize those structures.  The refcnt is set to 1.
  252  */
  253 struct vmspace *
  254 vmspace_alloc(min, max)
  255         vm_offset_t min, max;
  256 {
  257         struct vmspace *vm;
  258 
  259         vm = uma_zalloc(vmspace_zone, M_WAITOK);
  260         CTR1(KTR_VM, "vmspace_alloc: %p", vm);
  261         _vm_map_init(&vm->vm_map, min, max);
  262         vm->vm_map.pmap = vmspace_pmap(vm);             /* XXX */
  263         vm->vm_refcnt = 1;
  264         vm->vm_shm = NULL;
  265         vm->vm_swrss = 0;
  266         vm->vm_tsize = 0;
  267         vm->vm_dsize = 0;
  268         vm->vm_ssize = 0;
  269         vm->vm_taddr = 0;
  270         vm->vm_daddr = 0;
  271         vm->vm_maxsaddr = 0;
  272         return (vm);
  273 }
  274 
  275 void
  276 vm_init2(void)
  277 {
  278         uma_zone_set_obj(kmapentzone, &kmapentobj, lmin(cnt.v_page_count,
  279             (VM_MAX_KERNEL_ADDRESS - KERNBASE) / PAGE_SIZE) / 8 +
  280              maxproc * 2 + maxfiles);
  281         vmspace_zone = uma_zcreate("VMSPACE", sizeof(struct vmspace), NULL,
  282 #ifdef INVARIANTS
  283             vmspace_zdtor,
  284 #else
  285             NULL,
  286 #endif
  287             vmspace_zinit, vmspace_zfini, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  288         pmap_init2();
  289 }
  290 
  291 static __inline void
  292 vmspace_dofree(struct vmspace *vm)
  293 {
  294         CTR1(KTR_VM, "vmspace_free: %p", vm);
  295 
  296         /*
  297          * Make sure any SysV shm is freed, it might not have been in
  298          * exit1().
  299          */
  300         shmexit(vm);
  301 
  302         /*
  303          * Lock the map, to wait out all other references to it.
  304          * Delete all of the mappings and pages they hold, then call
  305          * the pmap module to reclaim anything left.
  306          */
  307         vm_map_lock(&vm->vm_map);
  308         (void) vm_map_delete(&vm->vm_map, vm->vm_map.min_offset,
  309             vm->vm_map.max_offset);
  310         vm_map_unlock(&vm->vm_map);
  311 
  312         uma_zfree(vmspace_zone, vm);
  313 }
  314 
  315 void
  316 vmspace_free(struct vmspace *vm)
  317 {
  318         int refcnt;
  319 
  320         if (vm->vm_refcnt == 0)
  321                 panic("vmspace_free: attempt to free already freed vmspace");
  322 
  323         do
  324                 refcnt = vm->vm_refcnt;
  325         while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
  326         if (refcnt == 1)
  327                 vmspace_dofree(vm);
  328 }
  329 
  330 void
  331 vmspace_exitfree(struct proc *p)
  332 {
  333         struct vmspace *vm;
  334 
  335         PROC_VMSPACE_LOCK(p);
  336         vm = p->p_vmspace;
  337         p->p_vmspace = NULL;
  338         PROC_VMSPACE_UNLOCK(p);
  339         KASSERT(vm == &vmspace0, ("vmspace_exitfree: wrong vmspace"));
  340         vmspace_free(vm);
  341 }
  342 
  343 void
  344 vmspace_exit(struct thread *td)
  345 {
  346         int refcnt;
  347         struct vmspace *vm;
  348         struct proc *p;
  349 
  350         /*
  351          * Release user portion of address space.
  352          * This releases references to vnodes,
  353          * which could cause I/O if the file has been unlinked.
  354          * Need to do this early enough that we can still sleep.
  355          *
  356          * The last exiting process to reach this point releases as
  357          * much of the environment as it can. vmspace_dofree() is the
  358          * slower fallback in case another process had a temporary
  359          * reference to the vmspace.
  360          */
  361 
  362         p = td->td_proc;
  363         vm = p->p_vmspace;
  364         atomic_add_int(&vmspace0.vm_refcnt, 1);
  365         do {
  366                 refcnt = vm->vm_refcnt;
  367                 if (refcnt > 1 && p->p_vmspace != &vmspace0) {
  368                         /* Switch now since other proc might free vmspace */
  369                         PROC_VMSPACE_LOCK(p);
  370                         p->p_vmspace = &vmspace0;
  371                         PROC_VMSPACE_UNLOCK(p);
  372                         pmap_activate(td);
  373                 }
  374         } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt - 1));
  375         if (refcnt == 1) {
  376                 if (p->p_vmspace != vm) {
  377                         /* vmspace not yet freed, switch back */
  378                         PROC_VMSPACE_LOCK(p);
  379                         p->p_vmspace = vm;
  380                         PROC_VMSPACE_UNLOCK(p);
  381                         pmap_activate(td);
  382                 }
  383                 pmap_remove_pages(vmspace_pmap(vm),
  384                                   vm_map_min(&vm->vm_map),
  385                                   vm_map_max(&vm->vm_map));
  386                 /* Switch now since this proc will free vmspace */
  387                 PROC_VMSPACE_LOCK(p);
  388                 p->p_vmspace = &vmspace0;
  389                 PROC_VMSPACE_UNLOCK(p);
  390                 pmap_activate(td);
  391                 vmspace_dofree(vm);
  392         }
  393 }
  394 
  395 /* Acquire reference to vmspace owned by another process. */
  396 
  397 struct vmspace *
  398 vmspace_acquire_ref(struct proc *p)
  399 {
  400         struct vmspace *vm;
  401         int refcnt;
  402 
  403         PROC_VMSPACE_LOCK(p);
  404         vm = p->p_vmspace;
  405         if (vm == NULL) {
  406                 PROC_VMSPACE_UNLOCK(p);
  407                 return (NULL);
  408         }
  409         do {
  410                 refcnt = vm->vm_refcnt;
  411                 if (refcnt <= 0) {      /* Avoid 0->1 transition */
  412                         PROC_VMSPACE_UNLOCK(p);
  413                         return (NULL);
  414                 }
  415         } while (!atomic_cmpset_int(&vm->vm_refcnt, refcnt, refcnt + 1));
  416         if (vm != p->p_vmspace) {
  417                 PROC_VMSPACE_UNLOCK(p);
  418                 vmspace_free(vm);
  419                 return (NULL);
  420         }
  421         PROC_VMSPACE_UNLOCK(p);
  422         return (vm);
  423 }
  424 
  425 void
  426 _vm_map_lock(vm_map_t map, const char *file, int line)
  427 {
  428 
  429         if (map->system_map)
  430                 _mtx_lock_flags(&map->system_mtx, 0, file, line);
  431         else
  432                 _sx_xlock(&map->lock, file, line);
  433         map->timestamp++;
  434 }
  435 
  436 void
  437 _vm_map_unlock(vm_map_t map, const char *file, int line)
  438 {
  439 
  440         if (map->system_map)
  441                 _mtx_unlock_flags(&map->system_mtx, 0, file, line);
  442         else
  443                 _sx_xunlock(&map->lock, file, line);
  444 }
  445 
  446 void
  447 _vm_map_lock_read(vm_map_t map, const char *file, int line)
  448 {
  449 
  450         if (map->system_map)
  451                 _mtx_lock_flags(&map->system_mtx, 0, file, line);
  452         else
  453                 _sx_xlock(&map->lock, file, line);
  454 }
  455 
  456 void
  457 _vm_map_unlock_read(vm_map_t map, const char *file, int line)
  458 {
  459 
  460         if (map->system_map)
  461                 _mtx_unlock_flags(&map->system_mtx, 0, file, line);
  462         else
  463                 _sx_xunlock(&map->lock, file, line);
  464 }
  465 
  466 int
  467 _vm_map_trylock(vm_map_t map, const char *file, int line)
  468 {
  469         int error;
  470 
  471         error = map->system_map ?
  472             !_mtx_trylock(&map->system_mtx, 0, file, line) :
  473             !_sx_try_xlock(&map->lock, file, line);
  474         if (error == 0)
  475                 map->timestamp++;
  476         return (error == 0);
  477 }
  478 
  479 int
  480 _vm_map_trylock_read(vm_map_t map, const char *file, int line)
  481 {
  482         int error;
  483 
  484         error = map->system_map ?
  485             !_mtx_trylock(&map->system_mtx, 0, file, line) :
  486             !_sx_try_xlock(&map->lock, file, line);
  487         return (error == 0);
  488 }
  489 
  490 int
  491 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line)
  492 {
  493 
  494 #ifdef INVARIANTS
  495         if (map->system_map) {
  496                 _mtx_assert(&map->system_mtx, MA_OWNED, file, line);
  497         } else
  498                 _sx_assert(&map->lock, SX_XLOCKED, file, line);
  499 #endif
  500         map->timestamp++;
  501         return (0);
  502 }
  503 
  504 void
  505 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
  506 {
  507 
  508 #ifdef INVARIANTS
  509         if (map->system_map) {
  510                 _mtx_assert(&map->system_mtx, MA_OWNED, file, line);
  511         } else
  512                 _sx_assert(&map->lock, SX_XLOCKED, file, line);
  513 #endif
  514 }
  515 
  516 /*
  517  *      vm_map_unlock_and_wait:
  518  */
  519 int
  520 vm_map_unlock_and_wait(vm_map_t map, boolean_t user_wait)
  521 {
  522 
  523         mtx_lock(&map_sleep_mtx);
  524         vm_map_unlock(map);
  525         return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", 0));
  526 }
  527 
  528 /*
  529  *      vm_map_wakeup:
  530  */
  531 void
  532 vm_map_wakeup(vm_map_t map)
  533 {
  534 
  535         /*
  536          * Acquire and release map_sleep_mtx to prevent a wakeup()
  537          * from being performed (and lost) between the vm_map_unlock()
  538          * and the msleep() in vm_map_unlock_and_wait().
  539          */
  540         mtx_lock(&map_sleep_mtx);
  541         mtx_unlock(&map_sleep_mtx);
  542         wakeup(&map->root);
  543 }
  544 
  545 long
  546 vmspace_resident_count(struct vmspace *vmspace)
  547 {
  548         return pmap_resident_count(vmspace_pmap(vmspace));
  549 }
  550 
  551 long
  552 vmspace_wired_count(struct vmspace *vmspace)
  553 {
  554         return pmap_wired_count(vmspace_pmap(vmspace));
  555 }
  556 
  557 /*
  558  *      vm_map_create:
  559  *
  560  *      Creates and returns a new empty VM map with
  561  *      the given physical map structure, and having
  562  *      the given lower and upper address bounds.
  563  */
  564 vm_map_t
  565 vm_map_create(pmap_t pmap, vm_offset_t min, vm_offset_t max)
  566 {
  567         vm_map_t result;
  568 
  569         result = uma_zalloc(mapzone, M_WAITOK);
  570         CTR1(KTR_VM, "vm_map_create: %p", result);
  571         _vm_map_init(result, min, max);
  572         result->pmap = pmap;
  573         return (result);
  574 }
  575 
  576 /*
  577  * Initialize an existing vm_map structure
  578  * such as that in the vmspace structure.
  579  * The pmap is set elsewhere.
  580  */
  581 static void
  582 _vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max)
  583 {
  584 
  585         map->header.next = map->header.prev = &map->header;
  586         map->needs_wakeup = FALSE;
  587         map->system_map = 0;
  588         map->min_offset = min;
  589         map->max_offset = max;
  590         map->flags = 0;
  591         map->root = NULL;
  592         map->timestamp = 0;
  593 }
  594 
  595 void
  596 vm_map_init(vm_map_t map, vm_offset_t min, vm_offset_t max)
  597 {
  598         _vm_map_init(map, min, max);
  599         mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK);
  600         sx_init(&map->lock, "user map");
  601 }
  602 
  603 /*
  604  *      vm_map_entry_dispose:   [ internal use only ]
  605  *
  606  *      Inverse of vm_map_entry_create.
  607  */
  608 static void
  609 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry)
  610 {
  611         uma_zfree(map->system_map ? kmapentzone : mapentzone, entry);
  612 }
  613 
  614 /*
  615  *      vm_map_entry_create:    [ internal use only ]
  616  *
  617  *      Allocates a VM map entry for insertion.
  618  *      No entry fields are filled in.
  619  */
  620 static vm_map_entry_t
  621 vm_map_entry_create(vm_map_t map)
  622 {
  623         vm_map_entry_t new_entry;
  624 
  625         if (map->system_map)
  626                 new_entry = uma_zalloc(kmapentzone, M_NOWAIT);
  627         else
  628                 new_entry = uma_zalloc(mapentzone, M_WAITOK);
  629         if (new_entry == NULL)
  630                 panic("vm_map_entry_create: kernel resources exhausted");
  631         return (new_entry);
  632 }
  633 
  634 /*
  635  *      vm_map_entry_set_behavior:
  636  *
  637  *      Set the expected access behavior, either normal, random, or
  638  *      sequential.
  639  */
  640 static __inline void
  641 vm_map_entry_set_behavior(vm_map_entry_t entry, u_char behavior)
  642 {
  643         entry->eflags = (entry->eflags & ~MAP_ENTRY_BEHAV_MASK) |
  644             (behavior & MAP_ENTRY_BEHAV_MASK);
  645 }
  646 
  647 /*
  648  *      vm_map_entry_set_max_free:
  649  *
  650  *      Set the max_free field in a vm_map_entry.
  651  */
  652 static __inline void
  653 vm_map_entry_set_max_free(vm_map_entry_t entry)
  654 {
  655 
  656         entry->max_free = entry->adj_free;
  657         if (entry->left != NULL && entry->left->max_free > entry->max_free)
  658                 entry->max_free = entry->left->max_free;
  659         if (entry->right != NULL && entry->right->max_free > entry->max_free)
  660                 entry->max_free = entry->right->max_free;
  661 }
  662 
  663 /*
  664  *      vm_map_entry_splay:
  665  *
  666  *      The Sleator and Tarjan top-down splay algorithm with the
  667  *      following variation.  Max_free must be computed bottom-up, so
  668  *      on the downward pass, maintain the left and right spines in
  669  *      reverse order.  Then, make a second pass up each side to fix
  670  *      the pointers and compute max_free.  The time bound is O(log n)
  671  *      amortized.
  672  *
  673  *      The new root is the vm_map_entry containing "addr", or else an
  674  *      adjacent entry (lower or higher) if addr is not in the tree.
  675  *
  676  *      The map must be locked, and leaves it so.
  677  *
  678  *      Returns: the new root.
  679  */
  680 static vm_map_entry_t
  681 vm_map_entry_splay(vm_offset_t addr, vm_map_entry_t root)
  682 {
  683         vm_map_entry_t llist, rlist;
  684         vm_map_entry_t ltree, rtree;
  685         vm_map_entry_t y;
  686 
  687         /* Special case of empty tree. */
  688         if (root == NULL)
  689                 return (root);
  690 
  691         /*
  692          * Pass One: Splay down the tree until we find addr or a NULL
  693          * pointer where addr would go.  llist and rlist are the two
  694          * sides in reverse order (bottom-up), with llist linked by
  695          * the right pointer and rlist linked by the left pointer in
  696          * the vm_map_entry.  Wait until Pass Two to set max_free on
  697          * the two spines.
  698          */
  699         llist = NULL;
  700         rlist = NULL;
  701         for (;;) {
  702                 /* root is never NULL in here. */
  703                 if (addr < root->start) {
  704                         y = root->left;
  705                         if (y == NULL)
  706                                 break;
  707                         if (addr < y->start && y->left != NULL) {
  708                                 /* Rotate right and put y on rlist. */
  709                                 root->left = y->right;
  710                                 y->right = root;
  711                                 vm_map_entry_set_max_free(root);
  712                                 root = y->left;
  713                                 y->left = rlist;
  714                                 rlist = y;
  715                         } else {
  716                                 /* Put root on rlist. */
  717                                 root->left = rlist;
  718                                 rlist = root;
  719                                 root = y;
  720                         }
  721                 } else {
  722                         y = root->right;
  723                         if (addr < root->end || y == NULL)
  724                                 break;
  725                         if (addr >= y->end && y->right != NULL) {
  726                                 /* Rotate left and put y on llist. */
  727                                 root->right = y->left;
  728                                 y->left = root;
  729                                 vm_map_entry_set_max_free(root);
  730                                 root = y->right;
  731                                 y->right = llist;
  732                                 llist = y;
  733                         } else {
  734                                 /* Put root on llist. */
  735                                 root->right = llist;
  736                                 llist = root;
  737                                 root = y;
  738                         }
  739                 }
  740         }
  741 
  742         /*
  743          * Pass Two: Walk back up the two spines, flip the pointers
  744          * and set max_free.  The subtrees of the root go at the
  745          * bottom of llist and rlist.
  746          */
  747         ltree = root->left;
  748         while (llist != NULL) {
  749                 y = llist->right;
  750                 llist->right = ltree;
  751                 vm_map_entry_set_max_free(llist);
  752                 ltree = llist;
  753                 llist = y;
  754         }
  755         rtree = root->right;
  756         while (rlist != NULL) {
  757                 y = rlist->left;
  758                 rlist->left = rtree;
  759                 vm_map_entry_set_max_free(rlist);
  760                 rtree = rlist;
  761                 rlist = y;
  762         }
  763 
  764         /*
  765          * Final assembly: add ltree and rtree as subtrees of root.
  766          */
  767         root->left = ltree;
  768         root->right = rtree;
  769         vm_map_entry_set_max_free(root);
  770 
  771         return (root);
  772 }
  773 
  774 /*
  775  *      vm_map_entry_{un,}link:
  776  *
  777  *      Insert/remove entries from maps.
  778  */
  779 static void
  780 vm_map_entry_link(vm_map_t map,
  781                   vm_map_entry_t after_where,
  782                   vm_map_entry_t entry)
  783 {
  784 
  785         CTR4(KTR_VM,
  786             "vm_map_entry_link: map %p, nentries %d, entry %p, after %p", map,
  787             map->nentries, entry, after_where);
  788         map->nentries++;
  789         entry->prev = after_where;
  790         entry->next = after_where->next;
  791         entry->next->prev = entry;
  792         after_where->next = entry;
  793 
  794         if (after_where != &map->header) {
  795                 if (after_where != map->root)
  796                         vm_map_entry_splay(after_where->start, map->root);
  797                 entry->right = after_where->right;
  798                 entry->left = after_where;
  799                 after_where->right = NULL;
  800                 after_where->adj_free = entry->start - after_where->end;
  801                 vm_map_entry_set_max_free(after_where);
  802         } else {
  803                 entry->right = map->root;
  804                 entry->left = NULL;
  805         }
  806         entry->adj_free = (entry->next == &map->header ? map->max_offset :
  807             entry->next->start) - entry->end;
  808         vm_map_entry_set_max_free(entry);
  809         map->root = entry;
  810 }
  811 
  812 static void
  813 vm_map_entry_unlink(vm_map_t map,
  814                     vm_map_entry_t entry)
  815 {
  816         vm_map_entry_t next, prev, root;
  817 
  818         if (entry != map->root)
  819                 vm_map_entry_splay(entry->start, map->root);
  820         if (entry->left == NULL)
  821                 root = entry->right;
  822         else {
  823                 root = vm_map_entry_splay(entry->start, entry->left);
  824                 root->right = entry->right;
  825                 root->adj_free = (entry->next == &map->header ? map->max_offset :
  826                     entry->next->start) - root->end;
  827                 vm_map_entry_set_max_free(root);
  828         }
  829         map->root = root;
  830 
  831         prev = entry->prev;
  832         next = entry->next;
  833         next->prev = prev;
  834         prev->next = next;
  835         map->nentries--;
  836         CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map,
  837             map->nentries, entry);
  838 }
  839 
  840 /*
  841  *      vm_map_entry_resize_free:
  842  *
  843  *      Recompute the amount of free space following a vm_map_entry
  844  *      and propagate that value up the tree.  Call this function after
  845  *      resizing a map entry in-place, that is, without a call to
  846  *      vm_map_entry_link() or _unlink().
  847  *
  848  *      The map must be locked, and leaves it so.
  849  */
  850 static void
  851 vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t entry)
  852 {
  853 
  854         /*
  855          * Using splay trees without parent pointers, propagating
  856          * max_free up the tree is done by moving the entry to the
  857          * root and making the change there.
  858          */
  859         if (entry != map->root)
  860                 map->root = vm_map_entry_splay(entry->start, map->root);
  861 
  862         entry->adj_free = (entry->next == &map->header ? map->max_offset :
  863             entry->next->start) - entry->end;
  864         vm_map_entry_set_max_free(entry);
  865 }
  866 
  867 /*
  868  *      vm_map_lookup_entry:    [ internal use only ]
  869  *
  870  *      Finds the map entry containing (or
  871  *      immediately preceding) the specified address
  872  *      in the given map; the entry is returned
  873  *      in the "entry" parameter.  The boolean
  874  *      result indicates whether the address is
  875  *      actually contained in the map.
  876  */
  877 boolean_t
  878 vm_map_lookup_entry(
  879         vm_map_t map,
  880         vm_offset_t address,
  881         vm_map_entry_t *entry)  /* OUT */
  882 {
  883         vm_map_entry_t cur;
  884 
  885         cur = vm_map_entry_splay(address, map->root);
  886         if (cur == NULL)
  887                 *entry = &map->header;
  888         else {
  889                 map->root = cur;
  890 
  891                 if (address >= cur->start) {
  892                         *entry = cur;
  893                         if (cur->end > address)
  894                                 return (TRUE);
  895                 } else
  896                         *entry = cur->prev;
  897         }
  898         return (FALSE);
  899 }
  900 
  901 /*
  902  *      vm_map_insert:
  903  *
  904  *      Inserts the given whole VM object into the target
  905  *      map at the specified address range.  The object's
  906  *      size should match that of the address range.
  907  *
  908  *      Requires that the map be locked, and leaves it so.
  909  *
  910  *      If object is non-NULL, ref count must be bumped by caller
  911  *      prior to making call to account for the new entry.
  912  */
  913 int
  914 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
  915               vm_offset_t start, vm_offset_t end, vm_prot_t prot, vm_prot_t max,
  916               int cow)
  917 {
  918         vm_map_entry_t new_entry;
  919         vm_map_entry_t prev_entry;
  920         vm_map_entry_t temp_entry;
  921         vm_eflags_t protoeflags;
  922 
  923         /*
  924          * Check that the start and end points are not bogus.
  925          */
  926         if ((start < map->min_offset) || (end > map->max_offset) ||
  927             (start >= end))
  928                 return (KERN_INVALID_ADDRESS);
  929 
  930         /*
  931          * Find the entry prior to the proposed starting address; if it's part
  932          * of an existing entry, this range is bogus.
  933          */
  934         if (vm_map_lookup_entry(map, start, &temp_entry))
  935                 return (KERN_NO_SPACE);
  936 
  937         prev_entry = temp_entry;
  938 
  939         /*
  940          * Assert that the next entry doesn't overlap the end point.
  941          */
  942         if ((prev_entry->next != &map->header) &&
  943             (prev_entry->next->start < end))
  944                 return (KERN_NO_SPACE);
  945 
  946         protoeflags = 0;
  947 
  948         if (cow & MAP_COPY_ON_WRITE)
  949                 protoeflags |= MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY;
  950 
  951         if (cow & MAP_NOFAULT) {
  952                 protoeflags |= MAP_ENTRY_NOFAULT;
  953 
  954                 KASSERT(object == NULL,
  955                         ("vm_map_insert: paradoxical MAP_NOFAULT request"));
  956         }
  957         if (cow & MAP_DISABLE_SYNCER)
  958                 protoeflags |= MAP_ENTRY_NOSYNC;
  959         if (cow & MAP_DISABLE_COREDUMP)
  960                 protoeflags |= MAP_ENTRY_NOCOREDUMP;
  961 
  962         if (object != NULL) {
  963                 /*
  964                  * OBJ_ONEMAPPING must be cleared unless this mapping
  965                  * is trivially proven to be the only mapping for any
  966                  * of the object's pages.  (Object granularity
  967                  * reference counting is insufficient to recognize
  968                  * aliases with precision.)
  969                  */
  970                 VM_OBJECT_LOCK(object);
  971                 if (object->ref_count > 1 || object->shadow_count != 0)
  972                         vm_object_clear_flag(object, OBJ_ONEMAPPING);
  973                 VM_OBJECT_UNLOCK(object);
  974         }
  975         else if ((prev_entry != &map->header) &&
  976                  (prev_entry->eflags == protoeflags) &&
  977                  (prev_entry->end == start) &&
  978                  (prev_entry->wired_count == 0) &&
  979                  ((prev_entry->object.vm_object == NULL) ||
  980                   vm_object_coalesce(prev_entry->object.vm_object,
  981                                      prev_entry->offset,
  982                                      (vm_size_t)(prev_entry->end - prev_entry->start),
  983                                      (vm_size_t)(end - prev_entry->end)))) {
  984                 /*
  985                  * We were able to extend the object.  Determine if we
  986                  * can extend the previous map entry to include the
  987                  * new range as well.
  988                  */
  989                 if ((prev_entry->inheritance == VM_INHERIT_DEFAULT) &&
  990                     (prev_entry->protection == prot) &&
  991                     (prev_entry->max_protection == max)) {
  992                         map->size += (end - prev_entry->end);
  993                         prev_entry->end = end;
  994                         vm_map_entry_resize_free(map, prev_entry);
  995                         vm_map_simplify_entry(map, prev_entry);
  996                         return (KERN_SUCCESS);
  997                 }
  998 
  999                 /*
 1000                  * If we can extend the object but cannot extend the
 1001                  * map entry, we have to create a new map entry.  We
 1002                  * must bump the ref count on the extended object to
 1003                  * account for it.  object may be NULL.
 1004                  */
 1005                 object = prev_entry->object.vm_object;
 1006                 offset = prev_entry->offset +
 1007                         (prev_entry->end - prev_entry->start);
 1008                 vm_object_reference(object);
 1009         }
 1010 
 1011         /*
 1012          * NOTE: if conditionals fail, object can be NULL here.  This occurs
 1013          * in things like the buffer map where we manage kva but do not manage
 1014          * backing objects.
 1015          */
 1016 
 1017         /*
 1018          * Create a new entry
 1019          */
 1020         new_entry = vm_map_entry_create(map);
 1021         new_entry->start = start;
 1022         new_entry->end = end;
 1023 
 1024         new_entry->eflags = protoeflags;
 1025         new_entry->object.vm_object = object;
 1026         new_entry->offset = offset;
 1027         new_entry->avail_ssize = 0;
 1028 
 1029         new_entry->inheritance = VM_INHERIT_DEFAULT;
 1030         new_entry->protection = prot;
 1031         new_entry->max_protection = max;
 1032         new_entry->wired_count = 0;
 1033 
 1034         /*
 1035          * Insert the new entry into the list
 1036          */
 1037         vm_map_entry_link(map, prev_entry, new_entry);
 1038         map->size += new_entry->end - new_entry->start;
 1039 
 1040 #if 0
 1041         /*
 1042          * Temporarily removed to avoid MAP_STACK panic, due to
 1043          * MAP_STACK being a huge hack.  Will be added back in
 1044          * when MAP_STACK (and the user stack mapping) is fixed.
 1045          */
 1046         /*
 1047          * It may be possible to simplify the entry
 1048          */
 1049         vm_map_simplify_entry(map, new_entry);
 1050 #endif
 1051 
 1052         if (cow & (MAP_PREFAULT|MAP_PREFAULT_PARTIAL)) {
 1053                 vm_map_pmap_enter(map, start, prot,
 1054                                     object, OFF_TO_IDX(offset), end - start,
 1055                                     cow & MAP_PREFAULT_PARTIAL);
 1056         }
 1057 
 1058         return (KERN_SUCCESS);
 1059 }
 1060 
 1061 /*
 1062  *      vm_map_findspace:
 1063  *
 1064  *      Find the first fit (lowest VM address) for "length" free bytes
 1065  *      beginning at address >= start in the given map.
 1066  *
 1067  *      In a vm_map_entry, "adj_free" is the amount of free space
 1068  *      adjacent (higher address) to this entry, and "max_free" is the
 1069  *      maximum amount of contiguous free space in its subtree.  This
 1070  *      allows finding a free region in one path down the tree, so
 1071  *      O(log n) amortized with splay trees.
 1072  *
 1073  *      The map must be locked, and leaves it so.
 1074  *
 1075  *      Returns: 0 on success, and starting address in *addr,
 1076  *               1 if insufficient space.
 1077  */
 1078 int
 1079 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length,
 1080     vm_offset_t *addr)  /* OUT */
 1081 {
 1082         vm_map_entry_t entry;
 1083         vm_offset_t end, st;
 1084 
 1085         /*
 1086          * Request must fit within min/max VM address and must avoid
 1087          * address wrap.
 1088          */
 1089         if (start < map->min_offset)
 1090                 start = map->min_offset;
 1091         if (start + length > map->max_offset || start + length < start)
 1092                 return (1);
 1093 
 1094         /* Empty tree means wide open address space. */
 1095         if (map->root == NULL) {
 1096                 *addr = start;
 1097                 goto found;
 1098         }
 1099 
 1100         /*
 1101          * After splay, if start comes before root node, then there
 1102          * must be a gap from start to the root.
 1103          */
 1104         map->root = vm_map_entry_splay(start, map->root);
 1105         if (start + length <= map->root->start) {
 1106                 *addr = start;
 1107                 goto found;
 1108         }
 1109 
 1110         /*
 1111          * Root is the last node that might begin its gap before
 1112          * start, and this is the last comparison where address
 1113          * wrap might be a problem.
 1114          */
 1115         st = (start > map->root->end) ? start : map->root->end;
 1116         if (length <= map->root->end + map->root->adj_free - st) {
 1117                 *addr = st;
 1118                 goto found;
 1119         }
 1120 
 1121         /* With max_free, can immediately tell if no solution. */
 1122         entry = map->root->right;
 1123         if (entry == NULL || length > entry->max_free)
 1124                 return (1);
 1125 
 1126         /*
 1127          * Search the right subtree in the order: left subtree, root,
 1128          * right subtree (first fit).  The previous splay implies that
 1129          * all regions in the right subtree have addresses > start.
 1130          */
 1131         while (entry != NULL) {
 1132                 if (entry->left != NULL && entry->left->max_free >= length)
 1133                         entry = entry->left;
 1134                 else if (entry->adj_free >= length) {
 1135                         *addr = entry->end;
 1136                         goto found;
 1137                 } else
 1138                         entry = entry->right;
 1139         }
 1140 
 1141         /* Can't get here, so panic if we do. */
 1142         panic("vm_map_findspace: max_free corrupt");
 1143 
 1144 found:
 1145         /* Expand the kernel pmap, if necessary. */
 1146         if (map == kernel_map) {
 1147                 end = round_page(*addr + length);
 1148                 if (end > kernel_vm_end)
 1149                         pmap_growkernel(end);
 1150         }
 1151         return (0);
 1152 }
 1153 
 1154 /*
 1155  *      vm_map_find finds an unallocated region in the target address
 1156  *      map with the given length.  The search is defined to be
 1157  *      first-fit from the specified address; the region found is
 1158  *      returned in the same parameter.
 1159  *
 1160  *      If object is non-NULL, ref count must be bumped by caller
 1161  *      prior to making call to account for the new entry.
 1162  */
 1163 int
 1164 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 1165             vm_offset_t *addr,  /* IN/OUT */
 1166             vm_size_t length, boolean_t find_space, vm_prot_t prot,
 1167             vm_prot_t max, int cow)
 1168 {
 1169         vm_offset_t start;
 1170         int result;
 1171 
 1172         start = *addr;
 1173         vm_map_lock(map);
 1174         if (find_space) {
 1175                 if (vm_map_findspace(map, start, length, addr)) {
 1176                         vm_map_unlock(map);
 1177                         return (KERN_NO_SPACE);
 1178                 }
 1179                 start = *addr;
 1180         }
 1181         result = vm_map_insert(map, object, offset,
 1182                 start, start + length, prot, max, cow);
 1183         vm_map_unlock(map);
 1184         return (result);
 1185 }
 1186 
 1187 /*
 1188  *      vm_map_simplify_entry:
 1189  *
 1190  *      Simplify the given map entry by merging with either neighbor.  This
 1191  *      routine also has the ability to merge with both neighbors.
 1192  *
 1193  *      The map must be locked.
 1194  *
 1195  *      This routine guarentees that the passed entry remains valid (though
 1196  *      possibly extended).  When merging, this routine may delete one or
 1197  *      both neighbors.
 1198  */
 1199 void
 1200 vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry)
 1201 {
 1202         vm_map_entry_t next, prev;
 1203         vm_size_t prevsize, esize;
 1204 
 1205         if (entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_IS_SUB_MAP))
 1206                 return;
 1207 
 1208         prev = entry->prev;
 1209         if (prev != &map->header) {
 1210                 prevsize = prev->end - prev->start;
 1211                 if ( (prev->end == entry->start) &&
 1212                      (prev->object.vm_object == entry->object.vm_object) &&
 1213                      (!prev->object.vm_object ||
 1214                         (prev->offset + prevsize == entry->offset)) &&
 1215                      (prev->eflags == entry->eflags) &&
 1216                      (prev->protection == entry->protection) &&
 1217                      (prev->max_protection == entry->max_protection) &&
 1218                      (prev->inheritance == entry->inheritance) &&
 1219                      (prev->wired_count == entry->wired_count)) {
 1220                         vm_map_entry_unlink(map, prev);
 1221                         entry->start = prev->start;
 1222                         entry->offset = prev->offset;
 1223                         if (entry->prev != &map->header)
 1224                                 vm_map_entry_resize_free(map, entry->prev);
 1225                         if (prev->object.vm_object)
 1226                                 vm_object_deallocate(prev->object.vm_object);
 1227                         vm_map_entry_dispose(map, prev);
 1228                 }
 1229         }
 1230 
 1231         next = entry->next;
 1232         if (next != &map->header) {
 1233                 esize = entry->end - entry->start;
 1234                 if ((entry->end == next->start) &&
 1235                     (next->object.vm_object == entry->object.vm_object) &&
 1236                      (!entry->object.vm_object ||
 1237                         (entry->offset + esize == next->offset)) &&
 1238                     (next->eflags == entry->eflags) &&
 1239                     (next->protection == entry->protection) &&
 1240                     (next->max_protection == entry->max_protection) &&
 1241                     (next->inheritance == entry->inheritance) &&
 1242                     (next->wired_count == entry->wired_count)) {
 1243                         vm_map_entry_unlink(map, next);
 1244                         entry->end = next->end;
 1245                         vm_map_entry_resize_free(map, entry);
 1246                         if (next->object.vm_object)
 1247                                 vm_object_deallocate(next->object.vm_object);
 1248                         vm_map_entry_dispose(map, next);
 1249                 }
 1250         }
 1251 }
 1252 /*
 1253  *      vm_map_clip_start:      [ internal use only ]
 1254  *
 1255  *      Asserts that the given entry begins at or after
 1256  *      the specified address; if necessary,
 1257  *      it splits the entry into two.
 1258  */
 1259 #define vm_map_clip_start(map, entry, startaddr) \
 1260 { \
 1261         if (startaddr > entry->start) \
 1262                 _vm_map_clip_start(map, entry, startaddr); \
 1263 }
 1264 
 1265 /*
 1266  *      This routine is called only when it is known that
 1267  *      the entry must be split.
 1268  */
 1269 static void
 1270 _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start)
 1271 {
 1272         vm_map_entry_t new_entry;
 1273 
 1274         /*
 1275          * Split off the front portion -- note that we must insert the new
 1276          * entry BEFORE this one, so that this entry has the specified
 1277          * starting address.
 1278          */
 1279         vm_map_simplify_entry(map, entry);
 1280 
 1281         /*
 1282          * If there is no object backing this entry, we might as well create
 1283          * one now.  If we defer it, an object can get created after the map
 1284          * is clipped, and individual objects will be created for the split-up
 1285          * map.  This is a bit of a hack, but is also about the best place to
 1286          * put this improvement.
 1287          */
 1288         if (entry->object.vm_object == NULL && !map->system_map) {
 1289                 vm_object_t object;
 1290                 object = vm_object_allocate(OBJT_DEFAULT,
 1291                                 atop(entry->end - entry->start));
 1292                 entry->object.vm_object = object;
 1293                 entry->offset = 0;
 1294         }
 1295 
 1296         new_entry = vm_map_entry_create(map);
 1297         *new_entry = *entry;
 1298 
 1299         new_entry->end = start;
 1300         entry->offset += (start - entry->start);
 1301         entry->start = start;
 1302 
 1303         vm_map_entry_link(map, entry->prev, new_entry);
 1304 
 1305         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 1306                 vm_object_reference(new_entry->object.vm_object);
 1307         }
 1308 }
 1309 
 1310 /*
 1311  *      vm_map_clip_end:        [ internal use only ]
 1312  *
 1313  *      Asserts that the given entry ends at or before
 1314  *      the specified address; if necessary,
 1315  *      it splits the entry into two.
 1316  */
 1317 #define vm_map_clip_end(map, entry, endaddr) \
 1318 { \
 1319         if ((endaddr) < (entry->end)) \
 1320                 _vm_map_clip_end((map), (entry), (endaddr)); \
 1321 }
 1322 
 1323 /*
 1324  *      This routine is called only when it is known that
 1325  *      the entry must be split.
 1326  */
 1327 static void
 1328 _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end)
 1329 {
 1330         vm_map_entry_t new_entry;
 1331 
 1332         /*
 1333          * If there is no object backing this entry, we might as well create
 1334          * one now.  If we defer it, an object can get created after the map
 1335          * is clipped, and individual objects will be created for the split-up
 1336          * map.  This is a bit of a hack, but is also about the best place to
 1337          * put this improvement.
 1338          */
 1339         if (entry->object.vm_object == NULL && !map->system_map) {
 1340                 vm_object_t object;
 1341                 object = vm_object_allocate(OBJT_DEFAULT,
 1342                                 atop(entry->end - entry->start));
 1343                 entry->object.vm_object = object;
 1344                 entry->offset = 0;
 1345         }
 1346 
 1347         /*
 1348          * Create a new entry and insert it AFTER the specified entry
 1349          */
 1350         new_entry = vm_map_entry_create(map);
 1351         *new_entry = *entry;
 1352 
 1353         new_entry->start = entry->end = end;
 1354         new_entry->offset += (end - entry->start);
 1355 
 1356         vm_map_entry_link(map, entry, new_entry);
 1357 
 1358         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 1359                 vm_object_reference(new_entry->object.vm_object);
 1360         }
 1361 }
 1362 
 1363 /*
 1364  *      VM_MAP_RANGE_CHECK:     [ internal use only ]
 1365  *
 1366  *      Asserts that the starting and ending region
 1367  *      addresses fall within the valid range of the map.
 1368  */
 1369 #define VM_MAP_RANGE_CHECK(map, start, end)             \
 1370                 {                                       \
 1371                 if (start < vm_map_min(map))            \
 1372                         start = vm_map_min(map);        \
 1373                 if (end > vm_map_max(map))              \
 1374                         end = vm_map_max(map);          \
 1375                 if (start > end)                        \
 1376                         start = end;                    \
 1377                 }
 1378 
 1379 /*
 1380  *      vm_map_submap:          [ kernel use only ]
 1381  *
 1382  *      Mark the given range as handled by a subordinate map.
 1383  *
 1384  *      This range must have been created with vm_map_find,
 1385  *      and no other operations may have been performed on this
 1386  *      range prior to calling vm_map_submap.
 1387  *
 1388  *      Only a limited number of operations can be performed
 1389  *      within this rage after calling vm_map_submap:
 1390  *              vm_fault
 1391  *      [Don't try vm_map_copy!]
 1392  *
 1393  *      To remove a submapping, one must first remove the
 1394  *      range from the superior map, and then destroy the
 1395  *      submap (if desired).  [Better yet, don't try it.]
 1396  */
 1397 int
 1398 vm_map_submap(
 1399         vm_map_t map,
 1400         vm_offset_t start,
 1401         vm_offset_t end,
 1402         vm_map_t submap)
 1403 {
 1404         vm_map_entry_t entry;
 1405         int result = KERN_INVALID_ARGUMENT;
 1406 
 1407         vm_map_lock(map);
 1408 
 1409         VM_MAP_RANGE_CHECK(map, start, end);
 1410 
 1411         if (vm_map_lookup_entry(map, start, &entry)) {
 1412                 vm_map_clip_start(map, entry, start);
 1413         } else
 1414                 entry = entry->next;
 1415 
 1416         vm_map_clip_end(map, entry, end);
 1417 
 1418         if ((entry->start == start) && (entry->end == end) &&
 1419             ((entry->eflags & MAP_ENTRY_COW) == 0) &&
 1420             (entry->object.vm_object == NULL)) {
 1421                 entry->object.sub_map = submap;
 1422                 entry->eflags |= MAP_ENTRY_IS_SUB_MAP;
 1423                 result = KERN_SUCCESS;
 1424         }
 1425         vm_map_unlock(map);
 1426 
 1427         return (result);
 1428 }
 1429 
 1430 /*
 1431  * The maximum number of pages to map
 1432  */
 1433 #define MAX_INIT_PT     96
 1434 
 1435 /*
 1436  *      vm_map_pmap_enter:
 1437  *
 1438  *      Preload read-only mappings for the given object into the specified
 1439  *      map.  This eliminates the soft faults on process startup and
 1440  *      immediately after an mmap(2).
 1441  */
 1442 void
 1443 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
 1444     vm_object_t object, vm_pindex_t pindex, vm_size_t size, int flags)
 1445 {
 1446         vm_offset_t tmpidx;
 1447         int psize;
 1448         vm_page_t p, mpte;
 1449         boolean_t are_queues_locked;
 1450 
 1451         if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
 1452                 return;
 1453         VM_OBJECT_LOCK(object);
 1454         if (object->type == OBJT_DEVICE) {
 1455                 pmap_object_init_pt(map->pmap, addr, object, pindex, size);
 1456                 goto unlock_return;
 1457         }
 1458 
 1459         psize = atop(size);
 1460 
 1461         if (object->type != OBJT_VNODE ||
 1462             ((flags & MAP_PREFAULT_PARTIAL) && (psize > MAX_INIT_PT) &&
 1463              (object->resident_page_count > MAX_INIT_PT))) {
 1464                 goto unlock_return;
 1465         }
 1466 
 1467         if (psize + pindex > object->size) {
 1468                 if (object->size < pindex)
 1469                         goto unlock_return;
 1470                 psize = object->size - pindex;
 1471         }
 1472 
 1473         are_queues_locked = FALSE;
 1474         mpte = NULL;
 1475 
 1476         if ((p = TAILQ_FIRST(&object->memq)) != NULL) {
 1477                 if (p->pindex < pindex) {
 1478                         p = vm_page_splay(pindex, object->root);
 1479                         if ((object->root = p)->pindex < pindex)
 1480                                 p = TAILQ_NEXT(p, listq);
 1481                 }
 1482         }
 1483         /*
 1484          * Assert: the variable p is either (1) the page with the
 1485          * least pindex greater than or equal to the parameter pindex
 1486          * or (2) NULL.
 1487          */
 1488         for (;
 1489              p != NULL && (tmpidx = p->pindex - pindex) < psize;
 1490              p = TAILQ_NEXT(p, listq)) {
 1491                 /*
 1492                  * don't allow an madvise to blow away our really
 1493                  * free pages allocating pv entries.
 1494                  */
 1495                 if ((flags & MAP_PREFAULT_MADVISE) &&
 1496                     cnt.v_free_count < cnt.v_free_reserved) {
 1497                         break;
 1498                 }
 1499                 if ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL &&
 1500                     (p->busy == 0) &&
 1501                     (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
 1502                         if (!are_queues_locked) {
 1503                                 are_queues_locked = TRUE;
 1504                                 vm_page_lock_queues();
 1505                         }
 1506                         if ((p->queue - p->pc) == PQ_CACHE)
 1507                                 vm_page_deactivate(p);
 1508                         mpte = pmap_enter_quick(map->pmap,
 1509                             addr + ptoa(tmpidx), p, prot, mpte);
 1510                 }
 1511         }
 1512         if (are_queues_locked)
 1513                 vm_page_unlock_queues();
 1514 unlock_return:
 1515         VM_OBJECT_UNLOCK(object);
 1516 }
 1517 
 1518 /*
 1519  *      vm_map_protect:
 1520  *
 1521  *      Sets the protection of the specified address
 1522  *      region in the target map.  If "set_max" is
 1523  *      specified, the maximum protection is to be set;
 1524  *      otherwise, only the current protection is affected.
 1525  */
 1526 int
 1527 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1528                vm_prot_t new_prot, boolean_t set_max)
 1529 {
 1530         vm_map_entry_t current;
 1531         vm_map_entry_t entry;
 1532 
 1533         vm_map_lock(map);
 1534 
 1535         VM_MAP_RANGE_CHECK(map, start, end);
 1536 
 1537         if (vm_map_lookup_entry(map, start, &entry)) {
 1538                 vm_map_clip_start(map, entry, start);
 1539         } else {
 1540                 entry = entry->next;
 1541         }
 1542 
 1543         /*
 1544          * Make a first pass to check for protection violations.
 1545          */
 1546         current = entry;
 1547         while ((current != &map->header) && (current->start < end)) {
 1548                 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
 1549                         vm_map_unlock(map);
 1550                         return (KERN_INVALID_ARGUMENT);
 1551                 }
 1552                 if ((new_prot & current->max_protection) != new_prot) {
 1553                         vm_map_unlock(map);
 1554                         return (KERN_PROTECTION_FAILURE);
 1555                 }
 1556                 current = current->next;
 1557         }
 1558 
 1559         /*
 1560          * Go back and fix up protections. [Note that clipping is not
 1561          * necessary the second time.]
 1562          */
 1563         current = entry;
 1564         while ((current != &map->header) && (current->start < end)) {
 1565                 vm_prot_t old_prot;
 1566 
 1567                 vm_map_clip_end(map, current, end);
 1568 
 1569                 old_prot = current->protection;
 1570                 if (set_max)
 1571                         current->protection =
 1572                             (current->max_protection = new_prot) &
 1573                             old_prot;
 1574                 else
 1575                         current->protection = new_prot;
 1576 
 1577                 /*
 1578                  * Update physical map if necessary. Worry about copy-on-write
 1579                  * here -- CHECK THIS XXX
 1580                  */
 1581                 if (current->protection != old_prot) {
 1582 #define MASK(entry)     (((entry)->eflags & MAP_ENTRY_COW) ? ~VM_PROT_WRITE : \
 1583                                                         VM_PROT_ALL)
 1584                         pmap_protect(map->pmap, current->start,
 1585                             current->end,
 1586                             current->protection & MASK(current));
 1587 #undef  MASK
 1588                 }
 1589                 vm_map_simplify_entry(map, current);
 1590                 current = current->next;
 1591         }
 1592         vm_map_unlock(map);
 1593         return (KERN_SUCCESS);
 1594 }
 1595 
 1596 /*
 1597  *      vm_map_madvise:
 1598  *
 1599  *      This routine traverses a processes map handling the madvise
 1600  *      system call.  Advisories are classified as either those effecting
 1601  *      the vm_map_entry structure, or those effecting the underlying
 1602  *      objects.
 1603  */
 1604 int
 1605 vm_map_madvise(
 1606         vm_map_t map,
 1607         vm_offset_t start,
 1608         vm_offset_t end,
 1609         int behav)
 1610 {
 1611         vm_map_entry_t current, entry;
 1612         int modify_map = 0;
 1613 
 1614         /*
 1615          * Some madvise calls directly modify the vm_map_entry, in which case
 1616          * we need to use an exclusive lock on the map and we need to perform
 1617          * various clipping operations.  Otherwise we only need a read-lock
 1618          * on the map.
 1619          */
 1620         switch(behav) {
 1621         case MADV_NORMAL:
 1622         case MADV_SEQUENTIAL:
 1623         case MADV_RANDOM:
 1624         case MADV_NOSYNC:
 1625         case MADV_AUTOSYNC:
 1626         case MADV_NOCORE:
 1627         case MADV_CORE:
 1628                 modify_map = 1;
 1629                 vm_map_lock(map);
 1630                 break;
 1631         case MADV_WILLNEED:
 1632         case MADV_DONTNEED:
 1633         case MADV_FREE:
 1634                 vm_map_lock_read(map);
 1635                 break;
 1636         default:
 1637                 return (KERN_INVALID_ARGUMENT);
 1638         }
 1639 
 1640         /*
 1641          * Locate starting entry and clip if necessary.
 1642          */
 1643         VM_MAP_RANGE_CHECK(map, start, end);
 1644 
 1645         if (vm_map_lookup_entry(map, start, &entry)) {
 1646                 if (modify_map)
 1647                         vm_map_clip_start(map, entry, start);
 1648         } else {
 1649                 entry = entry->next;
 1650         }
 1651 
 1652         if (modify_map) {
 1653                 /*
 1654                  * madvise behaviors that are implemented in the vm_map_entry.
 1655                  *
 1656                  * We clip the vm_map_entry so that behavioral changes are
 1657                  * limited to the specified address range.
 1658                  */
 1659                 for (current = entry;
 1660                      (current != &map->header) && (current->start < end);
 1661                      current = current->next
 1662                 ) {
 1663                         if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
 1664                                 continue;
 1665 
 1666                         vm_map_clip_end(map, current, end);
 1667 
 1668                         switch (behav) {
 1669                         case MADV_NORMAL:
 1670                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_NORMAL);
 1671                                 break;
 1672                         case MADV_SEQUENTIAL:
 1673                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_SEQUENTIAL);
 1674                                 break;
 1675                         case MADV_RANDOM:
 1676                                 vm_map_entry_set_behavior(current, MAP_ENTRY_BEHAV_RANDOM);
 1677                                 break;
 1678                         case MADV_NOSYNC:
 1679                                 current->eflags |= MAP_ENTRY_NOSYNC;
 1680                                 break;
 1681                         case MADV_AUTOSYNC:
 1682                                 current->eflags &= ~MAP_ENTRY_NOSYNC;
 1683                                 break;
 1684                         case MADV_NOCORE:
 1685                                 current->eflags |= MAP_ENTRY_NOCOREDUMP;
 1686                                 break;
 1687                         case MADV_CORE:
 1688                                 current->eflags &= ~MAP_ENTRY_NOCOREDUMP;
 1689                                 break;
 1690                         default:
 1691                                 break;
 1692                         }
 1693                         vm_map_simplify_entry(map, current);
 1694                 }
 1695                 vm_map_unlock(map);
 1696         } else {
 1697                 vm_pindex_t pindex;
 1698                 int count;
 1699 
 1700                 /*
 1701                  * madvise behaviors that are implemented in the underlying
 1702                  * vm_object.
 1703                  *
 1704                  * Since we don't clip the vm_map_entry, we have to clip
 1705                  * the vm_object pindex and count.
 1706                  */
 1707                 for (current = entry;
 1708                      (current != &map->header) && (current->start < end);
 1709                      current = current->next
 1710                 ) {
 1711                         vm_offset_t useStart;
 1712 
 1713                         if (current->eflags & MAP_ENTRY_IS_SUB_MAP)
 1714                                 continue;
 1715 
 1716                         pindex = OFF_TO_IDX(current->offset);
 1717                         count = atop(current->end - current->start);
 1718                         useStart = current->start;
 1719 
 1720                         if (current->start < start) {
 1721                                 pindex += atop(start - current->start);
 1722                                 count -= atop(start - current->start);
 1723                                 useStart = start;
 1724                         }
 1725                         if (current->end > end)
 1726                                 count -= atop(current->end - end);
 1727 
 1728                         if (count <= 0)
 1729                                 continue;
 1730 
 1731                         vm_object_madvise(current->object.vm_object,
 1732                                           pindex, count, behav);
 1733                         if (behav == MADV_WILLNEED) {
 1734                                 vm_map_pmap_enter(map,
 1735                                     useStart,
 1736                                     current->protection,
 1737                                     current->object.vm_object,
 1738                                     pindex,
 1739                                     (count << PAGE_SHIFT),
 1740                                     MAP_PREFAULT_MADVISE
 1741                                 );
 1742                         }
 1743                 }
 1744                 vm_map_unlock_read(map);
 1745         }
 1746         return (0);
 1747 }
 1748 
 1749 
 1750 /*
 1751  *      vm_map_inherit:
 1752  *
 1753  *      Sets the inheritance of the specified address
 1754  *      range in the target map.  Inheritance
 1755  *      affects how the map will be shared with
 1756  *      child maps at the time of vm_map_fork.
 1757  */
 1758 int
 1759 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1760                vm_inherit_t new_inheritance)
 1761 {
 1762         vm_map_entry_t entry;
 1763         vm_map_entry_t temp_entry;
 1764 
 1765         switch (new_inheritance) {
 1766         case VM_INHERIT_NONE:
 1767         case VM_INHERIT_COPY:
 1768         case VM_INHERIT_SHARE:
 1769                 break;
 1770         default:
 1771                 return (KERN_INVALID_ARGUMENT);
 1772         }
 1773         vm_map_lock(map);
 1774         VM_MAP_RANGE_CHECK(map, start, end);
 1775         if (vm_map_lookup_entry(map, start, &temp_entry)) {
 1776                 entry = temp_entry;
 1777                 vm_map_clip_start(map, entry, start);
 1778         } else
 1779                 entry = temp_entry->next;
 1780         while ((entry != &map->header) && (entry->start < end)) {
 1781                 vm_map_clip_end(map, entry, end);
 1782                 entry->inheritance = new_inheritance;
 1783                 vm_map_simplify_entry(map, entry);
 1784                 entry = entry->next;
 1785         }
 1786         vm_map_unlock(map);
 1787         return (KERN_SUCCESS);
 1788 }
 1789 
 1790 /*
 1791  *      vm_map_unwire:
 1792  *
 1793  *      Implements both kernel and user unwiring.
 1794  */
 1795 int
 1796 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1797     int flags)
 1798 {
 1799         vm_map_entry_t entry, first_entry, tmp_entry;
 1800         vm_offset_t saved_start;
 1801         unsigned int last_timestamp;
 1802         int rv;
 1803         boolean_t need_wakeup, result, user_unwire;
 1804 
 1805         user_unwire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
 1806         vm_map_lock(map);
 1807         VM_MAP_RANGE_CHECK(map, start, end);
 1808         if (!vm_map_lookup_entry(map, start, &first_entry)) {
 1809                 if (flags & VM_MAP_WIRE_HOLESOK)
 1810                         first_entry = first_entry->next;
 1811                 else {
 1812                         vm_map_unlock(map);
 1813                         return (KERN_INVALID_ADDRESS);
 1814                 }
 1815         }
 1816         last_timestamp = map->timestamp;
 1817         entry = first_entry;
 1818         while (entry != &map->header && entry->start < end) {
 1819                 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
 1820                         /*
 1821                          * We have not yet clipped the entry.
 1822                          */
 1823                         saved_start = (start >= entry->start) ? start :
 1824                             entry->start;
 1825                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 1826                         if (vm_map_unlock_and_wait(map, user_unwire)) {
 1827                                 /*
 1828                                  * Allow interruption of user unwiring?
 1829                                  */
 1830                         }
 1831                         vm_map_lock(map);
 1832                         if (last_timestamp+1 != map->timestamp) {
 1833                                 /*
 1834                                  * Look again for the entry because the map was
 1835                                  * modified while it was unlocked.
 1836                                  * Specifically, the entry may have been
 1837                                  * clipped, merged, or deleted.
 1838                                  */
 1839                                 if (!vm_map_lookup_entry(map, saved_start,
 1840                                     &tmp_entry)) {
 1841                                         if (flags & VM_MAP_WIRE_HOLESOK)
 1842                                                 tmp_entry = tmp_entry->next;
 1843                                         else {
 1844                                                 if (saved_start == start) {
 1845                                                         /*
 1846                                                          * First_entry has been deleted.
 1847                                                          */
 1848                                                         vm_map_unlock(map);
 1849                                                         return (KERN_INVALID_ADDRESS);
 1850                                                 }
 1851                                                 end = saved_start;
 1852                                                 rv = KERN_INVALID_ADDRESS;
 1853                                                 goto done;
 1854                                         }
 1855                                 }
 1856                                 if (entry == first_entry)
 1857                                         first_entry = tmp_entry;
 1858                                 else
 1859                                         first_entry = NULL;
 1860                                 entry = tmp_entry;
 1861                         }
 1862                         last_timestamp = map->timestamp;
 1863                         continue;
 1864                 }
 1865                 vm_map_clip_start(map, entry, start);
 1866                 vm_map_clip_end(map, entry, end);
 1867                 /*
 1868                  * Mark the entry in case the map lock is released.  (See
 1869                  * above.)
 1870                  */
 1871                 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
 1872                 /*
 1873                  * Check the map for holes in the specified region.
 1874                  * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
 1875                  */
 1876                 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
 1877                     (entry->end < end && (entry->next == &map->header ||
 1878                     entry->next->start > entry->end))) {
 1879                         end = entry->end;
 1880                         rv = KERN_INVALID_ADDRESS;
 1881                         goto done;
 1882                 }
 1883                 /*
 1884                  * If system unwiring, require that the entry is system wired.
 1885                  */
 1886                 if (!user_unwire &&
 1887                     vm_map_entry_system_wired_count(entry) == 0) {
 1888                         end = entry->end;
 1889                         rv = KERN_INVALID_ARGUMENT;
 1890                         goto done;
 1891                 }
 1892                 entry = entry->next;
 1893         }
 1894         rv = KERN_SUCCESS;
 1895 done:
 1896         need_wakeup = FALSE;
 1897         if (first_entry == NULL) {
 1898                 result = vm_map_lookup_entry(map, start, &first_entry);
 1899                 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
 1900                         first_entry = first_entry->next;
 1901                 else
 1902                         KASSERT(result, ("vm_map_unwire: lookup failed"));
 1903         }
 1904         entry = first_entry;
 1905         while (entry != &map->header && entry->start < end) {
 1906                 if (rv == KERN_SUCCESS && (!user_unwire ||
 1907                     (entry->eflags & MAP_ENTRY_USER_WIRED))) {
 1908                         if (user_unwire)
 1909                                 entry->eflags &= ~MAP_ENTRY_USER_WIRED;
 1910                         entry->wired_count--;
 1911                         if (entry->wired_count == 0) {
 1912                                 /*
 1913                                  * Retain the map lock.
 1914                                  */
 1915                                 vm_fault_unwire(map, entry->start, entry->end,
 1916                                     entry->object.vm_object != NULL &&
 1917                                     entry->object.vm_object->type == OBJT_DEVICE);
 1918                         }
 1919                 }
 1920                 KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
 1921                         ("vm_map_unwire: in-transition flag missing"));
 1922                 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
 1923                 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
 1924                         entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
 1925                         need_wakeup = TRUE;
 1926                 }
 1927                 vm_map_simplify_entry(map, entry);
 1928                 entry = entry->next;
 1929         }
 1930         vm_map_unlock(map);
 1931         if (need_wakeup)
 1932                 vm_map_wakeup(map);
 1933         return (rv);
 1934 }
 1935 
 1936 /*
 1937  *      vm_map_wire:
 1938  *
 1939  *      Implements both kernel and user wiring.
 1940  */
 1941 int
 1942 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 1943     int flags)
 1944 {
 1945         vm_map_entry_t entry, first_entry, tmp_entry;
 1946         vm_offset_t saved_end, saved_start;
 1947         unsigned int last_timestamp;
 1948         int rv;
 1949         boolean_t fictitious, need_wakeup, result, user_wire;
 1950 
 1951         user_wire = (flags & VM_MAP_WIRE_USER) ? TRUE : FALSE;
 1952         vm_map_lock(map);
 1953         VM_MAP_RANGE_CHECK(map, start, end);
 1954         if (!vm_map_lookup_entry(map, start, &first_entry)) {
 1955                 if (flags & VM_MAP_WIRE_HOLESOK)
 1956                         first_entry = first_entry->next;
 1957                 else {
 1958                         vm_map_unlock(map);
 1959                         return (KERN_INVALID_ADDRESS);
 1960                 }
 1961         }
 1962         last_timestamp = map->timestamp;
 1963         entry = first_entry;
 1964         while (entry != &map->header && entry->start < end) {
 1965                 if (entry->eflags & MAP_ENTRY_IN_TRANSITION) {
 1966                         /*
 1967                          * We have not yet clipped the entry.
 1968                          */
 1969                         saved_start = (start >= entry->start) ? start :
 1970                             entry->start;
 1971                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 1972                         if (vm_map_unlock_and_wait(map, user_wire)) {
 1973                                 /*
 1974                                  * Allow interruption of user wiring?
 1975                                  */
 1976                         }
 1977                         vm_map_lock(map);
 1978                         if (last_timestamp + 1 != map->timestamp) {
 1979                                 /*
 1980                                  * Look again for the entry because the map was
 1981                                  * modified while it was unlocked.
 1982                                  * Specifically, the entry may have been
 1983                                  * clipped, merged, or deleted.
 1984                                  */
 1985                                 if (!vm_map_lookup_entry(map, saved_start,
 1986                                     &tmp_entry)) {
 1987                                         if (flags & VM_MAP_WIRE_HOLESOK)
 1988                                                 tmp_entry = tmp_entry->next;
 1989                                         else {
 1990                                                 if (saved_start == start) {
 1991                                                         /*
 1992                                                          * first_entry has been deleted.
 1993                                                          */
 1994                                                         vm_map_unlock(map);
 1995                                                         return (KERN_INVALID_ADDRESS);
 1996                                                 }
 1997                                                 end = saved_start;
 1998                                                 rv = KERN_INVALID_ADDRESS;
 1999                                                 goto done;
 2000                                         }
 2001                                 }
 2002                                 if (entry == first_entry)
 2003                                         first_entry = tmp_entry;
 2004                                 else
 2005                                         first_entry = NULL;
 2006                                 entry = tmp_entry;
 2007                         }
 2008                         last_timestamp = map->timestamp;
 2009                         continue;
 2010                 }
 2011                 vm_map_clip_start(map, entry, start);
 2012                 vm_map_clip_end(map, entry, end);
 2013                 /*
 2014                  * Mark the entry in case the map lock is released.  (See
 2015                  * above.)
 2016                  */
 2017                 entry->eflags |= MAP_ENTRY_IN_TRANSITION;
 2018                 /*
 2019                  *
 2020                  */
 2021                 if (entry->wired_count == 0) {
 2022                         entry->wired_count++;
 2023                         saved_start = entry->start;
 2024                         saved_end = entry->end;
 2025                         fictitious = entry->object.vm_object != NULL &&
 2026                             entry->object.vm_object->type == OBJT_DEVICE;
 2027                         /*
 2028                          * Release the map lock, relying on the in-transition
 2029                          * mark.
 2030                          */
 2031                         vm_map_unlock(map);
 2032                         rv = vm_fault_wire(map, saved_start, saved_end,
 2033                             user_wire, fictitious);
 2034                         vm_map_lock(map);
 2035                         if (last_timestamp + 1 != map->timestamp) {
 2036                                 /*
 2037                                  * Look again for the entry because the map was
 2038                                  * modified while it was unlocked.  The entry
 2039                                  * may have been clipped, but NOT merged or
 2040                                  * deleted.
 2041                                  */
 2042                                 result = vm_map_lookup_entry(map, saved_start,
 2043                                     &tmp_entry);
 2044                                 KASSERT(result, ("vm_map_wire: lookup failed"));
 2045                                 if (entry == first_entry)
 2046                                         first_entry = tmp_entry;
 2047                                 else
 2048                                         first_entry = NULL;
 2049                                 entry = tmp_entry;
 2050                                 while (entry->end < saved_end) {
 2051                                         if (rv != KERN_SUCCESS) {
 2052                                                 KASSERT(entry->wired_count == 1,
 2053                                                     ("vm_map_wire: bad count"));
 2054                                                 entry->wired_count = -1;
 2055                                         }
 2056                                         entry = entry->next;
 2057                                 }
 2058                         }
 2059                         last_timestamp = map->timestamp;
 2060                         if (rv != KERN_SUCCESS) {
 2061                                 KASSERT(entry->wired_count == 1,
 2062                                     ("vm_map_wire: bad count"));
 2063                                 /*
 2064                                  * Assign an out-of-range value to represent
 2065                                  * the failure to wire this entry.
 2066                                  */
 2067                                 entry->wired_count = -1;
 2068                                 end = entry->end;
 2069                                 goto done;
 2070                         }
 2071                 } else if (!user_wire ||
 2072                            (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
 2073                         entry->wired_count++;
 2074                 }
 2075                 /*
 2076                  * Check the map for holes in the specified region.
 2077                  * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
 2078                  */
 2079                 if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
 2080                     (entry->end < end && (entry->next == &map->header ||
 2081                     entry->next->start > entry->end))) {
 2082                         end = entry->end;
 2083                         rv = KERN_INVALID_ADDRESS;
 2084                         goto done;
 2085                 }
 2086                 entry = entry->next;
 2087         }
 2088         rv = KERN_SUCCESS;
 2089 done:
 2090         need_wakeup = FALSE;
 2091         if (first_entry == NULL) {
 2092                 result = vm_map_lookup_entry(map, start, &first_entry);
 2093                 if (!result && (flags & VM_MAP_WIRE_HOLESOK))
 2094                         first_entry = first_entry->next;
 2095                 else
 2096                         KASSERT(result, ("vm_map_wire: lookup failed"));
 2097         }
 2098         entry = first_entry;
 2099         while (entry != &map->header && entry->start < end) {
 2100                 if (rv == KERN_SUCCESS) {
 2101                         if (user_wire)
 2102                                 entry->eflags |= MAP_ENTRY_USER_WIRED;
 2103                 } else if (entry->wired_count == -1) {
 2104                         /*
 2105                          * Wiring failed on this entry.  Thus, unwiring is
 2106                          * unnecessary.
 2107                          */
 2108                         entry->wired_count = 0;
 2109                 } else {
 2110                         if (!user_wire ||
 2111                             (entry->eflags & MAP_ENTRY_USER_WIRED) == 0)
 2112                                 entry->wired_count--;
 2113                         if (entry->wired_count == 0) {
 2114                                 /*
 2115                                  * Retain the map lock.
 2116                                  */
 2117                                 vm_fault_unwire(map, entry->start, entry->end,
 2118                                     entry->object.vm_object != NULL &&
 2119                                     entry->object.vm_object->type == OBJT_DEVICE);
 2120                         }
 2121                 }
 2122                 KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
 2123                         ("vm_map_wire: in-transition flag missing"));
 2124                 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
 2125                 if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
 2126                         entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
 2127                         need_wakeup = TRUE;
 2128                 }
 2129                 vm_map_simplify_entry(map, entry);
 2130                 entry = entry->next;
 2131         }
 2132         vm_map_unlock(map);
 2133         if (need_wakeup)
 2134                 vm_map_wakeup(map);
 2135         return (rv);
 2136 }
 2137 
 2138 /*
 2139  * vm_map_sync
 2140  *
 2141  * Push any dirty cached pages in the address range to their pager.
 2142  * If syncio is TRUE, dirty pages are written synchronously.
 2143  * If invalidate is TRUE, any cached pages are freed as well.
 2144  *
 2145  * If the size of the region from start to end is zero, we are
 2146  * supposed to flush all modified pages within the region containing
 2147  * start.  Unfortunately, a region can be split or coalesced with
 2148  * neighboring regions, making it difficult to determine what the
 2149  * original region was.  Therefore, we approximate this requirement by
 2150  * flushing the current region containing start.
 2151  *
 2152  * Returns an error if any part of the specified range is not mapped.
 2153  */
 2154 int
 2155 vm_map_sync(
 2156         vm_map_t map,
 2157         vm_offset_t start,
 2158         vm_offset_t end,
 2159         boolean_t syncio,
 2160         boolean_t invalidate)
 2161 {
 2162         vm_map_entry_t current;
 2163         vm_map_entry_t entry;
 2164         vm_size_t size;
 2165         vm_object_t object;
 2166         vm_ooffset_t offset;
 2167 
 2168         vm_map_lock_read(map);
 2169         VM_MAP_RANGE_CHECK(map, start, end);
 2170         if (!vm_map_lookup_entry(map, start, &entry)) {
 2171                 vm_map_unlock_read(map);
 2172                 return (KERN_INVALID_ADDRESS);
 2173         } else if (start == end) {
 2174                 start = entry->start;
 2175                 end = entry->end;
 2176         }
 2177         /*
 2178          * Make a first pass to check for user-wired memory and holes.
 2179          */
 2180         for (current = entry; current->start < end; current = current->next) {
 2181                 if (invalidate && (current->eflags & MAP_ENTRY_USER_WIRED)) {
 2182                         vm_map_unlock_read(map);
 2183                         return (KERN_INVALID_ARGUMENT);
 2184                 }
 2185                 if (end > current->end &&
 2186                     (current->next == &map->header ||
 2187                         current->end != current->next->start)) {
 2188                         vm_map_unlock_read(map);
 2189                         return (KERN_INVALID_ADDRESS);
 2190                 }
 2191         }
 2192 
 2193         if (invalidate) {
 2194                 VM_LOCK_GIANT();
 2195                 pmap_remove(map->pmap, start, end);
 2196                 VM_UNLOCK_GIANT();
 2197         }
 2198         /*
 2199          * Make a second pass, cleaning/uncaching pages from the indicated
 2200          * objects as we go.
 2201          */
 2202         for (current = entry; current->start < end; current = current->next) {
 2203                 offset = current->offset + (start - current->start);
 2204                 size = (end <= current->end ? end : current->end) - start;
 2205                 if (current->eflags & MAP_ENTRY_IS_SUB_MAP) {
 2206                         vm_map_t smap;
 2207                         vm_map_entry_t tentry;
 2208                         vm_size_t tsize;
 2209 
 2210                         smap = current->object.sub_map;
 2211                         vm_map_lock_read(smap);
 2212                         (void) vm_map_lookup_entry(smap, offset, &tentry);
 2213                         tsize = tentry->end - offset;
 2214                         if (tsize < size)
 2215                                 size = tsize;
 2216                         object = tentry->object.vm_object;
 2217                         offset = tentry->offset + (offset - tentry->start);
 2218                         vm_map_unlock_read(smap);
 2219                 } else {
 2220                         object = current->object.vm_object;
 2221                 }
 2222                 vm_object_sync(object, offset, size, syncio, invalidate);
 2223                 start += size;
 2224         }
 2225 
 2226         vm_map_unlock_read(map);
 2227         return (KERN_SUCCESS);
 2228 }
 2229 
 2230 /*
 2231  *      vm_map_entry_unwire:    [ internal use only ]
 2232  *
 2233  *      Make the region specified by this entry pageable.
 2234  *
 2235  *      The map in question should be locked.
 2236  *      [This is the reason for this routine's existence.]
 2237  */
 2238 static void
 2239 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry)
 2240 {
 2241         vm_fault_unwire(map, entry->start, entry->end,
 2242             entry->object.vm_object != NULL &&
 2243             entry->object.vm_object->type == OBJT_DEVICE);
 2244         entry->wired_count = 0;
 2245 }
 2246 
 2247 /*
 2248  *      vm_map_entry_delete:    [ internal use only ]
 2249  *
 2250  *      Deallocate the given entry from the target map.
 2251  */
 2252 static void
 2253 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry)
 2254 {
 2255         vm_object_t object;
 2256         vm_pindex_t offidxstart, offidxend, count;
 2257 
 2258         vm_map_entry_unlink(map, entry);
 2259         map->size -= entry->end - entry->start;
 2260 
 2261         if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2262             (object = entry->object.vm_object) != NULL) {
 2263                 count = OFF_TO_IDX(entry->end - entry->start);
 2264                 offidxstart = OFF_TO_IDX(entry->offset);
 2265                 offidxend = offidxstart + count;
 2266                 VM_OBJECT_LOCK(object);
 2267                 if (object->ref_count != 1 &&
 2268                     ((object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING ||
 2269                      object == kernel_object || object == kmem_object) &&
 2270                     (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
 2271                         vm_object_collapse(object);
 2272                         vm_object_page_remove(object, offidxstart, offidxend, FALSE);
 2273                         if (object->type == OBJT_SWAP)
 2274                                 swap_pager_freespace(object, offidxstart, count);
 2275                         if (offidxend >= object->size &&
 2276                             offidxstart < object->size)
 2277                                 object->size = offidxstart;
 2278                 }
 2279                 VM_OBJECT_UNLOCK(object);
 2280                 vm_object_deallocate(object);
 2281         }
 2282 
 2283         vm_map_entry_dispose(map, entry);
 2284 }
 2285 
 2286 /*
 2287  *      vm_map_delete:  [ internal use only ]
 2288  *
 2289  *      Deallocates the given address range from the target
 2290  *      map.
 2291  */
 2292 int
 2293 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
 2294 {
 2295         vm_map_entry_t entry;
 2296         vm_map_entry_t first_entry;
 2297 
 2298         /*
 2299          * Find the start of the region, and clip it
 2300          */
 2301         if (!vm_map_lookup_entry(map, start, &first_entry))
 2302                 entry = first_entry->next;
 2303         else {
 2304                 entry = first_entry;
 2305                 vm_map_clip_start(map, entry, start);
 2306         }
 2307 
 2308         /*
 2309          * Step through all entries in this region
 2310          */
 2311         while ((entry != &map->header) && (entry->start < end)) {
 2312                 vm_map_entry_t next;
 2313 
 2314                 /*
 2315                  * Wait for wiring or unwiring of an entry to complete.
 2316                  * Also wait for any system wirings to disappear on
 2317                  * user maps.
 2318                  */
 2319                 if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 ||
 2320                     (vm_map_pmap(map) != kernel_pmap &&
 2321                     vm_map_entry_system_wired_count(entry) != 0)) {
 2322                         unsigned int last_timestamp;
 2323                         vm_offset_t saved_start;
 2324                         vm_map_entry_t tmp_entry;
 2325 
 2326                         saved_start = entry->start;
 2327                         entry->eflags |= MAP_ENTRY_NEEDS_WAKEUP;
 2328                         last_timestamp = map->timestamp;
 2329                         (void) vm_map_unlock_and_wait(map, FALSE);
 2330                         vm_map_lock(map);
 2331                         if (last_timestamp + 1 != map->timestamp) {
 2332                                 /*
 2333                                  * Look again for the entry because the map was
 2334                                  * modified while it was unlocked.
 2335                                  * Specifically, the entry may have been
 2336                                  * clipped, merged, or deleted.
 2337                                  */
 2338                                 if (!vm_map_lookup_entry(map, saved_start,
 2339                                                          &tmp_entry))
 2340                                         entry = tmp_entry->next;
 2341                                 else {
 2342                                         entry = tmp_entry;
 2343                                         vm_map_clip_start(map, entry,
 2344                                                           saved_start);
 2345                                 }
 2346                         }
 2347                         continue;
 2348                 }
 2349                 vm_map_clip_end(map, entry, end);
 2350 
 2351                 next = entry->next;
 2352 
 2353                 /*
 2354                  * Unwire before removing addresses from the pmap; otherwise,
 2355                  * unwiring will put the entries back in the pmap.
 2356                  */
 2357                 if (entry->wired_count != 0) {
 2358                         vm_map_entry_unwire(map, entry);
 2359                 }
 2360 
 2361                 if (!map->system_map)
 2362                         VM_LOCK_GIANT();
 2363                 pmap_remove(map->pmap, entry->start, entry->end);
 2364                 if (!map->system_map)
 2365                         VM_UNLOCK_GIANT();
 2366 
 2367                 /*
 2368                  * Delete the entry (which may delete the object) only after
 2369                  * removing all pmap entries pointing to its pages.
 2370                  * (Otherwise, its page frames may be reallocated, and any
 2371                  * modify bits will be set in the wrong object!)
 2372                  */
 2373                 vm_map_entry_delete(map, entry);
 2374                 entry = next;
 2375         }
 2376         return (KERN_SUCCESS);
 2377 }
 2378 
 2379 /*
 2380  *      vm_map_remove:
 2381  *
 2382  *      Remove the given address range from the target map.
 2383  *      This is the exported form of vm_map_delete.
 2384  */
 2385 int
 2386 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end)
 2387 {
 2388         int result;
 2389 
 2390         vm_map_lock(map);
 2391         VM_MAP_RANGE_CHECK(map, start, end);
 2392         result = vm_map_delete(map, start, end);
 2393         vm_map_unlock(map);
 2394         return (result);
 2395 }
 2396 
 2397 /*
 2398  *      vm_map_check_protection:
 2399  *
 2400  *      Assert that the target map allows the specified privilege on the
 2401  *      entire address region given.  The entire region must be allocated.
 2402  *
 2403  *      WARNING!  This code does not and should not check whether the
 2404  *      contents of the region is accessible.  For example a smaller file
 2405  *      might be mapped into a larger address space.
 2406  *
 2407  *      NOTE!  This code is also called by munmap().
 2408  *
 2409  *      The map must be locked.  A read lock is sufficient.
 2410  */
 2411 boolean_t
 2412 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end,
 2413                         vm_prot_t protection)
 2414 {
 2415         vm_map_entry_t entry;
 2416         vm_map_entry_t tmp_entry;
 2417 
 2418         if (!vm_map_lookup_entry(map, start, &tmp_entry))
 2419                 return (FALSE);
 2420         entry = tmp_entry;
 2421 
 2422         while (start < end) {
 2423                 if (entry == &map->header)
 2424                         return (FALSE);
 2425                 /*
 2426                  * No holes allowed!
 2427                  */
 2428                 if (start < entry->start)
 2429                         return (FALSE);
 2430                 /*
 2431                  * Check protection associated with entry.
 2432                  */
 2433                 if ((entry->protection & protection) != protection)
 2434                         return (FALSE);
 2435                 /* go to next entry */
 2436                 start = entry->end;
 2437                 entry = entry->next;
 2438         }
 2439         return (TRUE);
 2440 }
 2441 
 2442 /*
 2443  *      vm_map_copy_entry:
 2444  *
 2445  *      Copies the contents of the source entry to the destination
 2446  *      entry.  The entries *must* be aligned properly.
 2447  */
 2448 static void
 2449 vm_map_copy_entry(
 2450         vm_map_t src_map,
 2451         vm_map_t dst_map,
 2452         vm_map_entry_t src_entry,
 2453         vm_map_entry_t dst_entry)
 2454 {
 2455         vm_object_t src_object;
 2456 
 2457         if ((dst_entry->eflags|src_entry->eflags) & MAP_ENTRY_IS_SUB_MAP)
 2458                 return;
 2459 
 2460         if (src_entry->wired_count == 0) {
 2461 
 2462                 /*
 2463                  * If the source entry is marked needs_copy, it is already
 2464                  * write-protected.
 2465                  */
 2466                 if ((src_entry->eflags & MAP_ENTRY_NEEDS_COPY) == 0) {
 2467                         pmap_protect(src_map->pmap,
 2468                             src_entry->start,
 2469                             src_entry->end,
 2470                             src_entry->protection & ~VM_PROT_WRITE);
 2471                 }
 2472 
 2473                 /*
 2474                  * Make a copy of the object.
 2475                  */
 2476                 if ((src_object = src_entry->object.vm_object) != NULL) {
 2477                         VM_OBJECT_LOCK(src_object);
 2478                         if ((src_object->handle == NULL) &&
 2479                                 (src_object->type == OBJT_DEFAULT ||
 2480                                  src_object->type == OBJT_SWAP)) {
 2481                                 vm_object_collapse(src_object);
 2482                                 if ((src_object->flags & (OBJ_NOSPLIT|OBJ_ONEMAPPING)) == OBJ_ONEMAPPING) {
 2483                                         vm_object_split(src_entry);
 2484                                         src_object = src_entry->object.vm_object;
 2485                                 }
 2486                         }
 2487                         vm_object_reference_locked(src_object);
 2488                         vm_object_clear_flag(src_object, OBJ_ONEMAPPING);
 2489                         VM_OBJECT_UNLOCK(src_object);
 2490                         dst_entry->object.vm_object = src_object;
 2491                         src_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
 2492                         dst_entry->eflags |= (MAP_ENTRY_COW|MAP_ENTRY_NEEDS_COPY);
 2493                         dst_entry->offset = src_entry->offset;
 2494                 } else {
 2495                         dst_entry->object.vm_object = NULL;
 2496                         dst_entry->offset = 0;
 2497                 }
 2498 
 2499                 pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start,
 2500                     dst_entry->end - dst_entry->start, src_entry->start);
 2501         } else {
 2502                 /*
 2503                  * Of course, wired down pages can't be set copy-on-write.
 2504                  * Cause wired pages to be copied into the new map by
 2505                  * simulating faults (the new pages are pageable)
 2506                  */
 2507                 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry);
 2508         }
 2509 }
 2510 
 2511 /*
 2512  * vmspace_map_entry_forked:
 2513  * Update the newly-forked vmspace each time a map entry is inherited
 2514  * or copied.  The values for vm_dsize and vm_tsize are approximate
 2515  * (and mostly-obsolete ideas in the face of mmap(2) et al.)
 2516  */
 2517 static void
 2518 vmspace_map_entry_forked(const struct vmspace *vm1, struct vmspace *vm2,
 2519     vm_map_entry_t entry)
 2520 {
 2521         vm_size_t entrysize;
 2522         vm_offset_t newend;
 2523 
 2524         entrysize = entry->end - entry->start;
 2525         vm2->vm_map.size += entrysize;
 2526         if (entry->eflags & (MAP_ENTRY_GROWS_DOWN | MAP_ENTRY_GROWS_UP)) {
 2527                 vm2->vm_ssize += btoc(entrysize);
 2528         } else if (entry->start >= (vm_offset_t)vm1->vm_daddr &&
 2529             entry->start < (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize)) {
 2530                 newend = MIN(entry->end,
 2531                     (vm_offset_t)vm1->vm_daddr + ctob(vm1->vm_dsize));
 2532                 vm2->vm_dsize += btoc(newend - entry->start);
 2533         } else if (entry->start >= (vm_offset_t)vm1->vm_taddr &&
 2534             entry->start < (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize)) {
 2535                 newend = MIN(entry->end,
 2536                     (vm_offset_t)vm1->vm_taddr + ctob(vm1->vm_tsize));
 2537                 vm2->vm_tsize += btoc(newend - entry->start);
 2538         }
 2539 }
 2540 
 2541 /*
 2542  * vmspace_fork:
 2543  * Create a new process vmspace structure and vm_map
 2544  * based on those of an existing process.  The new map
 2545  * is based on the old map, according to the inheritance
 2546  * values on the regions in that map.
 2547  *
 2548  * XXX It might be worth coalescing the entries added to the new vmspace.
 2549  *
 2550  * The source map must not be locked.
 2551  */
 2552 struct vmspace *
 2553 vmspace_fork(struct vmspace *vm1)
 2554 {
 2555         struct vmspace *vm2;
 2556         vm_map_t old_map = &vm1->vm_map;
 2557         vm_map_t new_map;
 2558         vm_map_entry_t old_entry;
 2559         vm_map_entry_t new_entry;
 2560         vm_object_t object;
 2561 
 2562         vm_map_lock(old_map);
 2563 
 2564         vm2 = vmspace_alloc(old_map->min_offset, old_map->max_offset);
 2565         vm2->vm_taddr = vm1->vm_taddr;
 2566         vm2->vm_daddr = vm1->vm_daddr;
 2567         vm2->vm_maxsaddr = vm1->vm_maxsaddr;
 2568         new_map = &vm2->vm_map; /* XXX */
 2569         new_map->timestamp = 1;
 2570 
 2571         /* Do not inherit the MAP_WIREFUTURE property. */
 2572         if ((new_map->flags & MAP_WIREFUTURE) == MAP_WIREFUTURE)
 2573                 new_map->flags &= ~MAP_WIREFUTURE;
 2574 
 2575         old_entry = old_map->header.next;
 2576 
 2577         while (old_entry != &old_map->header) {
 2578                 if (old_entry->eflags & MAP_ENTRY_IS_SUB_MAP)
 2579                         panic("vm_map_fork: encountered a submap");
 2580 
 2581                 switch (old_entry->inheritance) {
 2582                 case VM_INHERIT_NONE:
 2583                         break;
 2584 
 2585                 case VM_INHERIT_SHARE:
 2586                         /*
 2587                          * Clone the entry, creating the shared object if necessary.
 2588                          */
 2589                         object = old_entry->object.vm_object;
 2590                         if (object == NULL) {
 2591                                 object = vm_object_allocate(OBJT_DEFAULT,
 2592                                         atop(old_entry->end - old_entry->start));
 2593                                 old_entry->object.vm_object = object;
 2594                                 old_entry->offset = 0;
 2595                         }
 2596 
 2597                         /*
 2598                          * Add the reference before calling vm_object_shadow
 2599                          * to insure that a shadow object is created.
 2600                          */
 2601                         vm_object_reference(object);
 2602                         if (old_entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 2603                                 vm_object_shadow(&old_entry->object.vm_object,
 2604                                         &old_entry->offset,
 2605                                         atop(old_entry->end - old_entry->start));
 2606                                 old_entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
 2607                                 /* Transfer the second reference too. */
 2608                                 vm_object_reference(
 2609                                     old_entry->object.vm_object);
 2610                                 vm_object_deallocate(object);
 2611                                 object = old_entry->object.vm_object;
 2612                         }
 2613                         VM_OBJECT_LOCK(object);
 2614                         vm_object_clear_flag(object, OBJ_ONEMAPPING);
 2615                         VM_OBJECT_UNLOCK(object);
 2616 
 2617                         /*
 2618                          * Clone the entry, referencing the shared object.
 2619                          */
 2620                         new_entry = vm_map_entry_create(new_map);
 2621                         *new_entry = *old_entry;
 2622                         new_entry->eflags &= ~MAP_ENTRY_USER_WIRED;
 2623                         new_entry->wired_count = 0;
 2624 
 2625                         /*
 2626                          * Insert the entry into the new map -- we know we're
 2627                          * inserting at the end of the new map.
 2628                          */
 2629                         vm_map_entry_link(new_map, new_map->header.prev,
 2630                             new_entry);
 2631                         vmspace_map_entry_forked(vm1, vm2, new_entry);
 2632 
 2633                         /*
 2634                          * Update the physical map
 2635                          */
 2636                         pmap_copy(new_map->pmap, old_map->pmap,
 2637                             new_entry->start,
 2638                             (old_entry->end - old_entry->start),
 2639                             old_entry->start);
 2640                         break;
 2641 
 2642                 case VM_INHERIT_COPY:
 2643                         /*
 2644                          * Clone the entry and link into the map.
 2645                          */
 2646                         new_entry = vm_map_entry_create(new_map);
 2647                         *new_entry = *old_entry;
 2648                         new_entry->eflags &= ~MAP_ENTRY_USER_WIRED;
 2649                         new_entry->wired_count = 0;
 2650                         new_entry->object.vm_object = NULL;
 2651                         vm_map_entry_link(new_map, new_map->header.prev,
 2652                             new_entry);
 2653                         vmspace_map_entry_forked(vm1, vm2, new_entry);
 2654                         vm_map_copy_entry(old_map, new_map, old_entry,
 2655                             new_entry);
 2656                         break;
 2657                 }
 2658                 old_entry = old_entry->next;
 2659         }
 2660 
 2661         vm_map_unlock(old_map);
 2662 
 2663         return (vm2);
 2664 }
 2665 
 2666 int
 2667 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
 2668     vm_prot_t prot, vm_prot_t max, int cow)
 2669 {
 2670         vm_map_entry_t new_entry, prev_entry;
 2671         vm_offset_t bot, top;
 2672         vm_size_t init_ssize;
 2673         int orient, rv;
 2674         rlim_t vmemlim;
 2675 
 2676         /*
 2677          * The stack orientation is piggybacked with the cow argument.
 2678          * Extract it into orient and mask the cow argument so that we
 2679          * don't pass it around further.
 2680          * NOTE: We explicitly allow bi-directional stacks.
 2681          */
 2682         orient = cow & (MAP_STACK_GROWS_DOWN|MAP_STACK_GROWS_UP);
 2683         cow &= ~orient;
 2684         KASSERT(orient != 0, ("No stack grow direction"));
 2685 
 2686         if (addrbos < vm_map_min(map) || addrbos > map->max_offset)
 2687                 return (KERN_NO_SPACE);
 2688 
 2689         init_ssize = (max_ssize < sgrowsiz) ? max_ssize : sgrowsiz;
 2690 
 2691         PROC_LOCK(curthread->td_proc);
 2692         vmemlim = lim_cur(curthread->td_proc, RLIMIT_VMEM);
 2693         PROC_UNLOCK(curthread->td_proc);
 2694 
 2695         vm_map_lock(map);
 2696 
 2697         /* If addr is already mapped, no go */
 2698         if (vm_map_lookup_entry(map, addrbos, &prev_entry)) {
 2699                 vm_map_unlock(map);
 2700                 return (KERN_NO_SPACE);
 2701         }
 2702 
 2703         /* If we would blow our VMEM resource limit, no go */
 2704         if (map->size + init_ssize > vmemlim) {
 2705                 vm_map_unlock(map);
 2706                 return (KERN_NO_SPACE);
 2707         }
 2708 
 2709         /*
 2710          * If we can't accomodate max_ssize in the current mapping, no go.
 2711          * However, we need to be aware that subsequent user mappings might
 2712          * map into the space we have reserved for stack, and currently this
 2713          * space is not protected.
 2714          *
 2715          * Hopefully we will at least detect this condition when we try to
 2716          * grow the stack.
 2717          */
 2718         if ((prev_entry->next != &map->header) &&
 2719             (prev_entry->next->start < addrbos + max_ssize)) {
 2720                 vm_map_unlock(map);
 2721                 return (KERN_NO_SPACE);
 2722         }
 2723 
 2724         /*
 2725          * We initially map a stack of only init_ssize.  We will grow as
 2726          * needed later.  Depending on the orientation of the stack (i.e.
 2727          * the grow direction) we either map at the top of the range, the
 2728          * bottom of the range or in the middle.
 2729          *
 2730          * Note: we would normally expect prot and max to be VM_PROT_ALL,
 2731          * and cow to be 0.  Possibly we should eliminate these as input
 2732          * parameters, and just pass these values here in the insert call.
 2733          */
 2734         if (orient == MAP_STACK_GROWS_DOWN)
 2735                 bot = addrbos + max_ssize - init_ssize;
 2736         else if (orient == MAP_STACK_GROWS_UP)
 2737                 bot = addrbos;
 2738         else
 2739                 bot = round_page(addrbos + max_ssize/2 - init_ssize/2);
 2740         top = bot + init_ssize;
 2741         rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow);
 2742 
 2743         /* Now set the avail_ssize amount. */
 2744         if (rv == KERN_SUCCESS) {
 2745                 if (prev_entry != &map->header)
 2746                         vm_map_clip_end(map, prev_entry, bot);
 2747                 new_entry = prev_entry->next;
 2748                 if (new_entry->end != top || new_entry->start != bot)
 2749                         panic("Bad entry start/end for new stack entry");
 2750 
 2751                 new_entry->avail_ssize = max_ssize - init_ssize;
 2752                 if (orient & MAP_STACK_GROWS_DOWN)
 2753                         new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
 2754                 if (orient & MAP_STACK_GROWS_UP)
 2755                         new_entry->eflags |= MAP_ENTRY_GROWS_UP;
 2756         }
 2757 
 2758         vm_map_unlock(map);
 2759         return (rv);
 2760 }
 2761 
 2762 /* Attempts to grow a vm stack entry.  Returns KERN_SUCCESS if the
 2763  * desired address is already mapped, or if we successfully grow
 2764  * the stack.  Also returns KERN_SUCCESS if addr is outside the
 2765  * stack range (this is strange, but preserves compatibility with
 2766  * the grow function in vm_machdep.c).
 2767  */
 2768 int
 2769 vm_map_growstack(struct proc *p, vm_offset_t addr)
 2770 {
 2771         vm_map_entry_t next_entry, prev_entry;
 2772         vm_map_entry_t new_entry, stack_entry;
 2773         struct vmspace *vm = p->p_vmspace;
 2774         vm_map_t map = &vm->vm_map;
 2775         vm_offset_t end;
 2776         size_t grow_amount, max_grow;
 2777         rlim_t stacklim, vmemlim;
 2778         int is_procstack, rv;
 2779 
 2780 Retry:
 2781         PROC_LOCK(p);
 2782         stacklim = lim_cur(p, RLIMIT_STACK);
 2783         vmemlim = lim_cur(p, RLIMIT_VMEM);
 2784         PROC_UNLOCK(p);
 2785 
 2786         vm_map_lock_read(map);
 2787 
 2788         /* If addr is already in the entry range, no need to grow.*/
 2789         if (vm_map_lookup_entry(map, addr, &prev_entry)) {
 2790                 vm_map_unlock_read(map);
 2791                 return (KERN_SUCCESS);
 2792         }
 2793 
 2794         next_entry = prev_entry->next;
 2795         if (!(prev_entry->eflags & MAP_ENTRY_GROWS_UP)) {
 2796                 /*
 2797                  * This entry does not grow upwards. Since the address lies
 2798                  * beyond this entry, the next entry (if one exists) has to
 2799                  * be a downward growable entry. The entry list header is
 2800                  * never a growable entry, so it suffices to check the flags.
 2801                  */
 2802                 if (!(next_entry->eflags & MAP_ENTRY_GROWS_DOWN)) {
 2803                         vm_map_unlock_read(map);
 2804                         return (KERN_SUCCESS);
 2805                 }
 2806                 stack_entry = next_entry;
 2807         } else {
 2808                 /*
 2809                  * This entry grows upward. If the next entry does not at
 2810                  * least grow downwards, this is the entry we need to grow.
 2811                  * otherwise we have two possible choices and we have to
 2812                  * select one.
 2813                  */
 2814                 if (next_entry->eflags & MAP_ENTRY_GROWS_DOWN) {
 2815                         /*
 2816                          * We have two choices; grow the entry closest to
 2817                          * the address to minimize the amount of growth.
 2818                          */
 2819                         if (addr - prev_entry->end <= next_entry->start - addr)
 2820                                 stack_entry = prev_entry;
 2821                         else
 2822                                 stack_entry = next_entry;
 2823                 } else
 2824                         stack_entry = prev_entry;
 2825         }
 2826 
 2827         if (stack_entry == next_entry) {
 2828                 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_DOWN, ("foo"));
 2829                 KASSERT(addr < stack_entry->start, ("foo"));
 2830                 end = (prev_entry != &map->header) ? prev_entry->end :
 2831                     stack_entry->start - stack_entry->avail_ssize;
 2832                 grow_amount = roundup(stack_entry->start - addr, PAGE_SIZE);
 2833                 max_grow = stack_entry->start - end;
 2834         } else {
 2835                 KASSERT(stack_entry->eflags & MAP_ENTRY_GROWS_UP, ("foo"));
 2836                 KASSERT(addr >= stack_entry->end, ("foo"));
 2837                 end = (next_entry != &map->header) ? next_entry->start :
 2838                     stack_entry->end + stack_entry->avail_ssize;
 2839                 grow_amount = roundup(addr + 1 - stack_entry->end, PAGE_SIZE);
 2840                 max_grow = end - stack_entry->end;
 2841         }
 2842 
 2843         if (grow_amount > stack_entry->avail_ssize) {
 2844                 vm_map_unlock_read(map);
 2845                 return (KERN_NO_SPACE);
 2846         }
 2847 
 2848         /*
 2849          * If there is no longer enough space between the entries nogo, and
 2850          * adjust the available space.  Note: this  should only happen if the
 2851          * user has mapped into the stack area after the stack was created,
 2852          * and is probably an error.
 2853          *
 2854          * This also effectively destroys any guard page the user might have
 2855          * intended by limiting the stack size.
 2856          */
 2857         if (grow_amount > max_grow) {
 2858                 if (vm_map_lock_upgrade(map))
 2859                         goto Retry;
 2860 
 2861                 stack_entry->avail_ssize = max_grow;
 2862 
 2863                 vm_map_unlock(map);
 2864                 return (KERN_NO_SPACE);
 2865         }
 2866 
 2867         is_procstack = (addr >= (vm_offset_t)vm->vm_maxsaddr) ? 1 : 0;
 2868 
 2869         /*
 2870          * If this is the main process stack, see if we're over the stack
 2871          * limit.
 2872          */
 2873         if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
 2874                 vm_map_unlock_read(map);
 2875                 return (KERN_NO_SPACE);
 2876         }
 2877 
 2878         /* Round up the grow amount modulo SGROWSIZ */
 2879         grow_amount = roundup (grow_amount, sgrowsiz);
 2880         if (grow_amount > stack_entry->avail_ssize)
 2881                 grow_amount = stack_entry->avail_ssize;
 2882         if (is_procstack && (ctob(vm->vm_ssize) + grow_amount > stacklim)) {
 2883                 grow_amount = stacklim - ctob(vm->vm_ssize);
 2884         }
 2885 
 2886         /* If we would blow our VMEM resource limit, no go */
 2887         if (map->size + grow_amount > vmemlim) {
 2888                 vm_map_unlock_read(map);
 2889                 return (KERN_NO_SPACE);
 2890         }
 2891 
 2892         if (vm_map_lock_upgrade(map))
 2893                 goto Retry;
 2894 
 2895         if (stack_entry == next_entry) {
 2896                 /*
 2897                  * Growing downward.
 2898                  */
 2899                 /* Get the preliminary new entry start value */
 2900                 addr = stack_entry->start - grow_amount;
 2901 
 2902                 /*
 2903                  * If this puts us into the previous entry, cut back our
 2904                  * growth to the available space. Also, see the note above.
 2905                  */
 2906                 if (addr < end) {
 2907                         stack_entry->avail_ssize = max_grow;
 2908                         addr = end;
 2909                 }
 2910 
 2911                 rv = vm_map_insert(map, NULL, 0, addr, stack_entry->start,
 2912                     p->p_sysent->sv_stackprot, VM_PROT_ALL, 0);
 2913 
 2914                 /* Adjust the available stack space by the amount we grew. */
 2915                 if (rv == KERN_SUCCESS) {
 2916                         if (prev_entry != &map->header)
 2917                                 vm_map_clip_end(map, prev_entry, addr);
 2918                         new_entry = prev_entry->next;
 2919                         KASSERT(new_entry == stack_entry->prev, ("foo"));
 2920                         KASSERT(new_entry->end == stack_entry->start, ("foo"));
 2921                         KASSERT(new_entry->start == addr, ("foo"));
 2922                         grow_amount = new_entry->end - new_entry->start;
 2923                         new_entry->avail_ssize = stack_entry->avail_ssize -
 2924                             grow_amount;
 2925                         stack_entry->eflags &= ~MAP_ENTRY_GROWS_DOWN;
 2926                         new_entry->eflags |= MAP_ENTRY_GROWS_DOWN;
 2927                 }
 2928         } else {
 2929                 /*
 2930                  * Growing upward.
 2931                  */
 2932                 addr = stack_entry->end + grow_amount;
 2933 
 2934                 /*
 2935                  * If this puts us into the next entry, cut back our growth
 2936                  * to the available space. Also, see the note above.
 2937                  */
 2938                 if (addr > end) {
 2939                         stack_entry->avail_ssize = end - stack_entry->end;
 2940                         addr = end;
 2941                 }
 2942 
 2943                 grow_amount = addr - stack_entry->end;
 2944 
 2945                 /* Grow the underlying object if applicable. */
 2946                 if (stack_entry->object.vm_object == NULL ||
 2947                     vm_object_coalesce(stack_entry->object.vm_object,
 2948                     stack_entry->offset,
 2949                     (vm_size_t)(stack_entry->end - stack_entry->start),
 2950                     (vm_size_t)grow_amount)) {
 2951                         map->size += (addr - stack_entry->end);
 2952                         /* Update the current entry. */
 2953                         stack_entry->end = addr;
 2954                         stack_entry->avail_ssize -= grow_amount;
 2955                         vm_map_entry_resize_free(map, stack_entry);
 2956                         rv = KERN_SUCCESS;
 2957 
 2958                         if (next_entry != &map->header)
 2959                                 vm_map_clip_start(map, next_entry, addr);
 2960                 } else
 2961                         rv = KERN_FAILURE;
 2962         }
 2963 
 2964         if (rv == KERN_SUCCESS && is_procstack)
 2965                 vm->vm_ssize += btoc(grow_amount);
 2966 
 2967         vm_map_unlock(map);
 2968 
 2969         /*
 2970          * Heed the MAP_WIREFUTURE flag if it was set for this process.
 2971          */
 2972         if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE)) {
 2973                 vm_map_wire(map,
 2974                     (stack_entry == next_entry) ? addr : addr - grow_amount,
 2975                     (stack_entry == next_entry) ? stack_entry->start : addr,
 2976                     (p->p_flag & P_SYSTEM)
 2977                     ? VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES
 2978                     : VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
 2979         }
 2980 
 2981         return (rv);
 2982 }
 2983 
 2984 /*
 2985  * Unshare the specified VM space for exec.  If other processes are
 2986  * mapped to it, then create a new one.  The new vmspace is null.
 2987  */
 2988 void
 2989 vmspace_exec(struct proc *p, vm_offset_t minuser, vm_offset_t maxuser)
 2990 {
 2991         struct vmspace *oldvmspace = p->p_vmspace;
 2992         struct vmspace *newvmspace;
 2993 
 2994         newvmspace = vmspace_alloc(minuser, maxuser);
 2995         newvmspace->vm_swrss = oldvmspace->vm_swrss;
 2996         /*
 2997          * This code is written like this for prototype purposes.  The
 2998          * goal is to avoid running down the vmspace here, but let the
 2999          * other process's that are still using the vmspace to finally
 3000          * run it down.  Even though there is little or no chance of blocking
 3001          * here, it is a good idea to keep this form for future mods.
 3002          */
 3003         PROC_VMSPACE_LOCK(p);
 3004         p->p_vmspace = newvmspace;
 3005         PROC_VMSPACE_UNLOCK(p);
 3006         if (p == curthread->td_proc)            /* XXXKSE ? */
 3007                 pmap_activate(curthread);
 3008         vmspace_free(oldvmspace);
 3009 }
 3010 
 3011 /*
 3012  * Unshare the specified VM space for forcing COW.  This
 3013  * is called by rfork, for the (RFMEM|RFPROC) == 0 case.
 3014  */
 3015 void
 3016 vmspace_unshare(struct proc *p)
 3017 {
 3018         struct vmspace *oldvmspace = p->p_vmspace;
 3019         struct vmspace *newvmspace;
 3020 
 3021         if (oldvmspace->vm_refcnt == 1)
 3022                 return;
 3023         newvmspace = vmspace_fork(oldvmspace);
 3024         PROC_VMSPACE_LOCK(p);
 3025         p->p_vmspace = newvmspace;
 3026         PROC_VMSPACE_UNLOCK(p);
 3027         if (p == curthread->td_proc)            /* XXXKSE ? */
 3028                 pmap_activate(curthread);
 3029         vmspace_free(oldvmspace);
 3030 }
 3031 
 3032 /*
 3033  *      vm_map_lookup:
 3034  *
 3035  *      Finds the VM object, offset, and
 3036  *      protection for a given virtual address in the
 3037  *      specified map, assuming a page fault of the
 3038  *      type specified.
 3039  *
 3040  *      Leaves the map in question locked for read; return
 3041  *      values are guaranteed until a vm_map_lookup_done
 3042  *      call is performed.  Note that the map argument
 3043  *      is in/out; the returned map must be used in
 3044  *      the call to vm_map_lookup_done.
 3045  *
 3046  *      A handle (out_entry) is returned for use in
 3047  *      vm_map_lookup_done, to make that fast.
 3048  *
 3049  *      If a lookup is requested with "write protection"
 3050  *      specified, the map may be changed to perform virtual
 3051  *      copying operations, although the data referenced will
 3052  *      remain the same.
 3053  */
 3054 int
 3055 vm_map_lookup(vm_map_t *var_map,                /* IN/OUT */
 3056               vm_offset_t vaddr,
 3057               vm_prot_t fault_typea,
 3058               vm_map_entry_t *out_entry,        /* OUT */
 3059               vm_object_t *object,              /* OUT */
 3060               vm_pindex_t *pindex,              /* OUT */
 3061               vm_prot_t *out_prot,              /* OUT */
 3062               boolean_t *wired)                 /* OUT */
 3063 {
 3064         vm_map_entry_t entry;
 3065         vm_map_t map = *var_map;
 3066         vm_prot_t prot;
 3067         vm_prot_t fault_type = fault_typea;
 3068 
 3069 RetryLookup:;
 3070         /*
 3071          * Lookup the faulting address.
 3072          */
 3073 
 3074         vm_map_lock_read(map);
 3075 #define RETURN(why) \
 3076                 { \
 3077                 vm_map_unlock_read(map); \
 3078                 return (why); \
 3079                 }
 3080 
 3081         /*
 3082          * If the map has an interesting hint, try it before calling full
 3083          * blown lookup routine.
 3084          */
 3085         entry = map->root;
 3086         *out_entry = entry;
 3087         if (entry == NULL ||
 3088             (vaddr < entry->start) || (vaddr >= entry->end)) {
 3089                 /*
 3090                  * Entry was either not a valid hint, or the vaddr was not
 3091                  * contained in the entry, so do a full lookup.
 3092                  */
 3093                 if (!vm_map_lookup_entry(map, vaddr, out_entry))
 3094                         RETURN(KERN_INVALID_ADDRESS);
 3095 
 3096                 entry = *out_entry;
 3097         }
 3098 
 3099         /*
 3100          * Handle submaps.
 3101          */
 3102         if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 3103                 vm_map_t old_map = map;
 3104 
 3105                 *var_map = map = entry->object.sub_map;
 3106                 vm_map_unlock_read(old_map);
 3107                 goto RetryLookup;
 3108         }
 3109 
 3110         /*
 3111          * Check whether this task is allowed to have this page.
 3112          * Note the special case for MAP_ENTRY_COW
 3113          * pages with an override.  This is to implement a forced
 3114          * COW for debuggers.
 3115          */
 3116         if (fault_type & VM_PROT_OVERRIDE_WRITE)
 3117                 prot = entry->max_protection;
 3118         else
 3119                 prot = entry->protection;
 3120         fault_type &= (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE);
 3121         if ((fault_type & prot) != fault_type) {
 3122                         RETURN(KERN_PROTECTION_FAILURE);
 3123         }
 3124         if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
 3125             (entry->eflags & MAP_ENTRY_COW) &&
 3126             (fault_type & VM_PROT_WRITE) &&
 3127             (fault_typea & VM_PROT_OVERRIDE_WRITE) == 0) {
 3128                 RETURN(KERN_PROTECTION_FAILURE);
 3129         }
 3130 
 3131         /*
 3132          * If this page is not pageable, we have to get it for all possible
 3133          * accesses.
 3134          */
 3135         *wired = (entry->wired_count != 0);
 3136         if (*wired)
 3137                 prot = fault_type = entry->protection;
 3138 
 3139         /*
 3140          * If the entry was copy-on-write, we either ...
 3141          */
 3142         if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 3143                 /*
 3144                  * If we want to write the page, we may as well handle that
 3145                  * now since we've got the map locked.
 3146                  *
 3147                  * If we don't need to write the page, we just demote the
 3148                  * permissions allowed.
 3149                  */
 3150                 if (fault_type & VM_PROT_WRITE) {
 3151                         /*
 3152                          * Make a new object, and place it in the object
 3153                          * chain.  Note that no new references have appeared
 3154                          * -- one just moved from the map to the new
 3155                          * object.
 3156                          */
 3157                         if (vm_map_lock_upgrade(map))
 3158                                 goto RetryLookup;
 3159 
 3160                         vm_object_shadow(
 3161                             &entry->object.vm_object,
 3162                             &entry->offset,
 3163                             atop(entry->end - entry->start));
 3164                         entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
 3165 
 3166                         vm_map_lock_downgrade(map);
 3167                 } else {
 3168                         /*
 3169                          * We're attempting to read a copy-on-write page --
 3170                          * don't allow writes.
 3171                          */
 3172                         prot &= ~VM_PROT_WRITE;
 3173                 }
 3174         }
 3175 
 3176         /*
 3177          * Create an object if necessary.
 3178          */
 3179         if (entry->object.vm_object == NULL &&
 3180             !map->system_map) {
 3181                 if (vm_map_lock_upgrade(map))
 3182                         goto RetryLookup;
 3183                 entry->object.vm_object = vm_object_allocate(OBJT_DEFAULT,
 3184                     atop(entry->end - entry->start));
 3185                 entry->offset = 0;
 3186                 vm_map_lock_downgrade(map);
 3187         }
 3188 
 3189         /*
 3190          * Return the object/offset from this entry.  If the entry was
 3191          * copy-on-write or empty, it has been fixed up.
 3192          */
 3193         *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
 3194         *object = entry->object.vm_object;
 3195 
 3196         *out_prot = prot;
 3197         return (KERN_SUCCESS);
 3198 
 3199 #undef  RETURN
 3200 }
 3201 
 3202 /*
 3203  *      vm_map_lookup_locked:
 3204  *
 3205  *      Lookup the faulting address.  A version of vm_map_lookup that returns 
 3206  *      KERN_FAILURE instead of blocking on map lock or memory allocation.
 3207  */
 3208 int
 3209 vm_map_lookup_locked(vm_map_t *var_map,         /* IN/OUT */
 3210                      vm_offset_t vaddr,
 3211                      vm_prot_t fault_typea,
 3212                      vm_map_entry_t *out_entry, /* OUT */
 3213                      vm_object_t *object,       /* OUT */
 3214                      vm_pindex_t *pindex,       /* OUT */
 3215                      vm_prot_t *out_prot,       /* OUT */
 3216                      boolean_t *wired)          /* OUT */
 3217 {
 3218         vm_map_entry_t entry;
 3219         vm_map_t map = *var_map;
 3220         vm_prot_t prot;
 3221         vm_prot_t fault_type = fault_typea;
 3222 
 3223         /*
 3224          * If the map has an interesting hint, try it before calling full
 3225          * blown lookup routine.
 3226          */
 3227         entry = map->root;
 3228         *out_entry = entry;
 3229         if (entry == NULL ||
 3230             (vaddr < entry->start) || (vaddr >= entry->end)) {
 3231                 /*
 3232                  * Entry was either not a valid hint, or the vaddr was not
 3233                  * contained in the entry, so do a full lookup.
 3234                  */
 3235                 if (!vm_map_lookup_entry(map, vaddr, out_entry))
 3236                         return (KERN_INVALID_ADDRESS);
 3237 
 3238                 entry = *out_entry;
 3239         }
 3240 
 3241         /*
 3242          * Fail if the entry refers to a submap.
 3243          */
 3244         if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
 3245                 return (KERN_FAILURE);
 3246 
 3247         /*
 3248          * Check whether this task is allowed to have this page.
 3249          * Note the special case for MAP_ENTRY_COW
 3250          * pages with an override.  This is to implement a forced
 3251          * COW for debuggers.
 3252          */
 3253         if (fault_type & VM_PROT_OVERRIDE_WRITE)
 3254                 prot = entry->max_protection;
 3255         else
 3256                 prot = entry->protection;
 3257         fault_type &= VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE;
 3258         if ((fault_type & prot) != fault_type)
 3259                 return (KERN_PROTECTION_FAILURE);
 3260         if ((entry->eflags & MAP_ENTRY_USER_WIRED) &&
 3261             (entry->eflags & MAP_ENTRY_COW) &&
 3262             (fault_type & VM_PROT_WRITE) &&
 3263             (fault_typea & VM_PROT_OVERRIDE_WRITE) == 0)
 3264                 return (KERN_PROTECTION_FAILURE);
 3265 
 3266         /*
 3267          * If this page is not pageable, we have to get it for all possible
 3268          * accesses.
 3269          */
 3270         *wired = (entry->wired_count != 0);
 3271         if (*wired)
 3272                 prot = fault_type = entry->protection;
 3273 
 3274         if (entry->eflags & MAP_ENTRY_NEEDS_COPY) {
 3275                 /*
 3276                  * Fail if the entry was copy-on-write for a write fault.
 3277                  */
 3278                 if (fault_type & VM_PROT_WRITE)
 3279                         return (KERN_FAILURE);
 3280                 /*
 3281                  * We're attempting to read a copy-on-write page --
 3282                  * don't allow writes.
 3283                  */
 3284                 prot &= ~VM_PROT_WRITE;
 3285         }
 3286 
 3287         /*
 3288          * Fail if an object should be created.
 3289          */
 3290         if (entry->object.vm_object == NULL && !map->system_map)
 3291                 return (KERN_FAILURE);
 3292 
 3293         /*
 3294          * Return the object/offset from this entry.  If the entry was
 3295          * copy-on-write or empty, it has been fixed up.
 3296          */
 3297         *pindex = OFF_TO_IDX((vaddr - entry->start) + entry->offset);
 3298         *object = entry->object.vm_object;
 3299 
 3300         *out_prot = prot;
 3301         return (KERN_SUCCESS);
 3302 }
 3303 
 3304 /*
 3305  *      vm_map_lookup_done:
 3306  *
 3307  *      Releases locks acquired by a vm_map_lookup
 3308  *      (according to the handle returned by that lookup).
 3309  */
 3310 void
 3311 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry)
 3312 {
 3313         /*
 3314          * Unlock the main-level map
 3315          */
 3316         vm_map_unlock_read(map);
 3317 }
 3318 
 3319 #include "opt_ddb.h"
 3320 #ifdef DDB
 3321 #include <sys/kernel.h>
 3322 
 3323 #include <ddb/ddb.h>
 3324 
 3325 /*
 3326  *      vm_map_print:   [ debug ]
 3327  */
 3328 DB_SHOW_COMMAND(map, vm_map_print)
 3329 {
 3330         static int nlines;
 3331         /* XXX convert args. */
 3332         vm_map_t map = (vm_map_t)addr;
 3333         boolean_t full = have_addr;
 3334 
 3335         vm_map_entry_t entry;
 3336 
 3337         db_iprintf("Task map %p: pmap=%p, nentries=%d, version=%u\n",
 3338             (void *)map,
 3339             (void *)map->pmap, map->nentries, map->timestamp);
 3340         nlines++;
 3341 
 3342         if (!full && db_indent)
 3343                 return;
 3344 
 3345         db_indent += 2;
 3346         for (entry = map->header.next; entry != &map->header;
 3347             entry = entry->next) {
 3348                 db_iprintf("map entry %p: start=%p, end=%p\n",
 3349                     (void *)entry, (void *)entry->start, (void *)entry->end);
 3350                 nlines++;
 3351                 {
 3352                         static char *inheritance_name[4] =
 3353                         {"share", "copy", "none", "donate_copy"};
 3354 
 3355                         db_iprintf(" prot=%x/%x/%s",
 3356                             entry->protection,
 3357                             entry->max_protection,
 3358                             inheritance_name[(int)(unsigned char)entry->inheritance]);
 3359                         if (entry->wired_count != 0)
 3360                                 db_printf(", wired");
 3361                 }
 3362                 if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 3363                         db_printf(", share=%p, offset=0x%jx\n",
 3364                             (void *)entry->object.sub_map,
 3365                             (uintmax_t)entry->offset);
 3366                         nlines++;
 3367                         if ((entry->prev == &map->header) ||
 3368                             (entry->prev->object.sub_map !=
 3369                                 entry->object.sub_map)) {
 3370                                 db_indent += 2;
 3371                                 vm_map_print((db_expr_t)(intptr_t)
 3372                                              entry->object.sub_map,
 3373                                              full, 0, (char *)0);
 3374                                 db_indent -= 2;
 3375                         }
 3376                 } else {
 3377                         db_printf(", object=%p, offset=0x%jx",
 3378                             (void *)entry->object.vm_object,
 3379                             (uintmax_t)entry->offset);
 3380                         if (entry->eflags & MAP_ENTRY_COW)
 3381                                 db_printf(", copy (%s)",
 3382                                     (entry->eflags & MAP_ENTRY_NEEDS_COPY) ? "needed" : "done");
 3383                         db_printf("\n");
 3384                         nlines++;
 3385 
 3386                         if ((entry->prev == &map->header) ||
 3387                             (entry->prev->object.vm_object !=
 3388                                 entry->object.vm_object)) {
 3389                                 db_indent += 2;
 3390                                 vm_object_print((db_expr_t)(intptr_t)
 3391                                                 entry->object.vm_object,
 3392                                                 full, 0, (char *)0);
 3393                                 nlines += 4;
 3394                                 db_indent -= 2;
 3395                         }
 3396                 }
 3397         }
 3398         db_indent -= 2;
 3399         if (db_indent == 0)
 3400                 nlines = 0;
 3401 }
 3402 
 3403 
 3404 DB_SHOW_COMMAND(procvm, procvm)
 3405 {
 3406         struct proc *p;
 3407 
 3408         if (have_addr) {
 3409                 p = (struct proc *) addr;
 3410         } else {
 3411                 p = curproc;
 3412         }
 3413 
 3414         db_printf("p = %p, vmspace = %p, map = %p, pmap = %p\n",
 3415             (void *)p, (void *)p->p_vmspace, (void *)&p->p_vmspace->vm_map,
 3416             (void *)vmspace_pmap(p->p_vmspace));
 3417 
 3418         vm_map_print((db_expr_t)(intptr_t)&p->p_vmspace->vm_map, 1, 0, NULL);
 3419 }
 3420 
 3421 #endif /* DDB */

Cache object: 57e0176cda41a79d9c363e7d4fd5350c


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.