The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_object.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991, 1993
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * The Mach Operating System project at Carnegie-Mellon University.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 4. Neither the name of the University nor the names of its contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      from: @(#)vm_object.c   8.5 (Berkeley) 3/22/94
   33  *
   34  *
   35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   36  * All rights reserved.
   37  *
   38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   39  *
   40  * Permission to use, copy, modify and distribute this software and
   41  * its documentation is hereby granted, provided that both the copyright
   42  * notice and this permission notice appear in all copies of the
   43  * software, derivative works or modified versions, and any portions
   44  * thereof, and that both notices appear in supporting documentation.
   45  *
   46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   49  *
   50  * Carnegie Mellon requests users of this software to return to
   51  *
   52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   53  *  School of Computer Science
   54  *  Carnegie Mellon University
   55  *  Pittsburgh PA 15213-3890
   56  *
   57  * any improvements or extensions that they make and grant Carnegie the
   58  * rights to redistribute these changes.
   59  */
   60 
   61 /*
   62  *      Virtual memory object module.
   63  */
   64 
   65 #include <sys/cdefs.h>
   66 __FBSDID("$FreeBSD: releng/8.1/sys/vm/vm_object.c 208475 2010-05-23 21:57:45Z alc $");
   67 
   68 #include "opt_vm.h"
   69 
   70 #include <sys/param.h>
   71 #include <sys/systm.h>
   72 #include <sys/lock.h>
   73 #include <sys/mman.h>
   74 #include <sys/mount.h>
   75 #include <sys/kernel.h>
   76 #include <sys/sysctl.h>
   77 #include <sys/mutex.h>
   78 #include <sys/proc.h>           /* for curproc, pageproc */
   79 #include <sys/socket.h>
   80 #include <sys/resourcevar.h>
   81 #include <sys/vnode.h>
   82 #include <sys/vmmeter.h>
   83 #include <sys/sx.h>
   84 
   85 #include <vm/vm.h>
   86 #include <vm/vm_param.h>
   87 #include <vm/pmap.h>
   88 #include <vm/vm_map.h>
   89 #include <vm/vm_object.h>
   90 #include <vm/vm_page.h>
   91 #include <vm/vm_pageout.h>
   92 #include <vm/vm_pager.h>
   93 #include <vm/swap_pager.h>
   94 #include <vm/vm_kern.h>
   95 #include <vm/vm_extern.h>
   96 #include <vm/vm_reserv.h>
   97 #include <vm/uma.h>
   98 
   99 #define EASY_SCAN_FACTOR       8
  100 
  101 #define MSYNC_FLUSH_HARDSEQ     0x01
  102 #define MSYNC_FLUSH_SOFTSEQ     0x02
  103 
  104 /*
  105  * msync / VM object flushing optimizations
  106  */
  107 static int msync_flush_flags = MSYNC_FLUSH_HARDSEQ | MSYNC_FLUSH_SOFTSEQ;
  108 SYSCTL_INT(_vm, OID_AUTO, msync_flush_flags, CTLFLAG_RW, &msync_flush_flags, 0,
  109     "Enable sequential iteration optimization");
  110 
  111 static int old_msync;
  112 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
  113     "Use old (insecure) msync behavior");
  114 
  115 static void     vm_object_qcollapse(vm_object_t object);
  116 static int      vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags);
  117 static void     vm_object_vndeallocate(vm_object_t object);
  118 
  119 /*
  120  *      Virtual memory objects maintain the actual data
  121  *      associated with allocated virtual memory.  A given
  122  *      page of memory exists within exactly one object.
  123  *
  124  *      An object is only deallocated when all "references"
  125  *      are given up.  Only one "reference" to a given
  126  *      region of an object should be writeable.
  127  *
  128  *      Associated with each object is a list of all resident
  129  *      memory pages belonging to that object; this list is
  130  *      maintained by the "vm_page" module, and locked by the object's
  131  *      lock.
  132  *
  133  *      Each object also records a "pager" routine which is
  134  *      used to retrieve (and store) pages to the proper backing
  135  *      storage.  In addition, objects may be backed by other
  136  *      objects from which they were virtual-copied.
  137  *
  138  *      The only items within the object structure which are
  139  *      modified after time of creation are:
  140  *              reference count         locked by object's lock
  141  *              pager routine           locked by object's lock
  142  *
  143  */
  144 
  145 struct object_q vm_object_list;
  146 struct mtx vm_object_list_mtx;  /* lock for object list and count */
  147 
  148 struct vm_object kernel_object_store;
  149 struct vm_object kmem_object_store;
  150 
  151 SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD, 0, "VM object stats");
  152 
  153 static long object_collapses;
  154 SYSCTL_LONG(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
  155     &object_collapses, 0, "VM object collapses");
  156 
  157 static long object_bypasses;
  158 SYSCTL_LONG(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
  159     &object_bypasses, 0, "VM object bypasses");
  160 
  161 static uma_zone_t obj_zone;
  162 
  163 static int vm_object_zinit(void *mem, int size, int flags);
  164 
  165 #ifdef INVARIANTS
  166 static void vm_object_zdtor(void *mem, int size, void *arg);
  167 
  168 static void
  169 vm_object_zdtor(void *mem, int size, void *arg)
  170 {
  171         vm_object_t object;
  172 
  173         object = (vm_object_t)mem;
  174         KASSERT(TAILQ_EMPTY(&object->memq),
  175             ("object %p has resident pages",
  176             object));
  177 #if VM_NRESERVLEVEL > 0
  178         KASSERT(LIST_EMPTY(&object->rvq),
  179             ("object %p has reservations",
  180             object));
  181 #endif
  182         KASSERT(object->cache == NULL,
  183             ("object %p has cached pages",
  184             object));
  185         KASSERT(object->paging_in_progress == 0,
  186             ("object %p paging_in_progress = %d",
  187             object, object->paging_in_progress));
  188         KASSERT(object->resident_page_count == 0,
  189             ("object %p resident_page_count = %d",
  190             object, object->resident_page_count));
  191         KASSERT(object->shadow_count == 0,
  192             ("object %p shadow_count = %d",
  193             object, object->shadow_count));
  194 }
  195 #endif
  196 
  197 static int
  198 vm_object_zinit(void *mem, int size, int flags)
  199 {
  200         vm_object_t object;
  201 
  202         object = (vm_object_t)mem;
  203         bzero(&object->mtx, sizeof(object->mtx));
  204         VM_OBJECT_LOCK_INIT(object, "standard object");
  205 
  206         /* These are true for any object that has been freed */
  207         object->paging_in_progress = 0;
  208         object->resident_page_count = 0;
  209         object->shadow_count = 0;
  210         return (0);
  211 }
  212 
  213 void
  214 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
  215 {
  216 
  217         TAILQ_INIT(&object->memq);
  218         LIST_INIT(&object->shadow_head);
  219 
  220         object->root = NULL;
  221         object->type = type;
  222         object->size = size;
  223         object->generation = 1;
  224         object->ref_count = 1;
  225         object->memattr = VM_MEMATTR_DEFAULT;
  226         object->flags = 0;
  227         object->uip = NULL;
  228         object->charge = 0;
  229         if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
  230                 object->flags = OBJ_ONEMAPPING;
  231         object->pg_color = 0;
  232         object->handle = NULL;
  233         object->backing_object = NULL;
  234         object->backing_object_offset = (vm_ooffset_t) 0;
  235 #if VM_NRESERVLEVEL > 0
  236         LIST_INIT(&object->rvq);
  237 #endif
  238         object->cache = NULL;
  239 
  240         mtx_lock(&vm_object_list_mtx);
  241         TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
  242         mtx_unlock(&vm_object_list_mtx);
  243 }
  244 
  245 /*
  246  *      vm_object_init:
  247  *
  248  *      Initialize the VM objects module.
  249  */
  250 void
  251 vm_object_init(void)
  252 {
  253         TAILQ_INIT(&vm_object_list);
  254         mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
  255         
  256         VM_OBJECT_LOCK_INIT(&kernel_object_store, "kernel object");
  257         _vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
  258             kernel_object);
  259 #if VM_NRESERVLEVEL > 0
  260         kernel_object->flags |= OBJ_COLORED;
  261         kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
  262 #endif
  263 
  264         VM_OBJECT_LOCK_INIT(&kmem_object_store, "kmem object");
  265         _vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
  266             kmem_object);
  267 #if VM_NRESERVLEVEL > 0
  268         kmem_object->flags |= OBJ_COLORED;
  269         kmem_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
  270 #endif
  271 
  272         /*
  273          * The lock portion of struct vm_object must be type stable due
  274          * to vm_pageout_fallback_object_lock locking a vm object
  275          * without holding any references to it.
  276          */
  277         obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
  278 #ifdef INVARIANTS
  279             vm_object_zdtor,
  280 #else
  281             NULL,
  282 #endif
  283             vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE);
  284 }
  285 
  286 void
  287 vm_object_clear_flag(vm_object_t object, u_short bits)
  288 {
  289 
  290         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  291         object->flags &= ~bits;
  292 }
  293 
  294 /*
  295  *      Sets the default memory attribute for the specified object.  Pages
  296  *      that are allocated to this object are by default assigned this memory
  297  *      attribute.
  298  *
  299  *      Presently, this function must be called before any pages are allocated
  300  *      to the object.  In the future, this requirement may be relaxed for
  301  *      "default" and "swap" objects.
  302  */
  303 int
  304 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
  305 {
  306 
  307         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  308         switch (object->type) {
  309         case OBJT_DEFAULT:
  310         case OBJT_DEVICE:
  311         case OBJT_PHYS:
  312         case OBJT_SG:
  313         case OBJT_SWAP:
  314         case OBJT_VNODE:
  315                 if (!TAILQ_EMPTY(&object->memq))
  316                         return (KERN_FAILURE);
  317                 break;
  318         case OBJT_DEAD:
  319                 return (KERN_INVALID_ARGUMENT);
  320         }
  321         object->memattr = memattr;
  322         return (KERN_SUCCESS);
  323 }
  324 
  325 void
  326 vm_object_pip_add(vm_object_t object, short i)
  327 {
  328 
  329         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  330         object->paging_in_progress += i;
  331 }
  332 
  333 void
  334 vm_object_pip_subtract(vm_object_t object, short i)
  335 {
  336 
  337         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  338         object->paging_in_progress -= i;
  339 }
  340 
  341 void
  342 vm_object_pip_wakeup(vm_object_t object)
  343 {
  344 
  345         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  346         object->paging_in_progress--;
  347         if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
  348                 vm_object_clear_flag(object, OBJ_PIPWNT);
  349                 wakeup(object);
  350         }
  351 }
  352 
  353 void
  354 vm_object_pip_wakeupn(vm_object_t object, short i)
  355 {
  356 
  357         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  358         if (i)
  359                 object->paging_in_progress -= i;
  360         if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) {
  361                 vm_object_clear_flag(object, OBJ_PIPWNT);
  362                 wakeup(object);
  363         }
  364 }
  365 
  366 void
  367 vm_object_pip_wait(vm_object_t object, char *waitid)
  368 {
  369 
  370         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  371         while (object->paging_in_progress) {
  372                 object->flags |= OBJ_PIPWNT;
  373                 msleep(object, VM_OBJECT_MTX(object), PVM, waitid, 0);
  374         }
  375 }
  376 
  377 /*
  378  *      vm_object_allocate:
  379  *
  380  *      Returns a new object with the given size.
  381  */
  382 vm_object_t
  383 vm_object_allocate(objtype_t type, vm_pindex_t size)
  384 {
  385         vm_object_t object;
  386 
  387         object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
  388         _vm_object_allocate(type, size, object);
  389         return (object);
  390 }
  391 
  392 
  393 /*
  394  *      vm_object_reference:
  395  *
  396  *      Gets another reference to the given object.  Note: OBJ_DEAD
  397  *      objects can be referenced during final cleaning.
  398  */
  399 void
  400 vm_object_reference(vm_object_t object)
  401 {
  402         if (object == NULL)
  403                 return;
  404         VM_OBJECT_LOCK(object);
  405         vm_object_reference_locked(object);
  406         VM_OBJECT_UNLOCK(object);
  407 }
  408 
  409 /*
  410  *      vm_object_reference_locked:
  411  *
  412  *      Gets another reference to the given object.
  413  *
  414  *      The object must be locked.
  415  */
  416 void
  417 vm_object_reference_locked(vm_object_t object)
  418 {
  419         struct vnode *vp;
  420 
  421         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  422         object->ref_count++;
  423         if (object->type == OBJT_VNODE) {
  424                 vp = object->handle;
  425                 vref(vp);
  426         }
  427 }
  428 
  429 /*
  430  * Handle deallocating an object of type OBJT_VNODE.
  431  */
  432 static void
  433 vm_object_vndeallocate(vm_object_t object)
  434 {
  435         struct vnode *vp = (struct vnode *) object->handle;
  436 
  437         VFS_ASSERT_GIANT(vp->v_mount);
  438         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  439         KASSERT(object->type == OBJT_VNODE,
  440             ("vm_object_vndeallocate: not a vnode object"));
  441         KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
  442 #ifdef INVARIANTS
  443         if (object->ref_count == 0) {
  444                 vprint("vm_object_vndeallocate", vp);
  445                 panic("vm_object_vndeallocate: bad object reference count");
  446         }
  447 #endif
  448 
  449         object->ref_count--;
  450         if (object->ref_count == 0) {
  451                 mp_fixme("Unlocked vflag access.");
  452                 vp->v_vflag &= ~VV_TEXT;
  453         }
  454         VM_OBJECT_UNLOCK(object);
  455         /*
  456          * vrele may need a vop lock
  457          */
  458         vrele(vp);
  459 }
  460 
  461 /*
  462  *      vm_object_deallocate:
  463  *
  464  *      Release a reference to the specified object,
  465  *      gained either through a vm_object_allocate
  466  *      or a vm_object_reference call.  When all references
  467  *      are gone, storage associated with this object
  468  *      may be relinquished.
  469  *
  470  *      No object may be locked.
  471  */
  472 void
  473 vm_object_deallocate(vm_object_t object)
  474 {
  475         vm_object_t temp;
  476 
  477         while (object != NULL) {
  478                 int vfslocked;
  479 
  480                 vfslocked = 0;
  481         restart:
  482                 VM_OBJECT_LOCK(object);
  483                 if (object->type == OBJT_VNODE) {
  484                         struct vnode *vp = (struct vnode *) object->handle;
  485 
  486                         /*
  487                          * Conditionally acquire Giant for a vnode-backed
  488                          * object.  We have to be careful since the type of
  489                          * a vnode object can change while the object is
  490                          * unlocked.
  491                          */
  492                         if (VFS_NEEDSGIANT(vp->v_mount) && !vfslocked) {
  493                                 vfslocked = 1;
  494                                 if (!mtx_trylock(&Giant)) {
  495                                         VM_OBJECT_UNLOCK(object);
  496                                         mtx_lock(&Giant);
  497                                         goto restart;
  498                                 }
  499                         }
  500                         vm_object_vndeallocate(object);
  501                         VFS_UNLOCK_GIANT(vfslocked);
  502                         return;
  503                 } else
  504                         /*
  505                          * This is to handle the case that the object
  506                          * changed type while we dropped its lock to
  507                          * obtain Giant.
  508                          */
  509                         VFS_UNLOCK_GIANT(vfslocked);
  510 
  511                 KASSERT(object->ref_count != 0,
  512                         ("vm_object_deallocate: object deallocated too many times: %d", object->type));
  513 
  514                 /*
  515                  * If the reference count goes to 0 we start calling
  516                  * vm_object_terminate() on the object chain.
  517                  * A ref count of 1 may be a special case depending on the
  518                  * shadow count being 0 or 1.
  519                  */
  520                 object->ref_count--;
  521                 if (object->ref_count > 1) {
  522                         VM_OBJECT_UNLOCK(object);
  523                         return;
  524                 } else if (object->ref_count == 1) {
  525                         if (object->shadow_count == 0 &&
  526                             object->handle == NULL &&
  527                             (object->type == OBJT_DEFAULT ||
  528                              object->type == OBJT_SWAP)) {
  529                                 vm_object_set_flag(object, OBJ_ONEMAPPING);
  530                         } else if ((object->shadow_count == 1) &&
  531                             (object->handle == NULL) &&
  532                             (object->type == OBJT_DEFAULT ||
  533                              object->type == OBJT_SWAP)) {
  534                                 vm_object_t robject;
  535 
  536                                 robject = LIST_FIRST(&object->shadow_head);
  537                                 KASSERT(robject != NULL,
  538                                     ("vm_object_deallocate: ref_count: %d, shadow_count: %d",
  539                                          object->ref_count,
  540                                          object->shadow_count));
  541                                 if (!VM_OBJECT_TRYLOCK(robject)) {
  542                                         /*
  543                                          * Avoid a potential deadlock.
  544                                          */
  545                                         object->ref_count++;
  546                                         VM_OBJECT_UNLOCK(object);
  547                                         /*
  548                                          * More likely than not the thread
  549                                          * holding robject's lock has lower
  550                                          * priority than the current thread.
  551                                          * Let the lower priority thread run.
  552                                          */
  553                                         pause("vmo_de", 1);
  554                                         continue;
  555                                 }
  556                                 /*
  557                                  * Collapse object into its shadow unless its
  558                                  * shadow is dead.  In that case, object will
  559                                  * be deallocated by the thread that is
  560                                  * deallocating its shadow.
  561                                  */
  562                                 if ((robject->flags & OBJ_DEAD) == 0 &&
  563                                     (robject->handle == NULL) &&
  564                                     (robject->type == OBJT_DEFAULT ||
  565                                      robject->type == OBJT_SWAP)) {
  566 
  567                                         robject->ref_count++;
  568 retry:
  569                                         if (robject->paging_in_progress) {
  570                                                 VM_OBJECT_UNLOCK(object);
  571                                                 vm_object_pip_wait(robject,
  572                                                     "objde1");
  573                                                 temp = robject->backing_object;
  574                                                 if (object == temp) {
  575                                                         VM_OBJECT_LOCK(object);
  576                                                         goto retry;
  577                                                 }
  578                                         } else if (object->paging_in_progress) {
  579                                                 VM_OBJECT_UNLOCK(robject);
  580                                                 object->flags |= OBJ_PIPWNT;
  581                                                 msleep(object,
  582                                                     VM_OBJECT_MTX(object),
  583                                                     PDROP | PVM, "objde2", 0);
  584                                                 VM_OBJECT_LOCK(robject);
  585                                                 temp = robject->backing_object;
  586                                                 if (object == temp) {
  587                                                         VM_OBJECT_LOCK(object);
  588                                                         goto retry;
  589                                                 }
  590                                         } else
  591                                                 VM_OBJECT_UNLOCK(object);
  592 
  593                                         if (robject->ref_count == 1) {
  594                                                 robject->ref_count--;
  595                                                 object = robject;
  596                                                 goto doterm;
  597                                         }
  598                                         object = robject;
  599                                         vm_object_collapse(object);
  600                                         VM_OBJECT_UNLOCK(object);
  601                                         continue;
  602                                 }
  603                                 VM_OBJECT_UNLOCK(robject);
  604                         }
  605                         VM_OBJECT_UNLOCK(object);
  606                         return;
  607                 }
  608 doterm:
  609                 temp = object->backing_object;
  610                 if (temp != NULL) {
  611                         VM_OBJECT_LOCK(temp);
  612                         LIST_REMOVE(object, shadow_list);
  613                         temp->shadow_count--;
  614                         temp->generation++;
  615                         VM_OBJECT_UNLOCK(temp);
  616                         object->backing_object = NULL;
  617                 }
  618                 /*
  619                  * Don't double-terminate, we could be in a termination
  620                  * recursion due to the terminate having to sync data
  621                  * to disk.
  622                  */
  623                 if ((object->flags & OBJ_DEAD) == 0)
  624                         vm_object_terminate(object);
  625                 else
  626                         VM_OBJECT_UNLOCK(object);
  627                 object = temp;
  628         }
  629 }
  630 
  631 /*
  632  *      vm_object_destroy removes the object from the global object list
  633  *      and frees the space for the object.
  634  */
  635 void
  636 vm_object_destroy(vm_object_t object)
  637 {
  638 
  639         /*
  640          * Remove the object from the global object list.
  641          */
  642         mtx_lock(&vm_object_list_mtx);
  643         TAILQ_REMOVE(&vm_object_list, object, object_list);
  644         mtx_unlock(&vm_object_list_mtx);
  645 
  646         /*
  647          * Release the allocation charge.
  648          */
  649         if (object->uip != NULL) {
  650                 KASSERT(object->type == OBJT_DEFAULT ||
  651                     object->type == OBJT_SWAP,
  652                     ("vm_object_terminate: non-swap obj %p has uip",
  653                      object));
  654                 swap_release_by_uid(object->charge, object->uip);
  655                 object->charge = 0;
  656                 uifree(object->uip);
  657                 object->uip = NULL;
  658         }
  659 
  660         /*
  661          * Free the space for the object.
  662          */
  663         uma_zfree(obj_zone, object);
  664 }
  665 
  666 /*
  667  *      vm_object_terminate actually destroys the specified object, freeing
  668  *      up all previously used resources.
  669  *
  670  *      The object must be locked.
  671  *      This routine may block.
  672  */
  673 void
  674 vm_object_terminate(vm_object_t object)
  675 {
  676         vm_page_t p;
  677 
  678         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  679 
  680         /*
  681          * Make sure no one uses us.
  682          */
  683         vm_object_set_flag(object, OBJ_DEAD);
  684 
  685         /*
  686          * wait for the pageout daemon to be done with the object
  687          */
  688         vm_object_pip_wait(object, "objtrm");
  689 
  690         KASSERT(!object->paging_in_progress,
  691                 ("vm_object_terminate: pageout in progress"));
  692 
  693         /*
  694          * Clean and free the pages, as appropriate. All references to the
  695          * object are gone, so we don't need to lock it.
  696          */
  697         if (object->type == OBJT_VNODE) {
  698                 struct vnode *vp = (struct vnode *)object->handle;
  699 
  700                 /*
  701                  * Clean pages and flush buffers.
  702                  */
  703                 vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
  704                 VM_OBJECT_UNLOCK(object);
  705 
  706                 vinvalbuf(vp, V_SAVE, 0, 0);
  707 
  708                 VM_OBJECT_LOCK(object);
  709         }
  710 
  711         KASSERT(object->ref_count == 0, 
  712                 ("vm_object_terminate: object with references, ref_count=%d",
  713                 object->ref_count));
  714 
  715         /*
  716          * Now free any remaining pages. For internal objects, this also
  717          * removes them from paging queues. Don't free wired pages, just
  718          * remove them from the object. 
  719          */
  720         vm_page_lock_queues();
  721         while ((p = TAILQ_FIRST(&object->memq)) != NULL) {
  722                 KASSERT(!p->busy && (p->oflags & VPO_BUSY) == 0,
  723                         ("vm_object_terminate: freeing busy page %p "
  724                         "p->busy = %d, p->oflags %x\n", p, p->busy, p->oflags));
  725                 if (p->wire_count == 0) {
  726                         vm_page_free(p);
  727                         cnt.v_pfree++;
  728                 } else {
  729                         vm_page_remove(p);
  730                 }
  731         }
  732         vm_page_unlock_queues();
  733 
  734 #if VM_NRESERVLEVEL > 0
  735         if (__predict_false(!LIST_EMPTY(&object->rvq)))
  736                 vm_reserv_break_all(object);
  737 #endif
  738         if (__predict_false(object->cache != NULL))
  739                 vm_page_cache_free(object, 0, 0);
  740 
  741         /*
  742          * Let the pager know object is dead.
  743          */
  744         vm_pager_deallocate(object);
  745         VM_OBJECT_UNLOCK(object);
  746 
  747         vm_object_destroy(object);
  748 }
  749 
  750 /*
  751  *      vm_object_page_clean
  752  *
  753  *      Clean all dirty pages in the specified range of object.  Leaves page 
  754  *      on whatever queue it is currently on.   If NOSYNC is set then do not
  755  *      write out pages with VPO_NOSYNC set (originally comes from MAP_NOSYNC),
  756  *      leaving the object dirty.
  757  *
  758  *      When stuffing pages asynchronously, allow clustering.  XXX we need a
  759  *      synchronous clustering mode implementation.
  760  *
  761  *      Odd semantics: if start == end, we clean everything.
  762  *
  763  *      The object must be locked.
  764  */
  765 void
  766 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int flags)
  767 {
  768         vm_page_t p, np;
  769         vm_pindex_t tstart, tend;
  770         vm_pindex_t pi;
  771         int clearobjflags;
  772         int pagerflags;
  773         int curgeneration;
  774 
  775         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  776         if ((object->flags & OBJ_MIGHTBEDIRTY) == 0)
  777                 return;
  778         KASSERT(object->type == OBJT_VNODE, ("Not a vnode object"));
  779 
  780         pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
  781         pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
  782 
  783         vm_object_set_flag(object, OBJ_CLEANING);
  784 
  785         tstart = start;
  786         if (end == 0) {
  787                 tend = object->size;
  788         } else {
  789                 tend = end;
  790         }
  791 
  792         vm_page_lock_queues();
  793         /*
  794          * If the caller is smart and only msync()s a range he knows is
  795          * dirty, we may be able to avoid an object scan.  This results in
  796          * a phenominal improvement in performance.  We cannot do this
  797          * as a matter of course because the object may be huge - e.g.
  798          * the size might be in the gigabytes or terrabytes.
  799          */
  800         if (msync_flush_flags & MSYNC_FLUSH_HARDSEQ) {
  801                 vm_pindex_t tscan;
  802                 int scanlimit;
  803                 int scanreset;
  804 
  805                 scanreset = object->resident_page_count / EASY_SCAN_FACTOR;
  806                 if (scanreset < 16)
  807                         scanreset = 16;
  808                 pagerflags |= VM_PAGER_IGNORE_CLEANCHK;
  809 
  810                 scanlimit = scanreset;
  811                 tscan = tstart;
  812                 while (tscan < tend) {
  813                         curgeneration = object->generation;
  814                         p = vm_page_lookup(object, tscan);
  815                         if (p == NULL || p->valid == 0) {
  816                                 if (--scanlimit == 0)
  817                                         break;
  818                                 ++tscan;
  819                                 continue;
  820                         }
  821                         vm_page_test_dirty(p);
  822                         if (p->dirty == 0) {
  823                                 if (--scanlimit == 0)
  824                                         break;
  825                                 ++tscan;
  826                                 continue;
  827                         }
  828                         /*
  829                          * If we have been asked to skip nosync pages and 
  830                          * this is a nosync page, we can't continue.
  831                          */
  832                         if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) {
  833                                 if (--scanlimit == 0)
  834                                         break;
  835                                 ++tscan;
  836                                 continue;
  837                         }
  838                         scanlimit = scanreset;
  839 
  840                         /*
  841                          * This returns 0 if it was unable to busy the first
  842                          * page (i.e. had to sleep).
  843                          */
  844                         tscan += vm_object_page_collect_flush(object, p, curgeneration, pagerflags);
  845                 }
  846 
  847                 /*
  848                  * If everything was dirty and we flushed it successfully,
  849                  * and the requested range is not the entire object, we
  850                  * don't have to mess with CLEANCHK or MIGHTBEDIRTY and can
  851                  * return immediately.
  852                  */
  853                 if (tscan >= tend && (tstart || tend < object->size)) {
  854                         vm_page_unlock_queues();
  855                         vm_object_clear_flag(object, OBJ_CLEANING);
  856                         return;
  857                 }
  858                 pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK;
  859         }
  860 
  861         /*
  862          * Generally set CLEANCHK interlock and make the page read-only so
  863          * we can then clear the object flags.
  864          *
  865          * However, if this is a nosync mmap then the object is likely to 
  866          * stay dirty so do not mess with the page and do not clear the
  867          * object flags.
  868          */
  869         clearobjflags = 1;
  870         TAILQ_FOREACH(p, &object->memq, listq) {
  871                 p->oflags |= VPO_CLEANCHK;
  872                 if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC))
  873                         clearobjflags = 0;
  874                 else
  875                         pmap_remove_write(p);
  876         }
  877 
  878         if (clearobjflags && (tstart == 0) && (tend == object->size))
  879                 vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY);
  880 
  881 rescan:
  882         curgeneration = object->generation;
  883 
  884         for (p = TAILQ_FIRST(&object->memq); p; p = np) {
  885                 int n;
  886 
  887                 np = TAILQ_NEXT(p, listq);
  888 
  889 again:
  890                 pi = p->pindex;
  891                 if ((p->oflags & VPO_CLEANCHK) == 0 ||
  892                         (pi < tstart) || (pi >= tend) ||
  893                     p->valid == 0) {
  894                         p->oflags &= ~VPO_CLEANCHK;
  895                         continue;
  896                 }
  897 
  898                 vm_page_test_dirty(p);
  899                 if (p->dirty == 0) {
  900                         p->oflags &= ~VPO_CLEANCHK;
  901                         continue;
  902                 }
  903 
  904                 /*
  905                  * If we have been asked to skip nosync pages and this is a
  906                  * nosync page, skip it.  Note that the object flags were
  907                  * not cleared in this case so we do not have to set them.
  908                  */
  909                 if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) {
  910                         p->oflags &= ~VPO_CLEANCHK;
  911                         continue;
  912                 }
  913 
  914                 n = vm_object_page_collect_flush(object, p,
  915                         curgeneration, pagerflags);
  916                 if (n == 0)
  917                         goto rescan;
  918 
  919                 if (object->generation != curgeneration)
  920                         goto rescan;
  921 
  922                 /*
  923                  * Try to optimize the next page.  If we can't we pick up
  924                  * our (random) scan where we left off.
  925                  */
  926                 if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ) {
  927                         if ((p = vm_page_lookup(object, pi + n)) != NULL)
  928                                 goto again;
  929                 }
  930         }
  931         vm_page_unlock_queues();
  932 #if 0
  933         VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc);
  934 #endif
  935 
  936         vm_object_clear_flag(object, OBJ_CLEANING);
  937         return;
  938 }
  939 
  940 static int
  941 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags)
  942 {
  943         int runlen;
  944         int maxf;
  945         int chkb;
  946         int maxb;
  947         int i;
  948         vm_pindex_t pi;
  949         vm_page_t maf[vm_pageout_page_count];
  950         vm_page_t mab[vm_pageout_page_count];
  951         vm_page_t ma[vm_pageout_page_count];
  952 
  953         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  954         pi = p->pindex;
  955         while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) {
  956                 vm_page_lock_queues();
  957                 if (object->generation != curgeneration) {
  958                         return(0);
  959                 }
  960         }
  961         maxf = 0;
  962         for(i = 1; i < vm_pageout_page_count; i++) {
  963                 vm_page_t tp;
  964 
  965                 if ((tp = vm_page_lookup(object, pi + i)) != NULL) {
  966                         if ((tp->oflags & VPO_BUSY) ||
  967                                 ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
  968                                  (tp->oflags & VPO_CLEANCHK) == 0) ||
  969                                 (tp->busy != 0))
  970                                 break;
  971                         vm_page_test_dirty(tp);
  972                         if (tp->dirty == 0) {
  973                                 tp->oflags &= ~VPO_CLEANCHK;
  974                                 break;
  975                         }
  976                         maf[ i - 1 ] = tp;
  977                         maxf++;
  978                         continue;
  979                 }
  980                 break;
  981         }
  982 
  983         maxb = 0;
  984         chkb = vm_pageout_page_count -  maxf;
  985         if (chkb) {
  986                 for(i = 1; i < chkb;i++) {
  987                         vm_page_t tp;
  988 
  989                         if ((tp = vm_page_lookup(object, pi - i)) != NULL) {
  990                                 if ((tp->oflags & VPO_BUSY) ||
  991                                         ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
  992                                          (tp->oflags & VPO_CLEANCHK) == 0) ||
  993                                         (tp->busy != 0))
  994                                         break;
  995                                 vm_page_test_dirty(tp);
  996                                 if (tp->dirty == 0) {
  997                                         tp->oflags &= ~VPO_CLEANCHK;
  998                                         break;
  999                                 }
 1000                                 mab[ i - 1 ] = tp;
 1001                                 maxb++;
 1002                                 continue;
 1003                         }
 1004                         break;
 1005                 }
 1006         }
 1007 
 1008         for(i = 0; i < maxb; i++) {
 1009                 int index = (maxb - i) - 1;
 1010                 ma[index] = mab[i];
 1011                 ma[index]->oflags &= ~VPO_CLEANCHK;
 1012         }
 1013         p->oflags &= ~VPO_CLEANCHK;
 1014         ma[maxb] = p;
 1015         for(i = 0; i < maxf; i++) {
 1016                 int index = (maxb + i) + 1;
 1017                 ma[index] = maf[i];
 1018                 ma[index]->oflags &= ~VPO_CLEANCHK;
 1019         }
 1020         runlen = maxb + maxf + 1;
 1021 
 1022         vm_pageout_flush(ma, runlen, pagerflags);
 1023         for (i = 0; i < runlen; i++) {
 1024                 if (ma[i]->dirty) {
 1025                         pmap_remove_write(ma[i]);
 1026                         ma[i]->oflags |= VPO_CLEANCHK;
 1027 
 1028                         /*
 1029                          * maxf will end up being the actual number of pages
 1030                          * we wrote out contiguously, non-inclusive of the
 1031                          * first page.  We do not count look-behind pages.
 1032                          */
 1033                         if (i >= maxb + 1 && (maxf > i - maxb - 1))
 1034                                 maxf = i - maxb - 1;
 1035                 }
 1036         }
 1037         return(maxf + 1);
 1038 }
 1039 
 1040 /*
 1041  * Note that there is absolutely no sense in writing out
 1042  * anonymous objects, so we track down the vnode object
 1043  * to write out.
 1044  * We invalidate (remove) all pages from the address space
 1045  * for semantic correctness.
 1046  *
 1047  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
 1048  * may start out with a NULL object.
 1049  */
 1050 void
 1051 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
 1052     boolean_t syncio, boolean_t invalidate)
 1053 {
 1054         vm_object_t backing_object;
 1055         struct vnode *vp;
 1056         struct mount *mp;
 1057         int flags;
 1058 
 1059         if (object == NULL)
 1060                 return;
 1061         VM_OBJECT_LOCK(object);
 1062         while ((backing_object = object->backing_object) != NULL) {
 1063                 VM_OBJECT_LOCK(backing_object);
 1064                 offset += object->backing_object_offset;
 1065                 VM_OBJECT_UNLOCK(object);
 1066                 object = backing_object;
 1067                 if (object->size < OFF_TO_IDX(offset + size))
 1068                         size = IDX_TO_OFF(object->size) - offset;
 1069         }
 1070         /*
 1071          * Flush pages if writing is allowed, invalidate them
 1072          * if invalidation requested.  Pages undergoing I/O
 1073          * will be ignored by vm_object_page_remove().
 1074          *
 1075          * We cannot lock the vnode and then wait for paging
 1076          * to complete without deadlocking against vm_fault.
 1077          * Instead we simply call vm_object_page_remove() and
 1078          * allow it to block internally on a page-by-page
 1079          * basis when it encounters pages undergoing async
 1080          * I/O.
 1081          */
 1082         if (object->type == OBJT_VNODE &&
 1083             (object->flags & OBJ_MIGHTBEDIRTY) != 0) {
 1084                 int vfslocked;
 1085                 vp = object->handle;
 1086                 VM_OBJECT_UNLOCK(object);
 1087                 (void) vn_start_write(vp, &mp, V_WAIT);
 1088                 vfslocked = VFS_LOCK_GIANT(vp->v_mount);
 1089                 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 1090                 flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
 1091                 flags |= invalidate ? OBJPC_INVAL : 0;
 1092                 VM_OBJECT_LOCK(object);
 1093                 vm_object_page_clean(object,
 1094                     OFF_TO_IDX(offset),
 1095                     OFF_TO_IDX(offset + size + PAGE_MASK),
 1096                     flags);
 1097                 VM_OBJECT_UNLOCK(object);
 1098                 VOP_UNLOCK(vp, 0);
 1099                 VFS_UNLOCK_GIANT(vfslocked);
 1100                 vn_finished_write(mp);
 1101                 VM_OBJECT_LOCK(object);
 1102         }
 1103         if ((object->type == OBJT_VNODE ||
 1104              object->type == OBJT_DEVICE) && invalidate) {
 1105                 boolean_t purge;
 1106                 purge = old_msync || (object->type == OBJT_DEVICE);
 1107                 vm_object_page_remove(object,
 1108                     OFF_TO_IDX(offset),
 1109                     OFF_TO_IDX(offset + size + PAGE_MASK),
 1110                     purge ? FALSE : TRUE);
 1111         }
 1112         VM_OBJECT_UNLOCK(object);
 1113 }
 1114 
 1115 /*
 1116  *      vm_object_madvise:
 1117  *
 1118  *      Implements the madvise function at the object/page level.
 1119  *
 1120  *      MADV_WILLNEED   (any object)
 1121  *
 1122  *          Activate the specified pages if they are resident.
 1123  *
 1124  *      MADV_DONTNEED   (any object)
 1125  *
 1126  *          Deactivate the specified pages if they are resident.
 1127  *
 1128  *      MADV_FREE       (OBJT_DEFAULT/OBJT_SWAP objects,
 1129  *                       OBJ_ONEMAPPING only)
 1130  *
 1131  *          Deactivate and clean the specified pages if they are
 1132  *          resident.  This permits the process to reuse the pages
 1133  *          without faulting or the kernel to reclaim the pages
 1134  *          without I/O.
 1135  */
 1136 void
 1137 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, int count, int advise)
 1138 {
 1139         vm_pindex_t end, tpindex;
 1140         vm_object_t backing_object, tobject;
 1141         vm_page_t m;
 1142 
 1143         if (object == NULL)
 1144                 return;
 1145         VM_OBJECT_LOCK(object);
 1146         end = pindex + count;
 1147         /*
 1148          * Locate and adjust resident pages
 1149          */
 1150         for (; pindex < end; pindex += 1) {
 1151 relookup:
 1152                 tobject = object;
 1153                 tpindex = pindex;
 1154 shadowlookup:
 1155                 /*
 1156                  * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages
 1157                  * and those pages must be OBJ_ONEMAPPING.
 1158                  */
 1159                 if (advise == MADV_FREE) {
 1160                         if ((tobject->type != OBJT_DEFAULT &&
 1161                              tobject->type != OBJT_SWAP) ||
 1162                             (tobject->flags & OBJ_ONEMAPPING) == 0) {
 1163                                 goto unlock_tobject;
 1164                         }
 1165                 } else if (tobject->type == OBJT_PHYS)
 1166                         goto unlock_tobject;
 1167                 m = vm_page_lookup(tobject, tpindex);
 1168                 if (m == NULL && advise == MADV_WILLNEED) {
 1169                         /*
 1170                          * If the page is cached, reactivate it.
 1171                          */
 1172                         m = vm_page_alloc(tobject, tpindex, VM_ALLOC_IFCACHED |
 1173                             VM_ALLOC_NOBUSY);
 1174                 }
 1175                 if (m == NULL) {
 1176                         /*
 1177                          * There may be swap even if there is no backing page
 1178                          */
 1179                         if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
 1180                                 swap_pager_freespace(tobject, tpindex, 1);
 1181                         /*
 1182                          * next object
 1183                          */
 1184                         backing_object = tobject->backing_object;
 1185                         if (backing_object == NULL)
 1186                                 goto unlock_tobject;
 1187                         VM_OBJECT_LOCK(backing_object);
 1188                         tpindex += OFF_TO_IDX(tobject->backing_object_offset);
 1189                         if (tobject != object)
 1190                                 VM_OBJECT_UNLOCK(tobject);
 1191                         tobject = backing_object;
 1192                         goto shadowlookup;
 1193                 } else if (m->valid != VM_PAGE_BITS_ALL)
 1194                         goto unlock_tobject;
 1195                 /*
 1196                  * If the page is not in a normal state, skip it.
 1197                  */
 1198                 vm_page_lock_queues();
 1199                 if (m->hold_count != 0 || m->wire_count != 0) {
 1200                         vm_page_unlock_queues();
 1201                         goto unlock_tobject;
 1202                 }
 1203                 if ((m->oflags & VPO_BUSY) || m->busy) {
 1204                         if (advise == MADV_WILLNEED)
 1205                                 /*
 1206                                  * Reference the page before unlocking and
 1207                                  * sleeping so that the page daemon is less
 1208                                  * likely to reclaim it. 
 1209                                  */
 1210                                 vm_page_flag_set(m, PG_REFERENCED);
 1211                         vm_page_unlock_queues();
 1212                         if (object != tobject)
 1213                                 VM_OBJECT_UNLOCK(object);
 1214                         m->oflags |= VPO_WANTED;
 1215                         msleep(m, VM_OBJECT_MTX(tobject), PDROP | PVM, "madvpo",
 1216                             0);
 1217                         VM_OBJECT_LOCK(object);
 1218                         goto relookup;
 1219                 }
 1220                 if (advise == MADV_WILLNEED) {
 1221                         vm_page_activate(m);
 1222                 } else if (advise == MADV_DONTNEED) {
 1223                         vm_page_dontneed(m);
 1224                 } else if (advise == MADV_FREE) {
 1225                         /*
 1226                          * Mark the page clean.  This will allow the page
 1227                          * to be freed up by the system.  However, such pages
 1228                          * are often reused quickly by malloc()/free()
 1229                          * so we do not do anything that would cause
 1230                          * a page fault if we can help it.
 1231                          *
 1232                          * Specifically, we do not try to actually free
 1233                          * the page now nor do we try to put it in the
 1234                          * cache (which would cause a page fault on reuse).
 1235                          *
 1236                          * But we do make the page is freeable as we
 1237                          * can without actually taking the step of unmapping
 1238                          * it.
 1239                          */
 1240                         pmap_clear_modify(m);
 1241                         m->dirty = 0;
 1242                         m->act_count = 0;
 1243                         vm_page_dontneed(m);
 1244                 }
 1245                 vm_page_unlock_queues();
 1246                 if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
 1247                         swap_pager_freespace(tobject, tpindex, 1);
 1248 unlock_tobject:
 1249                 if (tobject != object)
 1250                         VM_OBJECT_UNLOCK(tobject);
 1251         }       
 1252         VM_OBJECT_UNLOCK(object);
 1253 }
 1254 
 1255 /*
 1256  *      vm_object_shadow:
 1257  *
 1258  *      Create a new object which is backed by the
 1259  *      specified existing object range.  The source
 1260  *      object reference is deallocated.
 1261  *
 1262  *      The new object and offset into that object
 1263  *      are returned in the source parameters.
 1264  */
 1265 void
 1266 vm_object_shadow(
 1267         vm_object_t *object,    /* IN/OUT */
 1268         vm_ooffset_t *offset,   /* IN/OUT */
 1269         vm_size_t length)
 1270 {
 1271         vm_object_t source;
 1272         vm_object_t result;
 1273 
 1274         source = *object;
 1275 
 1276         /*
 1277          * Don't create the new object if the old object isn't shared.
 1278          */
 1279         if (source != NULL) {
 1280                 VM_OBJECT_LOCK(source);
 1281                 if (source->ref_count == 1 &&
 1282                     source->handle == NULL &&
 1283                     (source->type == OBJT_DEFAULT ||
 1284                      source->type == OBJT_SWAP)) {
 1285                         VM_OBJECT_UNLOCK(source);
 1286                         return;
 1287                 }
 1288                 VM_OBJECT_UNLOCK(source);
 1289         }
 1290 
 1291         /*
 1292          * Allocate a new object with the given length.
 1293          */
 1294         result = vm_object_allocate(OBJT_DEFAULT, length);
 1295 
 1296         /*
 1297          * The new object shadows the source object, adding a reference to it.
 1298          * Our caller changes his reference to point to the new object,
 1299          * removing a reference to the source object.  Net result: no change
 1300          * of reference count.
 1301          *
 1302          * Try to optimize the result object's page color when shadowing
 1303          * in order to maintain page coloring consistency in the combined 
 1304          * shadowed object.
 1305          */
 1306         result->backing_object = source;
 1307         /*
 1308          * Store the offset into the source object, and fix up the offset into
 1309          * the new object.
 1310          */
 1311         result->backing_object_offset = *offset;
 1312         if (source != NULL) {
 1313                 VM_OBJECT_LOCK(source);
 1314                 LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
 1315                 source->shadow_count++;
 1316                 source->generation++;
 1317 #if VM_NRESERVLEVEL > 0
 1318                 result->flags |= source->flags & OBJ_COLORED;
 1319                 result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
 1320                     ((1 << (VM_NFREEORDER - 1)) - 1);
 1321 #endif
 1322                 VM_OBJECT_UNLOCK(source);
 1323         }
 1324 
 1325 
 1326         /*
 1327          * Return the new things
 1328          */
 1329         *offset = 0;
 1330         *object = result;
 1331 }
 1332 
 1333 /*
 1334  *      vm_object_split:
 1335  *
 1336  * Split the pages in a map entry into a new object.  This affords
 1337  * easier removal of unused pages, and keeps object inheritance from
 1338  * being a negative impact on memory usage.
 1339  */
 1340 void
 1341 vm_object_split(vm_map_entry_t entry)
 1342 {
 1343         vm_page_t m, m_next;
 1344         vm_object_t orig_object, new_object, source;
 1345         vm_pindex_t idx, offidxstart;
 1346         vm_size_t size;
 1347 
 1348         orig_object = entry->object.vm_object;
 1349         if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP)
 1350                 return;
 1351         if (orig_object->ref_count <= 1)
 1352                 return;
 1353         VM_OBJECT_UNLOCK(orig_object);
 1354 
 1355         offidxstart = OFF_TO_IDX(entry->offset);
 1356         size = atop(entry->end - entry->start);
 1357 
 1358         /*
 1359          * If swap_pager_copy() is later called, it will convert new_object
 1360          * into a swap object.
 1361          */
 1362         new_object = vm_object_allocate(OBJT_DEFAULT, size);
 1363 
 1364         /*
 1365          * At this point, the new object is still private, so the order in
 1366          * which the original and new objects are locked does not matter.
 1367          */
 1368         VM_OBJECT_LOCK(new_object);
 1369         VM_OBJECT_LOCK(orig_object);
 1370         source = orig_object->backing_object;
 1371         if (source != NULL) {
 1372                 VM_OBJECT_LOCK(source);
 1373                 if ((source->flags & OBJ_DEAD) != 0) {
 1374                         VM_OBJECT_UNLOCK(source);
 1375                         VM_OBJECT_UNLOCK(orig_object);
 1376                         VM_OBJECT_UNLOCK(new_object);
 1377                         vm_object_deallocate(new_object);
 1378                         VM_OBJECT_LOCK(orig_object);
 1379                         return;
 1380                 }
 1381                 LIST_INSERT_HEAD(&source->shadow_head,
 1382                                   new_object, shadow_list);
 1383                 source->shadow_count++;
 1384                 source->generation++;
 1385                 vm_object_reference_locked(source);     /* for new_object */
 1386                 vm_object_clear_flag(source, OBJ_ONEMAPPING);
 1387                 VM_OBJECT_UNLOCK(source);
 1388                 new_object->backing_object_offset = 
 1389                         orig_object->backing_object_offset + entry->offset;
 1390                 new_object->backing_object = source;
 1391         }
 1392         if (orig_object->uip != NULL) {
 1393                 new_object->uip = orig_object->uip;
 1394                 uihold(orig_object->uip);
 1395                 new_object->charge = ptoa(size);
 1396                 KASSERT(orig_object->charge >= ptoa(size),
 1397                     ("orig_object->charge < 0"));
 1398                 orig_object->charge -= ptoa(size);
 1399         }
 1400 retry:
 1401         if ((m = TAILQ_FIRST(&orig_object->memq)) != NULL) {
 1402                 if (m->pindex < offidxstart) {
 1403                         m = vm_page_splay(offidxstart, orig_object->root);
 1404                         if ((orig_object->root = m)->pindex < offidxstart)
 1405                                 m = TAILQ_NEXT(m, listq);
 1406                 }
 1407         }
 1408         vm_page_lock_queues();
 1409         for (; m != NULL && (idx = m->pindex - offidxstart) < size;
 1410             m = m_next) {
 1411                 m_next = TAILQ_NEXT(m, listq);
 1412 
 1413                 /*
 1414                  * We must wait for pending I/O to complete before we can
 1415                  * rename the page.
 1416                  *
 1417                  * We do not have to VM_PROT_NONE the page as mappings should
 1418                  * not be changed by this operation.
 1419                  */
 1420                 if ((m->oflags & VPO_BUSY) || m->busy) {
 1421                         vm_page_unlock_queues();
 1422                         VM_OBJECT_UNLOCK(new_object);
 1423                         m->oflags |= VPO_WANTED;
 1424                         msleep(m, VM_OBJECT_MTX(orig_object), PVM, "spltwt", 0);
 1425                         VM_OBJECT_LOCK(new_object);
 1426                         goto retry;
 1427                 }
 1428                 vm_page_rename(m, new_object, idx);
 1429                 /* page automatically made dirty by rename and cache handled */
 1430                 vm_page_busy(m);
 1431         }
 1432         vm_page_unlock_queues();
 1433         if (orig_object->type == OBJT_SWAP) {
 1434                 /*
 1435                  * swap_pager_copy() can sleep, in which case the orig_object's
 1436                  * and new_object's locks are released and reacquired. 
 1437                  */
 1438                 swap_pager_copy(orig_object, new_object, offidxstart, 0);
 1439 
 1440                 /*
 1441                  * Transfer any cached pages from orig_object to new_object.
 1442                  */
 1443                 if (__predict_false(orig_object->cache != NULL))
 1444                         vm_page_cache_transfer(orig_object, offidxstart,
 1445                             new_object);
 1446         }
 1447         VM_OBJECT_UNLOCK(orig_object);
 1448         TAILQ_FOREACH(m, &new_object->memq, listq)
 1449                 vm_page_wakeup(m);
 1450         VM_OBJECT_UNLOCK(new_object);
 1451         entry->object.vm_object = new_object;
 1452         entry->offset = 0LL;
 1453         vm_object_deallocate(orig_object);
 1454         VM_OBJECT_LOCK(new_object);
 1455 }
 1456 
 1457 #define OBSC_TEST_ALL_SHADOWED  0x0001
 1458 #define OBSC_COLLAPSE_NOWAIT    0x0002
 1459 #define OBSC_COLLAPSE_WAIT      0x0004
 1460 
 1461 static int
 1462 vm_object_backing_scan(vm_object_t object, int op)
 1463 {
 1464         int r = 1;
 1465         vm_page_t p;
 1466         vm_object_t backing_object;
 1467         vm_pindex_t backing_offset_index;
 1468 
 1469         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1470         VM_OBJECT_LOCK_ASSERT(object->backing_object, MA_OWNED);
 1471 
 1472         backing_object = object->backing_object;
 1473         backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
 1474 
 1475         /*
 1476          * Initial conditions
 1477          */
 1478         if (op & OBSC_TEST_ALL_SHADOWED) {
 1479                 /*
 1480                  * We do not want to have to test for the existence of cache
 1481                  * or swap pages in the backing object.  XXX but with the
 1482                  * new swapper this would be pretty easy to do.
 1483                  *
 1484                  * XXX what about anonymous MAP_SHARED memory that hasn't
 1485                  * been ZFOD faulted yet?  If we do not test for this, the
 1486                  * shadow test may succeed! XXX
 1487                  */
 1488                 if (backing_object->type != OBJT_DEFAULT) {
 1489                         return (0);
 1490                 }
 1491         }
 1492         if (op & OBSC_COLLAPSE_WAIT) {
 1493                 vm_object_set_flag(backing_object, OBJ_DEAD);
 1494         }
 1495 
 1496         /*
 1497          * Our scan
 1498          */
 1499         p = TAILQ_FIRST(&backing_object->memq);
 1500         while (p) {
 1501                 vm_page_t next = TAILQ_NEXT(p, listq);
 1502                 vm_pindex_t new_pindex = p->pindex - backing_offset_index;
 1503 
 1504                 if (op & OBSC_TEST_ALL_SHADOWED) {
 1505                         vm_page_t pp;
 1506 
 1507                         /*
 1508                          * Ignore pages outside the parent object's range
 1509                          * and outside the parent object's mapping of the 
 1510                          * backing object.
 1511                          *
 1512                          * note that we do not busy the backing object's
 1513                          * page.
 1514                          */
 1515                         if (
 1516                             p->pindex < backing_offset_index ||
 1517                             new_pindex >= object->size
 1518                         ) {
 1519                                 p = next;
 1520                                 continue;
 1521                         }
 1522 
 1523                         /*
 1524                          * See if the parent has the page or if the parent's
 1525                          * object pager has the page.  If the parent has the
 1526                          * page but the page is not valid, the parent's
 1527                          * object pager must have the page.
 1528                          *
 1529                          * If this fails, the parent does not completely shadow
 1530                          * the object and we might as well give up now.
 1531                          */
 1532 
 1533                         pp = vm_page_lookup(object, new_pindex);
 1534                         if (
 1535                             (pp == NULL || pp->valid == 0) &&
 1536                             !vm_pager_has_page(object, new_pindex, NULL, NULL)
 1537                         ) {
 1538                                 r = 0;
 1539                                 break;
 1540                         }
 1541                 }
 1542 
 1543                 /*
 1544                  * Check for busy page
 1545                  */
 1546                 if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
 1547                         vm_page_t pp;
 1548 
 1549                         if (op & OBSC_COLLAPSE_NOWAIT) {
 1550                                 if ((p->oflags & VPO_BUSY) ||
 1551                                     !p->valid || 
 1552                                     p->busy) {
 1553                                         p = next;
 1554                                         continue;
 1555                                 }
 1556                         } else if (op & OBSC_COLLAPSE_WAIT) {
 1557                                 if ((p->oflags & VPO_BUSY) || p->busy) {
 1558                                         VM_OBJECT_UNLOCK(object);
 1559                                         p->oflags |= VPO_WANTED;
 1560                                         msleep(p, VM_OBJECT_MTX(backing_object),
 1561                                             PDROP | PVM, "vmocol", 0);
 1562                                         VM_OBJECT_LOCK(object);
 1563                                         VM_OBJECT_LOCK(backing_object);
 1564                                         /*
 1565                                          * If we slept, anything could have
 1566                                          * happened.  Since the object is
 1567                                          * marked dead, the backing offset
 1568                                          * should not have changed so we
 1569                                          * just restart our scan.
 1570                                          */
 1571                                         p = TAILQ_FIRST(&backing_object->memq);
 1572                                         continue;
 1573                                 }
 1574                         }
 1575 
 1576                         KASSERT(
 1577                             p->object == backing_object,
 1578                             ("vm_object_backing_scan: object mismatch")
 1579                         );
 1580 
 1581                         /*
 1582                          * Destroy any associated swap
 1583                          */
 1584                         if (backing_object->type == OBJT_SWAP) {
 1585                                 swap_pager_freespace(
 1586                                     backing_object, 
 1587                                     p->pindex,
 1588                                     1
 1589                                 );
 1590                         }
 1591 
 1592                         if (
 1593                             p->pindex < backing_offset_index ||
 1594                             new_pindex >= object->size
 1595                         ) {
 1596                                 /*
 1597                                  * Page is out of the parent object's range, we 
 1598                                  * can simply destroy it. 
 1599                                  */
 1600                                 vm_page_lock_queues();
 1601                                 KASSERT(!pmap_page_is_mapped(p),
 1602                                     ("freeing mapped page %p", p));
 1603                                 if (p->wire_count == 0)
 1604                                         vm_page_free(p);
 1605                                 else
 1606                                         vm_page_remove(p);
 1607                                 vm_page_unlock_queues();
 1608                                 p = next;
 1609                                 continue;
 1610                         }
 1611 
 1612                         pp = vm_page_lookup(object, new_pindex);
 1613                         if (
 1614                             pp != NULL ||
 1615                             vm_pager_has_page(object, new_pindex, NULL, NULL)
 1616                         ) {
 1617                                 /*
 1618                                  * page already exists in parent OR swap exists
 1619                                  * for this location in the parent.  Destroy 
 1620                                  * the original page from the backing object.
 1621                                  *
 1622                                  * Leave the parent's page alone
 1623                                  */
 1624                                 vm_page_lock_queues();
 1625                                 KASSERT(!pmap_page_is_mapped(p),
 1626                                     ("freeing mapped page %p", p));
 1627                                 if (p->wire_count == 0)
 1628                                         vm_page_free(p);
 1629                                 else
 1630                                         vm_page_remove(p);
 1631                                 vm_page_unlock_queues();
 1632                                 p = next;
 1633                                 continue;
 1634                         }
 1635 
 1636 #if VM_NRESERVLEVEL > 0
 1637                         /*
 1638                          * Rename the reservation.
 1639                          */
 1640                         vm_reserv_rename(p, object, backing_object,
 1641                             backing_offset_index);
 1642 #endif
 1643 
 1644                         /*
 1645                          * Page does not exist in parent, rename the
 1646                          * page from the backing object to the main object. 
 1647                          *
 1648                          * If the page was mapped to a process, it can remain 
 1649                          * mapped through the rename.
 1650                          */
 1651                         vm_page_lock_queues();
 1652                         vm_page_rename(p, object, new_pindex);
 1653                         vm_page_unlock_queues();
 1654                         /* page automatically made dirty by rename */
 1655                 }
 1656                 p = next;
 1657         }
 1658         return (r);
 1659 }
 1660 
 1661 
 1662 /*
 1663  * this version of collapse allows the operation to occur earlier and
 1664  * when paging_in_progress is true for an object...  This is not a complete
 1665  * operation, but should plug 99.9% of the rest of the leaks.
 1666  */
 1667 static void
 1668 vm_object_qcollapse(vm_object_t object)
 1669 {
 1670         vm_object_t backing_object = object->backing_object;
 1671 
 1672         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1673         VM_OBJECT_LOCK_ASSERT(backing_object, MA_OWNED);
 1674 
 1675         if (backing_object->ref_count != 1)
 1676                 return;
 1677 
 1678         vm_object_backing_scan(object, OBSC_COLLAPSE_NOWAIT);
 1679 }
 1680 
 1681 /*
 1682  *      vm_object_collapse:
 1683  *
 1684  *      Collapse an object with the object backing it.
 1685  *      Pages in the backing object are moved into the
 1686  *      parent, and the backing object is deallocated.
 1687  */
 1688 void
 1689 vm_object_collapse(vm_object_t object)
 1690 {
 1691         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1692         
 1693         while (TRUE) {
 1694                 vm_object_t backing_object;
 1695 
 1696                 /*
 1697                  * Verify that the conditions are right for collapse:
 1698                  *
 1699                  * The object exists and the backing object exists.
 1700                  */
 1701                 if ((backing_object = object->backing_object) == NULL)
 1702                         break;
 1703 
 1704                 /*
 1705                  * we check the backing object first, because it is most likely
 1706                  * not collapsable.
 1707                  */
 1708                 VM_OBJECT_LOCK(backing_object);
 1709                 if (backing_object->handle != NULL ||
 1710                     (backing_object->type != OBJT_DEFAULT &&
 1711                      backing_object->type != OBJT_SWAP) ||
 1712                     (backing_object->flags & OBJ_DEAD) ||
 1713                     object->handle != NULL ||
 1714                     (object->type != OBJT_DEFAULT &&
 1715                      object->type != OBJT_SWAP) ||
 1716                     (object->flags & OBJ_DEAD)) {
 1717                         VM_OBJECT_UNLOCK(backing_object);
 1718                         break;
 1719                 }
 1720 
 1721                 if (
 1722                     object->paging_in_progress != 0 ||
 1723                     backing_object->paging_in_progress != 0
 1724                 ) {
 1725                         vm_object_qcollapse(object);
 1726                         VM_OBJECT_UNLOCK(backing_object);
 1727                         break;
 1728                 }
 1729                 /*
 1730                  * We know that we can either collapse the backing object (if
 1731                  * the parent is the only reference to it) or (perhaps) have
 1732                  * the parent bypass the object if the parent happens to shadow
 1733                  * all the resident pages in the entire backing object.
 1734                  *
 1735                  * This is ignoring pager-backed pages such as swap pages.
 1736                  * vm_object_backing_scan fails the shadowing test in this
 1737                  * case.
 1738                  */
 1739                 if (backing_object->ref_count == 1) {
 1740                         /*
 1741                          * If there is exactly one reference to the backing
 1742                          * object, we can collapse it into the parent.  
 1743                          */
 1744                         vm_object_backing_scan(object, OBSC_COLLAPSE_WAIT);
 1745 
 1746 #if VM_NRESERVLEVEL > 0
 1747                         /*
 1748                          * Break any reservations from backing_object.
 1749                          */
 1750                         if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
 1751                                 vm_reserv_break_all(backing_object);
 1752 #endif
 1753 
 1754                         /*
 1755                          * Move the pager from backing_object to object.
 1756                          */
 1757                         if (backing_object->type == OBJT_SWAP) {
 1758                                 /*
 1759                                  * swap_pager_copy() can sleep, in which case
 1760                                  * the backing_object's and object's locks are
 1761                                  * released and reacquired.
 1762                                  */
 1763                                 swap_pager_copy(
 1764                                     backing_object,
 1765                                     object,
 1766                                     OFF_TO_IDX(object->backing_object_offset), TRUE);
 1767 
 1768                                 /*
 1769                                  * Free any cached pages from backing_object.
 1770                                  */
 1771                                 if (__predict_false(backing_object->cache != NULL))
 1772                                         vm_page_cache_free(backing_object, 0, 0);
 1773                         }
 1774                         /*
 1775                          * Object now shadows whatever backing_object did.
 1776                          * Note that the reference to 
 1777                          * backing_object->backing_object moves from within 
 1778                          * backing_object to within object.
 1779                          */
 1780                         LIST_REMOVE(object, shadow_list);
 1781                         backing_object->shadow_count--;
 1782                         backing_object->generation++;
 1783                         if (backing_object->backing_object) {
 1784                                 VM_OBJECT_LOCK(backing_object->backing_object);
 1785                                 LIST_REMOVE(backing_object, shadow_list);
 1786                                 LIST_INSERT_HEAD(
 1787                                     &backing_object->backing_object->shadow_head,
 1788                                     object, shadow_list);
 1789                                 /*
 1790                                  * The shadow_count has not changed.
 1791                                  */
 1792                                 backing_object->backing_object->generation++;
 1793                                 VM_OBJECT_UNLOCK(backing_object->backing_object);
 1794                         }
 1795                         object->backing_object = backing_object->backing_object;
 1796                         object->backing_object_offset +=
 1797                             backing_object->backing_object_offset;
 1798 
 1799                         /*
 1800                          * Discard backing_object.
 1801                          *
 1802                          * Since the backing object has no pages, no pager left,
 1803                          * and no object references within it, all that is
 1804                          * necessary is to dispose of it.
 1805                          */
 1806                         KASSERT(backing_object->ref_count == 1, (
 1807 "backing_object %p was somehow re-referenced during collapse!",
 1808                             backing_object));
 1809                         VM_OBJECT_UNLOCK(backing_object);
 1810                         vm_object_destroy(backing_object);
 1811 
 1812                         object_collapses++;
 1813                 } else {
 1814                         vm_object_t new_backing_object;
 1815 
 1816                         /*
 1817                          * If we do not entirely shadow the backing object,
 1818                          * there is nothing we can do so we give up.
 1819                          */
 1820                         if (object->resident_page_count != object->size &&
 1821                             vm_object_backing_scan(object,
 1822                             OBSC_TEST_ALL_SHADOWED) == 0) {
 1823                                 VM_OBJECT_UNLOCK(backing_object);
 1824                                 break;
 1825                         }
 1826 
 1827                         /*
 1828                          * Make the parent shadow the next object in the
 1829                          * chain.  Deallocating backing_object will not remove
 1830                          * it, since its reference count is at least 2.
 1831                          */
 1832                         LIST_REMOVE(object, shadow_list);
 1833                         backing_object->shadow_count--;
 1834                         backing_object->generation++;
 1835 
 1836                         new_backing_object = backing_object->backing_object;
 1837                         if ((object->backing_object = new_backing_object) != NULL) {
 1838                                 VM_OBJECT_LOCK(new_backing_object);
 1839                                 LIST_INSERT_HEAD(
 1840                                     &new_backing_object->shadow_head,
 1841                                     object,
 1842                                     shadow_list
 1843                                 );
 1844                                 new_backing_object->shadow_count++;
 1845                                 new_backing_object->generation++;
 1846                                 vm_object_reference_locked(new_backing_object);
 1847                                 VM_OBJECT_UNLOCK(new_backing_object);
 1848                                 object->backing_object_offset +=
 1849                                         backing_object->backing_object_offset;
 1850                         }
 1851 
 1852                         /*
 1853                          * Drop the reference count on backing_object. Since
 1854                          * its ref_count was at least 2, it will not vanish.
 1855                          */
 1856                         backing_object->ref_count--;
 1857                         VM_OBJECT_UNLOCK(backing_object);
 1858                         object_bypasses++;
 1859                 }
 1860 
 1861                 /*
 1862                  * Try again with this object's new backing object.
 1863                  */
 1864         }
 1865 }
 1866 
 1867 /*
 1868  *      vm_object_page_remove:
 1869  *
 1870  *      For the given object, either frees or invalidates each of the
 1871  *      specified pages.  In general, a page is freed.  However, if a
 1872  *      page is wired for any reason other than the existence of a
 1873  *      managed, wired mapping, then it may be invalidated but not
 1874  *      removed from the object.  Pages are specified by the given
 1875  *      range ["start", "end") and Boolean "clean_only".  As a
 1876  *      special case, if "end" is zero, then the range extends from
 1877  *      "start" to the end of the object.  If "clean_only" is TRUE,
 1878  *      then only the non-dirty pages within the specified range are
 1879  *      affected.
 1880  *
 1881  *      In general, this operation should only be performed on objects
 1882  *      that contain managed pages.  There are two exceptions.  First,
 1883  *      it may be performed on the kernel and kmem objects.  Second,
 1884  *      it may be used by msync(..., MS_INVALIDATE) to invalidate
 1885  *      device-backed pages.  In both of these cases, "clean_only"
 1886  *      must be FALSE.
 1887  *
 1888  *      The object must be locked.
 1889  */
 1890 void
 1891 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
 1892     boolean_t clean_only)
 1893 {
 1894         vm_page_t p, next;
 1895         int wirings;
 1896 
 1897         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1898         if (object->resident_page_count == 0)
 1899                 goto skipmemq;
 1900 
 1901         /*
 1902          * Since physically-backed objects do not use managed pages, we can't
 1903          * remove pages from the object (we must instead remove the page
 1904          * references, and then destroy the object).
 1905          */
 1906         KASSERT(object->type != OBJT_PHYS || object == kernel_object ||
 1907             object == kmem_object,
 1908             ("attempt to remove pages from a physical object"));
 1909 
 1910         vm_object_pip_add(object, 1);
 1911 again:
 1912         if ((p = TAILQ_FIRST(&object->memq)) != NULL) {
 1913                 if (p->pindex < start) {
 1914                         p = vm_page_splay(start, object->root);
 1915                         if ((object->root = p)->pindex < start)
 1916                                 p = TAILQ_NEXT(p, listq);
 1917                 }
 1918         }
 1919         vm_page_lock_queues();
 1920         /*
 1921          * Assert: the variable p is either (1) the page with the
 1922          * least pindex greater than or equal to the parameter pindex
 1923          * or (2) NULL.
 1924          */
 1925         for (;
 1926              p != NULL && (p->pindex < end || end == 0);
 1927              p = next) {
 1928                 next = TAILQ_NEXT(p, listq);
 1929 
 1930                 /*
 1931                  * If the page is wired for any reason besides the
 1932                  * existence of managed, wired mappings, then it cannot
 1933                  * be freed.  For example, fictitious pages, which
 1934                  * represent device memory, are inherently wired and
 1935                  * cannot be freed.  They can, however, be invalidated
 1936                  * if "clean_only" is FALSE.
 1937                  */
 1938                 if ((wirings = p->wire_count) != 0 &&
 1939                     (wirings = pmap_page_wired_mappings(p)) != p->wire_count) {
 1940                         /* Fictitious pages do not have managed mappings. */
 1941                         if ((p->flags & PG_FICTITIOUS) == 0)
 1942                                 pmap_remove_all(p);
 1943                         /* Account for removal of managed, wired mappings. */
 1944                         p->wire_count -= wirings;
 1945                         if (!clean_only) {
 1946                                 p->valid = 0;
 1947                                 vm_page_undirty(p);
 1948                         }
 1949                         continue;
 1950                 }
 1951                 if (vm_page_sleep_if_busy(p, TRUE, "vmopar"))
 1952                         goto again;
 1953                 KASSERT((p->flags & PG_FICTITIOUS) == 0,
 1954                     ("vm_object_page_remove: page %p is fictitious", p));
 1955                 if (clean_only && p->valid) {
 1956                         pmap_remove_write(p);
 1957                         if (p->dirty)
 1958                                 continue;
 1959                 }
 1960                 pmap_remove_all(p);
 1961                 /* Account for removal of managed, wired mappings. */
 1962                 if (wirings != 0)
 1963                         p->wire_count -= wirings;
 1964                 vm_page_free(p);
 1965         }
 1966         vm_page_unlock_queues();
 1967         vm_object_pip_wakeup(object);
 1968 skipmemq:
 1969         if (__predict_false(object->cache != NULL))
 1970                 vm_page_cache_free(object, start, end);
 1971 }
 1972 
 1973 /*
 1974  *      Populate the specified range of the object with valid pages.  Returns
 1975  *      TRUE if the range is successfully populated and FALSE otherwise.
 1976  *
 1977  *      Note: This function should be optimized to pass a larger array of
 1978  *      pages to vm_pager_get_pages() before it is applied to a non-
 1979  *      OBJT_DEVICE object.
 1980  *
 1981  *      The object must be locked.
 1982  */
 1983 boolean_t
 1984 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
 1985 {
 1986         vm_page_t m, ma[1];
 1987         vm_pindex_t pindex;
 1988         int rv;
 1989 
 1990         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1991         for (pindex = start; pindex < end; pindex++) {
 1992                 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL |
 1993                     VM_ALLOC_RETRY);
 1994                 if (m->valid != VM_PAGE_BITS_ALL) {
 1995                         ma[0] = m;
 1996                         rv = vm_pager_get_pages(object, ma, 1, 0);
 1997                         m = vm_page_lookup(object, pindex);
 1998                         if (m == NULL)
 1999                                 break;
 2000                         if (rv != VM_PAGER_OK) {
 2001                                 vm_page_lock_queues();
 2002                                 vm_page_free(m);
 2003                                 vm_page_unlock_queues();
 2004                                 break;
 2005                         }
 2006                 }
 2007                 /*
 2008                  * Keep "m" busy because a subsequent iteration may unlock
 2009                  * the object.
 2010                  */
 2011         }
 2012         if (pindex > start) {
 2013                 m = vm_page_lookup(object, start);
 2014                 while (m != NULL && m->pindex < pindex) {
 2015                         vm_page_wakeup(m);
 2016                         m = TAILQ_NEXT(m, listq);
 2017                 }
 2018         }
 2019         return (pindex == end);
 2020 }
 2021 
 2022 /*
 2023  *      Routine:        vm_object_coalesce
 2024  *      Function:       Coalesces two objects backing up adjoining
 2025  *                      regions of memory into a single object.
 2026  *
 2027  *      returns TRUE if objects were combined.
 2028  *
 2029  *      NOTE:   Only works at the moment if the second object is NULL -
 2030  *              if it's not, which object do we lock first?
 2031  *
 2032  *      Parameters:
 2033  *              prev_object     First object to coalesce
 2034  *              prev_offset     Offset into prev_object
 2035  *              prev_size       Size of reference to prev_object
 2036  *              next_size       Size of reference to the second object
 2037  *              reserved        Indicator that extension region has
 2038  *                              swap accounted for
 2039  *
 2040  *      Conditions:
 2041  *      The object must *not* be locked.
 2042  */
 2043 boolean_t
 2044 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
 2045     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
 2046 {
 2047         vm_pindex_t next_pindex;
 2048 
 2049         if (prev_object == NULL)
 2050                 return (TRUE);
 2051         VM_OBJECT_LOCK(prev_object);
 2052         if (prev_object->type != OBJT_DEFAULT &&
 2053             prev_object->type != OBJT_SWAP) {
 2054                 VM_OBJECT_UNLOCK(prev_object);
 2055                 return (FALSE);
 2056         }
 2057 
 2058         /*
 2059          * Try to collapse the object first
 2060          */
 2061         vm_object_collapse(prev_object);
 2062 
 2063         /*
 2064          * Can't coalesce if: . more than one reference . paged out . shadows
 2065          * another object . has a copy elsewhere (any of which mean that the
 2066          * pages not mapped to prev_entry may be in use anyway)
 2067          */
 2068         if (prev_object->backing_object != NULL) {
 2069                 VM_OBJECT_UNLOCK(prev_object);
 2070                 return (FALSE);
 2071         }
 2072 
 2073         prev_size >>= PAGE_SHIFT;
 2074         next_size >>= PAGE_SHIFT;
 2075         next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
 2076 
 2077         if ((prev_object->ref_count > 1) &&
 2078             (prev_object->size != next_pindex)) {
 2079                 VM_OBJECT_UNLOCK(prev_object);
 2080                 return (FALSE);
 2081         }
 2082 
 2083         /*
 2084          * Account for the charge.
 2085          */
 2086         if (prev_object->uip != NULL) {
 2087 
 2088                 /*
 2089                  * If prev_object was charged, then this mapping,
 2090                  * althought not charged now, may become writable
 2091                  * later. Non-NULL uip in the object would prevent
 2092                  * swap reservation during enabling of the write
 2093                  * access, so reserve swap now. Failed reservation
 2094                  * cause allocation of the separate object for the map
 2095                  * entry, and swap reservation for this entry is
 2096                  * managed in appropriate time.
 2097                  */
 2098                 if (!reserved && !swap_reserve_by_uid(ptoa(next_size),
 2099                     prev_object->uip)) {
 2100                         return (FALSE);
 2101                 }
 2102                 prev_object->charge += ptoa(next_size);
 2103         }
 2104 
 2105         /*
 2106          * Remove any pages that may still be in the object from a previous
 2107          * deallocation.
 2108          */
 2109         if (next_pindex < prev_object->size) {
 2110                 vm_object_page_remove(prev_object,
 2111                                       next_pindex,
 2112                                       next_pindex + next_size, FALSE);
 2113                 if (prev_object->type == OBJT_SWAP)
 2114                         swap_pager_freespace(prev_object,
 2115                                              next_pindex, next_size);
 2116 #if 0
 2117                 if (prev_object->uip != NULL) {
 2118                         KASSERT(prev_object->charge >=
 2119                             ptoa(prev_object->size - next_pindex),
 2120                             ("object %p overcharged 1 %jx %jx", prev_object,
 2121                                 (uintmax_t)next_pindex, (uintmax_t)next_size));
 2122                         prev_object->charge -= ptoa(prev_object->size -
 2123                             next_pindex);
 2124                 }
 2125 #endif
 2126         }
 2127 
 2128         /*
 2129          * Extend the object if necessary.
 2130          */
 2131         if (next_pindex + next_size > prev_object->size)
 2132                 prev_object->size = next_pindex + next_size;
 2133 
 2134         VM_OBJECT_UNLOCK(prev_object);
 2135         return (TRUE);
 2136 }
 2137 
 2138 void
 2139 vm_object_set_writeable_dirty(vm_object_t object)
 2140 {
 2141 
 2142         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 2143         if (object->type != OBJT_VNODE ||
 2144             (object->flags & OBJ_MIGHTBEDIRTY) != 0)
 2145                 return;
 2146         vm_object_set_flag(object, OBJ_MIGHTBEDIRTY);
 2147 }
 2148 
 2149 #include "opt_ddb.h"
 2150 #ifdef DDB
 2151 #include <sys/kernel.h>
 2152 
 2153 #include <sys/cons.h>
 2154 
 2155 #include <ddb/ddb.h>
 2156 
 2157 static int
 2158 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
 2159 {
 2160         vm_map_t tmpm;
 2161         vm_map_entry_t tmpe;
 2162         vm_object_t obj;
 2163         int entcount;
 2164 
 2165         if (map == 0)
 2166                 return 0;
 2167 
 2168         if (entry == 0) {
 2169                 tmpe = map->header.next;
 2170                 entcount = map->nentries;
 2171                 while (entcount-- && (tmpe != &map->header)) {
 2172                         if (_vm_object_in_map(map, object, tmpe)) {
 2173                                 return 1;
 2174                         }
 2175                         tmpe = tmpe->next;
 2176                 }
 2177         } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 2178                 tmpm = entry->object.sub_map;
 2179                 tmpe = tmpm->header.next;
 2180                 entcount = tmpm->nentries;
 2181                 while (entcount-- && tmpe != &tmpm->header) {
 2182                         if (_vm_object_in_map(tmpm, object, tmpe)) {
 2183                                 return 1;
 2184                         }
 2185                         tmpe = tmpe->next;
 2186                 }
 2187         } else if ((obj = entry->object.vm_object) != NULL) {
 2188                 for (; obj; obj = obj->backing_object)
 2189                         if (obj == object) {
 2190                                 return 1;
 2191                         }
 2192         }
 2193         return 0;
 2194 }
 2195 
 2196 static int
 2197 vm_object_in_map(vm_object_t object)
 2198 {
 2199         struct proc *p;
 2200 
 2201         /* sx_slock(&allproc_lock); */
 2202         FOREACH_PROC_IN_SYSTEM(p) {
 2203                 if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
 2204                         continue;
 2205                 if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
 2206                         /* sx_sunlock(&allproc_lock); */
 2207                         return 1;
 2208                 }
 2209         }
 2210         /* sx_sunlock(&allproc_lock); */
 2211         if (_vm_object_in_map(kernel_map, object, 0))
 2212                 return 1;
 2213         if (_vm_object_in_map(kmem_map, object, 0))
 2214                 return 1;
 2215         if (_vm_object_in_map(pager_map, object, 0))
 2216                 return 1;
 2217         if (_vm_object_in_map(buffer_map, object, 0))
 2218                 return 1;
 2219         return 0;
 2220 }
 2221 
 2222 DB_SHOW_COMMAND(vmochk, vm_object_check)
 2223 {
 2224         vm_object_t object;
 2225 
 2226         /*
 2227          * make sure that internal objs are in a map somewhere
 2228          * and none have zero ref counts.
 2229          */
 2230         TAILQ_FOREACH(object, &vm_object_list, object_list) {
 2231                 if (object->handle == NULL &&
 2232                     (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
 2233                         if (object->ref_count == 0) {
 2234                                 db_printf("vmochk: internal obj has zero ref count: %ld\n",
 2235                                         (long)object->size);
 2236                         }
 2237                         if (!vm_object_in_map(object)) {
 2238                                 db_printf(
 2239                         "vmochk: internal obj is not in a map: "
 2240                         "ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
 2241                                     object->ref_count, (u_long)object->size, 
 2242                                     (u_long)object->size,
 2243                                     (void *)object->backing_object);
 2244                         }
 2245                 }
 2246         }
 2247 }
 2248 
 2249 /*
 2250  *      vm_object_print:        [ debug ]
 2251  */
 2252 DB_SHOW_COMMAND(object, vm_object_print_static)
 2253 {
 2254         /* XXX convert args. */
 2255         vm_object_t object = (vm_object_t)addr;
 2256         boolean_t full = have_addr;
 2257 
 2258         vm_page_t p;
 2259 
 2260         /* XXX count is an (unused) arg.  Avoid shadowing it. */
 2261 #define count   was_count
 2262 
 2263         int count;
 2264 
 2265         if (object == NULL)
 2266                 return;
 2267 
 2268         db_iprintf(
 2269             "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x uip %d charge %jx\n",
 2270             object, (int)object->type, (uintmax_t)object->size,
 2271             object->resident_page_count, object->ref_count, object->flags,
 2272             object->uip ? object->uip->ui_uid : -1, (uintmax_t)object->charge);
 2273         db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
 2274             object->shadow_count, 
 2275             object->backing_object ? object->backing_object->ref_count : 0,
 2276             object->backing_object, (uintmax_t)object->backing_object_offset);
 2277 
 2278         if (!full)
 2279                 return;
 2280 
 2281         db_indent += 2;
 2282         count = 0;
 2283         TAILQ_FOREACH(p, &object->memq, listq) {
 2284                 if (count == 0)
 2285                         db_iprintf("memory:=");
 2286                 else if (count == 6) {
 2287                         db_printf("\n");
 2288                         db_iprintf(" ...");
 2289                         count = 0;
 2290                 } else
 2291                         db_printf(",");
 2292                 count++;
 2293 
 2294                 db_printf("(off=0x%jx,page=0x%jx)",
 2295                     (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
 2296         }
 2297         if (count != 0)
 2298                 db_printf("\n");
 2299         db_indent -= 2;
 2300 }
 2301 
 2302 /* XXX. */
 2303 #undef count
 2304 
 2305 /* XXX need this non-static entry for calling from vm_map_print. */
 2306 void
 2307 vm_object_print(
 2308         /* db_expr_t */ long addr,
 2309         boolean_t have_addr,
 2310         /* db_expr_t */ long count,
 2311         char *modif)
 2312 {
 2313         vm_object_print_static(addr, have_addr, count, modif);
 2314 }
 2315 
 2316 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
 2317 {
 2318         vm_object_t object;
 2319         vm_pindex_t fidx;
 2320         vm_paddr_t pa;
 2321         vm_page_t m, prev_m;
 2322         int rcount, nl, c;
 2323 
 2324         nl = 0;
 2325         TAILQ_FOREACH(object, &vm_object_list, object_list) {
 2326                 db_printf("new object: %p\n", (void *)object);
 2327                 if (nl > 18) {
 2328                         c = cngetc();
 2329                         if (c != ' ')
 2330                                 return;
 2331                         nl = 0;
 2332                 }
 2333                 nl++;
 2334                 rcount = 0;
 2335                 fidx = 0;
 2336                 pa = -1;
 2337                 TAILQ_FOREACH(m, &object->memq, listq) {
 2338                         if (m->pindex > 128)
 2339                                 break;
 2340                         if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
 2341                             prev_m->pindex + 1 != m->pindex) {
 2342                                 if (rcount) {
 2343                                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
 2344                                                 (long)fidx, rcount, (long)pa);
 2345                                         if (nl > 18) {
 2346                                                 c = cngetc();
 2347                                                 if (c != ' ')
 2348                                                         return;
 2349                                                 nl = 0;
 2350                                         }
 2351                                         nl++;
 2352                                         rcount = 0;
 2353                                 }
 2354                         }                               
 2355                         if (rcount &&
 2356                                 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
 2357                                 ++rcount;
 2358                                 continue;
 2359                         }
 2360                         if (rcount) {
 2361                                 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
 2362                                         (long)fidx, rcount, (long)pa);
 2363                                 if (nl > 18) {
 2364                                         c = cngetc();
 2365                                         if (c != ' ')
 2366                                                 return;
 2367                                         nl = 0;
 2368                                 }
 2369                                 nl++;
 2370                         }
 2371                         fidx = m->pindex;
 2372                         pa = VM_PAGE_TO_PHYS(m);
 2373                         rcount = 1;
 2374                 }
 2375                 if (rcount) {
 2376                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
 2377                                 (long)fidx, rcount, (long)pa);
 2378                         if (nl > 18) {
 2379                                 c = cngetc();
 2380                                 if (c != ' ')
 2381                                         return;
 2382                                 nl = 0;
 2383                         }
 2384                         nl++;
 2385                 }
 2386         }
 2387 }
 2388 #endif /* DDB */

Cache object: 8754a735466c3e5cdf04c1784ccba683


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.