The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_object.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
    3  *
    4  * Copyright (c) 1991, 1993
    5  *      The Regents of the University of California.  All rights reserved.
    6  *
    7  * This code is derived from software contributed to Berkeley by
    8  * The Mach Operating System project at Carnegie-Mellon University.
    9  *
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions
   12  * are met:
   13  * 1. Redistributions of source code must retain the above copyright
   14  *    notice, this list of conditions and the following disclaimer.
   15  * 2. Redistributions in binary form must reproduce the above copyright
   16  *    notice, this list of conditions and the following disclaimer in the
   17  *    documentation and/or other materials provided with the distribution.
   18  * 3. Neither the name of the University nor the names of its contributors
   19  *    may be used to endorse or promote products derived from this software
   20  *    without specific prior written permission.
   21  *
   22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   32  * SUCH DAMAGE.
   33  *
   34  *      from: @(#)vm_object.c   8.5 (Berkeley) 3/22/94
   35  *
   36  *
   37  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   38  * All rights reserved.
   39  *
   40  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   41  *
   42  * Permission to use, copy, modify and distribute this software and
   43  * its documentation is hereby granted, provided that both the copyright
   44  * notice and this permission notice appear in all copies of the
   45  * software, derivative works or modified versions, and any portions
   46  * thereof, and that both notices appear in supporting documentation.
   47  *
   48  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   49  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   50  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   51  *
   52  * Carnegie Mellon requests users of this software to return to
   53  *
   54  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   55  *  School of Computer Science
   56  *  Carnegie Mellon University
   57  *  Pittsburgh PA 15213-3890
   58  *
   59  * any improvements or extensions that they make and grant Carnegie the
   60  * rights to redistribute these changes.
   61  */
   62 
   63 /*
   64  *      Virtual memory object module.
   65  */
   66 
   67 #include <sys/cdefs.h>
   68 __FBSDID("$FreeBSD$");
   69 
   70 #include "opt_vm.h"
   71 
   72 #include <sys/param.h>
   73 #include <sys/systm.h>
   74 #include <sys/blockcount.h>
   75 #include <sys/cpuset.h>
   76 #include <sys/limits.h>
   77 #include <sys/lock.h>
   78 #include <sys/mman.h>
   79 #include <sys/mount.h>
   80 #include <sys/kernel.h>
   81 #include <sys/pctrie.h>
   82 #include <sys/sysctl.h>
   83 #include <sys/mutex.h>
   84 #include <sys/proc.h>           /* for curproc, pageproc */
   85 #include <sys/refcount.h>
   86 #include <sys/socket.h>
   87 #include <sys/resourcevar.h>
   88 #include <sys/refcount.h>
   89 #include <sys/rwlock.h>
   90 #include <sys/user.h>
   91 #include <sys/vnode.h>
   92 #include <sys/vmmeter.h>
   93 #include <sys/sx.h>
   94 
   95 #include <vm/vm.h>
   96 #include <vm/vm_param.h>
   97 #include <vm/pmap.h>
   98 #include <vm/vm_map.h>
   99 #include <vm/vm_object.h>
  100 #include <vm/vm_page.h>
  101 #include <vm/vm_pageout.h>
  102 #include <vm/vm_pager.h>
  103 #include <vm/vm_phys.h>
  104 #include <vm/vm_pagequeue.h>
  105 #include <vm/swap_pager.h>
  106 #include <vm/vm_kern.h>
  107 #include <vm/vm_extern.h>
  108 #include <vm/vm_radix.h>
  109 #include <vm/vm_reserv.h>
  110 #include <vm/uma.h>
  111 
  112 static int old_msync;
  113 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
  114     "Use old (insecure) msync behavior");
  115 
  116 static int      vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
  117                     int pagerflags, int flags, boolean_t *allclean,
  118                     boolean_t *eio);
  119 static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
  120                     boolean_t *allclean);
  121 static void     vm_object_backing_remove(vm_object_t object);
  122 
  123 /*
  124  *      Virtual memory objects maintain the actual data
  125  *      associated with allocated virtual memory.  A given
  126  *      page of memory exists within exactly one object.
  127  *
  128  *      An object is only deallocated when all "references"
  129  *      are given up.  Only one "reference" to a given
  130  *      region of an object should be writeable.
  131  *
  132  *      Associated with each object is a list of all resident
  133  *      memory pages belonging to that object; this list is
  134  *      maintained by the "vm_page" module, and locked by the object's
  135  *      lock.
  136  *
  137  *      Each object also records a "pager" routine which is
  138  *      used to retrieve (and store) pages to the proper backing
  139  *      storage.  In addition, objects may be backed by other
  140  *      objects from which they were virtual-copied.
  141  *
  142  *      The only items within the object structure which are
  143  *      modified after time of creation are:
  144  *              reference count         locked by object's lock
  145  *              pager routine           locked by object's lock
  146  *
  147  */
  148 
  149 struct object_q vm_object_list;
  150 struct mtx vm_object_list_mtx;  /* lock for object list and count */
  151 
  152 struct vm_object kernel_object_store;
  153 
  154 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  155     "VM object stats");
  156 
  157 static COUNTER_U64_DEFINE_EARLY(object_collapses);
  158 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
  159     &object_collapses,
  160     "VM object collapses");
  161 
  162 static COUNTER_U64_DEFINE_EARLY(object_bypasses);
  163 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
  164     &object_bypasses,
  165     "VM object bypasses");
  166 
  167 static COUNTER_U64_DEFINE_EARLY(object_collapse_waits);
  168 SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapse_waits, CTLFLAG_RD,
  169     &object_collapse_waits,
  170     "Number of sleeps for collapse");
  171 
  172 static uma_zone_t obj_zone;
  173 
  174 static int vm_object_zinit(void *mem, int size, int flags);
  175 
  176 #ifdef INVARIANTS
  177 static void vm_object_zdtor(void *mem, int size, void *arg);
  178 
  179 static void
  180 vm_object_zdtor(void *mem, int size, void *arg)
  181 {
  182         vm_object_t object;
  183 
  184         object = (vm_object_t)mem;
  185         KASSERT(object->ref_count == 0,
  186             ("object %p ref_count = %d", object, object->ref_count));
  187         KASSERT(TAILQ_EMPTY(&object->memq),
  188             ("object %p has resident pages in its memq", object));
  189         KASSERT(vm_radix_is_empty(&object->rtree),
  190             ("object %p has resident pages in its trie", object));
  191 #if VM_NRESERVLEVEL > 0
  192         KASSERT(LIST_EMPTY(&object->rvq),
  193             ("object %p has reservations",
  194             object));
  195 #endif
  196         KASSERT(!vm_object_busied(object),
  197             ("object %p busy = %d", object, blockcount_read(&object->busy)));
  198         KASSERT(object->resident_page_count == 0,
  199             ("object %p resident_page_count = %d",
  200             object, object->resident_page_count));
  201         KASSERT(atomic_load_int(&object->shadow_count) == 0,
  202             ("object %p shadow_count = %d",
  203             object, atomic_load_int(&object->shadow_count)));
  204         KASSERT(object->type == OBJT_DEAD,
  205             ("object %p has non-dead type %d",
  206             object, object->type));
  207         KASSERT(object->charge == 0 && object->cred == NULL,
  208             ("object %p has non-zero charge %ju (%p)",
  209             object, (uintmax_t)object->charge, object->cred));
  210 }
  211 #endif
  212 
  213 static int
  214 vm_object_zinit(void *mem, int size, int flags)
  215 {
  216         vm_object_t object;
  217 
  218         object = (vm_object_t)mem;
  219         rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
  220 
  221         /* These are true for any object that has been freed */
  222         object->type = OBJT_DEAD;
  223         vm_radix_init(&object->rtree);
  224         refcount_init(&object->ref_count, 0);
  225         blockcount_init(&object->paging_in_progress);
  226         blockcount_init(&object->busy);
  227         object->resident_page_count = 0;
  228         atomic_store_int(&object->shadow_count, 0);
  229         object->flags = OBJ_DEAD;
  230 
  231         mtx_lock(&vm_object_list_mtx);
  232         TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
  233         mtx_unlock(&vm_object_list_mtx);
  234         return (0);
  235 }
  236 
  237 static void
  238 _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
  239     vm_object_t object, void *handle)
  240 {
  241 
  242         TAILQ_INIT(&object->memq);
  243         LIST_INIT(&object->shadow_head);
  244 
  245         object->type = type;
  246         object->flags = flags;
  247         if ((flags & OBJ_SWAP) != 0) {
  248                 pctrie_init(&object->un_pager.swp.swp_blks);
  249                 object->un_pager.swp.writemappings = 0;
  250         }
  251 
  252         /*
  253          * Ensure that swap_pager_swapoff() iteration over object_list
  254          * sees up to date type and pctrie head if it observed
  255          * non-dead object.
  256          */
  257         atomic_thread_fence_rel();
  258 
  259         object->pg_color = 0;
  260         object->size = size;
  261         object->domain.dr_policy = NULL;
  262         object->generation = 1;
  263         object->cleangeneration = 1;
  264         refcount_init(&object->ref_count, 1);
  265         object->memattr = VM_MEMATTR_DEFAULT;
  266         object->cred = NULL;
  267         object->charge = 0;
  268         object->handle = handle;
  269         object->backing_object = NULL;
  270         object->backing_object_offset = (vm_ooffset_t) 0;
  271 #if VM_NRESERVLEVEL > 0
  272         LIST_INIT(&object->rvq);
  273 #endif
  274         umtx_shm_object_init(object);
  275 }
  276 
  277 /*
  278  *      vm_object_init:
  279  *
  280  *      Initialize the VM objects module.
  281  */
  282 void
  283 vm_object_init(void)
  284 {
  285         TAILQ_INIT(&vm_object_list);
  286         mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
  287 
  288         rw_init(&kernel_object->lock, "kernel vm object");
  289         _vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
  290             VM_MIN_KERNEL_ADDRESS), OBJ_UNMANAGED, kernel_object, NULL);
  291 #if VM_NRESERVLEVEL > 0
  292         kernel_object->flags |= OBJ_COLORED;
  293         kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
  294 #endif
  295         kernel_object->un_pager.phys.ops = &default_phys_pg_ops;
  296 
  297         /*
  298          * The lock portion of struct vm_object must be type stable due
  299          * to vm_pageout_fallback_object_lock locking a vm object
  300          * without holding any references to it.
  301          *
  302          * paging_in_progress is valid always.  Lockless references to
  303          * the objects may acquire pip and then check OBJ_DEAD.
  304          */
  305         obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
  306 #ifdef INVARIANTS
  307             vm_object_zdtor,
  308 #else
  309             NULL,
  310 #endif
  311             vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
  312 
  313         vm_radix_zinit();
  314 }
  315 
  316 void
  317 vm_object_clear_flag(vm_object_t object, u_short bits)
  318 {
  319 
  320         VM_OBJECT_ASSERT_WLOCKED(object);
  321         object->flags &= ~bits;
  322 }
  323 
  324 /*
  325  *      Sets the default memory attribute for the specified object.  Pages
  326  *      that are allocated to this object are by default assigned this memory
  327  *      attribute.
  328  *
  329  *      Presently, this function must be called before any pages are allocated
  330  *      to the object.  In the future, this requirement may be relaxed for
  331  *      "default" and "swap" objects.
  332  */
  333 int
  334 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
  335 {
  336 
  337         VM_OBJECT_ASSERT_WLOCKED(object);
  338 
  339         if (object->type == OBJT_DEAD)
  340                 return (KERN_INVALID_ARGUMENT);
  341         if (!TAILQ_EMPTY(&object->memq))
  342                 return (KERN_FAILURE);
  343 
  344         object->memattr = memattr;
  345         return (KERN_SUCCESS);
  346 }
  347 
  348 void
  349 vm_object_pip_add(vm_object_t object, short i)
  350 {
  351 
  352         if (i > 0)
  353                 blockcount_acquire(&object->paging_in_progress, i);
  354 }
  355 
  356 void
  357 vm_object_pip_wakeup(vm_object_t object)
  358 {
  359 
  360         vm_object_pip_wakeupn(object, 1);
  361 }
  362 
  363 void
  364 vm_object_pip_wakeupn(vm_object_t object, short i)
  365 {
  366 
  367         if (i > 0)
  368                 blockcount_release(&object->paging_in_progress, i);
  369 }
  370 
  371 /*
  372  * Atomically drop the object lock and wait for pip to drain.  This protects
  373  * from sleep/wakeup races due to identity changes.  The lock is not re-acquired
  374  * on return.
  375  */
  376 static void
  377 vm_object_pip_sleep(vm_object_t object, const char *waitid)
  378 {
  379 
  380         (void)blockcount_sleep(&object->paging_in_progress, &object->lock,
  381             waitid, PVM | PDROP);
  382 }
  383 
  384 void
  385 vm_object_pip_wait(vm_object_t object, const char *waitid)
  386 {
  387 
  388         VM_OBJECT_ASSERT_WLOCKED(object);
  389 
  390         blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
  391             PVM);
  392 }
  393 
  394 void
  395 vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
  396 {
  397 
  398         VM_OBJECT_ASSERT_UNLOCKED(object);
  399 
  400         blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
  401 }
  402 
  403 /*
  404  *      vm_object_allocate:
  405  *
  406  *      Returns a new object with the given size.
  407  */
  408 vm_object_t
  409 vm_object_allocate(objtype_t type, vm_pindex_t size)
  410 {
  411         vm_object_t object;
  412         u_short flags;
  413 
  414         switch (type) {
  415         case OBJT_DEAD:
  416                 panic("vm_object_allocate: can't create OBJT_DEAD");
  417         case OBJT_SWAP:
  418                 flags = OBJ_COLORED | OBJ_SWAP;
  419                 break;
  420         case OBJT_DEVICE:
  421         case OBJT_SG:
  422                 flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
  423                 break;
  424         case OBJT_MGTDEVICE:
  425                 flags = OBJ_FICTITIOUS;
  426                 break;
  427         case OBJT_PHYS:
  428                 flags = OBJ_UNMANAGED;
  429                 break;
  430         case OBJT_VNODE:
  431                 flags = 0;
  432                 break;
  433         default:
  434                 panic("vm_object_allocate: type %d is undefined or dynamic",
  435                     type);
  436         }
  437         object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
  438         _vm_object_allocate(type, size, flags, object, NULL);
  439 
  440         return (object);
  441 }
  442 
  443 vm_object_t
  444 vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags)
  445 {
  446         vm_object_t object;
  447 
  448         MPASS(dyntype >= OBJT_FIRST_DYN /* && dyntype < nitems(pagertab) */);
  449         object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
  450         _vm_object_allocate(dyntype, size, flags, object, NULL);
  451 
  452         return (object);
  453 }
  454 
  455 /*
  456  *      vm_object_allocate_anon:
  457  *
  458  *      Returns a new default object of the given size and marked as
  459  *      anonymous memory for special split/collapse handling.  Color
  460  *      to be initialized by the caller.
  461  */
  462 vm_object_t
  463 vm_object_allocate_anon(vm_pindex_t size, vm_object_t backing_object,
  464     struct ucred *cred, vm_size_t charge)
  465 {
  466         vm_object_t handle, object;
  467 
  468         if (backing_object == NULL)
  469                 handle = NULL;
  470         else if ((backing_object->flags & OBJ_ANON) != 0)
  471                 handle = backing_object->handle;
  472         else
  473                 handle = backing_object;
  474         object = uma_zalloc(obj_zone, M_WAITOK);
  475         _vm_object_allocate(OBJT_SWAP, size,
  476             OBJ_ANON | OBJ_ONEMAPPING | OBJ_SWAP, object, handle);
  477         object->cred = cred;
  478         object->charge = cred != NULL ? charge : 0;
  479         return (object);
  480 }
  481 
  482 static void
  483 vm_object_reference_vnode(vm_object_t object)
  484 {
  485         u_int old;
  486 
  487         /*
  488          * vnode objects need the lock for the first reference
  489          * to serialize with vnode_object_deallocate().
  490          */
  491         if (!refcount_acquire_if_gt(&object->ref_count, 0)) {
  492                 VM_OBJECT_RLOCK(object);
  493                 old = refcount_acquire(&object->ref_count);
  494                 if (object->type == OBJT_VNODE && old == 0)
  495                         vref(object->handle);
  496                 VM_OBJECT_RUNLOCK(object);
  497         }
  498 }
  499 
  500 /*
  501  *      vm_object_reference:
  502  *
  503  *      Acquires a reference to the given object.
  504  */
  505 void
  506 vm_object_reference(vm_object_t object)
  507 {
  508 
  509         if (object == NULL)
  510                 return;
  511 
  512         if (object->type == OBJT_VNODE)
  513                 vm_object_reference_vnode(object);
  514         else
  515                 refcount_acquire(&object->ref_count);
  516         KASSERT((object->flags & OBJ_DEAD) == 0,
  517             ("vm_object_reference: Referenced dead object."));
  518 }
  519 
  520 /*
  521  *      vm_object_reference_locked:
  522  *
  523  *      Gets another reference to the given object.
  524  *
  525  *      The object must be locked.
  526  */
  527 void
  528 vm_object_reference_locked(vm_object_t object)
  529 {
  530         u_int old;
  531 
  532         VM_OBJECT_ASSERT_LOCKED(object);
  533         old = refcount_acquire(&object->ref_count);
  534         if (object->type == OBJT_VNODE && old == 0)
  535                 vref(object->handle);
  536         KASSERT((object->flags & OBJ_DEAD) == 0,
  537             ("vm_object_reference: Referenced dead object."));
  538 }
  539 
  540 /*
  541  * Handle deallocating an object of type OBJT_VNODE.
  542  */
  543 static void
  544 vm_object_deallocate_vnode(vm_object_t object)
  545 {
  546         struct vnode *vp = (struct vnode *) object->handle;
  547         bool last;
  548 
  549         KASSERT(object->type == OBJT_VNODE,
  550             ("vm_object_deallocate_vnode: not a vnode object"));
  551         KASSERT(vp != NULL, ("vm_object_deallocate_vnode: missing vp"));
  552 
  553         /* Object lock to protect handle lookup. */
  554         last = refcount_release(&object->ref_count);
  555         VM_OBJECT_RUNLOCK(object);
  556 
  557         if (!last)
  558                 return;
  559 
  560         if (!umtx_shm_vnobj_persistent)
  561                 umtx_shm_object_terminated(object);
  562 
  563         /* vrele may need the vnode lock. */
  564         vrele(vp);
  565 }
  566 
  567 /*
  568  * We dropped a reference on an object and discovered that it had a
  569  * single remaining shadow.  This is a sibling of the reference we
  570  * dropped.  Attempt to collapse the sibling and backing object.
  571  */
  572 static vm_object_t
  573 vm_object_deallocate_anon(vm_object_t backing_object)
  574 {
  575         vm_object_t object;
  576 
  577         /* Fetch the final shadow.  */
  578         object = LIST_FIRST(&backing_object->shadow_head);
  579         KASSERT(object != NULL &&
  580             atomic_load_int(&backing_object->shadow_count) == 1,
  581             ("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d",
  582             backing_object->ref_count,
  583             atomic_load_int(&backing_object->shadow_count)));
  584         KASSERT((object->flags & OBJ_ANON) != 0,
  585             ("invalid shadow object %p", object));
  586 
  587         if (!VM_OBJECT_TRYWLOCK(object)) {
  588                 /*
  589                  * Prevent object from disappearing since we do not have a
  590                  * reference.
  591                  */
  592                 vm_object_pip_add(object, 1);
  593                 VM_OBJECT_WUNLOCK(backing_object);
  594                 VM_OBJECT_WLOCK(object);
  595                 vm_object_pip_wakeup(object);
  596         } else
  597                 VM_OBJECT_WUNLOCK(backing_object);
  598 
  599         /*
  600          * Check for a collapse/terminate race with the last reference holder.
  601          */
  602         if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||
  603             !refcount_acquire_if_not_zero(&object->ref_count)) {
  604                 VM_OBJECT_WUNLOCK(object);
  605                 return (NULL);
  606         }
  607         backing_object = object->backing_object;
  608         if (backing_object != NULL && (backing_object->flags & OBJ_ANON) != 0)
  609                 vm_object_collapse(object);
  610         VM_OBJECT_WUNLOCK(object);
  611 
  612         return (object);
  613 }
  614 
  615 /*
  616  *      vm_object_deallocate:
  617  *
  618  *      Release a reference to the specified object,
  619  *      gained either through a vm_object_allocate
  620  *      or a vm_object_reference call.  When all references
  621  *      are gone, storage associated with this object
  622  *      may be relinquished.
  623  *
  624  *      No object may be locked.
  625  */
  626 void
  627 vm_object_deallocate(vm_object_t object)
  628 {
  629         vm_object_t temp;
  630         bool released;
  631 
  632         while (object != NULL) {
  633                 /*
  634                  * If the reference count goes to 0 we start calling
  635                  * vm_object_terminate() on the object chain.  A ref count
  636                  * of 1 may be a special case depending on the shadow count
  637                  * being 0 or 1.  These cases require a write lock on the
  638                  * object.
  639                  */
  640                 if ((object->flags & OBJ_ANON) == 0)
  641                         released = refcount_release_if_gt(&object->ref_count, 1);
  642                 else
  643                         released = refcount_release_if_gt(&object->ref_count, 2);
  644                 if (released)
  645                         return;
  646 
  647                 if (object->type == OBJT_VNODE) {
  648                         VM_OBJECT_RLOCK(object);
  649                         if (object->type == OBJT_VNODE) {
  650                                 vm_object_deallocate_vnode(object);
  651                                 return;
  652                         }
  653                         VM_OBJECT_RUNLOCK(object);
  654                 }
  655 
  656                 VM_OBJECT_WLOCK(object);
  657                 KASSERT(object->ref_count > 0,
  658                     ("vm_object_deallocate: object deallocated too many times: %d",
  659                     object->type));
  660 
  661                 /*
  662                  * If this is not the final reference to an anonymous
  663                  * object we may need to collapse the shadow chain.
  664                  */
  665                 if (!refcount_release(&object->ref_count)) {
  666                         if (object->ref_count > 1 ||
  667                             atomic_load_int(&object->shadow_count) == 0) {
  668                                 if ((object->flags & OBJ_ANON) != 0 &&
  669                                     object->ref_count == 1)
  670                                         vm_object_set_flag(object,
  671                                             OBJ_ONEMAPPING);
  672                                 VM_OBJECT_WUNLOCK(object);
  673                                 return;
  674                         }
  675 
  676                         /* Handle collapsing last ref on anonymous objects. */
  677                         object = vm_object_deallocate_anon(object);
  678                         continue;
  679                 }
  680 
  681                 /*
  682                  * Handle the final reference to an object.  We restart
  683                  * the loop with the backing object to avoid recursion.
  684                  */
  685                 umtx_shm_object_terminated(object);
  686                 temp = object->backing_object;
  687                 if (temp != NULL) {
  688                         KASSERT(object->type == OBJT_SWAP,
  689                             ("shadowed tmpfs v_object 2 %p", object));
  690                         vm_object_backing_remove(object);
  691                 }
  692 
  693                 KASSERT((object->flags & OBJ_DEAD) == 0,
  694                     ("vm_object_deallocate: Terminating dead object."));
  695                 vm_object_set_flag(object, OBJ_DEAD);
  696                 vm_object_terminate(object);
  697                 object = temp;
  698         }
  699 }
  700 
  701 void
  702 vm_object_destroy(vm_object_t object)
  703 {
  704         uma_zfree(obj_zone, object);
  705 }
  706 
  707 static void
  708 vm_object_sub_shadow(vm_object_t object)
  709 {
  710         KASSERT(object->shadow_count >= 1,
  711             ("object %p sub_shadow count zero", object));
  712         atomic_subtract_int(&object->shadow_count, 1);
  713 }
  714 
  715 static void
  716 vm_object_backing_remove_locked(vm_object_t object)
  717 {
  718         vm_object_t backing_object;
  719 
  720         backing_object = object->backing_object;
  721         VM_OBJECT_ASSERT_WLOCKED(object);
  722         VM_OBJECT_ASSERT_WLOCKED(backing_object);
  723 
  724         KASSERT((object->flags & OBJ_COLLAPSING) == 0,
  725             ("vm_object_backing_remove: Removing collapsing object."));
  726 
  727         vm_object_sub_shadow(backing_object);
  728         if ((object->flags & OBJ_SHADOWLIST) != 0) {
  729                 LIST_REMOVE(object, shadow_list);
  730                 vm_object_clear_flag(object, OBJ_SHADOWLIST);
  731         }
  732         object->backing_object = NULL;
  733 }
  734 
  735 static void
  736 vm_object_backing_remove(vm_object_t object)
  737 {
  738         vm_object_t backing_object;
  739 
  740         VM_OBJECT_ASSERT_WLOCKED(object);
  741 
  742         backing_object = object->backing_object;
  743         if ((object->flags & OBJ_SHADOWLIST) != 0) {
  744                 VM_OBJECT_WLOCK(backing_object);
  745                 vm_object_backing_remove_locked(object);
  746                 VM_OBJECT_WUNLOCK(backing_object);
  747         } else {
  748                 object->backing_object = NULL;
  749                 vm_object_sub_shadow(backing_object);
  750         }
  751 }
  752 
  753 static void
  754 vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
  755 {
  756 
  757         VM_OBJECT_ASSERT_WLOCKED(object);
  758 
  759         atomic_add_int(&backing_object->shadow_count, 1);
  760         if ((backing_object->flags & OBJ_ANON) != 0) {
  761                 VM_OBJECT_ASSERT_WLOCKED(backing_object);
  762                 LIST_INSERT_HEAD(&backing_object->shadow_head, object,
  763                     shadow_list);
  764                 vm_object_set_flag(object, OBJ_SHADOWLIST);
  765         }
  766         object->backing_object = backing_object;
  767 }
  768 
  769 static void
  770 vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
  771 {
  772 
  773         VM_OBJECT_ASSERT_WLOCKED(object);
  774 
  775         if ((backing_object->flags & OBJ_ANON) != 0) {
  776                 VM_OBJECT_WLOCK(backing_object);
  777                 vm_object_backing_insert_locked(object, backing_object);
  778                 VM_OBJECT_WUNLOCK(backing_object);
  779         } else {
  780                 object->backing_object = backing_object;
  781                 atomic_add_int(&backing_object->shadow_count, 1);
  782         }
  783 }
  784 
  785 /*
  786  * Insert an object into a backing_object's shadow list with an additional
  787  * reference to the backing_object added.
  788  */
  789 static void
  790 vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)
  791 {
  792 
  793         VM_OBJECT_ASSERT_WLOCKED(object);
  794 
  795         if ((backing_object->flags & OBJ_ANON) != 0) {
  796                 VM_OBJECT_WLOCK(backing_object);
  797                 KASSERT((backing_object->flags & OBJ_DEAD) == 0,
  798                     ("shadowing dead anonymous object"));
  799                 vm_object_reference_locked(backing_object);
  800                 vm_object_backing_insert_locked(object, backing_object);
  801                 vm_object_clear_flag(backing_object, OBJ_ONEMAPPING);
  802                 VM_OBJECT_WUNLOCK(backing_object);
  803         } else {
  804                 vm_object_reference(backing_object);
  805                 atomic_add_int(&backing_object->shadow_count, 1);
  806                 object->backing_object = backing_object;
  807         }
  808 }
  809 
  810 /*
  811  * Transfer a backing reference from backing_object to object.
  812  */
  813 static void
  814 vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)
  815 {
  816         vm_object_t new_backing_object;
  817 
  818         /*
  819          * Note that the reference to backing_object->backing_object
  820          * moves from within backing_object to within object.
  821          */
  822         vm_object_backing_remove_locked(object);
  823         new_backing_object = backing_object->backing_object;
  824         if (new_backing_object == NULL)
  825                 return;
  826         if ((new_backing_object->flags & OBJ_ANON) != 0) {
  827                 VM_OBJECT_WLOCK(new_backing_object);
  828                 vm_object_backing_remove_locked(backing_object);
  829                 vm_object_backing_insert_locked(object, new_backing_object);
  830                 VM_OBJECT_WUNLOCK(new_backing_object);
  831         } else {
  832                 /*
  833                  * shadow_count for new_backing_object is left
  834                  * unchanged, its reference provided by backing_object
  835                  * is replaced by object.
  836                  */
  837                 object->backing_object = new_backing_object;
  838                 backing_object->backing_object = NULL;
  839         }
  840 }
  841 
  842 /*
  843  * Wait for a concurrent collapse to settle.
  844  */
  845 static void
  846 vm_object_collapse_wait(vm_object_t object)
  847 {
  848 
  849         VM_OBJECT_ASSERT_WLOCKED(object);
  850 
  851         while ((object->flags & OBJ_COLLAPSING) != 0) {
  852                 vm_object_pip_wait(object, "vmcolwait");
  853                 counter_u64_add(object_collapse_waits, 1);
  854         }
  855 }
  856 
  857 /*
  858  * Waits for a backing object to clear a pending collapse and returns
  859  * it locked if it is an ANON object.
  860  */
  861 static vm_object_t
  862 vm_object_backing_collapse_wait(vm_object_t object)
  863 {
  864         vm_object_t backing_object;
  865 
  866         VM_OBJECT_ASSERT_WLOCKED(object);
  867 
  868         for (;;) {
  869                 backing_object = object->backing_object;
  870                 if (backing_object == NULL ||
  871                     (backing_object->flags & OBJ_ANON) == 0)
  872                         return (NULL);
  873                 VM_OBJECT_WLOCK(backing_object);
  874                 if ((backing_object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) == 0)
  875                         break;
  876                 VM_OBJECT_WUNLOCK(object);
  877                 vm_object_pip_sleep(backing_object, "vmbckwait");
  878                 counter_u64_add(object_collapse_waits, 1);
  879                 VM_OBJECT_WLOCK(object);
  880         }
  881         return (backing_object);
  882 }
  883 
  884 /*
  885  *      vm_object_terminate_pages removes any remaining pageable pages
  886  *      from the object and resets the object to an empty state.
  887  */
  888 static void
  889 vm_object_terminate_pages(vm_object_t object)
  890 {
  891         vm_page_t p, p_next;
  892 
  893         VM_OBJECT_ASSERT_WLOCKED(object);
  894 
  895         /*
  896          * Free any remaining pageable pages.  This also removes them from the
  897          * paging queues.  However, don't free wired pages, just remove them
  898          * from the object.  Rather than incrementally removing each page from
  899          * the object, the page and object are reset to any empty state. 
  900          */
  901         TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
  902                 vm_page_assert_unbusied(p);
  903                 KASSERT(p->object == object &&
  904                     (p->ref_count & VPRC_OBJREF) != 0,
  905                     ("vm_object_terminate_pages: page %p is inconsistent", p));
  906 
  907                 p->object = NULL;
  908                 if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) {
  909                         VM_CNT_INC(v_pfree);
  910                         vm_page_free(p);
  911                 }
  912         }
  913 
  914         /*
  915          * If the object contained any pages, then reset it to an empty state.
  916          * None of the object's fields, including "resident_page_count", were
  917          * modified by the preceding loop.
  918          */
  919         if (object->resident_page_count != 0) {
  920                 vm_radix_reclaim_allnodes(&object->rtree);
  921                 TAILQ_INIT(&object->memq);
  922                 object->resident_page_count = 0;
  923                 if (object->type == OBJT_VNODE)
  924                         vdrop(object->handle);
  925         }
  926 }
  927 
  928 /*
  929  *      vm_object_terminate actually destroys the specified object, freeing
  930  *      up all previously used resources.
  931  *
  932  *      The object must be locked.
  933  *      This routine may block.
  934  */
  935 void
  936 vm_object_terminate(vm_object_t object)
  937 {
  938 
  939         VM_OBJECT_ASSERT_WLOCKED(object);
  940         KASSERT((object->flags & OBJ_DEAD) != 0,
  941             ("terminating non-dead obj %p", object));
  942         KASSERT((object->flags & OBJ_COLLAPSING) == 0,
  943             ("terminating collapsing obj %p", object));
  944         KASSERT(object->backing_object == NULL,
  945             ("terminating shadow obj %p", object));
  946 
  947         /*
  948          * Wait for the pageout daemon and other current users to be
  949          * done with the object.  Note that new paging_in_progress
  950          * users can come after this wait, but they must check
  951          * OBJ_DEAD flag set (without unlocking the object), and avoid
  952          * the object being terminated.
  953          */
  954         vm_object_pip_wait(object, "objtrm");
  955 
  956         KASSERT(object->ref_count == 0,
  957             ("vm_object_terminate: object with references, ref_count=%d",
  958             object->ref_count));
  959 
  960         if ((object->flags & OBJ_PG_DTOR) == 0)
  961                 vm_object_terminate_pages(object);
  962 
  963 #if VM_NRESERVLEVEL > 0
  964         if (__predict_false(!LIST_EMPTY(&object->rvq)))
  965                 vm_reserv_break_all(object);
  966 #endif
  967 
  968         KASSERT(object->cred == NULL || (object->flags & OBJ_SWAP) != 0,
  969             ("%s: non-swap obj %p has cred", __func__, object));
  970 
  971         /*
  972          * Let the pager know object is dead.
  973          */
  974         vm_pager_deallocate(object);
  975         VM_OBJECT_WUNLOCK(object);
  976 
  977         vm_object_destroy(object);
  978 }
  979 
  980 /*
  981  * Make the page read-only so that we can clear the object flags.  However, if
  982  * this is a nosync mmap then the object is likely to stay dirty so do not
  983  * mess with the page and do not clear the object flags.  Returns TRUE if the
  984  * page should be flushed, and FALSE otherwise.
  985  */
  986 static boolean_t
  987 vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
  988 {
  989 
  990         vm_page_assert_busied(p);
  991 
  992         /*
  993          * If we have been asked to skip nosync pages and this is a
  994          * nosync page, skip it.  Note that the object flags were not
  995          * cleared in this case so we do not have to set them.
  996          */
  997         if ((flags & OBJPC_NOSYNC) != 0 && (p->a.flags & PGA_NOSYNC) != 0) {
  998                 *allclean = FALSE;
  999                 return (FALSE);
 1000         } else {
 1001                 pmap_remove_write(p);
 1002                 return (p->dirty != 0);
 1003         }
 1004 }
 1005 
 1006 /*
 1007  *      vm_object_page_clean
 1008  *
 1009  *      Clean all dirty pages in the specified range of object.  Leaves page 
 1010  *      on whatever queue it is currently on.   If NOSYNC is set then do not
 1011  *      write out pages with PGA_NOSYNC set (originally comes from MAP_NOSYNC),
 1012  *      leaving the object dirty.
 1013  *
 1014  *      For swap objects backing tmpfs regular files, do not flush anything,
 1015  *      but remove write protection on the mapped pages to update mtime through
 1016  *      mmaped writes.
 1017  *
 1018  *      When stuffing pages asynchronously, allow clustering.  XXX we need a
 1019  *      synchronous clustering mode implementation.
 1020  *
 1021  *      Odd semantics: if start == end, we clean everything.
 1022  *
 1023  *      The object must be locked.
 1024  *
 1025  *      Returns FALSE if some page from the range was not written, as
 1026  *      reported by the pager, and TRUE otherwise.
 1027  */
 1028 boolean_t
 1029 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
 1030     int flags)
 1031 {
 1032         vm_page_t np, p;
 1033         vm_pindex_t pi, tend, tstart;
 1034         int curgeneration, n, pagerflags;
 1035         boolean_t eio, res, allclean;
 1036 
 1037         VM_OBJECT_ASSERT_WLOCKED(object);
 1038 
 1039         if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)
 1040                 return (TRUE);
 1041 
 1042         pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
 1043             VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
 1044         pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
 1045 
 1046         tstart = OFF_TO_IDX(start);
 1047         tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
 1048         allclean = tstart == 0 && tend >= object->size;
 1049         res = TRUE;
 1050 
 1051 rescan:
 1052         curgeneration = object->generation;
 1053 
 1054         for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
 1055                 pi = p->pindex;
 1056                 if (pi >= tend)
 1057                         break;
 1058                 np = TAILQ_NEXT(p, listq);
 1059                 if (vm_page_none_valid(p))
 1060                         continue;
 1061                 if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
 1062                         if (object->generation != curgeneration &&
 1063                             (flags & OBJPC_SYNC) != 0)
 1064                                 goto rescan;
 1065                         np = vm_page_find_least(object, pi);
 1066                         continue;
 1067                 }
 1068                 if (!vm_object_page_remove_write(p, flags, &allclean)) {
 1069                         vm_page_xunbusy(p);
 1070                         continue;
 1071                 }
 1072                 if (object->type == OBJT_VNODE) {
 1073                         n = vm_object_page_collect_flush(object, p, pagerflags,
 1074                             flags, &allclean, &eio);
 1075                         if (eio) {
 1076                                 res = FALSE;
 1077                                 allclean = FALSE;
 1078                         }
 1079                         if (object->generation != curgeneration &&
 1080                             (flags & OBJPC_SYNC) != 0)
 1081                                 goto rescan;
 1082 
 1083                         /*
 1084                          * If the VOP_PUTPAGES() did a truncated write, so
 1085                          * that even the first page of the run is not fully
 1086                          * written, vm_pageout_flush() returns 0 as the run
 1087                          * length.  Since the condition that caused truncated
 1088                          * write may be permanent, e.g. exhausted free space,
 1089                          * accepting n == 0 would cause an infinite loop.
 1090                          *
 1091                          * Forwarding the iterator leaves the unwritten page
 1092                          * behind, but there is not much we can do there if
 1093                          * filesystem refuses to write it.
 1094                          */
 1095                         if (n == 0) {
 1096                                 n = 1;
 1097                                 allclean = FALSE;
 1098                         }
 1099                 } else {
 1100                         n = 1;
 1101                         vm_page_xunbusy(p);
 1102                 }
 1103                 np = vm_page_find_least(object, pi + n);
 1104         }
 1105 #if 0
 1106         VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
 1107 #endif
 1108 
 1109         /*
 1110          * Leave updating cleangeneration for tmpfs objects to tmpfs
 1111          * scan.  It needs to update mtime, which happens for other
 1112          * filesystems during page writeouts.
 1113          */
 1114         if (allclean && object->type == OBJT_VNODE)
 1115                 object->cleangeneration = curgeneration;
 1116         return (res);
 1117 }
 1118 
 1119 static int
 1120 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
 1121     int flags, boolean_t *allclean, boolean_t *eio)
 1122 {
 1123         vm_page_t ma[vm_pageout_page_count], p_first, tp;
 1124         int count, i, mreq, runlen;
 1125 
 1126         vm_page_lock_assert(p, MA_NOTOWNED);
 1127         vm_page_assert_xbusied(p);
 1128         VM_OBJECT_ASSERT_WLOCKED(object);
 1129 
 1130         count = 1;
 1131         mreq = 0;
 1132 
 1133         for (tp = p; count < vm_pageout_page_count; count++) {
 1134                 tp = vm_page_next(tp);
 1135                 if (tp == NULL || vm_page_tryxbusy(tp) == 0)
 1136                         break;
 1137                 if (!vm_object_page_remove_write(tp, flags, allclean)) {
 1138                         vm_page_xunbusy(tp);
 1139                         break;
 1140                 }
 1141         }
 1142 
 1143         for (p_first = p; count < vm_pageout_page_count; count++) {
 1144                 tp = vm_page_prev(p_first);
 1145                 if (tp == NULL || vm_page_tryxbusy(tp) == 0)
 1146                         break;
 1147                 if (!vm_object_page_remove_write(tp, flags, allclean)) {
 1148                         vm_page_xunbusy(tp);
 1149                         break;
 1150                 }
 1151                 p_first = tp;
 1152                 mreq++;
 1153         }
 1154 
 1155         for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
 1156                 ma[i] = tp;
 1157 
 1158         vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
 1159         return (runlen);
 1160 }
 1161 
 1162 /*
 1163  * Note that there is absolutely no sense in writing out
 1164  * anonymous objects, so we track down the vnode object
 1165  * to write out.
 1166  * We invalidate (remove) all pages from the address space
 1167  * for semantic correctness.
 1168  *
 1169  * If the backing object is a device object with unmanaged pages, then any
 1170  * mappings to the specified range of pages must be removed before this
 1171  * function is called.
 1172  *
 1173  * Note: certain anonymous maps, such as MAP_NOSYNC maps,
 1174  * may start out with a NULL object.
 1175  */
 1176 boolean_t
 1177 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
 1178     boolean_t syncio, boolean_t invalidate)
 1179 {
 1180         vm_object_t backing_object;
 1181         struct vnode *vp;
 1182         struct mount *mp;
 1183         int error, flags, fsync_after;
 1184         boolean_t res;
 1185 
 1186         if (object == NULL)
 1187                 return (TRUE);
 1188         res = TRUE;
 1189         error = 0;
 1190         VM_OBJECT_WLOCK(object);
 1191         while ((backing_object = object->backing_object) != NULL) {
 1192                 VM_OBJECT_WLOCK(backing_object);
 1193                 offset += object->backing_object_offset;
 1194                 VM_OBJECT_WUNLOCK(object);
 1195                 object = backing_object;
 1196                 if (object->size < OFF_TO_IDX(offset + size))
 1197                         size = IDX_TO_OFF(object->size) - offset;
 1198         }
 1199         /*
 1200          * Flush pages if writing is allowed, invalidate them
 1201          * if invalidation requested.  Pages undergoing I/O
 1202          * will be ignored by vm_object_page_remove().
 1203          *
 1204          * We cannot lock the vnode and then wait for paging
 1205          * to complete without deadlocking against vm_fault.
 1206          * Instead we simply call vm_object_page_remove() and
 1207          * allow it to block internally on a page-by-page
 1208          * basis when it encounters pages undergoing async
 1209          * I/O.
 1210          */
 1211         if (object->type == OBJT_VNODE &&
 1212             vm_object_mightbedirty(object) != 0 &&
 1213             ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
 1214                 VM_OBJECT_WUNLOCK(object);
 1215                 (void)vn_start_write(vp, &mp, V_WAIT);
 1216                 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 1217                 if (syncio && !invalidate && offset == 0 &&
 1218                     atop(size) == object->size) {
 1219                         /*
 1220                          * If syncing the whole mapping of the file,
 1221                          * it is faster to schedule all the writes in
 1222                          * async mode, also allowing the clustering,
 1223                          * and then wait for i/o to complete.
 1224                          */
 1225                         flags = 0;
 1226                         fsync_after = TRUE;
 1227                 } else {
 1228                         flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
 1229                         flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
 1230                         fsync_after = FALSE;
 1231                 }
 1232                 VM_OBJECT_WLOCK(object);
 1233                 res = vm_object_page_clean(object, offset, offset + size,
 1234                     flags);
 1235                 VM_OBJECT_WUNLOCK(object);
 1236                 if (fsync_after) {
 1237                         for (;;) {
 1238                                 error = VOP_FSYNC(vp, MNT_WAIT, curthread);
 1239                                 if (error != ERELOOKUP)
 1240                                         break;
 1241 
 1242                                 /*
 1243                                  * Allow SU/bufdaemon to handle more
 1244                                  * dependencies in the meantime.
 1245                                  */
 1246                                 VOP_UNLOCK(vp);
 1247                                 vn_finished_write(mp);
 1248 
 1249                                 (void)vn_start_write(vp, &mp, V_WAIT);
 1250                                 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 1251                         }
 1252                 }
 1253                 VOP_UNLOCK(vp);
 1254                 vn_finished_write(mp);
 1255                 if (error != 0)
 1256                         res = FALSE;
 1257                 VM_OBJECT_WLOCK(object);
 1258         }
 1259         if ((object->type == OBJT_VNODE ||
 1260              object->type == OBJT_DEVICE) && invalidate) {
 1261                 if (object->type == OBJT_DEVICE)
 1262                         /*
 1263                          * The option OBJPR_NOTMAPPED must be passed here
 1264                          * because vm_object_page_remove() cannot remove
 1265                          * unmanaged mappings.
 1266                          */
 1267                         flags = OBJPR_NOTMAPPED;
 1268                 else if (old_msync)
 1269                         flags = 0;
 1270                 else
 1271                         flags = OBJPR_CLEANONLY;
 1272                 vm_object_page_remove(object, OFF_TO_IDX(offset),
 1273                     OFF_TO_IDX(offset + size + PAGE_MASK), flags);
 1274         }
 1275         VM_OBJECT_WUNLOCK(object);
 1276         return (res);
 1277 }
 1278 
 1279 /*
 1280  * Determine whether the given advice can be applied to the object.  Advice is
 1281  * not applied to unmanaged pages since they never belong to page queues, and
 1282  * since MADV_FREE is destructive, it can apply only to anonymous pages that
 1283  * have been mapped at most once.
 1284  */
 1285 static bool
 1286 vm_object_advice_applies(vm_object_t object, int advice)
 1287 {
 1288 
 1289         if ((object->flags & OBJ_UNMANAGED) != 0)
 1290                 return (false);
 1291         if (advice != MADV_FREE)
 1292                 return (true);
 1293         return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==
 1294             (OBJ_ONEMAPPING | OBJ_ANON));
 1295 }
 1296 
 1297 static void
 1298 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
 1299     vm_size_t size)
 1300 {
 1301 
 1302         if (advice == MADV_FREE)
 1303                 vm_pager_freespace(object, pindex, size);
 1304 }
 1305 
 1306 /*
 1307  *      vm_object_madvise:
 1308  *
 1309  *      Implements the madvise function at the object/page level.
 1310  *
 1311  *      MADV_WILLNEED   (any object)
 1312  *
 1313  *          Activate the specified pages if they are resident.
 1314  *
 1315  *      MADV_DONTNEED   (any object)
 1316  *
 1317  *          Deactivate the specified pages if they are resident.
 1318  *
 1319  *      MADV_FREE       (OBJT_SWAP objects, OBJ_ONEMAPPING only)
 1320  *
 1321  *          Deactivate and clean the specified pages if they are
 1322  *          resident.  This permits the process to reuse the pages
 1323  *          without faulting or the kernel to reclaim the pages
 1324  *          without I/O.
 1325  */
 1326 void
 1327 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
 1328     int advice)
 1329 {
 1330         vm_pindex_t tpindex;
 1331         vm_object_t backing_object, tobject;
 1332         vm_page_t m, tm;
 1333 
 1334         if (object == NULL)
 1335                 return;
 1336 
 1337 relookup:
 1338         VM_OBJECT_WLOCK(object);
 1339         if (!vm_object_advice_applies(object, advice)) {
 1340                 VM_OBJECT_WUNLOCK(object);
 1341                 return;
 1342         }
 1343         for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
 1344                 tobject = object;
 1345 
 1346                 /*
 1347                  * If the next page isn't resident in the top-level object, we
 1348                  * need to search the shadow chain.  When applying MADV_FREE, we
 1349                  * take care to release any swap space used to store
 1350                  * non-resident pages.
 1351                  */
 1352                 if (m == NULL || pindex < m->pindex) {
 1353                         /*
 1354                          * Optimize a common case: if the top-level object has
 1355                          * no backing object, we can skip over the non-resident
 1356                          * range in constant time.
 1357                          */
 1358                         if (object->backing_object == NULL) {
 1359                                 tpindex = (m != NULL && m->pindex < end) ?
 1360                                     m->pindex : end;
 1361                                 vm_object_madvise_freespace(object, advice,
 1362                                     pindex, tpindex - pindex);
 1363                                 if ((pindex = tpindex) == end)
 1364                                         break;
 1365                                 goto next_page;
 1366                         }
 1367 
 1368                         tpindex = pindex;
 1369                         do {
 1370                                 vm_object_madvise_freespace(tobject, advice,
 1371                                     tpindex, 1);
 1372                                 /*
 1373                                  * Prepare to search the next object in the
 1374                                  * chain.
 1375                                  */
 1376                                 backing_object = tobject->backing_object;
 1377                                 if (backing_object == NULL)
 1378                                         goto next_pindex;
 1379                                 VM_OBJECT_WLOCK(backing_object);
 1380                                 tpindex +=
 1381                                     OFF_TO_IDX(tobject->backing_object_offset);
 1382                                 if (tobject != object)
 1383                                         VM_OBJECT_WUNLOCK(tobject);
 1384                                 tobject = backing_object;
 1385                                 if (!vm_object_advice_applies(tobject, advice))
 1386                                         goto next_pindex;
 1387                         } while ((tm = vm_page_lookup(tobject, tpindex)) ==
 1388                             NULL);
 1389                 } else {
 1390 next_page:
 1391                         tm = m;
 1392                         m = TAILQ_NEXT(m, listq);
 1393                 }
 1394 
 1395                 /*
 1396                  * If the page is not in a normal state, skip it.  The page
 1397                  * can not be invalidated while the object lock is held.
 1398                  */
 1399                 if (!vm_page_all_valid(tm) || vm_page_wired(tm))
 1400                         goto next_pindex;
 1401                 KASSERT((tm->flags & PG_FICTITIOUS) == 0,
 1402                     ("vm_object_madvise: page %p is fictitious", tm));
 1403                 KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
 1404                     ("vm_object_madvise: page %p is not managed", tm));
 1405                 if (vm_page_tryxbusy(tm) == 0) {
 1406                         if (object != tobject)
 1407                                 VM_OBJECT_WUNLOCK(object);
 1408                         if (advice == MADV_WILLNEED) {
 1409                                 /*
 1410                                  * Reference the page before unlocking and
 1411                                  * sleeping so that the page daemon is less
 1412                                  * likely to reclaim it.
 1413                                  */
 1414                                 vm_page_aflag_set(tm, PGA_REFERENCED);
 1415                         }
 1416                         if (!vm_page_busy_sleep(tm, "madvpo", 0))
 1417                                 VM_OBJECT_WUNLOCK(tobject);
 1418                         goto relookup;
 1419                 }
 1420                 vm_page_advise(tm, advice);
 1421                 vm_page_xunbusy(tm);
 1422                 vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
 1423 next_pindex:
 1424                 if (tobject != object)
 1425                         VM_OBJECT_WUNLOCK(tobject);
 1426         }
 1427         VM_OBJECT_WUNLOCK(object);
 1428 }
 1429 
 1430 /*
 1431  *      vm_object_shadow:
 1432  *
 1433  *      Create a new object which is backed by the
 1434  *      specified existing object range.  The source
 1435  *      object reference is deallocated.
 1436  *
 1437  *      The new object and offset into that object
 1438  *      are returned in the source parameters.
 1439  */
 1440 void
 1441 vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,
 1442     struct ucred *cred, bool shared)
 1443 {
 1444         vm_object_t source;
 1445         vm_object_t result;
 1446 
 1447         source = *object;
 1448 
 1449         /*
 1450          * Don't create the new object if the old object isn't shared.
 1451          *
 1452          * If we hold the only reference we can guarantee that it won't
 1453          * increase while we have the map locked.  Otherwise the race is
 1454          * harmless and we will end up with an extra shadow object that
 1455          * will be collapsed later.
 1456          */
 1457         if (source != NULL && source->ref_count == 1 &&
 1458             (source->flags & OBJ_ANON) != 0)
 1459                 return;
 1460 
 1461         /*
 1462          * Allocate a new object with the given length.
 1463          */
 1464         result = vm_object_allocate_anon(atop(length), source, cred, length);
 1465 
 1466         /*
 1467          * Store the offset into the source object, and fix up the offset into
 1468          * the new object.
 1469          */
 1470         result->backing_object_offset = *offset;
 1471 
 1472         if (shared || source != NULL) {
 1473                 VM_OBJECT_WLOCK(result);
 1474 
 1475                 /*
 1476                  * The new object shadows the source object, adding a
 1477                  * reference to it.  Our caller changes his reference
 1478                  * to point to the new object, removing a reference to
 1479                  * the source object.  Net result: no change of
 1480                  * reference count, unless the caller needs to add one
 1481                  * more reference due to forking a shared map entry.
 1482                  */
 1483                 if (shared) {
 1484                         vm_object_reference_locked(result);
 1485                         vm_object_clear_flag(result, OBJ_ONEMAPPING);
 1486                 }
 1487 
 1488                 /*
 1489                  * Try to optimize the result object's page color when
 1490                  * shadowing in order to maintain page coloring
 1491                  * consistency in the combined shadowed object.
 1492                  */
 1493                 if (source != NULL) {
 1494                         vm_object_backing_insert(result, source);
 1495                         result->domain = source->domain;
 1496 #if VM_NRESERVLEVEL > 0
 1497                         vm_object_set_flag(result,
 1498                             (source->flags & OBJ_COLORED));
 1499                         result->pg_color = (source->pg_color +
 1500                             OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER -
 1501                             1)) - 1);
 1502 #endif
 1503                 }
 1504                 VM_OBJECT_WUNLOCK(result);
 1505         }
 1506 
 1507         /*
 1508          * Return the new things
 1509          */
 1510         *offset = 0;
 1511         *object = result;
 1512 }
 1513 
 1514 /*
 1515  *      vm_object_split:
 1516  *
 1517  * Split the pages in a map entry into a new object.  This affords
 1518  * easier removal of unused pages, and keeps object inheritance from
 1519  * being a negative impact on memory usage.
 1520  */
 1521 void
 1522 vm_object_split(vm_map_entry_t entry)
 1523 {
 1524         vm_page_t m, m_next;
 1525         vm_object_t orig_object, new_object, backing_object;
 1526         vm_pindex_t idx, offidxstart;
 1527         vm_size_t size;
 1528 
 1529         orig_object = entry->object.vm_object;
 1530         KASSERT((orig_object->flags & OBJ_ONEMAPPING) != 0,
 1531             ("vm_object_split:  Splitting object with multiple mappings."));
 1532         if ((orig_object->flags & OBJ_ANON) == 0)
 1533                 return;
 1534         if (orig_object->ref_count <= 1)
 1535                 return;
 1536         VM_OBJECT_WUNLOCK(orig_object);
 1537 
 1538         offidxstart = OFF_TO_IDX(entry->offset);
 1539         size = atop(entry->end - entry->start);
 1540 
 1541         new_object = vm_object_allocate_anon(size, orig_object,
 1542             orig_object->cred, ptoa(size));
 1543 
 1544         /*
 1545          * We must wait for the orig_object to complete any in-progress
 1546          * collapse so that the swap blocks are stable below.  The
 1547          * additional reference on backing_object by new object will
 1548          * prevent further collapse operations until split completes.
 1549          */
 1550         VM_OBJECT_WLOCK(orig_object);
 1551         vm_object_collapse_wait(orig_object);
 1552 
 1553         /*
 1554          * At this point, the new object is still private, so the order in
 1555          * which the original and new objects are locked does not matter.
 1556          */
 1557         VM_OBJECT_WLOCK(new_object);
 1558         new_object->domain = orig_object->domain;
 1559         backing_object = orig_object->backing_object;
 1560         if (backing_object != NULL) {
 1561                 vm_object_backing_insert_ref(new_object, backing_object);
 1562                 new_object->backing_object_offset = 
 1563                     orig_object->backing_object_offset + entry->offset;
 1564         }
 1565         if (orig_object->cred != NULL) {
 1566                 crhold(orig_object->cred);
 1567                 KASSERT(orig_object->charge >= ptoa(size),
 1568                     ("orig_object->charge < 0"));
 1569                 orig_object->charge -= ptoa(size);
 1570         }
 1571 
 1572         /*
 1573          * Mark the split operation so that swap_pager_getpages() knows
 1574          * that the object is in transition.
 1575          */
 1576         vm_object_set_flag(orig_object, OBJ_SPLIT);
 1577 #ifdef INVARIANTS
 1578         idx = 0;
 1579 #endif
 1580 retry:
 1581         m = vm_page_find_least(orig_object, offidxstart);
 1582         KASSERT(m == NULL || idx <= m->pindex - offidxstart,
 1583             ("%s: object %p was repopulated", __func__, orig_object));
 1584         for (; m != NULL && (idx = m->pindex - offidxstart) < size;
 1585             m = m_next) {
 1586                 m_next = TAILQ_NEXT(m, listq);
 1587 
 1588                 /*
 1589                  * We must wait for pending I/O to complete before we can
 1590                  * rename the page.
 1591                  *
 1592                  * We do not have to VM_PROT_NONE the page as mappings should
 1593                  * not be changed by this operation.
 1594                  */
 1595                 if (vm_page_tryxbusy(m) == 0) {
 1596                         VM_OBJECT_WUNLOCK(new_object);
 1597                         if (vm_page_busy_sleep(m, "spltwt", 0))
 1598                                 VM_OBJECT_WLOCK(orig_object);
 1599                         VM_OBJECT_WLOCK(new_object);
 1600                         goto retry;
 1601                 }
 1602 
 1603                 /*
 1604                  * The page was left invalid.  Likely placed there by
 1605                  * an incomplete fault.  Just remove and ignore.
 1606                  */
 1607                 if (vm_page_none_valid(m)) {
 1608                         if (vm_page_remove(m))
 1609                                 vm_page_free(m);
 1610                         continue;
 1611                 }
 1612 
 1613                 /* vm_page_rename() will dirty the page. */
 1614                 if (vm_page_rename(m, new_object, idx)) {
 1615                         vm_page_xunbusy(m);
 1616                         VM_OBJECT_WUNLOCK(new_object);
 1617                         VM_OBJECT_WUNLOCK(orig_object);
 1618                         vm_radix_wait();
 1619                         VM_OBJECT_WLOCK(orig_object);
 1620                         VM_OBJECT_WLOCK(new_object);
 1621                         goto retry;
 1622                 }
 1623 
 1624 #if VM_NRESERVLEVEL > 0
 1625                 /*
 1626                  * If some of the reservation's allocated pages remain with
 1627                  * the original object, then transferring the reservation to
 1628                  * the new object is neither particularly beneficial nor
 1629                  * particularly harmful as compared to leaving the reservation
 1630                  * with the original object.  If, however, all of the
 1631                  * reservation's allocated pages are transferred to the new
 1632                  * object, then transferring the reservation is typically
 1633                  * beneficial.  Determining which of these two cases applies
 1634                  * would be more costly than unconditionally renaming the
 1635                  * reservation.
 1636                  */
 1637                 vm_reserv_rename(m, new_object, orig_object, offidxstart);
 1638 #endif
 1639         }
 1640 
 1641         /*
 1642          * swap_pager_copy() can sleep, in which case the orig_object's
 1643          * and new_object's locks are released and reacquired.
 1644          */
 1645         swap_pager_copy(orig_object, new_object, offidxstart, 0);
 1646 
 1647         TAILQ_FOREACH(m, &new_object->memq, listq)
 1648                 vm_page_xunbusy(m);
 1649 
 1650         vm_object_clear_flag(orig_object, OBJ_SPLIT);
 1651         VM_OBJECT_WUNLOCK(orig_object);
 1652         VM_OBJECT_WUNLOCK(new_object);
 1653         entry->object.vm_object = new_object;
 1654         entry->offset = 0LL;
 1655         vm_object_deallocate(orig_object);
 1656         VM_OBJECT_WLOCK(new_object);
 1657 }
 1658 
 1659 static vm_page_t
 1660 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
 1661 {
 1662         vm_object_t backing_object;
 1663 
 1664         VM_OBJECT_ASSERT_WLOCKED(object);
 1665         backing_object = object->backing_object;
 1666         VM_OBJECT_ASSERT_WLOCKED(backing_object);
 1667 
 1668         KASSERT(p == NULL || p->object == object || p->object == backing_object,
 1669             ("invalid ownership %p %p %p", p, object, backing_object));
 1670         /* The page is only NULL when rename fails. */
 1671         if (p == NULL) {
 1672                 VM_OBJECT_WUNLOCK(object);
 1673                 VM_OBJECT_WUNLOCK(backing_object);
 1674                 vm_radix_wait();
 1675                 VM_OBJECT_WLOCK(object);
 1676         } else if (p->object == object) {
 1677                 VM_OBJECT_WUNLOCK(backing_object);
 1678                 if (vm_page_busy_sleep(p, "vmocol", 0))
 1679                         VM_OBJECT_WLOCK(object);
 1680         } else {
 1681                 VM_OBJECT_WUNLOCK(object);
 1682                 if (!vm_page_busy_sleep(p, "vmocol", 0))
 1683                         VM_OBJECT_WUNLOCK(backing_object);
 1684                 VM_OBJECT_WLOCK(object);
 1685         }
 1686         VM_OBJECT_WLOCK(backing_object);
 1687         return (TAILQ_FIRST(&backing_object->memq));
 1688 }
 1689 
 1690 static bool
 1691 vm_object_scan_all_shadowed(vm_object_t object)
 1692 {
 1693         vm_object_t backing_object;
 1694         vm_page_t p, pp;
 1695         vm_pindex_t backing_offset_index, new_pindex, pi, ps;
 1696 
 1697         VM_OBJECT_ASSERT_WLOCKED(object);
 1698         VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
 1699 
 1700         backing_object = object->backing_object;
 1701 
 1702         if ((backing_object->flags & OBJ_ANON) == 0)
 1703                 return (false);
 1704 
 1705         pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
 1706         p = vm_page_find_least(backing_object, pi);
 1707         ps = swap_pager_find_least(backing_object, pi);
 1708 
 1709         /*
 1710          * Only check pages inside the parent object's range and
 1711          * inside the parent object's mapping of the backing object.
 1712          */
 1713         for (;; pi++) {
 1714                 if (p != NULL && p->pindex < pi)
 1715                         p = TAILQ_NEXT(p, listq);
 1716                 if (ps < pi)
 1717                         ps = swap_pager_find_least(backing_object, pi);
 1718                 if (p == NULL && ps >= backing_object->size)
 1719                         break;
 1720                 else if (p == NULL)
 1721                         pi = ps;
 1722                 else
 1723                         pi = MIN(p->pindex, ps);
 1724 
 1725                 new_pindex = pi - backing_offset_index;
 1726                 if (new_pindex >= object->size)
 1727                         break;
 1728 
 1729                 if (p != NULL) {
 1730                         /*
 1731                          * If the backing object page is busy a
 1732                          * grandparent or older page may still be
 1733                          * undergoing CoW.  It is not safe to collapse
 1734                          * the backing object until it is quiesced.
 1735                          */
 1736                         if (vm_page_tryxbusy(p) == 0)
 1737                                 return (false);
 1738 
 1739                         /*
 1740                          * We raced with the fault handler that left
 1741                          * newly allocated invalid page on the object
 1742                          * queue and retried.
 1743                          */
 1744                         if (!vm_page_all_valid(p))
 1745                                 goto unbusy_ret;
 1746                 }
 1747 
 1748                 /*
 1749                  * See if the parent has the page or if the parent's object
 1750                  * pager has the page.  If the parent has the page but the page
 1751                  * is not valid, the parent's object pager must have the page.
 1752                  *
 1753                  * If this fails, the parent does not completely shadow the
 1754                  * object and we might as well give up now.
 1755                  */
 1756                 pp = vm_page_lookup(object, new_pindex);
 1757 
 1758                 /*
 1759                  * The valid check here is stable due to object lock
 1760                  * being required to clear valid and initiate paging.
 1761                  * Busy of p disallows fault handler to validate pp.
 1762                  */
 1763                 if ((pp == NULL || vm_page_none_valid(pp)) &&
 1764                     !vm_pager_has_page(object, new_pindex, NULL, NULL))
 1765                         goto unbusy_ret;
 1766                 if (p != NULL)
 1767                         vm_page_xunbusy(p);
 1768         }
 1769         return (true);
 1770 
 1771 unbusy_ret:
 1772         if (p != NULL)
 1773                 vm_page_xunbusy(p);
 1774         return (false);
 1775 }
 1776 
 1777 static void
 1778 vm_object_collapse_scan(vm_object_t object)
 1779 {
 1780         vm_object_t backing_object;
 1781         vm_page_t next, p, pp;
 1782         vm_pindex_t backing_offset_index, new_pindex;
 1783 
 1784         VM_OBJECT_ASSERT_WLOCKED(object);
 1785         VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
 1786 
 1787         backing_object = object->backing_object;
 1788         backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
 1789 
 1790         /*
 1791          * Our scan
 1792          */
 1793         for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
 1794                 next = TAILQ_NEXT(p, listq);
 1795                 new_pindex = p->pindex - backing_offset_index;
 1796 
 1797                 /*
 1798                  * Check for busy page
 1799                  */
 1800                 if (vm_page_tryxbusy(p) == 0) {
 1801                         next = vm_object_collapse_scan_wait(object, p);
 1802                         continue;
 1803                 }
 1804 
 1805                 KASSERT(object->backing_object == backing_object,
 1806                     ("vm_object_collapse_scan: backing object mismatch %p != %p",
 1807                     object->backing_object, backing_object));
 1808                 KASSERT(p->object == backing_object,
 1809                     ("vm_object_collapse_scan: object mismatch %p != %p",
 1810                     p->object, backing_object));
 1811 
 1812                 if (p->pindex < backing_offset_index ||
 1813                     new_pindex >= object->size) {
 1814                         vm_pager_freespace(backing_object, p->pindex, 1);
 1815 
 1816                         KASSERT(!pmap_page_is_mapped(p),
 1817                             ("freeing mapped page %p", p));
 1818                         if (vm_page_remove(p))
 1819                                 vm_page_free(p);
 1820                         continue;
 1821                 }
 1822 
 1823                 if (!vm_page_all_valid(p)) {
 1824                         KASSERT(!pmap_page_is_mapped(p),
 1825                             ("freeing mapped page %p", p));
 1826                         if (vm_page_remove(p))
 1827                                 vm_page_free(p);
 1828                         continue;
 1829                 }
 1830 
 1831                 pp = vm_page_lookup(object, new_pindex);
 1832                 if (pp != NULL && vm_page_tryxbusy(pp) == 0) {
 1833                         vm_page_xunbusy(p);
 1834                         /*
 1835                          * The page in the parent is busy and possibly not
 1836                          * (yet) valid.  Until its state is finalized by the
 1837                          * busy bit owner, we can't tell whether it shadows the
 1838                          * original page.
 1839                          */
 1840                         next = vm_object_collapse_scan_wait(object, pp);
 1841                         continue;
 1842                 }
 1843 
 1844                 if (pp != NULL && vm_page_none_valid(pp)) {
 1845                         /*
 1846                          * The page was invalid in the parent.  Likely placed
 1847                          * there by an incomplete fault.  Just remove and
 1848                          * ignore.  p can replace it.
 1849                          */
 1850                         if (vm_page_remove(pp))
 1851                                 vm_page_free(pp);
 1852                         pp = NULL;
 1853                 }
 1854 
 1855                 if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
 1856                         NULL)) {
 1857                         /*
 1858                          * The page already exists in the parent OR swap exists
 1859                          * for this location in the parent.  Leave the parent's
 1860                          * page alone.  Destroy the original page from the
 1861                          * backing object.
 1862                          */
 1863                         vm_pager_freespace(backing_object, p->pindex, 1);
 1864                         KASSERT(!pmap_page_is_mapped(p),
 1865                             ("freeing mapped page %p", p));
 1866                         if (vm_page_remove(p))
 1867                                 vm_page_free(p);
 1868                         if (pp != NULL)
 1869                                 vm_page_xunbusy(pp);
 1870                         continue;
 1871                 }
 1872 
 1873                 /*
 1874                  * Page does not exist in parent, rename the page from the
 1875                  * backing object to the main object.
 1876                  *
 1877                  * If the page was mapped to a process, it can remain mapped
 1878                  * through the rename.  vm_page_rename() will dirty the page.
 1879                  */
 1880                 if (vm_page_rename(p, object, new_pindex)) {
 1881                         vm_page_xunbusy(p);
 1882                         next = vm_object_collapse_scan_wait(object, NULL);
 1883                         continue;
 1884                 }
 1885 
 1886                 /* Use the old pindex to free the right page. */
 1887                 vm_pager_freespace(backing_object, new_pindex +
 1888                     backing_offset_index, 1);
 1889 
 1890 #if VM_NRESERVLEVEL > 0
 1891                 /*
 1892                  * Rename the reservation.
 1893                  */
 1894                 vm_reserv_rename(p, object, backing_object,
 1895                     backing_offset_index);
 1896 #endif
 1897                 vm_page_xunbusy(p);
 1898         }
 1899         return;
 1900 }
 1901 
 1902 /*
 1903  *      vm_object_collapse:
 1904  *
 1905  *      Collapse an object with the object backing it.
 1906  *      Pages in the backing object are moved into the
 1907  *      parent, and the backing object is deallocated.
 1908  */
 1909 void
 1910 vm_object_collapse(vm_object_t object)
 1911 {
 1912         vm_object_t backing_object, new_backing_object;
 1913 
 1914         VM_OBJECT_ASSERT_WLOCKED(object);
 1915 
 1916         while (TRUE) {
 1917                 KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,
 1918                     ("collapsing invalid object"));
 1919 
 1920                 /*
 1921                  * Wait for the backing_object to finish any pending
 1922                  * collapse so that the caller sees the shortest possible
 1923                  * shadow chain.
 1924                  */
 1925                 backing_object = vm_object_backing_collapse_wait(object);
 1926                 if (backing_object == NULL)
 1927                         return;
 1928 
 1929                 KASSERT(object->ref_count > 0 &&
 1930                     object->ref_count > atomic_load_int(&object->shadow_count),
 1931                     ("collapse with invalid ref %d or shadow %d count.",
 1932                     object->ref_count, atomic_load_int(&object->shadow_count)));
 1933                 KASSERT((backing_object->flags &
 1934                     (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
 1935                     ("vm_object_collapse: Backing object already collapsing."));
 1936                 KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
 1937                     ("vm_object_collapse: object is already collapsing."));
 1938 
 1939                 /*
 1940                  * We know that we can either collapse the backing object if
 1941                  * the parent is the only reference to it, or (perhaps) have
 1942                  * the parent bypass the object if the parent happens to shadow
 1943                  * all the resident pages in the entire backing object.
 1944                  */
 1945                 if (backing_object->ref_count == 1) {
 1946                         KASSERT(atomic_load_int(&backing_object->shadow_count)
 1947                             == 1,
 1948                             ("vm_object_collapse: shadow_count: %d",
 1949                             atomic_load_int(&backing_object->shadow_count)));
 1950                         vm_object_pip_add(object, 1);
 1951                         vm_object_set_flag(object, OBJ_COLLAPSING);
 1952                         vm_object_pip_add(backing_object, 1);
 1953                         vm_object_set_flag(backing_object, OBJ_DEAD);
 1954 
 1955                         /*
 1956                          * If there is exactly one reference to the backing
 1957                          * object, we can collapse it into the parent.
 1958                          */
 1959                         vm_object_collapse_scan(object);
 1960 
 1961 #if VM_NRESERVLEVEL > 0
 1962                         /*
 1963                          * Break any reservations from backing_object.
 1964                          */
 1965                         if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
 1966                                 vm_reserv_break_all(backing_object);
 1967 #endif
 1968 
 1969                         /*
 1970                          * Move the pager from backing_object to object.
 1971                          *
 1972                          * swap_pager_copy() can sleep, in which case the
 1973                          * backing_object's and object's locks are released and
 1974                          * reacquired.
 1975                          */
 1976                         swap_pager_copy(backing_object, object,
 1977                             OFF_TO_IDX(object->backing_object_offset), TRUE);
 1978 
 1979                         /*
 1980                          * Object now shadows whatever backing_object did.
 1981                          */
 1982                         vm_object_clear_flag(object, OBJ_COLLAPSING);
 1983                         vm_object_backing_transfer(object, backing_object);
 1984                         object->backing_object_offset +=
 1985                             backing_object->backing_object_offset;
 1986                         VM_OBJECT_WUNLOCK(object);
 1987                         vm_object_pip_wakeup(object);
 1988 
 1989                         /*
 1990                          * Discard backing_object.
 1991                          *
 1992                          * Since the backing object has no pages, no pager left,
 1993                          * and no object references within it, all that is
 1994                          * necessary is to dispose of it.
 1995                          */
 1996                         KASSERT(backing_object->ref_count == 1, (
 1997 "backing_object %p was somehow re-referenced during collapse!",
 1998                             backing_object));
 1999                         vm_object_pip_wakeup(backing_object);
 2000                         (void)refcount_release(&backing_object->ref_count);
 2001                         vm_object_terminate(backing_object);
 2002                         counter_u64_add(object_collapses, 1);
 2003                         VM_OBJECT_WLOCK(object);
 2004                 } else {
 2005                         /*
 2006                          * If we do not entirely shadow the backing object,
 2007                          * there is nothing we can do so we give up.
 2008                          *
 2009                          * The object lock and backing_object lock must not
 2010                          * be dropped during this sequence.
 2011                          */
 2012                         if (!vm_object_scan_all_shadowed(object)) {
 2013                                 VM_OBJECT_WUNLOCK(backing_object);
 2014                                 break;
 2015                         }
 2016 
 2017                         /*
 2018                          * Make the parent shadow the next object in the
 2019                          * chain.  Deallocating backing_object will not remove
 2020                          * it, since its reference count is at least 2.
 2021                          */
 2022                         vm_object_backing_remove_locked(object);
 2023                         new_backing_object = backing_object->backing_object;
 2024                         if (new_backing_object != NULL) {
 2025                                 vm_object_backing_insert_ref(object,
 2026                                     new_backing_object);
 2027                                 object->backing_object_offset +=
 2028                                     backing_object->backing_object_offset;
 2029                         }
 2030 
 2031                         /*
 2032                          * Drop the reference count on backing_object. Since
 2033                          * its ref_count was at least 2, it will not vanish.
 2034                          */
 2035                         (void)refcount_release(&backing_object->ref_count);
 2036                         KASSERT(backing_object->ref_count >= 1, (
 2037 "backing_object %p was somehow dereferenced during collapse!",
 2038                             backing_object));
 2039                         VM_OBJECT_WUNLOCK(backing_object);
 2040                         counter_u64_add(object_bypasses, 1);
 2041                 }
 2042 
 2043                 /*
 2044                  * Try again with this object's new backing object.
 2045                  */
 2046         }
 2047 }
 2048 
 2049 /*
 2050  *      vm_object_page_remove:
 2051  *
 2052  *      For the given object, either frees or invalidates each of the
 2053  *      specified pages.  In general, a page is freed.  However, if a page is
 2054  *      wired for any reason other than the existence of a managed, wired
 2055  *      mapping, then it may be invalidated but not removed from the object.
 2056  *      Pages are specified by the given range ["start", "end") and the option
 2057  *      OBJPR_CLEANONLY.  As a special case, if "end" is zero, then the range
 2058  *      extends from "start" to the end of the object.  If the option
 2059  *      OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
 2060  *      specified range are affected.  If the option OBJPR_NOTMAPPED is
 2061  *      specified, then the pages within the specified range must have no
 2062  *      mappings.  Otherwise, if this option is not specified, any mappings to
 2063  *      the specified pages are removed before the pages are freed or
 2064  *      invalidated.
 2065  *
 2066  *      In general, this operation should only be performed on objects that
 2067  *      contain managed pages.  There are, however, two exceptions.  First, it
 2068  *      is performed on the kernel and kmem objects by vm_map_entry_delete().
 2069  *      Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
 2070  *      backed pages.  In both of these cases, the option OBJPR_CLEANONLY must
 2071  *      not be specified and the option OBJPR_NOTMAPPED must be specified.
 2072  *
 2073  *      The object must be locked.
 2074  */
 2075 void
 2076 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
 2077     int options)
 2078 {
 2079         vm_page_t p, next;
 2080 
 2081         VM_OBJECT_ASSERT_WLOCKED(object);
 2082         KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
 2083             (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
 2084             ("vm_object_page_remove: illegal options for object %p", object));
 2085         if (object->resident_page_count == 0)
 2086                 return;
 2087         vm_object_pip_add(object, 1);
 2088 again:
 2089         p = vm_page_find_least(object, start);
 2090 
 2091         /*
 2092          * Here, the variable "p" is either (1) the page with the least pindex
 2093          * greater than or equal to the parameter "start" or (2) NULL. 
 2094          */
 2095         for (; p != NULL && (p->pindex < end || end == 0); p = next) {
 2096                 next = TAILQ_NEXT(p, listq);
 2097 
 2098                 /*
 2099                  * Skip invalid pages if asked to do so.  Try to avoid acquiring
 2100                  * the busy lock, as some consumers rely on this to avoid
 2101                  * deadlocks.
 2102                  *
 2103                  * A thread may concurrently transition the page from invalid to
 2104                  * valid using only the busy lock, so the result of this check
 2105                  * is immediately stale.  It is up to consumers to handle this,
 2106                  * for instance by ensuring that all invalid->valid transitions
 2107                  * happen with a mutex held, as may be possible for a
 2108                  * filesystem.
 2109                  */
 2110                 if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p))
 2111                         continue;
 2112 
 2113                 /*
 2114                  * If the page is wired for any reason besides the existence
 2115                  * of managed, wired mappings, then it cannot be freed.  For
 2116                  * example, fictitious pages, which represent device memory,
 2117                  * are inherently wired and cannot be freed.  They can,
 2118                  * however, be invalidated if the option OBJPR_CLEANONLY is
 2119                  * not specified.
 2120                  */
 2121                 if (vm_page_tryxbusy(p) == 0) {
 2122                         if (vm_page_busy_sleep(p, "vmopar", 0))
 2123                                 VM_OBJECT_WLOCK(object);
 2124                         goto again;
 2125                 }
 2126                 if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p)) {
 2127                         vm_page_xunbusy(p);
 2128                         continue;
 2129                 }
 2130                 if (vm_page_wired(p)) {
 2131 wired:
 2132                         if ((options & OBJPR_NOTMAPPED) == 0 &&
 2133                             object->ref_count != 0)
 2134                                 pmap_remove_all(p);
 2135                         if ((options & OBJPR_CLEANONLY) == 0) {
 2136                                 vm_page_invalid(p);
 2137                                 vm_page_undirty(p);
 2138                         }
 2139                         vm_page_xunbusy(p);
 2140                         continue;
 2141                 }
 2142                 KASSERT((p->flags & PG_FICTITIOUS) == 0,
 2143                     ("vm_object_page_remove: page %p is fictitious", p));
 2144                 if ((options & OBJPR_CLEANONLY) != 0 &&
 2145                     !vm_page_none_valid(p)) {
 2146                         if ((options & OBJPR_NOTMAPPED) == 0 &&
 2147                             object->ref_count != 0 &&
 2148                             !vm_page_try_remove_write(p))
 2149                                 goto wired;
 2150                         if (p->dirty != 0) {
 2151                                 vm_page_xunbusy(p);
 2152                                 continue;
 2153                         }
 2154                 }
 2155                 if ((options & OBJPR_NOTMAPPED) == 0 &&
 2156                     object->ref_count != 0 && !vm_page_try_remove_all(p))
 2157                         goto wired;
 2158                 vm_page_free(p);
 2159         }
 2160         vm_object_pip_wakeup(object);
 2161 
 2162         vm_pager_freespace(object, start, (end == 0 ? object->size : end) -
 2163             start);
 2164 }
 2165 
 2166 /*
 2167  *      vm_object_page_noreuse:
 2168  *
 2169  *      For the given object, attempt to move the specified pages to
 2170  *      the head of the inactive queue.  This bypasses regular LRU
 2171  *      operation and allows the pages to be reused quickly under memory
 2172  *      pressure.  If a page is wired for any reason, then it will not
 2173  *      be queued.  Pages are specified by the range ["start", "end").
 2174  *      As a special case, if "end" is zero, then the range extends from
 2175  *      "start" to the end of the object.
 2176  *
 2177  *      This operation should only be performed on objects that
 2178  *      contain non-fictitious, managed pages.
 2179  *
 2180  *      The object must be locked.
 2181  */
 2182 void
 2183 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
 2184 {
 2185         vm_page_t p, next;
 2186 
 2187         VM_OBJECT_ASSERT_LOCKED(object);
 2188         KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
 2189             ("vm_object_page_noreuse: illegal object %p", object));
 2190         if (object->resident_page_count == 0)
 2191                 return;
 2192         p = vm_page_find_least(object, start);
 2193 
 2194         /*
 2195          * Here, the variable "p" is either (1) the page with the least pindex
 2196          * greater than or equal to the parameter "start" or (2) NULL. 
 2197          */
 2198         for (; p != NULL && (p->pindex < end || end == 0); p = next) {
 2199                 next = TAILQ_NEXT(p, listq);
 2200                 vm_page_deactivate_noreuse(p);
 2201         }
 2202 }
 2203 
 2204 /*
 2205  *      Populate the specified range of the object with valid pages.  Returns
 2206  *      TRUE if the range is successfully populated and FALSE otherwise.
 2207  *
 2208  *      Note: This function should be optimized to pass a larger array of
 2209  *      pages to vm_pager_get_pages() before it is applied to a non-
 2210  *      OBJT_DEVICE object.
 2211  *
 2212  *      The object must be locked.
 2213  */
 2214 boolean_t
 2215 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
 2216 {
 2217         vm_page_t m;
 2218         vm_pindex_t pindex;
 2219         int rv;
 2220 
 2221         VM_OBJECT_ASSERT_WLOCKED(object);
 2222         for (pindex = start; pindex < end; pindex++) {
 2223                 rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);
 2224                 if (rv != VM_PAGER_OK)
 2225                         break;
 2226 
 2227                 /*
 2228                  * Keep "m" busy because a subsequent iteration may unlock
 2229                  * the object.
 2230                  */
 2231         }
 2232         if (pindex > start) {
 2233                 m = vm_page_lookup(object, start);
 2234                 while (m != NULL && m->pindex < pindex) {
 2235                         vm_page_xunbusy(m);
 2236                         m = TAILQ_NEXT(m, listq);
 2237                 }
 2238         }
 2239         return (pindex == end);
 2240 }
 2241 
 2242 /*
 2243  *      Routine:        vm_object_coalesce
 2244  *      Function:       Coalesces two objects backing up adjoining
 2245  *                      regions of memory into a single object.
 2246  *
 2247  *      returns TRUE if objects were combined.
 2248  *
 2249  *      NOTE:   Only works at the moment if the second object is NULL -
 2250  *              if it's not, which object do we lock first?
 2251  *
 2252  *      Parameters:
 2253  *              prev_object     First object to coalesce
 2254  *              prev_offset     Offset into prev_object
 2255  *              prev_size       Size of reference to prev_object
 2256  *              next_size       Size of reference to the second object
 2257  *              reserved        Indicator that extension region has
 2258  *                              swap accounted for
 2259  *
 2260  *      Conditions:
 2261  *      The object must *not* be locked.
 2262  */
 2263 boolean_t
 2264 vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
 2265     vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
 2266 {
 2267         vm_pindex_t next_pindex;
 2268 
 2269         if (prev_object == NULL)
 2270                 return (TRUE);
 2271         if ((prev_object->flags & OBJ_ANON) == 0)
 2272                 return (FALSE);
 2273 
 2274         VM_OBJECT_WLOCK(prev_object);
 2275         /*
 2276          * Try to collapse the object first.
 2277          */
 2278         vm_object_collapse(prev_object);
 2279 
 2280         /*
 2281          * Can't coalesce if: . more than one reference . paged out . shadows
 2282          * another object . has a copy elsewhere (any of which mean that the
 2283          * pages not mapped to prev_entry may be in use anyway)
 2284          */
 2285         if (prev_object->backing_object != NULL) {
 2286                 VM_OBJECT_WUNLOCK(prev_object);
 2287                 return (FALSE);
 2288         }
 2289 
 2290         prev_size >>= PAGE_SHIFT;
 2291         next_size >>= PAGE_SHIFT;
 2292         next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
 2293 
 2294         if (prev_object->ref_count > 1 &&
 2295             prev_object->size != next_pindex &&
 2296             (prev_object->flags & OBJ_ONEMAPPING) == 0) {
 2297                 VM_OBJECT_WUNLOCK(prev_object);
 2298                 return (FALSE);
 2299         }
 2300 
 2301         /*
 2302          * Account for the charge.
 2303          */
 2304         if (prev_object->cred != NULL) {
 2305                 /*
 2306                  * If prev_object was charged, then this mapping,
 2307                  * although not charged now, may become writable
 2308                  * later. Non-NULL cred in the object would prevent
 2309                  * swap reservation during enabling of the write
 2310                  * access, so reserve swap now. Failed reservation
 2311                  * cause allocation of the separate object for the map
 2312                  * entry, and swap reservation for this entry is
 2313                  * managed in appropriate time.
 2314                  */
 2315                 if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
 2316                     prev_object->cred)) {
 2317                         VM_OBJECT_WUNLOCK(prev_object);
 2318                         return (FALSE);
 2319                 }
 2320                 prev_object->charge += ptoa(next_size);
 2321         }
 2322 
 2323         /*
 2324          * Remove any pages that may still be in the object from a previous
 2325          * deallocation.
 2326          */
 2327         if (next_pindex < prev_object->size) {
 2328                 vm_object_page_remove(prev_object, next_pindex, next_pindex +
 2329                     next_size, 0);
 2330 #if 0
 2331                 if (prev_object->cred != NULL) {
 2332                         KASSERT(prev_object->charge >=
 2333                             ptoa(prev_object->size - next_pindex),
 2334                             ("object %p overcharged 1 %jx %jx", prev_object,
 2335                                 (uintmax_t)next_pindex, (uintmax_t)next_size));
 2336                         prev_object->charge -= ptoa(prev_object->size -
 2337                             next_pindex);
 2338                 }
 2339 #endif
 2340         }
 2341 
 2342         /*
 2343          * Extend the object if necessary.
 2344          */
 2345         if (next_pindex + next_size > prev_object->size)
 2346                 prev_object->size = next_pindex + next_size;
 2347 
 2348         VM_OBJECT_WUNLOCK(prev_object);
 2349         return (TRUE);
 2350 }
 2351 
 2352 void
 2353 vm_object_set_writeable_dirty_(vm_object_t object)
 2354 {
 2355         atomic_add_int(&object->generation, 1);
 2356 }
 2357 
 2358 bool
 2359 vm_object_mightbedirty_(vm_object_t object)
 2360 {
 2361         return (object->generation != object->cleangeneration);
 2362 }
 2363 
 2364 /*
 2365  *      vm_object_unwire:
 2366  *
 2367  *      For each page offset within the specified range of the given object,
 2368  *      find the highest-level page in the shadow chain and unwire it.  A page
 2369  *      must exist at every page offset, and the highest-level page must be
 2370  *      wired.
 2371  */
 2372 void
 2373 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
 2374     uint8_t queue)
 2375 {
 2376         vm_object_t tobject, t1object;
 2377         vm_page_t m, tm;
 2378         vm_pindex_t end_pindex, pindex, tpindex;
 2379         int depth, locked_depth;
 2380 
 2381         KASSERT((offset & PAGE_MASK) == 0,
 2382             ("vm_object_unwire: offset is not page aligned"));
 2383         KASSERT((length & PAGE_MASK) == 0,
 2384             ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
 2385         /* The wired count of a fictitious page never changes. */
 2386         if ((object->flags & OBJ_FICTITIOUS) != 0)
 2387                 return;
 2388         pindex = OFF_TO_IDX(offset);
 2389         end_pindex = pindex + atop(length);
 2390 again:
 2391         locked_depth = 1;
 2392         VM_OBJECT_RLOCK(object);
 2393         m = vm_page_find_least(object, pindex);
 2394         while (pindex < end_pindex) {
 2395                 if (m == NULL || pindex < m->pindex) {
 2396                         /*
 2397                          * The first object in the shadow chain doesn't
 2398                          * contain a page at the current index.  Therefore,
 2399                          * the page must exist in a backing object.
 2400                          */
 2401                         tobject = object;
 2402                         tpindex = pindex;
 2403                         depth = 0;
 2404                         do {
 2405                                 tpindex +=
 2406                                     OFF_TO_IDX(tobject->backing_object_offset);
 2407                                 tobject = tobject->backing_object;
 2408                                 KASSERT(tobject != NULL,
 2409                                     ("vm_object_unwire: missing page"));
 2410                                 if ((tobject->flags & OBJ_FICTITIOUS) != 0)
 2411                                         goto next_page;
 2412                                 depth++;
 2413                                 if (depth == locked_depth) {
 2414                                         locked_depth++;
 2415                                         VM_OBJECT_RLOCK(tobject);
 2416                                 }
 2417                         } while ((tm = vm_page_lookup(tobject, tpindex)) ==
 2418                             NULL);
 2419                 } else {
 2420                         tm = m;
 2421                         m = TAILQ_NEXT(m, listq);
 2422                 }
 2423                 if (vm_page_trysbusy(tm) == 0) {
 2424                         for (tobject = object; locked_depth >= 1;
 2425                             locked_depth--) {
 2426                                 t1object = tobject->backing_object;
 2427                                 if (tm->object != tobject)
 2428                                         VM_OBJECT_RUNLOCK(tobject);
 2429                                 tobject = t1object;
 2430                         }
 2431                         tobject = tm->object;
 2432                         if (!vm_page_busy_sleep(tm, "unwbo",
 2433                             VM_ALLOC_IGN_SBUSY))
 2434                                 VM_OBJECT_RUNLOCK(tobject);
 2435                         goto again;
 2436                 }
 2437                 vm_page_unwire(tm, queue);
 2438                 vm_page_sunbusy(tm);
 2439 next_page:
 2440                 pindex++;
 2441         }
 2442         /* Release the accumulated object locks. */
 2443         for (tobject = object; locked_depth >= 1; locked_depth--) {
 2444                 t1object = tobject->backing_object;
 2445                 VM_OBJECT_RUNLOCK(tobject);
 2446                 tobject = t1object;
 2447         }
 2448 }
 2449 
 2450 /*
 2451  * Return the vnode for the given object, or NULL if none exists.
 2452  * For tmpfs objects, the function may return NULL if there is
 2453  * no vnode allocated at the time of the call.
 2454  */
 2455 struct vnode *
 2456 vm_object_vnode(vm_object_t object)
 2457 {
 2458         struct vnode *vp;
 2459 
 2460         VM_OBJECT_ASSERT_LOCKED(object);
 2461         vm_pager_getvp(object, &vp, NULL);
 2462         return (vp);
 2463 }
 2464 
 2465 /*
 2466  * Busy the vm object.  This prevents new pages belonging to the object from
 2467  * becoming busy.  Existing pages persist as busy.  Callers are responsible
 2468  * for checking page state before proceeding.
 2469  */
 2470 void
 2471 vm_object_busy(vm_object_t obj)
 2472 {
 2473 
 2474         VM_OBJECT_ASSERT_LOCKED(obj);
 2475 
 2476         blockcount_acquire(&obj->busy, 1);
 2477         /* The fence is required to order loads of page busy. */
 2478         atomic_thread_fence_acq_rel();
 2479 }
 2480 
 2481 void
 2482 vm_object_unbusy(vm_object_t obj)
 2483 {
 2484 
 2485         blockcount_release(&obj->busy, 1);
 2486 }
 2487 
 2488 void
 2489 vm_object_busy_wait(vm_object_t obj, const char *wmesg)
 2490 {
 2491 
 2492         VM_OBJECT_ASSERT_UNLOCKED(obj);
 2493 
 2494         (void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
 2495 }
 2496 
 2497 /*
 2498  * This function aims to determine if the object is mapped,
 2499  * specifically, if it is referenced by a vm_map_entry.  Because
 2500  * objects occasionally acquire transient references that do not
 2501  * represent a mapping, the method used here is inexact.  However, it
 2502  * has very low overhead and is good enough for the advisory
 2503  * vm.vmtotal sysctl.
 2504  */
 2505 bool
 2506 vm_object_is_active(vm_object_t obj)
 2507 {
 2508 
 2509         return (obj->ref_count > atomic_load_int(&obj->shadow_count));
 2510 }
 2511 
 2512 static int
 2513 vm_object_list_handler(struct sysctl_req *req, bool swap_only)
 2514 {
 2515         struct kinfo_vmobject *kvo;
 2516         char *fullpath, *freepath;
 2517         struct vnode *vp;
 2518         struct vattr va;
 2519         vm_object_t obj;
 2520         vm_page_t m;
 2521         u_long sp;
 2522         int count, error;
 2523 
 2524         if (req->oldptr == NULL) {
 2525                 /*
 2526                  * If an old buffer has not been provided, generate an
 2527                  * estimate of the space needed for a subsequent call.
 2528                  */
 2529                 mtx_lock(&vm_object_list_mtx);
 2530                 count = 0;
 2531                 TAILQ_FOREACH(obj, &vm_object_list, object_list) {
 2532                         if (obj->type == OBJT_DEAD)
 2533                                 continue;
 2534                         count++;
 2535                 }
 2536                 mtx_unlock(&vm_object_list_mtx);
 2537                 return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
 2538                     count * 11 / 10));
 2539         }
 2540 
 2541         kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK | M_ZERO);
 2542         error = 0;
 2543 
 2544         /*
 2545          * VM objects are type stable and are never removed from the
 2546          * list once added.  This allows us to safely read obj->object_list
 2547          * after reacquiring the VM object lock.
 2548          */
 2549         mtx_lock(&vm_object_list_mtx);
 2550         TAILQ_FOREACH(obj, &vm_object_list, object_list) {
 2551                 if (obj->type == OBJT_DEAD ||
 2552                     (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0))
 2553                         continue;
 2554                 VM_OBJECT_RLOCK(obj);
 2555                 if (obj->type == OBJT_DEAD ||
 2556                     (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
 2557                         VM_OBJECT_RUNLOCK(obj);
 2558                         continue;
 2559                 }
 2560                 mtx_unlock(&vm_object_list_mtx);
 2561                 kvo->kvo_size = ptoa(obj->size);
 2562                 kvo->kvo_resident = obj->resident_page_count;
 2563                 kvo->kvo_ref_count = obj->ref_count;
 2564                 kvo->kvo_shadow_count = atomic_load_int(&obj->shadow_count);
 2565                 kvo->kvo_memattr = obj->memattr;
 2566                 kvo->kvo_active = 0;
 2567                 kvo->kvo_inactive = 0;
 2568                 if (!swap_only) {
 2569                         TAILQ_FOREACH(m, &obj->memq, listq) {
 2570                                 /*
 2571                                  * A page may belong to the object but be
 2572                                  * dequeued and set to PQ_NONE while the
 2573                                  * object lock is not held.  This makes the
 2574                                  * reads of m->queue below racy, and we do not
 2575                                  * count pages set to PQ_NONE.  However, this
 2576                                  * sysctl is only meant to give an
 2577                                  * approximation of the system anyway.
 2578                                  */
 2579                                 if (m->a.queue == PQ_ACTIVE)
 2580                                         kvo->kvo_active++;
 2581                                 else if (m->a.queue == PQ_INACTIVE)
 2582                                         kvo->kvo_inactive++;
 2583                         }
 2584                 }
 2585 
 2586                 kvo->kvo_vn_fileid = 0;
 2587                 kvo->kvo_vn_fsid = 0;
 2588                 kvo->kvo_vn_fsid_freebsd11 = 0;
 2589                 freepath = NULL;
 2590                 fullpath = "";
 2591                 vp = NULL;
 2592                 kvo->kvo_type = vm_object_kvme_type(obj, swap_only ? NULL : &vp);
 2593                 if (vp != NULL) {
 2594                         vref(vp);
 2595                 } else if ((obj->flags & OBJ_ANON) != 0) {
 2596                         MPASS(kvo->kvo_type == KVME_TYPE_SWAP);
 2597                         kvo->kvo_me = (uintptr_t)obj;
 2598                         /* tmpfs objs are reported as vnodes */
 2599                         kvo->kvo_backing_obj = (uintptr_t)obj->backing_object;
 2600                         sp = swap_pager_swapped_pages(obj);
 2601                         kvo->kvo_swapped = sp > UINT32_MAX ? UINT32_MAX : sp;
 2602                 }
 2603                 VM_OBJECT_RUNLOCK(obj);
 2604                 if (vp != NULL) {
 2605                         vn_fullpath(vp, &fullpath, &freepath);
 2606                         vn_lock(vp, LK_SHARED | LK_RETRY);
 2607                         if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
 2608                                 kvo->kvo_vn_fileid = va.va_fileid;
 2609                                 kvo->kvo_vn_fsid = va.va_fsid;
 2610                                 kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
 2611                                                                 /* truncate */
 2612                         }
 2613                         vput(vp);
 2614                 }
 2615 
 2616                 strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
 2617                 if (freepath != NULL)
 2618                         free(freepath, M_TEMP);
 2619 
 2620                 /* Pack record size down */
 2621                 kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
 2622                     + strlen(kvo->kvo_path) + 1;
 2623                 kvo->kvo_structsize = roundup(kvo->kvo_structsize,
 2624                     sizeof(uint64_t));
 2625                 error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
 2626                 maybe_yield();
 2627                 mtx_lock(&vm_object_list_mtx);
 2628                 if (error)
 2629                         break;
 2630         }
 2631         mtx_unlock(&vm_object_list_mtx);
 2632         free(kvo, M_TEMP);
 2633         return (error);
 2634 }
 2635 
 2636 static int
 2637 sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
 2638 {
 2639         return (vm_object_list_handler(req, false));
 2640 }
 2641 
 2642 SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
 2643     CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
 2644     "List of VM objects");
 2645 
 2646 static int
 2647 sysctl_vm_object_list_swap(SYSCTL_HANDLER_ARGS)
 2648 {
 2649         return (vm_object_list_handler(req, true));
 2650 }
 2651 
 2652 /*
 2653  * This sysctl returns list of the anonymous or swap objects. Intent
 2654  * is to provide stripped optimized list useful to analyze swap use.
 2655  * Since technically non-swap (default) objects participate in the
 2656  * shadow chains, and are converted to swap type as needed by swap
 2657  * pager, we must report them.
 2658  */
 2659 SYSCTL_PROC(_vm, OID_AUTO, swap_objects,
 2660     CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE, NULL, 0,
 2661     sysctl_vm_object_list_swap, "S,kinfo_vmobject",
 2662     "List of swap VM objects");
 2663 
 2664 #include "opt_ddb.h"
 2665 #ifdef DDB
 2666 #include <sys/kernel.h>
 2667 
 2668 #include <sys/cons.h>
 2669 
 2670 #include <ddb/ddb.h>
 2671 
 2672 static int
 2673 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
 2674 {
 2675         vm_map_t tmpm;
 2676         vm_map_entry_t tmpe;
 2677         vm_object_t obj;
 2678 
 2679         if (map == 0)
 2680                 return 0;
 2681 
 2682         if (entry == 0) {
 2683                 VM_MAP_ENTRY_FOREACH(tmpe, map) {
 2684                         if (_vm_object_in_map(map, object, tmpe)) {
 2685                                 return 1;
 2686                         }
 2687                 }
 2688         } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
 2689                 tmpm = entry->object.sub_map;
 2690                 VM_MAP_ENTRY_FOREACH(tmpe, tmpm) {
 2691                         if (_vm_object_in_map(tmpm, object, tmpe)) {
 2692                                 return 1;
 2693                         }
 2694                 }
 2695         } else if ((obj = entry->object.vm_object) != NULL) {
 2696                 for (; obj; obj = obj->backing_object)
 2697                         if (obj == object) {
 2698                                 return 1;
 2699                         }
 2700         }
 2701         return 0;
 2702 }
 2703 
 2704 static int
 2705 vm_object_in_map(vm_object_t object)
 2706 {
 2707         struct proc *p;
 2708 
 2709         /* sx_slock(&allproc_lock); */
 2710         FOREACH_PROC_IN_SYSTEM(p) {
 2711                 if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
 2712                         continue;
 2713                 if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
 2714                         /* sx_sunlock(&allproc_lock); */
 2715                         return 1;
 2716                 }
 2717         }
 2718         /* sx_sunlock(&allproc_lock); */
 2719         if (_vm_object_in_map(kernel_map, object, 0))
 2720                 return 1;
 2721         return 0;
 2722 }
 2723 
 2724 DB_SHOW_COMMAND_FLAGS(vmochk, vm_object_check, DB_CMD_MEMSAFE)
 2725 {
 2726         vm_object_t object;
 2727 
 2728         /*
 2729          * make sure that internal objs are in a map somewhere
 2730          * and none have zero ref counts.
 2731          */
 2732         TAILQ_FOREACH(object, &vm_object_list, object_list) {
 2733                 if ((object->flags & OBJ_ANON) != 0) {
 2734                         if (object->ref_count == 0) {
 2735                                 db_printf("vmochk: internal obj has zero ref count: %ld\n",
 2736                                         (long)object->size);
 2737                         }
 2738                         if (!vm_object_in_map(object)) {
 2739                                 db_printf(
 2740                         "vmochk: internal obj is not in a map: "
 2741                         "ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
 2742                                     object->ref_count, (u_long)object->size, 
 2743                                     (u_long)object->size,
 2744                                     (void *)object->backing_object);
 2745                         }
 2746                 }
 2747                 if (db_pager_quit)
 2748                         return;
 2749         }
 2750 }
 2751 
 2752 /*
 2753  *      vm_object_print:        [ debug ]
 2754  */
 2755 DB_SHOW_COMMAND(object, vm_object_print_static)
 2756 {
 2757         /* XXX convert args. */
 2758         vm_object_t object = (vm_object_t)addr;
 2759         boolean_t full = have_addr;
 2760 
 2761         vm_page_t p;
 2762 
 2763         /* XXX count is an (unused) arg.  Avoid shadowing it. */
 2764 #define count   was_count
 2765 
 2766         int count;
 2767 
 2768         if (object == NULL)
 2769                 return;
 2770 
 2771         db_iprintf(
 2772             "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
 2773             object, (int)object->type, (uintmax_t)object->size,
 2774             object->resident_page_count, object->ref_count, object->flags,
 2775             object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
 2776         db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
 2777             atomic_load_int(&object->shadow_count),
 2778             object->backing_object ? object->backing_object->ref_count : 0,
 2779             object->backing_object, (uintmax_t)object->backing_object_offset);
 2780 
 2781         if (!full)
 2782                 return;
 2783 
 2784         db_indent += 2;
 2785         count = 0;
 2786         TAILQ_FOREACH(p, &object->memq, listq) {
 2787                 if (count == 0)
 2788                         db_iprintf("memory:=");
 2789                 else if (count == 6) {
 2790                         db_printf("\n");
 2791                         db_iprintf(" ...");
 2792                         count = 0;
 2793                 } else
 2794                         db_printf(",");
 2795                 count++;
 2796 
 2797                 db_printf("(off=0x%jx,page=0x%jx)",
 2798                     (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
 2799 
 2800                 if (db_pager_quit)
 2801                         break;
 2802         }
 2803         if (count != 0)
 2804                 db_printf("\n");
 2805         db_indent -= 2;
 2806 }
 2807 
 2808 /* XXX. */
 2809 #undef count
 2810 
 2811 /* XXX need this non-static entry for calling from vm_map_print. */
 2812 void
 2813 vm_object_print(
 2814         /* db_expr_t */ long addr,
 2815         boolean_t have_addr,
 2816         /* db_expr_t */ long count,
 2817         char *modif)
 2818 {
 2819         vm_object_print_static(addr, have_addr, count, modif);
 2820 }
 2821 
 2822 DB_SHOW_COMMAND_FLAGS(vmopag, vm_object_print_pages, DB_CMD_MEMSAFE)
 2823 {
 2824         vm_object_t object;
 2825         vm_pindex_t fidx;
 2826         vm_paddr_t pa;
 2827         vm_page_t m, prev_m;
 2828         int rcount;
 2829 
 2830         TAILQ_FOREACH(object, &vm_object_list, object_list) {
 2831                 db_printf("new object: %p\n", (void *)object);
 2832                 if (db_pager_quit)
 2833                         return;
 2834 
 2835                 rcount = 0;
 2836                 fidx = 0;
 2837                 pa = -1;
 2838                 TAILQ_FOREACH(m, &object->memq, listq) {
 2839                         if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
 2840                             prev_m->pindex + 1 != m->pindex) {
 2841                                 if (rcount) {
 2842                                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
 2843                                                 (long)fidx, rcount, (long)pa);
 2844                                         if (db_pager_quit)
 2845                                                 return;
 2846                                         rcount = 0;
 2847                                 }
 2848                         }                               
 2849                         if (rcount &&
 2850                                 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
 2851                                 ++rcount;
 2852                                 continue;
 2853                         }
 2854                         if (rcount) {
 2855                                 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
 2856                                         (long)fidx, rcount, (long)pa);
 2857                                 if (db_pager_quit)
 2858                                         return;
 2859                         }
 2860                         fidx = m->pindex;
 2861                         pa = VM_PAGE_TO_PHYS(m);
 2862                         rcount = 1;
 2863                 }
 2864                 if (rcount) {
 2865                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
 2866                                 (long)fidx, rcount, (long)pa);
 2867                         if (db_pager_quit)
 2868                                 return;
 2869                 }
 2870         }
 2871 }
 2872 #endif /* DDB */

Cache object: c02f8cfbbba8f3bb9be5a47fdfa3772d


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.