The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_page.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * The Mach Operating System project at Carnegie-Mellon University.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 4. Neither the name of the University nor the names of its contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      from: @(#)vm_page.c     7.4 (Berkeley) 5/7/91
   33  */
   34 
   35 /*-
   36  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   37  * All rights reserved.
   38  *
   39  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   40  *
   41  * Permission to use, copy, modify and distribute this software and
   42  * its documentation is hereby granted, provided that both the copyright
   43  * notice and this permission notice appear in all copies of the
   44  * software, derivative works or modified versions, and any portions
   45  * thereof, and that both notices appear in supporting documentation.
   46  *
   47  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   48  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   49  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   50  *
   51  * Carnegie Mellon requests users of this software to return to
   52  *
   53  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   54  *  School of Computer Science
   55  *  Carnegie Mellon University
   56  *  Pittsburgh PA 15213-3890
   57  *
   58  * any improvements or extensions that they make and grant Carnegie the
   59  * rights to redistribute these changes.
   60  */
   61 
   62 /*
   63  *                      GENERAL RULES ON VM_PAGE MANIPULATION
   64  *
   65  *      - a pageq mutex is required when adding or removing a page from a
   66  *        page queue (vm_page_queue[]), regardless of other mutexes or the
   67  *        busy state of a page.
   68  *
   69  *      - a hash chain mutex is required when associating or disassociating
   70  *        a page from the VM PAGE CACHE hash table (vm_page_buckets),
   71  *        regardless of other mutexes or the busy state of a page.
   72  *
   73  *      - either a hash chain mutex OR a busied page is required in order
   74  *        to modify the page flags.  A hash chain mutex must be obtained in
   75  *        order to busy a page.  A page's flags cannot be modified by a
   76  *        hash chain mutex if the page is marked busy.
   77  *
   78  *      - The object memq mutex is held when inserting or removing
   79  *        pages from an object (vm_page_insert() or vm_page_remove()).  This
   80  *        is different from the object's main mutex.
   81  *
   82  *      Generally speaking, you have to be aware of side effects when running
   83  *      vm_page ops.  A vm_page_lookup() will return with the hash chain
   84  *      locked, whether it was able to lookup the page or not.  vm_page_free(),
   85  *      vm_page_cache(), vm_page_activate(), and a number of other routines
   86  *      will release the hash chain mutex for you.  Intermediate manipulation
   87  *      routines such as vm_page_flag_set() expect the hash chain to be held
   88  *      on entry and the hash chain will remain held on return.
   89  *
   90  *      pageq scanning can only occur with the pageq in question locked.
   91  *      We have a known bottleneck with the active queue, but the cache
   92  *      and free queues are actually arrays already. 
   93  */
   94 
   95 /*
   96  *      Resident memory management module.
   97  */
   98 
   99 #include <sys/cdefs.h>
  100 __FBSDID("$FreeBSD: src/sys/vm/vm_page.c,v 1.304.2.2 2005/11/13 08:44:25 alc Exp $");
  101 
  102 #include <sys/param.h>
  103 #include <sys/systm.h>
  104 #include <sys/lock.h>
  105 #include <sys/kernel.h>
  106 #include <sys/malloc.h>
  107 #include <sys/mutex.h>
  108 #include <sys/proc.h>
  109 #include <sys/sysctl.h>
  110 #include <sys/vmmeter.h>
  111 #include <sys/vnode.h>
  112 
  113 #include <vm/vm.h>
  114 #include <vm/vm_param.h>
  115 #include <vm/vm_kern.h>
  116 #include <vm/vm_object.h>
  117 #include <vm/vm_page.h>
  118 #include <vm/vm_pageout.h>
  119 #include <vm/vm_pager.h>
  120 #include <vm/vm_extern.h>
  121 #include <vm/uma.h>
  122 #include <vm/uma_int.h>
  123 
  124 /*
  125  *      Associated with page of user-allocatable memory is a
  126  *      page structure.
  127  */
  128 
  129 struct mtx vm_page_queue_mtx;
  130 struct mtx vm_page_queue_free_mtx;
  131 
  132 vm_page_t vm_page_array = 0;
  133 int vm_page_array_size = 0;
  134 long first_page = 0;
  135 int vm_page_zero_count = 0;
  136 
  137 static int boot_pages = UMA_BOOT_PAGES;
  138 TUNABLE_INT("vm.boot_pages", &boot_pages);
  139 SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0,
  140         "number of pages allocated for bootstrapping the VM system");
  141 
  142 /*
  143  *      vm_set_page_size:
  144  *
  145  *      Sets the page size, perhaps based upon the memory
  146  *      size.  Must be called before any use of page-size
  147  *      dependent functions.
  148  */
  149 void
  150 vm_set_page_size(void)
  151 {
  152         if (cnt.v_page_size == 0)
  153                 cnt.v_page_size = PAGE_SIZE;
  154         if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0)
  155                 panic("vm_set_page_size: page size not a power of two");
  156 }
  157 
  158 /*
  159  *      vm_page_startup:
  160  *
  161  *      Initializes the resident memory module.
  162  *
  163  *      Allocates memory for the page cells, and
  164  *      for the object/offset-to-page hash table headers.
  165  *      Each page cell is initialized and placed on the free list.
  166  */
  167 vm_offset_t
  168 vm_page_startup(vm_offset_t vaddr)
  169 {
  170         vm_offset_t mapped;
  171         vm_size_t npages;
  172         vm_paddr_t page_range;
  173         vm_paddr_t new_end;
  174         int i;
  175         vm_paddr_t pa;
  176         int nblocks;
  177         vm_paddr_t last_pa;
  178 
  179         /* the biggest memory array is the second group of pages */
  180         vm_paddr_t end;
  181         vm_paddr_t biggestsize;
  182         int biggestone;
  183 
  184         vm_paddr_t total;
  185 
  186         total = 0;
  187         biggestsize = 0;
  188         biggestone = 0;
  189         nblocks = 0;
  190         vaddr = round_page(vaddr);
  191 
  192         for (i = 0; phys_avail[i + 1]; i += 2) {
  193                 phys_avail[i] = round_page(phys_avail[i]);
  194                 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
  195         }
  196 
  197         for (i = 0; phys_avail[i + 1]; i += 2) {
  198                 vm_paddr_t size = phys_avail[i + 1] - phys_avail[i];
  199 
  200                 if (size > biggestsize) {
  201                         biggestone = i;
  202                         biggestsize = size;
  203                 }
  204                 ++nblocks;
  205                 total += size;
  206         }
  207 
  208         end = phys_avail[biggestone+1];
  209 
  210         /*
  211          * Initialize the locks.
  212          */
  213         mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF |
  214             MTX_RECURSE);
  215         mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL,
  216             MTX_SPIN);
  217 
  218         /*
  219          * Initialize the queue headers for the free queue, the active queue
  220          * and the inactive queue.
  221          */
  222         vm_pageq_init();
  223 
  224         /*
  225          * Allocate memory for use when boot strapping the kernel memory
  226          * allocator.
  227          */
  228         new_end = end - (boot_pages * UMA_SLAB_SIZE);
  229         new_end = trunc_page(new_end);
  230         mapped = pmap_map(&vaddr, new_end, end,
  231             VM_PROT_READ | VM_PROT_WRITE);
  232         bzero((void *)mapped, end - new_end);
  233         uma_startup((void *)mapped, boot_pages);
  234 
  235         /*
  236          * Compute the number of pages of memory that will be available for
  237          * use (taking into account the overhead of a page structure per
  238          * page).
  239          */
  240         first_page = phys_avail[0] / PAGE_SIZE;
  241         page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page;
  242         npages = (total - (page_range * sizeof(struct vm_page)) -
  243             (end - new_end)) / PAGE_SIZE;
  244         end = new_end;
  245 
  246         /*
  247          * Reserve an unmapped guard page to trap access to vm_page_array[-1].
  248          */
  249         vaddr += PAGE_SIZE;
  250 
  251         /*
  252          * Initialize the mem entry structures now, and put them in the free
  253          * queue.
  254          */
  255         new_end = trunc_page(end - page_range * sizeof(struct vm_page));
  256         mapped = pmap_map(&vaddr, new_end, end,
  257             VM_PROT_READ | VM_PROT_WRITE);
  258         vm_page_array = (vm_page_t) mapped;
  259         phys_avail[biggestone + 1] = new_end;
  260 
  261         /*
  262          * Clear all of the page structures
  263          */
  264         bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
  265         vm_page_array_size = page_range;
  266 
  267         /*
  268          * Construct the free queue(s) in descending order (by physical
  269          * address) so that the first 16MB of physical memory is allocated
  270          * last rather than first.  On large-memory machines, this avoids
  271          * the exhaustion of low physical memory before isa_dma_init has run.
  272          */
  273         cnt.v_page_count = 0;
  274         cnt.v_free_count = 0;
  275         for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) {
  276                 pa = phys_avail[i];
  277                 last_pa = phys_avail[i + 1];
  278                 while (pa < last_pa && npages-- > 0) {
  279                         vm_pageq_add_new_page(pa);
  280                         pa += PAGE_SIZE;
  281                 }
  282         }
  283         return (vaddr);
  284 }
  285 
  286 void
  287 vm_page_flag_set(vm_page_t m, unsigned short bits)
  288 {
  289 
  290         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  291         m->flags |= bits;
  292 } 
  293 
  294 void
  295 vm_page_flag_clear(vm_page_t m, unsigned short bits)
  296 {
  297 
  298         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  299         m->flags &= ~bits;
  300 }
  301 
  302 void
  303 vm_page_busy(vm_page_t m)
  304 {
  305 
  306         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  307         KASSERT((m->flags & PG_BUSY) == 0,
  308             ("vm_page_busy: page already busy!!!"));
  309         vm_page_flag_set(m, PG_BUSY);
  310 }
  311 
  312 /*
  313  *      vm_page_flash:
  314  *
  315  *      wakeup anyone waiting for the page.
  316  */
  317 void
  318 vm_page_flash(vm_page_t m)
  319 {
  320 
  321         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  322         if (m->flags & PG_WANTED) {
  323                 vm_page_flag_clear(m, PG_WANTED);
  324                 wakeup(m);
  325         }
  326 }
  327 
  328 /*
  329  *      vm_page_wakeup:
  330  *
  331  *      clear the PG_BUSY flag and wakeup anyone waiting for the
  332  *      page.
  333  *
  334  */
  335 void
  336 vm_page_wakeup(vm_page_t m)
  337 {
  338 
  339         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  340         KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!"));
  341         vm_page_flag_clear(m, PG_BUSY);
  342         vm_page_flash(m);
  343 }
  344 
  345 void
  346 vm_page_io_start(vm_page_t m)
  347 {
  348 
  349         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  350         m->busy++;
  351 }
  352 
  353 void
  354 vm_page_io_finish(vm_page_t m)
  355 {
  356 
  357         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  358         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  359         m->busy--;
  360         if (m->busy == 0)
  361                 vm_page_flash(m);
  362 }
  363 
  364 /*
  365  * Keep page from being freed by the page daemon
  366  * much of the same effect as wiring, except much lower
  367  * overhead and should be used only for *very* temporary
  368  * holding ("wiring").
  369  */
  370 void
  371 vm_page_hold(vm_page_t mem)
  372 {
  373 
  374         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  375         mem->hold_count++;
  376 }
  377 
  378 void
  379 vm_page_unhold(vm_page_t mem)
  380 {
  381 
  382         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  383         --mem->hold_count;
  384         KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
  385         if (mem->hold_count == 0 && mem->queue == PQ_HOLD)
  386                 vm_page_free_toq(mem);
  387 }
  388 
  389 /*
  390  *      vm_page_free:
  391  *
  392  *      Free a page
  393  *
  394  *      The clearing of PG_ZERO is a temporary safety until the code can be
  395  *      reviewed to determine that PG_ZERO is being properly cleared on
  396  *      write faults or maps.  PG_ZERO was previously cleared in
  397  *      vm_page_alloc().
  398  */
  399 void
  400 vm_page_free(vm_page_t m)
  401 {
  402         vm_page_flag_clear(m, PG_ZERO);
  403         vm_page_free_toq(m);
  404         vm_page_zero_idle_wakeup();
  405 }
  406 
  407 /*
  408  *      vm_page_free_zero:
  409  *
  410  *      Free a page to the zerod-pages queue
  411  */
  412 void
  413 vm_page_free_zero(vm_page_t m)
  414 {
  415         vm_page_flag_set(m, PG_ZERO);
  416         vm_page_free_toq(m);
  417 }
  418 
  419 /*
  420  *      vm_page_sleep_if_busy:
  421  *
  422  *      Sleep and release the page queues lock if PG_BUSY is set or,
  423  *      if also_m_busy is TRUE, busy is non-zero.  Returns TRUE if the
  424  *      thread slept and the page queues lock was released.
  425  *      Otherwise, retains the page queues lock and returns FALSE.
  426  */
  427 int
  428 vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
  429 {
  430         vm_object_t object;
  431 
  432         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  433         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  434         if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
  435                 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
  436                 /*
  437                  * It's possible that while we sleep, the page will get
  438                  * unbusied and freed.  If we are holding the object
  439                  * lock, we will assume we hold a reference to the object
  440                  * such that even if m->object changes, we can re-lock
  441                  * it.
  442                  */
  443                 object = m->object;
  444                 VM_OBJECT_UNLOCK(object);
  445                 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
  446                 VM_OBJECT_LOCK(object);
  447                 return (TRUE);
  448         }
  449         return (FALSE);
  450 }
  451 
  452 /*
  453  *      vm_page_dirty:
  454  *
  455  *      make page all dirty
  456  */
  457 void
  458 vm_page_dirty(vm_page_t m)
  459 {
  460         KASSERT(m->queue - m->pc != PQ_CACHE,
  461             ("vm_page_dirty: page in cache!"));
  462         KASSERT(m->queue - m->pc != PQ_FREE,
  463             ("vm_page_dirty: page is free!"));
  464         m->dirty = VM_PAGE_BITS_ALL;
  465 }
  466 
  467 /*
  468  *      vm_page_splay:
  469  *
  470  *      Implements Sleator and Tarjan's top-down splay algorithm.  Returns
  471  *      the vm_page containing the given pindex.  If, however, that
  472  *      pindex is not found in the vm_object, returns a vm_page that is
  473  *      adjacent to the pindex, coming before or after it.
  474  */
  475 vm_page_t
  476 vm_page_splay(vm_pindex_t pindex, vm_page_t root)
  477 {
  478         struct vm_page dummy;
  479         vm_page_t lefttreemax, righttreemin, y;
  480 
  481         if (root == NULL)
  482                 return (root);
  483         lefttreemax = righttreemin = &dummy;
  484         for (;; root = y) {
  485                 if (pindex < root->pindex) {
  486                         if ((y = root->left) == NULL)
  487                                 break;
  488                         if (pindex < y->pindex) {
  489                                 /* Rotate right. */
  490                                 root->left = y->right;
  491                                 y->right = root;
  492                                 root = y;
  493                                 if ((y = root->left) == NULL)
  494                                         break;
  495                         }
  496                         /* Link into the new root's right tree. */
  497                         righttreemin->left = root;
  498                         righttreemin = root;
  499                 } else if (pindex > root->pindex) {
  500                         if ((y = root->right) == NULL)
  501                                 break;
  502                         if (pindex > y->pindex) {
  503                                 /* Rotate left. */
  504                                 root->right = y->left;
  505                                 y->left = root;
  506                                 root = y;
  507                                 if ((y = root->right) == NULL)
  508                                         break;
  509                         }
  510                         /* Link into the new root's left tree. */
  511                         lefttreemax->right = root;
  512                         lefttreemax = root;
  513                 } else
  514                         break;
  515         }
  516         /* Assemble the new root. */
  517         lefttreemax->right = root->left;
  518         righttreemin->left = root->right;
  519         root->left = dummy.right;
  520         root->right = dummy.left;
  521         return (root);
  522 }
  523 
  524 /*
  525  *      vm_page_insert:         [ internal use only ]
  526  *
  527  *      Inserts the given mem entry into the object and object list.
  528  *
  529  *      The pagetables are not updated but will presumably fault the page
  530  *      in if necessary, or if a kernel page the caller will at some point
  531  *      enter the page into the kernel's pmap.  We are not allowed to block
  532  *      here so we *can't* do this anyway.
  533  *
  534  *      The object and page must be locked.
  535  *      This routine may not block.
  536  */
  537 void
  538 vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
  539 {
  540         vm_page_t root;
  541 
  542         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  543         if (m->object != NULL)
  544                 panic("vm_page_insert: page already inserted");
  545 
  546         /*
  547          * Record the object/offset pair in this page
  548          */
  549         m->object = object;
  550         m->pindex = pindex;
  551 
  552         /*
  553          * Now link into the object's ordered list of backed pages.
  554          */
  555         root = object->root;
  556         if (root == NULL) {
  557                 m->left = NULL;
  558                 m->right = NULL;
  559                 TAILQ_INSERT_TAIL(&object->memq, m, listq);
  560         } else {
  561                 root = vm_page_splay(pindex, root);
  562                 if (pindex < root->pindex) {
  563                         m->left = root->left;
  564                         m->right = root;
  565                         root->left = NULL;
  566                         TAILQ_INSERT_BEFORE(root, m, listq);
  567                 } else if (pindex == root->pindex)
  568                         panic("vm_page_insert: offset already allocated");
  569                 else {
  570                         m->right = root->right;
  571                         m->left = root;
  572                         root->right = NULL;
  573                         TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
  574                 }
  575         }
  576         object->root = m;
  577         object->generation++;
  578 
  579         /*
  580          * show that the object has one more resident page.
  581          */
  582         object->resident_page_count++;
  583         /*
  584          * Hold the vnode until the last page is released.
  585          */
  586         if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
  587                 vhold((struct vnode *)object->handle);
  588 
  589         /*
  590          * Since we are inserting a new and possibly dirty page,
  591          * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags.
  592          */
  593         if (m->flags & PG_WRITEABLE)
  594                 vm_object_set_writeable_dirty(object);
  595 }
  596 
  597 /*
  598  *      vm_page_remove:
  599  *                              NOTE: used by device pager as well -wfj
  600  *
  601  *      Removes the given mem entry from the object/offset-page
  602  *      table and the object page list, but do not invalidate/terminate
  603  *      the backing store.
  604  *
  605  *      The object and page must be locked.
  606  *      The underlying pmap entry (if any) is NOT removed here.
  607  *      This routine may not block.
  608  */
  609 void
  610 vm_page_remove(vm_page_t m)
  611 {
  612         vm_object_t object;
  613         vm_page_t root;
  614 
  615         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  616         if ((object = m->object) == NULL)
  617                 return;
  618         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  619         if (m->flags & PG_BUSY) {
  620                 vm_page_flag_clear(m, PG_BUSY);
  621                 vm_page_flash(m);
  622         }
  623 
  624         /*
  625          * Now remove from the object's list of backed pages.
  626          */
  627         if (m != object->root)
  628                 vm_page_splay(m->pindex, object->root);
  629         if (m->left == NULL)
  630                 root = m->right;
  631         else {
  632                 root = vm_page_splay(m->pindex, m->left);
  633                 root->right = m->right;
  634         }
  635         object->root = root;
  636         TAILQ_REMOVE(&object->memq, m, listq);
  637 
  638         /*
  639          * And show that the object has one fewer resident page.
  640          */
  641         object->resident_page_count--;
  642         object->generation++;
  643         /*
  644          * The vnode may now be recycled.
  645          */
  646         if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
  647                 vdrop((struct vnode *)object->handle);
  648 
  649         m->object = NULL;
  650 }
  651 
  652 /*
  653  *      vm_page_lookup:
  654  *
  655  *      Returns the page associated with the object/offset
  656  *      pair specified; if none is found, NULL is returned.
  657  *
  658  *      The object must be locked.
  659  *      This routine may not block.
  660  *      This is a critical path routine
  661  */
  662 vm_page_t
  663 vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
  664 {
  665         vm_page_t m;
  666 
  667         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  668         if ((m = object->root) != NULL && m->pindex != pindex) {
  669                 m = vm_page_splay(pindex, m);
  670                 if ((object->root = m)->pindex != pindex)
  671                         m = NULL;
  672         }
  673         return (m);
  674 }
  675 
  676 /*
  677  *      vm_page_rename:
  678  *
  679  *      Move the given memory entry from its
  680  *      current object to the specified target object/offset.
  681  *
  682  *      The object must be locked.
  683  *      This routine may not block.
  684  *
  685  *      Note: swap associated with the page must be invalidated by the move.  We
  686  *            have to do this for several reasons:  (1) we aren't freeing the
  687  *            page, (2) we are dirtying the page, (3) the VM system is probably
  688  *            moving the page from object A to B, and will then later move
  689  *            the backing store from A to B and we can't have a conflict.
  690  *
  691  *      Note: we *always* dirty the page.  It is necessary both for the
  692  *            fact that we moved it, and because we may be invalidating
  693  *            swap.  If the page is on the cache, we have to deactivate it
  694  *            or vm_page_dirty() will panic.  Dirty pages are not allowed
  695  *            on the cache.
  696  */
  697 void
  698 vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
  699 {
  700 
  701         vm_page_remove(m);
  702         vm_page_insert(m, new_object, new_pindex);
  703         if (m->queue - m->pc == PQ_CACHE)
  704                 vm_page_deactivate(m);
  705         vm_page_dirty(m);
  706 }
  707 
  708 /*
  709  *      vm_page_select_cache:
  710  *
  711  *      Move a page of the given color from the cache queue to the free
  712  *      queue.  As pages might be found, but are not applicable, they are
  713  *      deactivated.
  714  *
  715  *      This routine may not block.
  716  */
  717 vm_page_t
  718 vm_page_select_cache(int color)
  719 {
  720         vm_object_t object;
  721         vm_page_t m;
  722         boolean_t was_trylocked;
  723 
  724         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  725         while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) {
  726                 KASSERT(m->dirty == 0, ("Found dirty cache page %p", m));
  727                 KASSERT(!pmap_page_is_mapped(m),
  728                     ("Found mapped cache page %p", m));
  729                 KASSERT((m->flags & PG_UNMANAGED) == 0,
  730                     ("Found unmanaged cache page %p", m));
  731                 KASSERT(m->wire_count == 0, ("Found wired cache page %p", m));
  732                 if (m->hold_count == 0 && (object = m->object,
  733                     (was_trylocked = VM_OBJECT_TRYLOCK(object)) ||
  734                     VM_OBJECT_LOCKED(object))) {
  735                         KASSERT((m->flags & PG_BUSY) == 0 && m->busy == 0,
  736                             ("Found busy cache page %p", m));
  737                         vm_page_free(m);
  738                         if (was_trylocked)
  739                                 VM_OBJECT_UNLOCK(object);
  740                         break;
  741                 }
  742                 vm_page_deactivate(m);
  743         }
  744         return (m);
  745 }
  746 
  747 /*
  748  *      vm_page_alloc:
  749  *
  750  *      Allocate and return a memory cell associated
  751  *      with this VM object/offset pair.
  752  *
  753  *      page_req classes:
  754  *      VM_ALLOC_NORMAL         normal process request
  755  *      VM_ALLOC_SYSTEM         system *really* needs a page
  756  *      VM_ALLOC_INTERRUPT      interrupt time request
  757  *      VM_ALLOC_ZERO           zero page
  758  *
  759  *      This routine may not block.
  760  *
  761  *      Additional special handling is required when called from an
  762  *      interrupt (VM_ALLOC_INTERRUPT).  We are not allowed to mess with
  763  *      the page cache in this case.
  764  */
  765 vm_page_t
  766 vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
  767 {
  768         vm_page_t m = NULL;
  769         int color, flags, page_req;
  770 
  771         page_req = req & VM_ALLOC_CLASS_MASK;
  772         KASSERT(curthread->td_intr_nesting_level == 0 ||
  773             page_req == VM_ALLOC_INTERRUPT,
  774             ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
  775 
  776         if ((req & VM_ALLOC_NOOBJ) == 0) {
  777                 KASSERT(object != NULL,
  778                     ("vm_page_alloc: NULL object."));
  779                 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  780                 color = (pindex + object->pg_color) & PQ_L2_MASK;
  781         } else
  782                 color = pindex & PQ_L2_MASK;
  783 
  784         /*
  785          * The pager is allowed to eat deeper into the free page list.
  786          */
  787         if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
  788                 page_req = VM_ALLOC_SYSTEM;
  789         };
  790 
  791 loop:
  792         mtx_lock_spin(&vm_page_queue_free_mtx);
  793         if (cnt.v_free_count > cnt.v_free_reserved ||
  794             (page_req == VM_ALLOC_SYSTEM && 
  795              cnt.v_cache_count == 0 && 
  796              cnt.v_free_count > cnt.v_interrupt_free_min) ||
  797             (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) {
  798                 /*
  799                  * Allocate from the free queue if the number of free pages
  800                  * exceeds the minimum for the request class.
  801                  */
  802                 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
  803         } else if (page_req != VM_ALLOC_INTERRUPT) {
  804                 mtx_unlock_spin(&vm_page_queue_free_mtx);
  805                 /*
  806                  * Allocatable from cache (non-interrupt only).  On success,
  807                  * we must free the page and try again, thus ensuring that
  808                  * cnt.v_*_free_min counters are replenished.
  809                  */
  810                 vm_page_lock_queues();
  811                 if ((m = vm_page_select_cache(color)) == NULL) {
  812 #if defined(DIAGNOSTIC)
  813                         if (cnt.v_cache_count > 0)
  814                                 printf("vm_page_alloc(NORMAL): missing pages on cache queue: %d\n", cnt.v_cache_count);
  815 #endif
  816                         vm_page_unlock_queues();
  817                         atomic_add_int(&vm_pageout_deficit, 1);
  818                         pagedaemon_wakeup();
  819                         return (NULL);
  820                 }
  821                 vm_page_unlock_queues();
  822                 goto loop;
  823         } else {
  824                 /*
  825                  * Not allocatable from cache from interrupt, give up.
  826                  */
  827                 mtx_unlock_spin(&vm_page_queue_free_mtx);
  828                 atomic_add_int(&vm_pageout_deficit, 1);
  829                 pagedaemon_wakeup();
  830                 return (NULL);
  831         }
  832 
  833         /*
  834          *  At this point we had better have found a good page.
  835          */
  836 
  837         KASSERT(
  838             m != NULL,
  839             ("vm_page_alloc(): missing page on free queue")
  840         );
  841 
  842         /*
  843          * Remove from free queue
  844          */
  845         vm_pageq_remove_nowakeup(m);
  846 
  847         /*
  848          * Initialize structure.  Only the PG_ZERO flag is inherited.
  849          */
  850         flags = PG_BUSY;
  851         if (m->flags & PG_ZERO) {
  852                 vm_page_zero_count--;
  853                 if (req & VM_ALLOC_ZERO)
  854                         flags = PG_ZERO | PG_BUSY;
  855         }
  856         if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
  857                 flags &= ~PG_BUSY;
  858         m->flags = flags;
  859         if (req & VM_ALLOC_WIRED) {
  860                 atomic_add_int(&cnt.v_wire_count, 1);
  861                 m->wire_count = 1;
  862         } else
  863                 m->wire_count = 0;
  864         m->hold_count = 0;
  865         m->act_count = 0;
  866         m->busy = 0;
  867         m->valid = 0;
  868         KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
  869         mtx_unlock_spin(&vm_page_queue_free_mtx);
  870 
  871         if ((req & VM_ALLOC_NOOBJ) == 0)
  872                 vm_page_insert(m, object, pindex);
  873         else
  874                 m->pindex = pindex;
  875 
  876         /*
  877          * Don't wakeup too often - wakeup the pageout daemon when
  878          * we would be nearly out of memory.
  879          */
  880         if (vm_paging_needed())
  881                 pagedaemon_wakeup();
  882 
  883         return (m);
  884 }
  885 
  886 /*
  887  *      vm_wait:        (also see VM_WAIT macro)
  888  *
  889  *      Block until free pages are available for allocation
  890  *      - Called in various places before memory allocations.
  891  */
  892 void
  893 vm_wait(void)
  894 {
  895 
  896         vm_page_lock_queues();
  897         if (curproc == pageproc) {
  898                 vm_pageout_pages_needed = 1;
  899                 msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx,
  900                     PDROP | PSWP, "VMWait", 0);
  901         } else {
  902                 if (!vm_pages_needed) {
  903                         vm_pages_needed = 1;
  904                         wakeup(&vm_pages_needed);
  905                 }
  906                 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
  907                     "vmwait", 0);
  908         }
  909 }
  910 
  911 /*
  912  *      vm_waitpfault:  (also see VM_WAITPFAULT macro)
  913  *
  914  *      Block until free pages are available for allocation
  915  *      - Called only in vm_fault so that processes page faulting
  916  *        can be easily tracked.
  917  *      - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
  918  *        processes will be able to grab memory first.  Do not change
  919  *        this balance without careful testing first.
  920  */
  921 void
  922 vm_waitpfault(void)
  923 {
  924 
  925         vm_page_lock_queues();
  926         if (!vm_pages_needed) {
  927                 vm_pages_needed = 1;
  928                 wakeup(&vm_pages_needed);
  929         }
  930         msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER,
  931             "pfault", 0);
  932 }
  933 
  934 /*
  935  *      vm_page_activate:
  936  *
  937  *      Put the specified page on the active list (if appropriate).
  938  *      Ensure that act_count is at least ACT_INIT but do not otherwise
  939  *      mess with it.
  940  *
  941  *      The page queues must be locked.
  942  *      This routine may not block.
  943  */
  944 void
  945 vm_page_activate(vm_page_t m)
  946 {
  947 
  948         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  949         if (m->queue != PQ_ACTIVE) {
  950                 if ((m->queue - m->pc) == PQ_CACHE)
  951                         cnt.v_reactivated++;
  952                 vm_pageq_remove(m);
  953                 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
  954                         if (m->act_count < ACT_INIT)
  955                                 m->act_count = ACT_INIT;
  956                         vm_pageq_enqueue(PQ_ACTIVE, m);
  957                 }
  958         } else {
  959                 if (m->act_count < ACT_INIT)
  960                         m->act_count = ACT_INIT;
  961         }
  962 }
  963 
  964 /*
  965  *      vm_page_free_wakeup:
  966  *
  967  *      Helper routine for vm_page_free_toq() and vm_page_cache().  This
  968  *      routine is called when a page has been added to the cache or free
  969  *      queues.
  970  *
  971  *      The page queues must be locked.
  972  *      This routine may not block.
  973  */
  974 static __inline void
  975 vm_page_free_wakeup(void)
  976 {
  977 
  978         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  979         /*
  980          * if pageout daemon needs pages, then tell it that there are
  981          * some free.
  982          */
  983         if (vm_pageout_pages_needed &&
  984             cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
  985                 wakeup(&vm_pageout_pages_needed);
  986                 vm_pageout_pages_needed = 0;
  987         }
  988         /*
  989          * wakeup processes that are waiting on memory if we hit a
  990          * high water mark. And wakeup scheduler process if we have
  991          * lots of memory. this process will swapin processes.
  992          */
  993         if (vm_pages_needed && !vm_page_count_min()) {
  994                 vm_pages_needed = 0;
  995                 wakeup(&cnt.v_free_count);
  996         }
  997 }
  998 
  999 /*
 1000  *      vm_page_free_toq:
 1001  *
 1002  *      Returns the given page to the PQ_FREE list,
 1003  *      disassociating it with any VM object.
 1004  *
 1005  *      Object and page must be locked prior to entry.
 1006  *      This routine may not block.
 1007  */
 1008 
 1009 void
 1010 vm_page_free_toq(vm_page_t m)
 1011 {
 1012         struct vpgqueues *pq;
 1013 
 1014         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1015         cnt.v_tfree++;
 1016 
 1017         if (m->busy || ((m->queue - m->pc) == PQ_FREE)) {
 1018                 printf(
 1019                 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n",
 1020                     (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0,
 1021                     m->hold_count);
 1022                 if ((m->queue - m->pc) == PQ_FREE)
 1023                         panic("vm_page_free: freeing free page");
 1024                 else
 1025                         panic("vm_page_free: freeing busy page");
 1026         }
 1027 
 1028         /*
 1029          * unqueue, then remove page.  Note that we cannot destroy
 1030          * the page here because we do not want to call the pager's
 1031          * callback routine until after we've put the page on the
 1032          * appropriate free queue.
 1033          */
 1034         vm_pageq_remove_nowakeup(m);
 1035         vm_page_remove(m);
 1036 
 1037         /*
 1038          * If fictitious remove object association and
 1039          * return, otherwise delay object association removal.
 1040          */
 1041         if ((m->flags & PG_FICTITIOUS) != 0) {
 1042                 return;
 1043         }
 1044 
 1045         m->valid = 0;
 1046         vm_page_undirty(m);
 1047 
 1048         if (m->wire_count != 0) {
 1049                 if (m->wire_count > 1) {
 1050                         panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
 1051                                 m->wire_count, (long)m->pindex);
 1052                 }
 1053                 panic("vm_page_free: freeing wired page");
 1054         }
 1055 
 1056         /*
 1057          * Clear the UNMANAGED flag when freeing an unmanaged page.
 1058          */
 1059         if (m->flags & PG_UNMANAGED) {
 1060                 m->flags &= ~PG_UNMANAGED;
 1061         }
 1062 
 1063         if (m->hold_count != 0) {
 1064                 m->flags &= ~PG_ZERO;
 1065                 m->queue = PQ_HOLD;
 1066         } else
 1067                 m->queue = PQ_FREE + m->pc;
 1068         pq = &vm_page_queues[m->queue];
 1069         mtx_lock_spin(&vm_page_queue_free_mtx);
 1070         pq->lcnt++;
 1071         ++(*pq->cnt);
 1072 
 1073         /*
 1074          * Put zero'd pages on the end ( where we look for zero'd pages
 1075          * first ) and non-zerod pages at the head.
 1076          */
 1077         if (m->flags & PG_ZERO) {
 1078                 TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
 1079                 ++vm_page_zero_count;
 1080         } else {
 1081                 TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
 1082         }
 1083         mtx_unlock_spin(&vm_page_queue_free_mtx);
 1084         vm_page_free_wakeup();
 1085 }
 1086 
 1087 /*
 1088  *      vm_page_unmanage:
 1089  *
 1090  *      Prevent PV management from being done on the page.  The page is
 1091  *      removed from the paging queues as if it were wired, and as a 
 1092  *      consequence of no longer being managed the pageout daemon will not
 1093  *      touch it (since there is no way to locate the pte mappings for the
 1094  *      page).  madvise() calls that mess with the pmap will also no longer
 1095  *      operate on the page.
 1096  *
 1097  *      Beyond that the page is still reasonably 'normal'.  Freeing the page
 1098  *      will clear the flag.
 1099  *
 1100  *      This routine is used by OBJT_PHYS objects - objects using unswappable
 1101  *      physical memory as backing store rather then swap-backed memory and
 1102  *      will eventually be extended to support 4MB unmanaged physical 
 1103  *      mappings.
 1104  */
 1105 void
 1106 vm_page_unmanage(vm_page_t m)
 1107 {
 1108 
 1109         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1110         if ((m->flags & PG_UNMANAGED) == 0) {
 1111                 if (m->wire_count == 0)
 1112                         vm_pageq_remove(m);
 1113         }
 1114         vm_page_flag_set(m, PG_UNMANAGED);
 1115 }
 1116 
 1117 /*
 1118  *      vm_page_wire:
 1119  *
 1120  *      Mark this page as wired down by yet
 1121  *      another map, removing it from paging queues
 1122  *      as necessary.
 1123  *
 1124  *      The page queues must be locked.
 1125  *      This routine may not block.
 1126  */
 1127 void
 1128 vm_page_wire(vm_page_t m)
 1129 {
 1130 
 1131         /*
 1132          * Only bump the wire statistics if the page is not already wired,
 1133          * and only unqueue the page if it is on some queue (if it is unmanaged
 1134          * it is already off the queues).
 1135          */
 1136         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1137         if (m->flags & PG_FICTITIOUS)
 1138                 return;
 1139         if (m->wire_count == 0) {
 1140                 if ((m->flags & PG_UNMANAGED) == 0)
 1141                         vm_pageq_remove(m);
 1142                 atomic_add_int(&cnt.v_wire_count, 1);
 1143         }
 1144         m->wire_count++;
 1145         KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
 1146 }
 1147 
 1148 /*
 1149  *      vm_page_unwire:
 1150  *
 1151  *      Release one wiring of this page, potentially
 1152  *      enabling it to be paged again.
 1153  *
 1154  *      Many pages placed on the inactive queue should actually go
 1155  *      into the cache, but it is difficult to figure out which.  What
 1156  *      we do instead, if the inactive target is well met, is to put
 1157  *      clean pages at the head of the inactive queue instead of the tail.
 1158  *      This will cause them to be moved to the cache more quickly and
 1159  *      if not actively re-referenced, freed more quickly.  If we just
 1160  *      stick these pages at the end of the inactive queue, heavy filesystem
 1161  *      meta-data accesses can cause an unnecessary paging load on memory bound 
 1162  *      processes.  This optimization causes one-time-use metadata to be
 1163  *      reused more quickly.
 1164  *
 1165  *      BUT, if we are in a low-memory situation we have no choice but to
 1166  *      put clean pages on the cache queue.
 1167  *
 1168  *      A number of routines use vm_page_unwire() to guarantee that the page
 1169  *      will go into either the inactive or active queues, and will NEVER
 1170  *      be placed in the cache - for example, just after dirtying a page.
 1171  *      dirty pages in the cache are not allowed.
 1172  *
 1173  *      The page queues must be locked.
 1174  *      This routine may not block.
 1175  */
 1176 void
 1177 vm_page_unwire(vm_page_t m, int activate)
 1178 {
 1179 
 1180         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1181         if (m->flags & PG_FICTITIOUS)
 1182                 return;
 1183         if (m->wire_count > 0) {
 1184                 m->wire_count--;
 1185                 if (m->wire_count == 0) {
 1186                         atomic_subtract_int(&cnt.v_wire_count, 1);
 1187                         if (m->flags & PG_UNMANAGED) {
 1188                                 ;
 1189                         } else if (activate)
 1190                                 vm_pageq_enqueue(PQ_ACTIVE, m);
 1191                         else {
 1192                                 vm_page_flag_clear(m, PG_WINATCFLS);
 1193                                 vm_pageq_enqueue(PQ_INACTIVE, m);
 1194                         }
 1195                 }
 1196         } else {
 1197                 panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
 1198         }
 1199 }
 1200 
 1201 
 1202 /*
 1203  * Move the specified page to the inactive queue.  If the page has
 1204  * any associated swap, the swap is deallocated.
 1205  *
 1206  * Normally athead is 0 resulting in LRU operation.  athead is set
 1207  * to 1 if we want this page to be 'as if it were placed in the cache',
 1208  * except without unmapping it from the process address space.
 1209  *
 1210  * This routine may not block.
 1211  */
 1212 static __inline void
 1213 _vm_page_deactivate(vm_page_t m, int athead)
 1214 {
 1215 
 1216         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1217 
 1218         /*
 1219          * Ignore if already inactive.
 1220          */
 1221         if (m->queue == PQ_INACTIVE)
 1222                 return;
 1223         if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
 1224                 if ((m->queue - m->pc) == PQ_CACHE)
 1225                         cnt.v_reactivated++;
 1226                 vm_page_flag_clear(m, PG_WINATCFLS);
 1227                 vm_pageq_remove(m);
 1228                 if (athead)
 1229                         TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 1230                 else
 1231                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 1232                 m->queue = PQ_INACTIVE;
 1233                 vm_page_queues[PQ_INACTIVE].lcnt++;
 1234                 cnt.v_inactive_count++;
 1235         }
 1236 }
 1237 
 1238 void
 1239 vm_page_deactivate(vm_page_t m)
 1240 {
 1241     _vm_page_deactivate(m, 0);
 1242 }
 1243 
 1244 /*
 1245  * vm_page_try_to_cache:
 1246  *
 1247  * Returns 0 on failure, 1 on success
 1248  */
 1249 int
 1250 vm_page_try_to_cache(vm_page_t m)
 1251 {
 1252 
 1253         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1254         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1255         if (m->dirty || m->hold_count || m->busy || m->wire_count ||
 1256             (m->flags & (PG_BUSY|PG_UNMANAGED))) {
 1257                 return (0);
 1258         }
 1259         pmap_remove_all(m);
 1260         if (m->dirty)
 1261                 return (0);
 1262         vm_page_cache(m);
 1263         return (1);
 1264 }
 1265 
 1266 /*
 1267  * vm_page_try_to_free()
 1268  *
 1269  *      Attempt to free the page.  If we cannot free it, we do nothing.
 1270  *      1 is returned on success, 0 on failure.
 1271  */
 1272 int
 1273 vm_page_try_to_free(vm_page_t m)
 1274 {
 1275 
 1276         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1277         if (m->object != NULL)
 1278                 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1279         if (m->dirty || m->hold_count || m->busy || m->wire_count ||
 1280             (m->flags & (PG_BUSY|PG_UNMANAGED))) {
 1281                 return (0);
 1282         }
 1283         pmap_remove_all(m);
 1284         if (m->dirty)
 1285                 return (0);
 1286         vm_page_free(m);
 1287         return (1);
 1288 }
 1289 
 1290 /*
 1291  * vm_page_cache
 1292  *
 1293  * Put the specified page onto the page cache queue (if appropriate).
 1294  *
 1295  * This routine may not block.
 1296  */
 1297 void
 1298 vm_page_cache(vm_page_t m)
 1299 {
 1300 
 1301         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1302         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1303         if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy ||
 1304             m->hold_count || m->wire_count) {
 1305                 printf("vm_page_cache: attempting to cache busy page\n");
 1306                 return;
 1307         }
 1308         if ((m->queue - m->pc) == PQ_CACHE)
 1309                 return;
 1310 
 1311         /*
 1312          * Remove all pmaps and indicate that the page is not
 1313          * writeable or mapped.
 1314          */
 1315         pmap_remove_all(m);
 1316         if (m->dirty != 0) {
 1317                 panic("vm_page_cache: caching a dirty page, pindex: %ld",
 1318                         (long)m->pindex);
 1319         }
 1320         vm_pageq_remove_nowakeup(m);
 1321         vm_pageq_enqueue(PQ_CACHE + m->pc, m);
 1322         vm_page_free_wakeup();
 1323 }
 1324 
 1325 /*
 1326  * vm_page_dontneed
 1327  *
 1328  *      Cache, deactivate, or do nothing as appropriate.  This routine
 1329  *      is typically used by madvise() MADV_DONTNEED.
 1330  *
 1331  *      Generally speaking we want to move the page into the cache so
 1332  *      it gets reused quickly.  However, this can result in a silly syndrome
 1333  *      due to the page recycling too quickly.  Small objects will not be
 1334  *      fully cached.  On the otherhand, if we move the page to the inactive
 1335  *      queue we wind up with a problem whereby very large objects 
 1336  *      unnecessarily blow away our inactive and cache queues.
 1337  *
 1338  *      The solution is to move the pages based on a fixed weighting.  We
 1339  *      either leave them alone, deactivate them, or move them to the cache,
 1340  *      where moving them to the cache has the highest weighting.
 1341  *      By forcing some pages into other queues we eventually force the
 1342  *      system to balance the queues, potentially recovering other unrelated
 1343  *      space from active.  The idea is to not force this to happen too
 1344  *      often.
 1345  */
 1346 void
 1347 vm_page_dontneed(vm_page_t m)
 1348 {
 1349         static int dnweight;
 1350         int dnw;
 1351         int head;
 1352 
 1353         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1354         dnw = ++dnweight;
 1355 
 1356         /*
 1357          * occassionally leave the page alone
 1358          */
 1359         if ((dnw & 0x01F0) == 0 ||
 1360             m->queue == PQ_INACTIVE || 
 1361             m->queue - m->pc == PQ_CACHE
 1362         ) {
 1363                 if (m->act_count >= ACT_INIT)
 1364                         --m->act_count;
 1365                 return;
 1366         }
 1367 
 1368         if (m->dirty == 0 && pmap_is_modified(m))
 1369                 vm_page_dirty(m);
 1370 
 1371         if (m->dirty || (dnw & 0x0070) == 0) {
 1372                 /*
 1373                  * Deactivate the page 3 times out of 32.
 1374                  */
 1375                 head = 0;
 1376         } else {
 1377                 /*
 1378                  * Cache the page 28 times out of every 32.  Note that
 1379                  * the page is deactivated instead of cached, but placed
 1380                  * at the head of the queue instead of the tail.
 1381                  */
 1382                 head = 1;
 1383         }
 1384         _vm_page_deactivate(m, head);
 1385 }
 1386 
 1387 /*
 1388  * Grab a page, waiting until we are waken up due to the page
 1389  * changing state.  We keep on waiting, if the page continues
 1390  * to be in the object.  If the page doesn't exist, first allocate it
 1391  * and then conditionally zero it.
 1392  *
 1393  * This routine may block.
 1394  */
 1395 vm_page_t
 1396 vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
 1397 {
 1398         vm_page_t m;
 1399 
 1400         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1401 retrylookup:
 1402         if ((m = vm_page_lookup(object, pindex)) != NULL) {
 1403                 vm_page_lock_queues();
 1404                 if (m->busy || (m->flags & PG_BUSY)) {
 1405                         vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
 1406                         VM_OBJECT_UNLOCK(object);
 1407                         msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
 1408                         VM_OBJECT_LOCK(object);
 1409                         if ((allocflags & VM_ALLOC_RETRY) == 0)
 1410                                 return (NULL);
 1411                         goto retrylookup;
 1412                 } else {
 1413                         if (allocflags & VM_ALLOC_WIRED)
 1414                                 vm_page_wire(m);
 1415                         if ((allocflags & VM_ALLOC_NOBUSY) == 0)
 1416                                 vm_page_busy(m);
 1417                         vm_page_unlock_queues();
 1418                         return (m);
 1419                 }
 1420         }
 1421         m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
 1422         if (m == NULL) {
 1423                 VM_OBJECT_UNLOCK(object);
 1424                 VM_WAIT;
 1425                 VM_OBJECT_LOCK(object);
 1426                 if ((allocflags & VM_ALLOC_RETRY) == 0)
 1427                         return (NULL);
 1428                 goto retrylookup;
 1429         }
 1430         if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
 1431                 pmap_zero_page(m);
 1432         return (m);
 1433 }
 1434 
 1435 /*
 1436  * Mapping function for valid bits or for dirty bits in
 1437  * a page.  May not block.
 1438  *
 1439  * Inputs are required to range within a page.
 1440  */
 1441 __inline int
 1442 vm_page_bits(int base, int size)
 1443 {
 1444         int first_bit;
 1445         int last_bit;
 1446 
 1447         KASSERT(
 1448             base + size <= PAGE_SIZE,
 1449             ("vm_page_bits: illegal base/size %d/%d", base, size)
 1450         );
 1451 
 1452         if (size == 0)          /* handle degenerate case */
 1453                 return (0);
 1454 
 1455         first_bit = base >> DEV_BSHIFT;
 1456         last_bit = (base + size - 1) >> DEV_BSHIFT;
 1457 
 1458         return ((2 << last_bit) - (1 << first_bit));
 1459 }
 1460 
 1461 /*
 1462  *      vm_page_set_validclean:
 1463  *
 1464  *      Sets portions of a page valid and clean.  The arguments are expected
 1465  *      to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
 1466  *      of any partial chunks touched by the range.  The invalid portion of
 1467  *      such chunks will be zero'd.
 1468  *
 1469  *      This routine may not block.
 1470  *
 1471  *      (base + size) must be less then or equal to PAGE_SIZE.
 1472  */
 1473 void
 1474 vm_page_set_validclean(vm_page_t m, int base, int size)
 1475 {
 1476         int pagebits;
 1477         int frag;
 1478         int endoff;
 1479 
 1480         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1481         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1482         if (size == 0)  /* handle degenerate case */
 1483                 return;
 1484 
 1485         /*
 1486          * If the base is not DEV_BSIZE aligned and the valid
 1487          * bit is clear, we have to zero out a portion of the
 1488          * first block.
 1489          */
 1490         if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
 1491             (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
 1492                 pmap_zero_page_area(m, frag, base - frag);
 1493 
 1494         /*
 1495          * If the ending offset is not DEV_BSIZE aligned and the 
 1496          * valid bit is clear, we have to zero out a portion of
 1497          * the last block.
 1498          */
 1499         endoff = base + size;
 1500         if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
 1501             (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
 1502                 pmap_zero_page_area(m, endoff,
 1503                     DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
 1504 
 1505         /*
 1506          * Set valid, clear dirty bits.  If validating the entire
 1507          * page we can safely clear the pmap modify bit.  We also
 1508          * use this opportunity to clear the PG_NOSYNC flag.  If a process
 1509          * takes a write fault on a MAP_NOSYNC memory area the flag will
 1510          * be set again.
 1511          *
 1512          * We set valid bits inclusive of any overlap, but we can only
 1513          * clear dirty bits for DEV_BSIZE chunks that are fully within
 1514          * the range.
 1515          */
 1516         pagebits = vm_page_bits(base, size);
 1517         m->valid |= pagebits;
 1518 #if 0   /* NOT YET */
 1519         if ((frag = base & (DEV_BSIZE - 1)) != 0) {
 1520                 frag = DEV_BSIZE - frag;
 1521                 base += frag;
 1522                 size -= frag;
 1523                 if (size < 0)
 1524                         size = 0;
 1525         }
 1526         pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
 1527 #endif
 1528         m->dirty &= ~pagebits;
 1529         if (base == 0 && size == PAGE_SIZE) {
 1530                 pmap_clear_modify(m);
 1531                 vm_page_flag_clear(m, PG_NOSYNC);
 1532         }
 1533 }
 1534 
 1535 void
 1536 vm_page_clear_dirty(vm_page_t m, int base, int size)
 1537 {
 1538 
 1539         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1540         m->dirty &= ~vm_page_bits(base, size);
 1541 }
 1542 
 1543 /*
 1544  *      vm_page_set_invalid:
 1545  *
 1546  *      Invalidates DEV_BSIZE'd chunks within a page.  Both the
 1547  *      valid and dirty bits for the effected areas are cleared.
 1548  *
 1549  *      May not block.
 1550  */
 1551 void
 1552 vm_page_set_invalid(vm_page_t m, int base, int size)
 1553 {
 1554         int bits;
 1555 
 1556         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1557         bits = vm_page_bits(base, size);
 1558         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1559         m->valid &= ~bits;
 1560         m->dirty &= ~bits;
 1561         m->object->generation++;
 1562 }
 1563 
 1564 /*
 1565  * vm_page_zero_invalid()
 1566  *
 1567  *      The kernel assumes that the invalid portions of a page contain 
 1568  *      garbage, but such pages can be mapped into memory by user code.
 1569  *      When this occurs, we must zero out the non-valid portions of the
 1570  *      page so user code sees what it expects.
 1571  *
 1572  *      Pages are most often semi-valid when the end of a file is mapped 
 1573  *      into memory and the file's size is not page aligned.
 1574  */
 1575 void
 1576 vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
 1577 {
 1578         int b;
 1579         int i;
 1580 
 1581         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1582         /*
 1583          * Scan the valid bits looking for invalid sections that
 1584          * must be zerod.  Invalid sub-DEV_BSIZE'd areas ( where the
 1585          * valid bit may be set ) have already been zerod by
 1586          * vm_page_set_validclean().
 1587          */
 1588         for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
 1589                 if (i == (PAGE_SIZE / DEV_BSIZE) || 
 1590                     (m->valid & (1 << i))
 1591                 ) {
 1592                         if (i > b) {
 1593                                 pmap_zero_page_area(m, 
 1594                                     b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
 1595                         }
 1596                         b = i + 1;
 1597                 }
 1598         }
 1599 
 1600         /*
 1601          * setvalid is TRUE when we can safely set the zero'd areas
 1602          * as being valid.  We can do this if there are no cache consistancy
 1603          * issues.  e.g. it is ok to do with UFS, but not ok to do with NFS.
 1604          */
 1605         if (setvalid)
 1606                 m->valid = VM_PAGE_BITS_ALL;
 1607 }
 1608 
 1609 /*
 1610  *      vm_page_is_valid:
 1611  *
 1612  *      Is (partial) page valid?  Note that the case where size == 0
 1613  *      will return FALSE in the degenerate case where the page is
 1614  *      entirely invalid, and TRUE otherwise.
 1615  *
 1616  *      May not block.
 1617  */
 1618 int
 1619 vm_page_is_valid(vm_page_t m, int base, int size)
 1620 {
 1621         int bits = vm_page_bits(base, size);
 1622 
 1623         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1624         if (m->valid && ((m->valid & bits) == bits))
 1625                 return 1;
 1626         else
 1627                 return 0;
 1628 }
 1629 
 1630 /*
 1631  * update dirty bits from pmap/mmu.  May not block.
 1632  */
 1633 void
 1634 vm_page_test_dirty(vm_page_t m)
 1635 {
 1636         if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
 1637                 vm_page_dirty(m);
 1638         }
 1639 }
 1640 
 1641 int so_zerocp_fullpage = 0;
 1642 
 1643 void
 1644 vm_page_cowfault(vm_page_t m)
 1645 {
 1646         vm_page_t mnew;
 1647         vm_object_t object;
 1648         vm_pindex_t pindex;
 1649 
 1650         object = m->object;
 1651         pindex = m->pindex;
 1652 
 1653  retry_alloc:
 1654         pmap_remove_all(m);
 1655         vm_page_remove(m);
 1656         mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
 1657         if (mnew == NULL) {
 1658                 vm_page_insert(m, object, pindex);
 1659                 vm_page_unlock_queues();
 1660                 VM_OBJECT_UNLOCK(object);
 1661                 VM_WAIT;
 1662                 VM_OBJECT_LOCK(object);
 1663                 vm_page_lock_queues();
 1664                 goto retry_alloc;
 1665         }
 1666 
 1667         if (m->cow == 0) {
 1668                 /* 
 1669                  * check to see if we raced with an xmit complete when 
 1670                  * waiting to allocate a page.  If so, put things back 
 1671                  * the way they were 
 1672                  */
 1673                 vm_page_free(mnew);
 1674                 vm_page_insert(m, object, pindex);
 1675         } else { /* clear COW & copy page */
 1676                 if (!so_zerocp_fullpage)
 1677                         pmap_copy_page(m, mnew);
 1678                 mnew->valid = VM_PAGE_BITS_ALL;
 1679                 vm_page_dirty(mnew);
 1680                 vm_page_flag_clear(mnew, PG_BUSY);
 1681                 mnew->wire_count = m->wire_count - m->cow;
 1682                 m->wire_count = m->cow;
 1683         }
 1684 }
 1685 
 1686 void 
 1687 vm_page_cowclear(vm_page_t m)
 1688 {
 1689 
 1690         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1691         if (m->cow) {
 1692                 m->cow--;
 1693                 /* 
 1694                  * let vm_fault add back write permission  lazily
 1695                  */
 1696         } 
 1697         /*
 1698          *  sf_buf_free() will free the page, so we needn't do it here
 1699          */ 
 1700 }
 1701 
 1702 void
 1703 vm_page_cowsetup(vm_page_t m)
 1704 {
 1705 
 1706         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1707         m->cow++;
 1708         pmap_page_protect(m, VM_PROT_READ);
 1709 }
 1710 
 1711 #include "opt_ddb.h"
 1712 #ifdef DDB
 1713 #include <sys/kernel.h>
 1714 
 1715 #include <ddb/ddb.h>
 1716 
 1717 DB_SHOW_COMMAND(page, vm_page_print_page_info)
 1718 {
 1719         db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
 1720         db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
 1721         db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
 1722         db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
 1723         db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
 1724         db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
 1725         db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
 1726         db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
 1727         db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
 1728         db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
 1729 }
 1730 
 1731 DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
 1732 {
 1733         int i;
 1734         db_printf("PQ_FREE:");
 1735         for (i = 0; i < PQ_L2_SIZE; i++) {
 1736                 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
 1737         }
 1738         db_printf("\n");
 1739                 
 1740         db_printf("PQ_CACHE:");
 1741         for (i = 0; i < PQ_L2_SIZE; i++) {
 1742                 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
 1743         }
 1744         db_printf("\n");
 1745 
 1746         db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
 1747                 vm_page_queues[PQ_ACTIVE].lcnt,
 1748                 vm_page_queues[PQ_INACTIVE].lcnt);
 1749 }
 1750 #endif /* DDB */

Cache object: fdb0d423c55e21e18462836e2f5e8c32


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.