The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_page.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  *
    5  * This code is derived from software contributed to Berkeley by
    6  * The Mach Operating System project at Carnegie-Mellon University.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  * 4. Neither the name of the University nor the names of its contributors
   17  *    may be used to endorse or promote products derived from this software
   18  *    without specific prior written permission.
   19  *
   20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   30  * SUCH DAMAGE.
   31  *
   32  *      from: @(#)vm_page.c     7.4 (Berkeley) 5/7/91
   33  */
   34 
   35 /*
   36  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
   37  * All rights reserved.
   38  *
   39  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
   40  *
   41  * Permission to use, copy, modify and distribute this software and
   42  * its documentation is hereby granted, provided that both the copyright
   43  * notice and this permission notice appear in all copies of the
   44  * software, derivative works or modified versions, and any portions
   45  * thereof, and that both notices appear in supporting documentation.
   46  *
   47  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
   48  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
   49  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
   50  *
   51  * Carnegie Mellon requests users of this software to return to
   52  *
   53  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
   54  *  School of Computer Science
   55  *  Carnegie Mellon University
   56  *  Pittsburgh PA 15213-3890
   57  *
   58  * any improvements or extensions that they make and grant Carnegie the
   59  * rights to redistribute these changes.
   60  */
   61 
   62 /*
   63  *                      GENERAL RULES ON VM_PAGE MANIPULATION
   64  *
   65  *      - a pageq mutex is required when adding or removing a page from a
   66  *        page queue (vm_page_queue[]), regardless of other mutexes or the
   67  *        busy state of a page.
   68  *
   69  *      - a hash chain mutex is required when associating or disassociating
   70  *        a page from the VM PAGE CACHE hash table (vm_page_buckets),
   71  *        regardless of other mutexes or the busy state of a page.
   72  *
   73  *      - either a hash chain mutex OR a busied page is required in order
   74  *        to modify the page flags.  A hash chain mutex must be obtained in
   75  *        order to busy a page.  A page's flags cannot be modified by a
   76  *        hash chain mutex if the page is marked busy.
   77  *
   78  *      - The object memq mutex is held when inserting or removing
   79  *        pages from an object (vm_page_insert() or vm_page_remove()).  This
   80  *        is different from the object's main mutex.
   81  *
   82  *      Generally speaking, you have to be aware of side effects when running
   83  *      vm_page ops.  A vm_page_lookup() will return with the hash chain
   84  *      locked, whether it was able to lookup the page or not.  vm_page_free(),
   85  *      vm_page_cache(), vm_page_activate(), and a number of other routines
   86  *      will release the hash chain mutex for you.  Intermediate manipulation
   87  *      routines such as vm_page_flag_set() expect the hash chain to be held
   88  *      on entry and the hash chain will remain held on return.
   89  *
   90  *      pageq scanning can only occur with the pageq in question locked.
   91  *      We have a known bottleneck with the active queue, but the cache
   92  *      and free queues are actually arrays already. 
   93  */
   94 
   95 /*
   96  *      Resident memory management module.
   97  */
   98 
   99 #include <sys/cdefs.h>
  100 __FBSDID("$FreeBSD: releng/5.3/sys/vm/vm_page.c 132852 2004-07-29 18:56:31Z alc $");
  101 
  102 #include <sys/param.h>
  103 #include <sys/systm.h>
  104 #include <sys/lock.h>
  105 #include <sys/malloc.h>
  106 #include <sys/mutex.h>
  107 #include <sys/proc.h>
  108 #include <sys/vmmeter.h>
  109 #include <sys/vnode.h>
  110 
  111 #include <vm/vm.h>
  112 #include <vm/vm_param.h>
  113 #include <vm/vm_kern.h>
  114 #include <vm/vm_object.h>
  115 #include <vm/vm_page.h>
  116 #include <vm/vm_pageout.h>
  117 #include <vm/vm_pager.h>
  118 #include <vm/vm_extern.h>
  119 #include <vm/uma.h>
  120 #include <vm/uma_int.h>
  121 
  122 /*
  123  *      Associated with page of user-allocatable memory is a
  124  *      page structure.
  125  */
  126 
  127 struct mtx vm_page_queue_mtx;
  128 struct mtx vm_page_queue_free_mtx;
  129 
  130 vm_page_t vm_page_array = 0;
  131 int vm_page_array_size = 0;
  132 long first_page = 0;
  133 int vm_page_zero_count = 0;
  134 
  135 /*
  136  *      vm_set_page_size:
  137  *
  138  *      Sets the page size, perhaps based upon the memory
  139  *      size.  Must be called before any use of page-size
  140  *      dependent functions.
  141  */
  142 void
  143 vm_set_page_size(void)
  144 {
  145         if (cnt.v_page_size == 0)
  146                 cnt.v_page_size = PAGE_SIZE;
  147         if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0)
  148                 panic("vm_set_page_size: page size not a power of two");
  149 }
  150 
  151 /*
  152  *      vm_page_startup:
  153  *
  154  *      Initializes the resident memory module.
  155  *
  156  *      Allocates memory for the page cells, and
  157  *      for the object/offset-to-page hash table headers.
  158  *      Each page cell is initialized and placed on the free list.
  159  */
  160 vm_offset_t
  161 vm_page_startup(vm_offset_t vaddr)
  162 {
  163         vm_offset_t mapped;
  164         vm_size_t npages;
  165         vm_paddr_t page_range;
  166         vm_paddr_t new_end;
  167         int i;
  168         vm_paddr_t pa;
  169         int nblocks;
  170         vm_paddr_t last_pa;
  171 
  172         /* the biggest memory array is the second group of pages */
  173         vm_paddr_t end;
  174         vm_paddr_t biggestsize;
  175         int biggestone;
  176 
  177         vm_paddr_t total;
  178         vm_size_t bootpages;
  179 
  180         total = 0;
  181         biggestsize = 0;
  182         biggestone = 0;
  183         nblocks = 0;
  184         vaddr = round_page(vaddr);
  185 
  186         for (i = 0; phys_avail[i + 1]; i += 2) {
  187                 phys_avail[i] = round_page(phys_avail[i]);
  188                 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
  189         }
  190 
  191         for (i = 0; phys_avail[i + 1]; i += 2) {
  192                 vm_paddr_t size = phys_avail[i + 1] - phys_avail[i];
  193 
  194                 if (size > biggestsize) {
  195                         biggestone = i;
  196                         biggestsize = size;
  197                 }
  198                 ++nblocks;
  199                 total += size;
  200         }
  201 
  202         end = phys_avail[biggestone+1];
  203 
  204         /*
  205          * Initialize the locks.
  206          */
  207         mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF |
  208             MTX_RECURSE);
  209         mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL,
  210             MTX_SPIN);
  211 
  212         /*
  213          * Initialize the queue headers for the free queue, the active queue
  214          * and the inactive queue.
  215          */
  216         vm_pageq_init();
  217 
  218         /*
  219          * Allocate memory for use when boot strapping the kernel memory
  220          * allocator.
  221          */
  222         bootpages = UMA_BOOT_PAGES * UMA_SLAB_SIZE;
  223         new_end = end - bootpages;
  224         new_end = trunc_page(new_end);
  225         mapped = pmap_map(&vaddr, new_end, end,
  226             VM_PROT_READ | VM_PROT_WRITE);
  227         bzero((caddr_t) mapped, end - new_end);
  228         uma_startup((caddr_t)mapped);
  229 
  230         /*
  231          * Compute the number of pages of memory that will be available for
  232          * use (taking into account the overhead of a page structure per
  233          * page).
  234          */
  235         first_page = phys_avail[0] / PAGE_SIZE;
  236         page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page;
  237         npages = (total - (page_range * sizeof(struct vm_page)) -
  238             (end - new_end)) / PAGE_SIZE;
  239         end = new_end;
  240 
  241         /*
  242          * Reserve an unmapped guard page to trap access to vm_page_array[-1].
  243          */
  244         vaddr += PAGE_SIZE;
  245 
  246         /*
  247          * Initialize the mem entry structures now, and put them in the free
  248          * queue.
  249          */
  250         new_end = trunc_page(end - page_range * sizeof(struct vm_page));
  251         mapped = pmap_map(&vaddr, new_end, end,
  252             VM_PROT_READ | VM_PROT_WRITE);
  253         vm_page_array = (vm_page_t) mapped;
  254         phys_avail[biggestone + 1] = new_end;
  255 
  256         /*
  257          * Clear all of the page structures
  258          */
  259         bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
  260         vm_page_array_size = page_range;
  261 
  262         /*
  263          * Construct the free queue(s) in descending order (by physical
  264          * address) so that the first 16MB of physical memory is allocated
  265          * last rather than first.  On large-memory machines, this avoids
  266          * the exhaustion of low physical memory before isa_dmainit has run.
  267          */
  268         cnt.v_page_count = 0;
  269         cnt.v_free_count = 0;
  270         for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) {
  271                 pa = phys_avail[i];
  272                 last_pa = phys_avail[i + 1];
  273                 while (pa < last_pa && npages-- > 0) {
  274                         vm_pageq_add_new_page(pa);
  275                         pa += PAGE_SIZE;
  276                 }
  277         }
  278         return (vaddr);
  279 }
  280 
  281 void
  282 vm_page_flag_set(vm_page_t m, unsigned short bits)
  283 {
  284 
  285         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  286         m->flags |= bits;
  287 } 
  288 
  289 void
  290 vm_page_flag_clear(vm_page_t m, unsigned short bits)
  291 {
  292 
  293         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  294         m->flags &= ~bits;
  295 }
  296 
  297 void
  298 vm_page_busy(vm_page_t m)
  299 {
  300         KASSERT((m->flags & PG_BUSY) == 0,
  301             ("vm_page_busy: page already busy!!!"));
  302         vm_page_flag_set(m, PG_BUSY);
  303 }
  304 
  305 /*
  306  *      vm_page_flash:
  307  *
  308  *      wakeup anyone waiting for the page.
  309  */
  310 void
  311 vm_page_flash(vm_page_t m)
  312 {
  313         if (m->flags & PG_WANTED) {
  314                 vm_page_flag_clear(m, PG_WANTED);
  315                 wakeup(m);
  316         }
  317 }
  318 
  319 /*
  320  *      vm_page_wakeup:
  321  *
  322  *      clear the PG_BUSY flag and wakeup anyone waiting for the
  323  *      page.
  324  *
  325  */
  326 void
  327 vm_page_wakeup(vm_page_t m)
  328 {
  329         KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!"));
  330         vm_page_flag_clear(m, PG_BUSY);
  331         vm_page_flash(m);
  332 }
  333 
  334 void
  335 vm_page_io_start(vm_page_t m)
  336 {
  337 
  338         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  339         m->busy++;
  340 }
  341 
  342 void
  343 vm_page_io_finish(vm_page_t m)
  344 {
  345 
  346         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  347         m->busy--;
  348         if (m->busy == 0)
  349                 vm_page_flash(m);
  350 }
  351 
  352 /*
  353  * Keep page from being freed by the page daemon
  354  * much of the same effect as wiring, except much lower
  355  * overhead and should be used only for *very* temporary
  356  * holding ("wiring").
  357  */
  358 void
  359 vm_page_hold(vm_page_t mem)
  360 {
  361 
  362         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  363         mem->hold_count++;
  364 }
  365 
  366 void
  367 vm_page_unhold(vm_page_t mem)
  368 {
  369 
  370         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  371         --mem->hold_count;
  372         KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
  373         if (mem->hold_count == 0 && mem->queue == PQ_HOLD)
  374                 vm_page_free_toq(mem);
  375 }
  376 
  377 /*
  378  *      vm_page_free:
  379  *
  380  *      Free a page
  381  *
  382  *      The clearing of PG_ZERO is a temporary safety until the code can be
  383  *      reviewed to determine that PG_ZERO is being properly cleared on
  384  *      write faults or maps.  PG_ZERO was previously cleared in
  385  *      vm_page_alloc().
  386  */
  387 void
  388 vm_page_free(vm_page_t m)
  389 {
  390         vm_page_flag_clear(m, PG_ZERO);
  391         vm_page_free_toq(m);
  392         vm_page_zero_idle_wakeup();
  393 }
  394 
  395 /*
  396  *      vm_page_free_zero:
  397  *
  398  *      Free a page to the zerod-pages queue
  399  */
  400 void
  401 vm_page_free_zero(vm_page_t m)
  402 {
  403         vm_page_flag_set(m, PG_ZERO);
  404         vm_page_free_toq(m);
  405 }
  406 
  407 /*
  408  *      vm_page_sleep_if_busy:
  409  *
  410  *      Sleep and release the page queues lock if PG_BUSY is set or,
  411  *      if also_m_busy is TRUE, busy is non-zero.  Returns TRUE if the
  412  *      thread slept and the page queues lock was released.
  413  *      Otherwise, retains the page queues lock and returns FALSE.
  414  */
  415 int
  416 vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
  417 {
  418         vm_object_t object;
  419         int is_object_locked;
  420 
  421         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  422         if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
  423                 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
  424                 /*
  425                  * It's possible that while we sleep, the page will get
  426                  * unbusied and freed.  If we are holding the object
  427                  * lock, we will assume we hold a reference to the object
  428                  * such that even if m->object changes, we can re-lock
  429                  * it.
  430                  *
  431                  * Remove mtx_owned() after vm_object locking is finished.
  432                  */
  433                 object = m->object;
  434                 if ((is_object_locked = object != NULL &&
  435                      mtx_owned(&object->mtx)))
  436                         mtx_unlock(&object->mtx);
  437                 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
  438                 if (is_object_locked)
  439                         mtx_lock(&object->mtx);
  440                 return (TRUE);
  441         }
  442         return (FALSE);
  443 }
  444 
  445 /*
  446  *      vm_page_dirty:
  447  *
  448  *      make page all dirty
  449  */
  450 void
  451 vm_page_dirty(vm_page_t m)
  452 {
  453         KASSERT(m->queue - m->pc != PQ_CACHE,
  454             ("vm_page_dirty: page in cache!"));
  455         KASSERT(m->queue - m->pc != PQ_FREE,
  456             ("vm_page_dirty: page is free!"));
  457         m->dirty = VM_PAGE_BITS_ALL;
  458 }
  459 
  460 /*
  461  *      vm_page_splay:
  462  *
  463  *      Implements Sleator and Tarjan's top-down splay algorithm.  Returns
  464  *      the vm_page containing the given pindex.  If, however, that
  465  *      pindex is not found in the vm_object, returns a vm_page that is
  466  *      adjacent to the pindex, coming before or after it.
  467  */
  468 vm_page_t
  469 vm_page_splay(vm_pindex_t pindex, vm_page_t root)
  470 {
  471         struct vm_page dummy;
  472         vm_page_t lefttreemax, righttreemin, y;
  473 
  474         if (root == NULL)
  475                 return (root);
  476         lefttreemax = righttreemin = &dummy;
  477         for (;; root = y) {
  478                 if (pindex < root->pindex) {
  479                         if ((y = root->left) == NULL)
  480                                 break;
  481                         if (pindex < y->pindex) {
  482                                 /* Rotate right. */
  483                                 root->left = y->right;
  484                                 y->right = root;
  485                                 root = y;
  486                                 if ((y = root->left) == NULL)
  487                                         break;
  488                         }
  489                         /* Link into the new root's right tree. */
  490                         righttreemin->left = root;
  491                         righttreemin = root;
  492                 } else if (pindex > root->pindex) {
  493                         if ((y = root->right) == NULL)
  494                                 break;
  495                         if (pindex > y->pindex) {
  496                                 /* Rotate left. */
  497                                 root->right = y->left;
  498                                 y->left = root;
  499                                 root = y;
  500                                 if ((y = root->right) == NULL)
  501                                         break;
  502                         }
  503                         /* Link into the new root's left tree. */
  504                         lefttreemax->right = root;
  505                         lefttreemax = root;
  506                 } else
  507                         break;
  508         }
  509         /* Assemble the new root. */
  510         lefttreemax->right = root->left;
  511         righttreemin->left = root->right;
  512         root->left = dummy.right;
  513         root->right = dummy.left;
  514         return (root);
  515 }
  516 
  517 /*
  518  *      vm_page_insert:         [ internal use only ]
  519  *
  520  *      Inserts the given mem entry into the object and object list.
  521  *
  522  *      The pagetables are not updated but will presumably fault the page
  523  *      in if necessary, or if a kernel page the caller will at some point
  524  *      enter the page into the kernel's pmap.  We are not allowed to block
  525  *      here so we *can't* do this anyway.
  526  *
  527  *      The object and page must be locked.
  528  *      This routine may not block.
  529  */
  530 void
  531 vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
  532 {
  533         vm_page_t root;
  534 
  535         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  536         if (m->object != NULL)
  537                 panic("vm_page_insert: page already inserted");
  538 
  539         /*
  540          * Record the object/offset pair in this page
  541          */
  542         m->object = object;
  543         m->pindex = pindex;
  544 
  545         /*
  546          * Now link into the object's ordered list of backed pages.
  547          */
  548         root = object->root;
  549         if (root == NULL) {
  550                 m->left = NULL;
  551                 m->right = NULL;
  552                 TAILQ_INSERT_TAIL(&object->memq, m, listq);
  553         } else {
  554                 root = vm_page_splay(pindex, root);
  555                 if (pindex < root->pindex) {
  556                         m->left = root->left;
  557                         m->right = root;
  558                         root->left = NULL;
  559                         TAILQ_INSERT_BEFORE(root, m, listq);
  560                 } else if (pindex == root->pindex)
  561                         panic("vm_page_insert: offset already allocated");
  562                 else {
  563                         m->right = root->right;
  564                         m->left = root;
  565                         root->right = NULL;
  566                         TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
  567                 }
  568         }
  569         object->root = m;
  570         object->generation++;
  571 
  572         /*
  573          * show that the object has one more resident page.
  574          */
  575         object->resident_page_count++;
  576 
  577         /*
  578          * Since we are inserting a new and possibly dirty page,
  579          * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags.
  580          */
  581         if (m->flags & PG_WRITEABLE)
  582                 vm_object_set_writeable_dirty(object);
  583 }
  584 
  585 /*
  586  *      vm_page_remove:
  587  *                              NOTE: used by device pager as well -wfj
  588  *
  589  *      Removes the given mem entry from the object/offset-page
  590  *      table and the object page list, but do not invalidate/terminate
  591  *      the backing store.
  592  *
  593  *      The object and page must be locked.
  594  *      The underlying pmap entry (if any) is NOT removed here.
  595  *      This routine may not block.
  596  */
  597 void
  598 vm_page_remove(vm_page_t m)
  599 {
  600         vm_object_t object;
  601         vm_page_t root;
  602 
  603         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  604         if (m->object == NULL)
  605                 return;
  606         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  607         if ((m->flags & PG_BUSY) == 0) {
  608                 panic("vm_page_remove: page not busy");
  609         }
  610 
  611         /*
  612          * Basically destroy the page.
  613          */
  614         vm_page_wakeup(m);
  615 
  616         object = m->object;
  617 
  618         /*
  619          * Now remove from the object's list of backed pages.
  620          */
  621         if (m != object->root)
  622                 vm_page_splay(m->pindex, object->root);
  623         if (m->left == NULL)
  624                 root = m->right;
  625         else {
  626                 root = vm_page_splay(m->pindex, m->left);
  627                 root->right = m->right;
  628         }
  629         object->root = root;
  630         TAILQ_REMOVE(&object->memq, m, listq);
  631 
  632         /*
  633          * And show that the object has one fewer resident page.
  634          */
  635         object->resident_page_count--;
  636         object->generation++;
  637 
  638         m->object = NULL;
  639 }
  640 
  641 /*
  642  *      vm_page_lookup:
  643  *
  644  *      Returns the page associated with the object/offset
  645  *      pair specified; if none is found, NULL is returned.
  646  *
  647  *      The object must be locked.
  648  *      This routine may not block.
  649  *      This is a critical path routine
  650  */
  651 vm_page_t
  652 vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
  653 {
  654         vm_page_t m;
  655 
  656         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  657         if ((m = object->root) != NULL && m->pindex != pindex) {
  658                 m = vm_page_splay(pindex, m);
  659                 if ((object->root = m)->pindex != pindex)
  660                         m = NULL;
  661         }
  662         return (m);
  663 }
  664 
  665 /*
  666  *      vm_page_rename:
  667  *
  668  *      Move the given memory entry from its
  669  *      current object to the specified target object/offset.
  670  *
  671  *      The object must be locked.
  672  *      This routine may not block.
  673  *
  674  *      Note: swap associated with the page must be invalidated by the move.  We
  675  *            have to do this for several reasons:  (1) we aren't freeing the
  676  *            page, (2) we are dirtying the page, (3) the VM system is probably
  677  *            moving the page from object A to B, and will then later move
  678  *            the backing store from A to B and we can't have a conflict.
  679  *
  680  *      Note: we *always* dirty the page.  It is necessary both for the
  681  *            fact that we moved it, and because we may be invalidating
  682  *            swap.  If the page is on the cache, we have to deactivate it
  683  *            or vm_page_dirty() will panic.  Dirty pages are not allowed
  684  *            on the cache.
  685  */
  686 void
  687 vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
  688 {
  689 
  690         vm_page_remove(m);
  691         vm_page_insert(m, new_object, new_pindex);
  692         if (m->queue - m->pc == PQ_CACHE)
  693                 vm_page_deactivate(m);
  694         vm_page_dirty(m);
  695 }
  696 
  697 /*
  698  *      vm_page_select_cache:
  699  *
  700  *      Find a page on the cache queue with color optimization.  As pages
  701  *      might be found, but not applicable, they are deactivated.  This
  702  *      keeps us from using potentially busy cached pages.
  703  *
  704  *      This routine may not block.
  705  */
  706 vm_page_t
  707 vm_page_select_cache(int color)
  708 {
  709         vm_page_t m;
  710 
  711         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  712         while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) {
  713                 if ((m->flags & PG_BUSY) == 0 && m->busy == 0 &&
  714                     m->hold_count == 0 && (VM_OBJECT_TRYLOCK(m->object) ||
  715                     VM_OBJECT_LOCKED(m->object))) {
  716                         KASSERT(m->dirty == 0,
  717                             ("Found dirty cache page %p", m));
  718                         KASSERT(!pmap_page_is_mapped(m),
  719                             ("Found mapped cache page %p", m));
  720                         KASSERT((m->flags & PG_UNMANAGED) == 0,
  721                             ("Found unmanaged cache page %p", m));
  722                         KASSERT(m->wire_count == 0,
  723                             ("Found wired cache page %p", m));
  724                         break;
  725                 }
  726                 vm_page_deactivate(m);
  727         }
  728         return (m);
  729 }
  730 
  731 /*
  732  *      vm_page_alloc:
  733  *
  734  *      Allocate and return a memory cell associated
  735  *      with this VM object/offset pair.
  736  *
  737  *      page_req classes:
  738  *      VM_ALLOC_NORMAL         normal process request
  739  *      VM_ALLOC_SYSTEM         system *really* needs a page
  740  *      VM_ALLOC_INTERRUPT      interrupt time request
  741  *      VM_ALLOC_ZERO           zero page
  742  *
  743  *      This routine may not block.
  744  *
  745  *      Additional special handling is required when called from an
  746  *      interrupt (VM_ALLOC_INTERRUPT).  We are not allowed to mess with
  747  *      the page cache in this case.
  748  */
  749 vm_page_t
  750 vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
  751 {
  752         vm_object_t m_object;
  753         vm_page_t m = NULL;
  754         int color, flags, page_req;
  755 
  756         page_req = req & VM_ALLOC_CLASS_MASK;
  757 
  758         if ((req & VM_ALLOC_NOOBJ) == 0) {
  759                 KASSERT(object != NULL,
  760                     ("vm_page_alloc: NULL object."));
  761                 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  762                 color = (pindex + object->pg_color) & PQ_L2_MASK;
  763         } else
  764                 color = pindex & PQ_L2_MASK;
  765 
  766         /*
  767          * The pager is allowed to eat deeper into the free page list.
  768          */
  769         if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
  770                 page_req = VM_ALLOC_SYSTEM;
  771         };
  772 
  773 loop:
  774         mtx_lock_spin(&vm_page_queue_free_mtx);
  775         if (cnt.v_free_count > cnt.v_free_reserved ||
  776             (page_req == VM_ALLOC_SYSTEM && 
  777              cnt.v_cache_count == 0 && 
  778              cnt.v_free_count > cnt.v_interrupt_free_min) ||
  779             (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) {
  780                 /*
  781                  * Allocate from the free queue if the number of free pages
  782                  * exceeds the minimum for the request class.
  783                  */
  784                 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
  785         } else if (page_req != VM_ALLOC_INTERRUPT) {
  786                 mtx_unlock_spin(&vm_page_queue_free_mtx);
  787                 /*
  788                  * Allocatable from cache (non-interrupt only).  On success,
  789                  * we must free the page and try again, thus ensuring that
  790                  * cnt.v_*_free_min counters are replenished.
  791                  */
  792                 vm_page_lock_queues();
  793                 if ((m = vm_page_select_cache(color)) == NULL) {
  794 #if defined(DIAGNOSTIC)
  795                         if (cnt.v_cache_count > 0)
  796                                 printf("vm_page_alloc(NORMAL): missing pages on cache queue: %d\n", cnt.v_cache_count);
  797 #endif
  798                         vm_page_unlock_queues();
  799                         atomic_add_int(&vm_pageout_deficit, 1);
  800                         pagedaemon_wakeup();
  801                         return (NULL);
  802                 }
  803                 m_object = m->object;
  804                 VM_OBJECT_LOCK_ASSERT(m_object, MA_OWNED);
  805                 vm_page_busy(m);
  806                 vm_page_free(m);
  807                 vm_page_unlock_queues();
  808                 if (m_object != object)
  809                         VM_OBJECT_UNLOCK(m_object);
  810                 goto loop;
  811         } else {
  812                 /*
  813                  * Not allocatable from cache from interrupt, give up.
  814                  */
  815                 mtx_unlock_spin(&vm_page_queue_free_mtx);
  816                 atomic_add_int(&vm_pageout_deficit, 1);
  817                 pagedaemon_wakeup();
  818                 return (NULL);
  819         }
  820 
  821         /*
  822          *  At this point we had better have found a good page.
  823          */
  824 
  825         KASSERT(
  826             m != NULL,
  827             ("vm_page_alloc(): missing page on free queue")
  828         );
  829 
  830         /*
  831          * Remove from free queue
  832          */
  833         vm_pageq_remove_nowakeup(m);
  834 
  835         /*
  836          * Initialize structure.  Only the PG_ZERO flag is inherited.
  837          */
  838         flags = PG_BUSY;
  839         if (m->flags & PG_ZERO) {
  840                 vm_page_zero_count--;
  841                 if (req & VM_ALLOC_ZERO)
  842                         flags = PG_ZERO | PG_BUSY;
  843         }
  844         if (req & VM_ALLOC_NOOBJ)
  845                 flags &= ~PG_BUSY;
  846         m->flags = flags;
  847         if (req & VM_ALLOC_WIRED) {
  848                 atomic_add_int(&cnt.v_wire_count, 1);
  849                 m->wire_count = 1;
  850         } else
  851                 m->wire_count = 0;
  852         m->hold_count = 0;
  853         m->act_count = 0;
  854         m->busy = 0;
  855         m->valid = 0;
  856         KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
  857         mtx_unlock_spin(&vm_page_queue_free_mtx);
  858 
  859         if ((req & VM_ALLOC_NOOBJ) == 0)
  860                 vm_page_insert(m, object, pindex);
  861         else
  862                 m->pindex = pindex;
  863 
  864         /*
  865          * Don't wakeup too often - wakeup the pageout daemon when
  866          * we would be nearly out of memory.
  867          */
  868         if (vm_paging_needed())
  869                 pagedaemon_wakeup();
  870 
  871         return (m);
  872 }
  873 
  874 /*
  875  *      vm_wait:        (also see VM_WAIT macro)
  876  *
  877  *      Block until free pages are available for allocation
  878  *      - Called in various places before memory allocations.
  879  */
  880 void
  881 vm_wait(void)
  882 {
  883 
  884         vm_page_lock_queues();
  885         if (curproc == pageproc) {
  886                 vm_pageout_pages_needed = 1;
  887                 msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx,
  888                     PDROP | PSWP, "VMWait", 0);
  889         } else {
  890                 if (!vm_pages_needed) {
  891                         vm_pages_needed = 1;
  892                         wakeup(&vm_pages_needed);
  893                 }
  894                 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
  895                     "vmwait", 0);
  896         }
  897 }
  898 
  899 /*
  900  *      vm_waitpfault:  (also see VM_WAITPFAULT macro)
  901  *
  902  *      Block until free pages are available for allocation
  903  *      - Called only in vm_fault so that processes page faulting
  904  *        can be easily tracked.
  905  *      - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
  906  *        processes will be able to grab memory first.  Do not change
  907  *        this balance without careful testing first.
  908  */
  909 void
  910 vm_waitpfault(void)
  911 {
  912 
  913         vm_page_lock_queues();
  914         if (!vm_pages_needed) {
  915                 vm_pages_needed = 1;
  916                 wakeup(&vm_pages_needed);
  917         }
  918         msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER,
  919             "pfault", 0);
  920 }
  921 
  922 /*
  923  *      vm_page_activate:
  924  *
  925  *      Put the specified page on the active list (if appropriate).
  926  *      Ensure that act_count is at least ACT_INIT but do not otherwise
  927  *      mess with it.
  928  *
  929  *      The page queues must be locked.
  930  *      This routine may not block.
  931  */
  932 void
  933 vm_page_activate(vm_page_t m)
  934 {
  935 
  936         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  937         if (m->queue != PQ_ACTIVE) {
  938                 if ((m->queue - m->pc) == PQ_CACHE)
  939                         cnt.v_reactivated++;
  940                 vm_pageq_remove(m);
  941                 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
  942                         if (m->act_count < ACT_INIT)
  943                                 m->act_count = ACT_INIT;
  944                         vm_pageq_enqueue(PQ_ACTIVE, m);
  945                 }
  946         } else {
  947                 if (m->act_count < ACT_INIT)
  948                         m->act_count = ACT_INIT;
  949         }
  950 }
  951 
  952 /*
  953  *      vm_page_free_wakeup:
  954  *
  955  *      Helper routine for vm_page_free_toq() and vm_page_cache().  This
  956  *      routine is called when a page has been added to the cache or free
  957  *      queues.
  958  *
  959  *      The page queues must be locked.
  960  *      This routine may not block.
  961  */
  962 static __inline void
  963 vm_page_free_wakeup(void)
  964 {
  965 
  966         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  967         /*
  968          * if pageout daemon needs pages, then tell it that there are
  969          * some free.
  970          */
  971         if (vm_pageout_pages_needed &&
  972             cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
  973                 wakeup(&vm_pageout_pages_needed);
  974                 vm_pageout_pages_needed = 0;
  975         }
  976         /*
  977          * wakeup processes that are waiting on memory if we hit a
  978          * high water mark. And wakeup scheduler process if we have
  979          * lots of memory. this process will swapin processes.
  980          */
  981         if (vm_pages_needed && !vm_page_count_min()) {
  982                 vm_pages_needed = 0;
  983                 wakeup(&cnt.v_free_count);
  984         }
  985 }
  986 
  987 /*
  988  *      vm_page_free_toq:
  989  *
  990  *      Returns the given page to the PQ_FREE list,
  991  *      disassociating it with any VM object.
  992  *
  993  *      Object and page must be locked prior to entry.
  994  *      This routine may not block.
  995  */
  996 
  997 void
  998 vm_page_free_toq(vm_page_t m)
  999 {
 1000         struct vpgqueues *pq;
 1001         vm_object_t object = m->object;
 1002 
 1003         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1004         cnt.v_tfree++;
 1005 
 1006         if (m->busy || ((m->queue - m->pc) == PQ_FREE)) {
 1007                 printf(
 1008                 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n",
 1009                     (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0,
 1010                     m->hold_count);
 1011                 if ((m->queue - m->pc) == PQ_FREE)
 1012                         panic("vm_page_free: freeing free page");
 1013                 else
 1014                         panic("vm_page_free: freeing busy page");
 1015         }
 1016 
 1017         /*
 1018          * unqueue, then remove page.  Note that we cannot destroy
 1019          * the page here because we do not want to call the pager's
 1020          * callback routine until after we've put the page on the
 1021          * appropriate free queue.
 1022          */
 1023         vm_pageq_remove_nowakeup(m);
 1024         vm_page_remove(m);
 1025 
 1026         /*
 1027          * If fictitious remove object association and
 1028          * return, otherwise delay object association removal.
 1029          */
 1030         if ((m->flags & PG_FICTITIOUS) != 0) {
 1031                 return;
 1032         }
 1033 
 1034         m->valid = 0;
 1035         vm_page_undirty(m);
 1036 
 1037         if (m->wire_count != 0) {
 1038                 if (m->wire_count > 1) {
 1039                         panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
 1040                                 m->wire_count, (long)m->pindex);
 1041                 }
 1042                 panic("vm_page_free: freeing wired page");
 1043         }
 1044 
 1045         /*
 1046          * If we've exhausted the object's resident pages we want to free
 1047          * it up.
 1048          */
 1049         if (object && 
 1050             (object->type == OBJT_VNODE) &&
 1051             ((object->flags & OBJ_DEAD) == 0)
 1052         ) {
 1053                 struct vnode *vp = (struct vnode *)object->handle;
 1054 
 1055                 if (vp) {
 1056                         VI_LOCK(vp);
 1057                         if (VSHOULDFREE(vp))
 1058                                 vfree(vp);
 1059                         VI_UNLOCK(vp);
 1060                 }
 1061         }
 1062 
 1063         /*
 1064          * Clear the UNMANAGED flag when freeing an unmanaged page.
 1065          */
 1066         if (m->flags & PG_UNMANAGED) {
 1067                 m->flags &= ~PG_UNMANAGED;
 1068         }
 1069 
 1070         if (m->hold_count != 0) {
 1071                 m->flags &= ~PG_ZERO;
 1072                 m->queue = PQ_HOLD;
 1073         } else
 1074                 m->queue = PQ_FREE + m->pc;
 1075         pq = &vm_page_queues[m->queue];
 1076         mtx_lock_spin(&vm_page_queue_free_mtx);
 1077         pq->lcnt++;
 1078         ++(*pq->cnt);
 1079 
 1080         /*
 1081          * Put zero'd pages on the end ( where we look for zero'd pages
 1082          * first ) and non-zerod pages at the head.
 1083          */
 1084         if (m->flags & PG_ZERO) {
 1085                 TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
 1086                 ++vm_page_zero_count;
 1087         } else {
 1088                 TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
 1089         }
 1090         mtx_unlock_spin(&vm_page_queue_free_mtx);
 1091         vm_page_free_wakeup();
 1092 }
 1093 
 1094 /*
 1095  *      vm_page_unmanage:
 1096  *
 1097  *      Prevent PV management from being done on the page.  The page is
 1098  *      removed from the paging queues as if it were wired, and as a 
 1099  *      consequence of no longer being managed the pageout daemon will not
 1100  *      touch it (since there is no way to locate the pte mappings for the
 1101  *      page).  madvise() calls that mess with the pmap will also no longer
 1102  *      operate on the page.
 1103  *
 1104  *      Beyond that the page is still reasonably 'normal'.  Freeing the page
 1105  *      will clear the flag.
 1106  *
 1107  *      This routine is used by OBJT_PHYS objects - objects using unswappable
 1108  *      physical memory as backing store rather then swap-backed memory and
 1109  *      will eventually be extended to support 4MB unmanaged physical 
 1110  *      mappings.
 1111  */
 1112 void
 1113 vm_page_unmanage(vm_page_t m)
 1114 {
 1115 
 1116         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1117         if ((m->flags & PG_UNMANAGED) == 0) {
 1118                 if (m->wire_count == 0)
 1119                         vm_pageq_remove(m);
 1120         }
 1121         vm_page_flag_set(m, PG_UNMANAGED);
 1122 }
 1123 
 1124 /*
 1125  *      vm_page_wire:
 1126  *
 1127  *      Mark this page as wired down by yet
 1128  *      another map, removing it from paging queues
 1129  *      as necessary.
 1130  *
 1131  *      The page queues must be locked.
 1132  *      This routine may not block.
 1133  */
 1134 void
 1135 vm_page_wire(vm_page_t m)
 1136 {
 1137 
 1138         /*
 1139          * Only bump the wire statistics if the page is not already wired,
 1140          * and only unqueue the page if it is on some queue (if it is unmanaged
 1141          * it is already off the queues).
 1142          */
 1143         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1144         if (m->flags & PG_FICTITIOUS)
 1145                 return;
 1146         if (m->wire_count == 0) {
 1147                 if ((m->flags & PG_UNMANAGED) == 0)
 1148                         vm_pageq_remove(m);
 1149                 atomic_add_int(&cnt.v_wire_count, 1);
 1150         }
 1151         m->wire_count++;
 1152         KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
 1153 }
 1154 
 1155 /*
 1156  *      vm_page_unwire:
 1157  *
 1158  *      Release one wiring of this page, potentially
 1159  *      enabling it to be paged again.
 1160  *
 1161  *      Many pages placed on the inactive queue should actually go
 1162  *      into the cache, but it is difficult to figure out which.  What
 1163  *      we do instead, if the inactive target is well met, is to put
 1164  *      clean pages at the head of the inactive queue instead of the tail.
 1165  *      This will cause them to be moved to the cache more quickly and
 1166  *      if not actively re-referenced, freed more quickly.  If we just
 1167  *      stick these pages at the end of the inactive queue, heavy filesystem
 1168  *      meta-data accesses can cause an unnecessary paging load on memory bound 
 1169  *      processes.  This optimization causes one-time-use metadata to be
 1170  *      reused more quickly.
 1171  *
 1172  *      BUT, if we are in a low-memory situation we have no choice but to
 1173  *      put clean pages on the cache queue.
 1174  *
 1175  *      A number of routines use vm_page_unwire() to guarantee that the page
 1176  *      will go into either the inactive or active queues, and will NEVER
 1177  *      be placed in the cache - for example, just after dirtying a page.
 1178  *      dirty pages in the cache are not allowed.
 1179  *
 1180  *      The page queues must be locked.
 1181  *      This routine may not block.
 1182  */
 1183 void
 1184 vm_page_unwire(vm_page_t m, int activate)
 1185 {
 1186 
 1187         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1188         if (m->flags & PG_FICTITIOUS)
 1189                 return;
 1190         if (m->wire_count > 0) {
 1191                 m->wire_count--;
 1192                 if (m->wire_count == 0) {
 1193                         atomic_subtract_int(&cnt.v_wire_count, 1);
 1194                         if (m->flags & PG_UNMANAGED) {
 1195                                 ;
 1196                         } else if (activate)
 1197                                 vm_pageq_enqueue(PQ_ACTIVE, m);
 1198                         else {
 1199                                 vm_page_flag_clear(m, PG_WINATCFLS);
 1200                                 vm_pageq_enqueue(PQ_INACTIVE, m);
 1201                         }
 1202                 }
 1203         } else {
 1204                 panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
 1205         }
 1206 }
 1207 
 1208 
 1209 /*
 1210  * Move the specified page to the inactive queue.  If the page has
 1211  * any associated swap, the swap is deallocated.
 1212  *
 1213  * Normally athead is 0 resulting in LRU operation.  athead is set
 1214  * to 1 if we want this page to be 'as if it were placed in the cache',
 1215  * except without unmapping it from the process address space.
 1216  *
 1217  * This routine may not block.
 1218  */
 1219 static __inline void
 1220 _vm_page_deactivate(vm_page_t m, int athead)
 1221 {
 1222 
 1223         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1224 
 1225         /*
 1226          * Ignore if already inactive.
 1227          */
 1228         if (m->queue == PQ_INACTIVE)
 1229                 return;
 1230         if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
 1231                 if ((m->queue - m->pc) == PQ_CACHE)
 1232                         cnt.v_reactivated++;
 1233                 vm_page_flag_clear(m, PG_WINATCFLS);
 1234                 vm_pageq_remove(m);
 1235                 if (athead)
 1236                         TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 1237                 else
 1238                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 1239                 m->queue = PQ_INACTIVE;
 1240                 vm_page_queues[PQ_INACTIVE].lcnt++;
 1241                 cnt.v_inactive_count++;
 1242         }
 1243 }
 1244 
 1245 void
 1246 vm_page_deactivate(vm_page_t m)
 1247 {
 1248     _vm_page_deactivate(m, 0);
 1249 }
 1250 
 1251 /*
 1252  * vm_page_try_to_cache:
 1253  *
 1254  * Returns 0 on failure, 1 on success
 1255  */
 1256 int
 1257 vm_page_try_to_cache(vm_page_t m)
 1258 {
 1259 
 1260         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1261         if (m->dirty || m->hold_count || m->busy || m->wire_count ||
 1262             (m->flags & (PG_BUSY|PG_UNMANAGED))) {
 1263                 return (0);
 1264         }
 1265         pmap_remove_all(m);
 1266         if (m->dirty)
 1267                 return (0);
 1268         vm_page_cache(m);
 1269         return (1);
 1270 }
 1271 
 1272 /*
 1273  * vm_page_try_to_free()
 1274  *
 1275  *      Attempt to free the page.  If we cannot free it, we do nothing.
 1276  *      1 is returned on success, 0 on failure.
 1277  */
 1278 int
 1279 vm_page_try_to_free(vm_page_t m)
 1280 {
 1281 
 1282         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1283         if (m->object != NULL)
 1284                 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1285         if (m->dirty || m->hold_count || m->busy || m->wire_count ||
 1286             (m->flags & (PG_BUSY|PG_UNMANAGED))) {
 1287                 return (0);
 1288         }
 1289         pmap_remove_all(m);
 1290         if (m->dirty)
 1291                 return (0);
 1292         vm_page_busy(m);
 1293         vm_page_free(m);
 1294         return (1);
 1295 }
 1296 
 1297 /*
 1298  * vm_page_cache
 1299  *
 1300  * Put the specified page onto the page cache queue (if appropriate).
 1301  *
 1302  * This routine may not block.
 1303  */
 1304 void
 1305 vm_page_cache(vm_page_t m)
 1306 {
 1307 
 1308         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1309         if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy ||
 1310             m->hold_count || m->wire_count) {
 1311                 printf("vm_page_cache: attempting to cache busy page\n");
 1312                 return;
 1313         }
 1314         if ((m->queue - m->pc) == PQ_CACHE)
 1315                 return;
 1316 
 1317         /*
 1318          * Remove all pmaps and indicate that the page is not
 1319          * writeable or mapped.
 1320          */
 1321         pmap_remove_all(m);
 1322         if (m->dirty != 0) {
 1323                 panic("vm_page_cache: caching a dirty page, pindex: %ld",
 1324                         (long)m->pindex);
 1325         }
 1326         vm_pageq_remove_nowakeup(m);
 1327         vm_pageq_enqueue(PQ_CACHE + m->pc, m);
 1328         vm_page_free_wakeup();
 1329 }
 1330 
 1331 /*
 1332  * vm_page_dontneed
 1333  *
 1334  *      Cache, deactivate, or do nothing as appropriate.  This routine
 1335  *      is typically used by madvise() MADV_DONTNEED.
 1336  *
 1337  *      Generally speaking we want to move the page into the cache so
 1338  *      it gets reused quickly.  However, this can result in a silly syndrome
 1339  *      due to the page recycling too quickly.  Small objects will not be
 1340  *      fully cached.  On the otherhand, if we move the page to the inactive
 1341  *      queue we wind up with a problem whereby very large objects 
 1342  *      unnecessarily blow away our inactive and cache queues.
 1343  *
 1344  *      The solution is to move the pages based on a fixed weighting.  We
 1345  *      either leave them alone, deactivate them, or move them to the cache,
 1346  *      where moving them to the cache has the highest weighting.
 1347  *      By forcing some pages into other queues we eventually force the
 1348  *      system to balance the queues, potentially recovering other unrelated
 1349  *      space from active.  The idea is to not force this to happen too
 1350  *      often.
 1351  */
 1352 void
 1353 vm_page_dontneed(vm_page_t m)
 1354 {
 1355         static int dnweight;
 1356         int dnw;
 1357         int head;
 1358 
 1359         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1360         dnw = ++dnweight;
 1361 
 1362         /*
 1363          * occassionally leave the page alone
 1364          */
 1365         if ((dnw & 0x01F0) == 0 ||
 1366             m->queue == PQ_INACTIVE || 
 1367             m->queue - m->pc == PQ_CACHE
 1368         ) {
 1369                 if (m->act_count >= ACT_INIT)
 1370                         --m->act_count;
 1371                 return;
 1372         }
 1373 
 1374         if (m->dirty == 0 && pmap_is_modified(m))
 1375                 vm_page_dirty(m);
 1376 
 1377         if (m->dirty || (dnw & 0x0070) == 0) {
 1378                 /*
 1379                  * Deactivate the page 3 times out of 32.
 1380                  */
 1381                 head = 0;
 1382         } else {
 1383                 /*
 1384                  * Cache the page 28 times out of every 32.  Note that
 1385                  * the page is deactivated instead of cached, but placed
 1386                  * at the head of the queue instead of the tail.
 1387                  */
 1388                 head = 1;
 1389         }
 1390         _vm_page_deactivate(m, head);
 1391 }
 1392 
 1393 /*
 1394  * Grab a page, waiting until we are waken up due to the page
 1395  * changing state.  We keep on waiting, if the page continues
 1396  * to be in the object.  If the page doesn't exist, first allocate it
 1397  * and then conditionally zero it.
 1398  *
 1399  * This routine may block.
 1400  */
 1401 vm_page_t
 1402 vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
 1403 {
 1404         vm_page_t m;
 1405 
 1406         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1407 retrylookup:
 1408         if ((m = vm_page_lookup(object, pindex)) != NULL) {
 1409                 vm_page_lock_queues();
 1410                 if (m->busy || (m->flags & PG_BUSY)) {
 1411                         vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
 1412                         VM_OBJECT_UNLOCK(object);
 1413                         msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
 1414                         VM_OBJECT_LOCK(object);
 1415                         if ((allocflags & VM_ALLOC_RETRY) == 0)
 1416                                 return (NULL);
 1417                         goto retrylookup;
 1418                 } else {
 1419                         if (allocflags & VM_ALLOC_WIRED)
 1420                                 vm_page_wire(m);
 1421                         vm_page_busy(m);
 1422                         vm_page_unlock_queues();
 1423                         return (m);
 1424                 }
 1425         }
 1426         m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
 1427         if (m == NULL) {
 1428                 VM_OBJECT_UNLOCK(object);
 1429                 VM_WAIT;
 1430                 VM_OBJECT_LOCK(object);
 1431                 if ((allocflags & VM_ALLOC_RETRY) == 0)
 1432                         return (NULL);
 1433                 goto retrylookup;
 1434         }
 1435         if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
 1436                 pmap_zero_page(m);
 1437         return (m);
 1438 }
 1439 
 1440 /*
 1441  * Mapping function for valid bits or for dirty bits in
 1442  * a page.  May not block.
 1443  *
 1444  * Inputs are required to range within a page.
 1445  */
 1446 __inline int
 1447 vm_page_bits(int base, int size)
 1448 {
 1449         int first_bit;
 1450         int last_bit;
 1451 
 1452         KASSERT(
 1453             base + size <= PAGE_SIZE,
 1454             ("vm_page_bits: illegal base/size %d/%d", base, size)
 1455         );
 1456 
 1457         if (size == 0)          /* handle degenerate case */
 1458                 return (0);
 1459 
 1460         first_bit = base >> DEV_BSHIFT;
 1461         last_bit = (base + size - 1) >> DEV_BSHIFT;
 1462 
 1463         return ((2 << last_bit) - (1 << first_bit));
 1464 }
 1465 
 1466 /*
 1467  *      vm_page_set_validclean:
 1468  *
 1469  *      Sets portions of a page valid and clean.  The arguments are expected
 1470  *      to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
 1471  *      of any partial chunks touched by the range.  The invalid portion of
 1472  *      such chunks will be zero'd.
 1473  *
 1474  *      This routine may not block.
 1475  *
 1476  *      (base + size) must be less then or equal to PAGE_SIZE.
 1477  */
 1478 void
 1479 vm_page_set_validclean(vm_page_t m, int base, int size)
 1480 {
 1481         int pagebits;
 1482         int frag;
 1483         int endoff;
 1484 
 1485         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1486         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1487         if (size == 0)  /* handle degenerate case */
 1488                 return;
 1489 
 1490         /*
 1491          * If the base is not DEV_BSIZE aligned and the valid
 1492          * bit is clear, we have to zero out a portion of the
 1493          * first block.
 1494          */
 1495         if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
 1496             (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
 1497                 pmap_zero_page_area(m, frag, base - frag);
 1498 
 1499         /*
 1500          * If the ending offset is not DEV_BSIZE aligned and the 
 1501          * valid bit is clear, we have to zero out a portion of
 1502          * the last block.
 1503          */
 1504         endoff = base + size;
 1505         if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
 1506             (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
 1507                 pmap_zero_page_area(m, endoff,
 1508                     DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
 1509 
 1510         /*
 1511          * Set valid, clear dirty bits.  If validating the entire
 1512          * page we can safely clear the pmap modify bit.  We also
 1513          * use this opportunity to clear the PG_NOSYNC flag.  If a process
 1514          * takes a write fault on a MAP_NOSYNC memory area the flag will
 1515          * be set again.
 1516          *
 1517          * We set valid bits inclusive of any overlap, but we can only
 1518          * clear dirty bits for DEV_BSIZE chunks that are fully within
 1519          * the range.
 1520          */
 1521         pagebits = vm_page_bits(base, size);
 1522         m->valid |= pagebits;
 1523 #if 0   /* NOT YET */
 1524         if ((frag = base & (DEV_BSIZE - 1)) != 0) {
 1525                 frag = DEV_BSIZE - frag;
 1526                 base += frag;
 1527                 size -= frag;
 1528                 if (size < 0)
 1529                         size = 0;
 1530         }
 1531         pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
 1532 #endif
 1533         m->dirty &= ~pagebits;
 1534         if (base == 0 && size == PAGE_SIZE) {
 1535                 pmap_clear_modify(m);
 1536                 vm_page_flag_clear(m, PG_NOSYNC);
 1537         }
 1538 }
 1539 
 1540 void
 1541 vm_page_clear_dirty(vm_page_t m, int base, int size)
 1542 {
 1543 
 1544         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1545         m->dirty &= ~vm_page_bits(base, size);
 1546 }
 1547 
 1548 /*
 1549  *      vm_page_set_invalid:
 1550  *
 1551  *      Invalidates DEV_BSIZE'd chunks within a page.  Both the
 1552  *      valid and dirty bits for the effected areas are cleared.
 1553  *
 1554  *      May not block.
 1555  */
 1556 void
 1557 vm_page_set_invalid(vm_page_t m, int base, int size)
 1558 {
 1559         int bits;
 1560 
 1561         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1562         bits = vm_page_bits(base, size);
 1563         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1564         m->valid &= ~bits;
 1565         m->dirty &= ~bits;
 1566         m->object->generation++;
 1567 }
 1568 
 1569 /*
 1570  * vm_page_zero_invalid()
 1571  *
 1572  *      The kernel assumes that the invalid portions of a page contain 
 1573  *      garbage, but such pages can be mapped into memory by user code.
 1574  *      When this occurs, we must zero out the non-valid portions of the
 1575  *      page so user code sees what it expects.
 1576  *
 1577  *      Pages are most often semi-valid when the end of a file is mapped 
 1578  *      into memory and the file's size is not page aligned.
 1579  */
 1580 void
 1581 vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
 1582 {
 1583         int b;
 1584         int i;
 1585 
 1586         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1587         /*
 1588          * Scan the valid bits looking for invalid sections that
 1589          * must be zerod.  Invalid sub-DEV_BSIZE'd areas ( where the
 1590          * valid bit may be set ) have already been zerod by
 1591          * vm_page_set_validclean().
 1592          */
 1593         for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
 1594                 if (i == (PAGE_SIZE / DEV_BSIZE) || 
 1595                     (m->valid & (1 << i))
 1596                 ) {
 1597                         if (i > b) {
 1598                                 pmap_zero_page_area(m, 
 1599                                     b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
 1600                         }
 1601                         b = i + 1;
 1602                 }
 1603         }
 1604 
 1605         /*
 1606          * setvalid is TRUE when we can safely set the zero'd areas
 1607          * as being valid.  We can do this if there are no cache consistancy
 1608          * issues.  e.g. it is ok to do with UFS, but not ok to do with NFS.
 1609          */
 1610         if (setvalid)
 1611                 m->valid = VM_PAGE_BITS_ALL;
 1612 }
 1613 
 1614 /*
 1615  *      vm_page_is_valid:
 1616  *
 1617  *      Is (partial) page valid?  Note that the case where size == 0
 1618  *      will return FALSE in the degenerate case where the page is
 1619  *      entirely invalid, and TRUE otherwise.
 1620  *
 1621  *      May not block.
 1622  */
 1623 int
 1624 vm_page_is_valid(vm_page_t m, int base, int size)
 1625 {
 1626         int bits = vm_page_bits(base, size);
 1627 
 1628         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1629         if (m->valid && ((m->valid & bits) == bits))
 1630                 return 1;
 1631         else
 1632                 return 0;
 1633 }
 1634 
 1635 /*
 1636  * update dirty bits from pmap/mmu.  May not block.
 1637  */
 1638 void
 1639 vm_page_test_dirty(vm_page_t m)
 1640 {
 1641         if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
 1642                 vm_page_dirty(m);
 1643         }
 1644 }
 1645 
 1646 int so_zerocp_fullpage = 0;
 1647 
 1648 void
 1649 vm_page_cowfault(vm_page_t m)
 1650 {
 1651         vm_page_t mnew;
 1652         vm_object_t object;
 1653         vm_pindex_t pindex;
 1654 
 1655         object = m->object;
 1656         pindex = m->pindex;
 1657         vm_page_busy(m);
 1658 
 1659  retry_alloc:
 1660         vm_page_remove(m);
 1661         mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
 1662         if (mnew == NULL) {
 1663                 vm_page_insert(m, object, pindex);
 1664                 vm_page_unlock_queues();
 1665                 VM_OBJECT_UNLOCK(object);
 1666                 VM_WAIT;
 1667                 VM_OBJECT_LOCK(object);
 1668                 vm_page_lock_queues();
 1669                 goto retry_alloc;
 1670         }
 1671 
 1672         if (m->cow == 0) {
 1673                 /* 
 1674                  * check to see if we raced with an xmit complete when 
 1675                  * waiting to allocate a page.  If so, put things back 
 1676                  * the way they were 
 1677                  */
 1678                 vm_page_busy(mnew);
 1679                 vm_page_free(mnew);
 1680                 vm_page_insert(m, object, pindex);
 1681         } else { /* clear COW & copy page */
 1682                 if (!so_zerocp_fullpage)
 1683                         pmap_copy_page(m, mnew);
 1684                 mnew->valid = VM_PAGE_BITS_ALL;
 1685                 vm_page_dirty(mnew);
 1686                 vm_page_flag_clear(mnew, PG_BUSY);
 1687         }
 1688 }
 1689 
 1690 void 
 1691 vm_page_cowclear(vm_page_t m)
 1692 {
 1693 
 1694         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1695         if (m->cow) {
 1696                 m->cow--;
 1697                 /* 
 1698                  * let vm_fault add back write permission  lazily
 1699                  */
 1700         } 
 1701         /*
 1702          *  sf_buf_free() will free the page, so we needn't do it here
 1703          */ 
 1704 }
 1705 
 1706 void
 1707 vm_page_cowsetup(vm_page_t m)
 1708 {
 1709 
 1710         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1711         m->cow++;
 1712         pmap_page_protect(m, VM_PROT_READ);
 1713 }
 1714 
 1715 #include "opt_ddb.h"
 1716 #ifdef DDB
 1717 #include <sys/kernel.h>
 1718 
 1719 #include <ddb/ddb.h>
 1720 
 1721 DB_SHOW_COMMAND(page, vm_page_print_page_info)
 1722 {
 1723         db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
 1724         db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
 1725         db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
 1726         db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
 1727         db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
 1728         db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
 1729         db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
 1730         db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
 1731         db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
 1732         db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
 1733 }
 1734 
 1735 DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
 1736 {
 1737         int i;
 1738         db_printf("PQ_FREE:");
 1739         for (i = 0; i < PQ_L2_SIZE; i++) {
 1740                 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
 1741         }
 1742         db_printf("\n");
 1743                 
 1744         db_printf("PQ_CACHE:");
 1745         for (i = 0; i < PQ_L2_SIZE; i++) {
 1746                 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
 1747         }
 1748         db_printf("\n");
 1749 
 1750         db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
 1751                 vm_page_queues[PQ_ACTIVE].lcnt,
 1752                 vm_page_queues[PQ_INACTIVE].lcnt);
 1753 }
 1754 #endif /* DDB */

Cache object: 34ac27f618fc51623b9f4abbf7f776fb


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.