The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/swap_pager.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Matthew Dillon,
    3  * Copyright (c) 1994 John S. Dyson
    4  * Copyright (c) 1990 University of Utah.
    5  * Copyright (c) 1982, 1986, 1989, 1993
    6  *      The Regents of the University of California.  All rights reserved.
    7  *
    8  * This code is derived from software contributed to Berkeley by
    9  * the Systems Programming Group of the University of Utah Computer
   10  * Science Department.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. All advertising materials mentioning features or use of this software
   21  *    must display the following acknowledgement:
   22  *      This product includes software developed by the University of
   23  *      California, Berkeley and its contributors.
   24  * 4. Neither the name of the University nor the names of its contributors
   25  *    may be used to endorse or promote products derived from this software
   26  *    without specific prior written permission.
   27  *
   28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   38  * SUCH DAMAGE.
   39  *
   40  *                              New Swap System
   41  *                              Matthew Dillon
   42  *
   43  * Radix Bitmap 'blists'.
   44  *
   45  *      - The new swapper uses the new radix bitmap code.  This should scale
   46  *        to arbitrarily small or arbitrarily large swap spaces and an almost
   47  *        arbitrary degree of fragmentation.
   48  *
   49  * Features:
   50  *
   51  *      - on the fly reallocation of swap during putpages.  The new system
   52  *        does not try to keep previously allocated swap blocks for dirty
   53  *        pages.  
   54  *
   55  *      - on the fly deallocation of swap
   56  *
   57  *      - No more garbage collection required.  Unnecessarily allocated swap
   58  *        blocks only exist for dirty vm_page_t's now and these are already
   59  *        cycled (in a high-load system) by the pager.  We also do on-the-fly
   60  *        removal of invalidated swap blocks when a page is destroyed
   61  *        or renamed.
   62  *
   63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
   64  *
   65  *      @(#)swap_pager.c        8.9 (Berkeley) 3/21/94
   66  *      @(#)vm_swap.c   8.5 (Berkeley) 2/17/94
   67  */
   68 
   69 #include <sys/cdefs.h>
   70 __FBSDID("$FreeBSD: releng/9.0/sys/vm/swap_pager.c 225617 2011-09-16 13:58:51Z kmacy $");
   71 
   72 #include "opt_swap.h"
   73 #include "opt_vm.h"
   74 
   75 #include <sys/param.h>
   76 #include <sys/systm.h>
   77 #include <sys/conf.h>
   78 #include <sys/kernel.h>
   79 #include <sys/priv.h>
   80 #include <sys/proc.h>
   81 #include <sys/bio.h>
   82 #include <sys/buf.h>
   83 #include <sys/disk.h>
   84 #include <sys/fcntl.h>
   85 #include <sys/mount.h>
   86 #include <sys/namei.h>
   87 #include <sys/vnode.h>
   88 #include <sys/malloc.h>
   89 #include <sys/racct.h>
   90 #include <sys/resource.h>
   91 #include <sys/resourcevar.h>
   92 #include <sys/sysctl.h>
   93 #include <sys/sysproto.h>
   94 #include <sys/blist.h>
   95 #include <sys/lock.h>
   96 #include <sys/sx.h>
   97 #include <sys/vmmeter.h>
   98 
   99 #include <security/mac/mac_framework.h>
  100 
  101 #include <vm/vm.h>
  102 #include <vm/pmap.h>
  103 #include <vm/vm_map.h>
  104 #include <vm/vm_kern.h>
  105 #include <vm/vm_object.h>
  106 #include <vm/vm_page.h>
  107 #include <vm/vm_pager.h>
  108 #include <vm/vm_pageout.h>
  109 #include <vm/vm_param.h>
  110 #include <vm/swap_pager.h>
  111 #include <vm/vm_extern.h>
  112 #include <vm/uma.h>
  113 
  114 #include <geom/geom.h>
  115 
  116 /*
  117  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
  118  * or 32 pages per allocation.
  119  * The 32-page limit is due to the radix code (kern/subr_blist.c).
  120  */
  121 #ifndef MAX_PAGEOUT_CLUSTER
  122 #define MAX_PAGEOUT_CLUSTER 16
  123 #endif
  124 
  125 #if !defined(SWB_NPAGES)
  126 #define SWB_NPAGES      MAX_PAGEOUT_CLUSTER
  127 #endif
  128 
  129 /*
  130  * The swblock structure maps an object and a small, fixed-size range
  131  * of page indices to disk addresses within a swap area.
  132  * The collection of these mappings is implemented as a hash table.
  133  * Unused disk addresses within a swap area are allocated and managed
  134  * using a blist.
  135  */
  136 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
  137 #define SWAP_META_PAGES         (SWB_NPAGES * 2)
  138 #define SWAP_META_MASK          (SWAP_META_PAGES - 1)
  139 
  140 struct swblock {
  141         struct swblock  *swb_hnext;
  142         vm_object_t     swb_object;
  143         vm_pindex_t     swb_index;
  144         int             swb_count;
  145         daddr_t         swb_pages[SWAP_META_PAGES];
  146 };
  147 
  148 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
  149 static struct mtx sw_dev_mtx;
  150 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
  151 static struct swdevt *swdevhd;  /* Allocate from here next */
  152 static int nswapdev;            /* Number of swap devices */
  153 int swap_pager_avail;
  154 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
  155 
  156 static vm_ooffset_t swap_total;
  157 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0, 
  158     "Total amount of available swap storage.");
  159 static vm_ooffset_t swap_reserved;
  160 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0, 
  161     "Amount of swap storage needed to back all allocated anonymous memory.");
  162 static int overcommit = 0;
  163 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0, 
  164     "Configure virtual memory overcommit behavior. See tuning(7) "
  165     "for details.");
  166 
  167 /* bits from overcommit */
  168 #define SWAP_RESERVE_FORCE_ON           (1 << 0)
  169 #define SWAP_RESERVE_RLIMIT_ON          (1 << 1)
  170 #define SWAP_RESERVE_ALLOW_NONWIRED     (1 << 2)
  171 
  172 int
  173 swap_reserve(vm_ooffset_t incr)
  174 {
  175 
  176         return (swap_reserve_by_cred(incr, curthread->td_ucred));
  177 }
  178 
  179 int
  180 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
  181 {
  182         vm_ooffset_t r, s;
  183         int res, error;
  184         static int curfail;
  185         static struct timeval lastfail;
  186         struct uidinfo *uip;
  187         
  188         uip = cred->cr_ruidinfo;
  189 
  190         if (incr & PAGE_MASK)
  191                 panic("swap_reserve: & PAGE_MASK");
  192 
  193 #ifdef RACCT
  194         PROC_LOCK(curproc);
  195         error = racct_add(curproc, RACCT_SWAP, incr);
  196         PROC_UNLOCK(curproc);
  197         if (error != 0)
  198                 return (0);
  199 #endif
  200 
  201         res = 0;
  202         mtx_lock(&sw_dev_mtx);
  203         r = swap_reserved + incr;
  204         if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
  205                 s = cnt.v_page_count - cnt.v_free_reserved - cnt.v_wire_count;
  206                 s *= PAGE_SIZE;
  207         } else
  208                 s = 0;
  209         s += swap_total;
  210         if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
  211             (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
  212                 res = 1;
  213                 swap_reserved = r;
  214         }
  215         mtx_unlock(&sw_dev_mtx);
  216 
  217         if (res) {
  218                 PROC_LOCK(curproc);
  219                 UIDINFO_VMSIZE_LOCK(uip);
  220                 if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
  221                     uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
  222                     priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
  223                         res = 0;
  224                 else
  225                         uip->ui_vmsize += incr;
  226                 UIDINFO_VMSIZE_UNLOCK(uip);
  227                 PROC_UNLOCK(curproc);
  228                 if (!res) {
  229                         mtx_lock(&sw_dev_mtx);
  230                         swap_reserved -= incr;
  231                         mtx_unlock(&sw_dev_mtx);
  232                 }
  233         }
  234         if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
  235                 printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
  236                     curproc->p_pid, uip->ui_uid, incr);
  237         }
  238 
  239 #ifdef RACCT
  240         if (!res) {
  241                 PROC_LOCK(curproc);
  242                 racct_sub(curproc, RACCT_SWAP, incr);
  243                 PROC_UNLOCK(curproc);
  244         }
  245 #endif
  246 
  247         return (res);
  248 }
  249 
  250 void
  251 swap_reserve_force(vm_ooffset_t incr)
  252 {
  253         struct uidinfo *uip;
  254 
  255         mtx_lock(&sw_dev_mtx);
  256         swap_reserved += incr;
  257         mtx_unlock(&sw_dev_mtx);
  258 
  259 #ifdef RACCT
  260         PROC_LOCK(curproc);
  261         racct_add_force(curproc, RACCT_SWAP, incr);
  262         PROC_UNLOCK(curproc);
  263 #endif
  264 
  265         uip = curthread->td_ucred->cr_ruidinfo;
  266         PROC_LOCK(curproc);
  267         UIDINFO_VMSIZE_LOCK(uip);
  268         uip->ui_vmsize += incr;
  269         UIDINFO_VMSIZE_UNLOCK(uip);
  270         PROC_UNLOCK(curproc);
  271 }
  272 
  273 void
  274 swap_release(vm_ooffset_t decr)
  275 {
  276         struct ucred *cred;
  277 
  278         PROC_LOCK(curproc);
  279         cred = curthread->td_ucred;
  280         swap_release_by_cred(decr, cred);
  281         PROC_UNLOCK(curproc);
  282 }
  283 
  284 void
  285 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
  286 {
  287         struct uidinfo *uip;
  288         
  289         uip = cred->cr_ruidinfo;
  290 
  291         if (decr & PAGE_MASK)
  292                 panic("swap_release: & PAGE_MASK");
  293 
  294         mtx_lock(&sw_dev_mtx);
  295         if (swap_reserved < decr)
  296                 panic("swap_reserved < decr");
  297         swap_reserved -= decr;
  298         mtx_unlock(&sw_dev_mtx);
  299 
  300         UIDINFO_VMSIZE_LOCK(uip);
  301         if (uip->ui_vmsize < decr)
  302                 printf("negative vmsize for uid = %d\n", uip->ui_uid);
  303         uip->ui_vmsize -= decr;
  304         UIDINFO_VMSIZE_UNLOCK(uip);
  305 
  306         racct_sub_cred(cred, RACCT_SWAP, decr);
  307 }
  308 
  309 static void swapdev_strategy(struct buf *, struct swdevt *sw);
  310 
  311 #define SWM_FREE        0x02    /* free, period                 */
  312 #define SWM_POP         0x04    /* pop out                      */
  313 
  314 int swap_pager_full = 2;        /* swap space exhaustion (task killing) */
  315 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
  316 static int nsw_rcount;          /* free read buffers                    */
  317 static int nsw_wcount_sync;     /* limit write buffers / synchronous    */
  318 static int nsw_wcount_async;    /* limit write buffers / asynchronous   */
  319 static int nsw_wcount_async_max;/* assigned maximum                     */
  320 static int nsw_cluster_max;     /* maximum VOP I/O allowed              */
  321 
  322 static struct swblock **swhash;
  323 static int swhash_mask;
  324 static struct mtx swhash_mtx;
  325 
  326 static int swap_async_max = 4;  /* maximum in-progress async I/O's      */
  327 static struct sx sw_alloc_sx;
  328 
  329 
  330 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
  331         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
  332 
  333 /*
  334  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
  335  * of searching a named list by hashing it just a little.
  336  */
  337 
  338 #define NOBJLISTS               8
  339 
  340 #define NOBJLIST(handle)        \
  341         (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
  342 
  343 static struct mtx sw_alloc_mtx; /* protect list manipulation */ 
  344 static struct pagerlst  swap_pager_object_list[NOBJLISTS];
  345 static uma_zone_t       swap_zone;
  346 static struct vm_object swap_zone_obj;
  347 
  348 /*
  349  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
  350  * calls hooked from other parts of the VM system and do not appear here.
  351  * (see vm/swap_pager.h).
  352  */
  353 static vm_object_t
  354                 swap_pager_alloc(void *handle, vm_ooffset_t size,
  355                     vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
  356 static void     swap_pager_dealloc(vm_object_t object);
  357 static int      swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
  358 static void     swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
  359 static boolean_t
  360                 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
  361 static void     swap_pager_init(void);
  362 static void     swap_pager_unswapped(vm_page_t);
  363 static void     swap_pager_swapoff(struct swdevt *sp);
  364 
  365 struct pagerops swappagerops = {
  366         .pgo_init =     swap_pager_init,        /* early system initialization of pager */
  367         .pgo_alloc =    swap_pager_alloc,       /* allocate an OBJT_SWAP object         */
  368         .pgo_dealloc =  swap_pager_dealloc,     /* deallocate an OBJT_SWAP object       */
  369         .pgo_getpages = swap_pager_getpages,    /* pagein                               */
  370         .pgo_putpages = swap_pager_putpages,    /* pageout                              */
  371         .pgo_haspage =  swap_pager_haspage,     /* get backing store status for page    */
  372         .pgo_pageunswapped = swap_pager_unswapped,      /* remove swap related to page          */
  373 };
  374 
  375 /*
  376  * dmmax is in page-sized chunks with the new swap system.  It was
  377  * dev-bsized chunks in the old.  dmmax is always a power of 2.
  378  *
  379  * swap_*() routines are externally accessible.  swp_*() routines are
  380  * internal.
  381  */
  382 static int dmmax;
  383 static int nswap_lowat = 128;   /* in pages, swap_pager_almost_full warn */
  384 static int nswap_hiwat = 512;   /* in pages, swap_pager_almost_full warn */
  385 
  386 SYSCTL_INT(_vm, OID_AUTO, dmmax,
  387         CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
  388 
  389 static void     swp_sizecheck(void);
  390 static void     swp_pager_async_iodone(struct buf *bp);
  391 static int      swapongeom(struct thread *, struct vnode *);
  392 static int      swaponvp(struct thread *, struct vnode *, u_long);
  393 static int      swapoff_one(struct swdevt *sp, struct ucred *cred);
  394 
  395 /*
  396  * Swap bitmap functions
  397  */
  398 static void     swp_pager_freeswapspace(daddr_t blk, int npages);
  399 static daddr_t  swp_pager_getswapspace(int npages);
  400 
  401 /*
  402  * Metadata functions
  403  */
  404 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
  405 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
  406 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
  407 static void swp_pager_meta_free_all(vm_object_t);
  408 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
  409 
  410 static void
  411 swp_pager_free_nrpage(vm_page_t m)
  412 {
  413 
  414         vm_page_lock(m);
  415         if (m->wire_count == 0)
  416                 vm_page_free(m);
  417         vm_page_unlock(m);
  418 }
  419 
  420 /*
  421  * SWP_SIZECHECK() -    update swap_pager_full indication
  422  *      
  423  *      update the swap_pager_almost_full indication and warn when we are
  424  *      about to run out of swap space, using lowat/hiwat hysteresis.
  425  *
  426  *      Clear swap_pager_full ( task killing ) indication when lowat is met.
  427  *
  428  *      No restrictions on call
  429  *      This routine may not block.
  430  */
  431 static void
  432 swp_sizecheck(void)
  433 {
  434 
  435         if (swap_pager_avail < nswap_lowat) {
  436                 if (swap_pager_almost_full == 0) {
  437                         printf("swap_pager: out of swap space\n");
  438                         swap_pager_almost_full = 1;
  439                 }
  440         } else {
  441                 swap_pager_full = 0;
  442                 if (swap_pager_avail > nswap_hiwat)
  443                         swap_pager_almost_full = 0;
  444         }
  445 }
  446 
  447 /*
  448  * SWP_PAGER_HASH() -   hash swap meta data
  449  *
  450  *      This is an helper function which hashes the swapblk given
  451  *      the object and page index.  It returns a pointer to a pointer
  452  *      to the object, or a pointer to a NULL pointer if it could not
  453  *      find a swapblk.
  454  */
  455 static struct swblock **
  456 swp_pager_hash(vm_object_t object, vm_pindex_t index)
  457 {
  458         struct swblock **pswap;
  459         struct swblock *swap;
  460 
  461         index &= ~(vm_pindex_t)SWAP_META_MASK;
  462         pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
  463         while ((swap = *pswap) != NULL) {
  464                 if (swap->swb_object == object &&
  465                     swap->swb_index == index
  466                 ) {
  467                         break;
  468                 }
  469                 pswap = &swap->swb_hnext;
  470         }
  471         return (pswap);
  472 }
  473 
  474 /*
  475  * SWAP_PAGER_INIT() -  initialize the swap pager!
  476  *
  477  *      Expected to be started from system init.  NOTE:  This code is run 
  478  *      before much else so be careful what you depend on.  Most of the VM
  479  *      system has yet to be initialized at this point.
  480  */
  481 static void
  482 swap_pager_init(void)
  483 {
  484         /*
  485          * Initialize object lists
  486          */
  487         int i;
  488 
  489         for (i = 0; i < NOBJLISTS; ++i)
  490                 TAILQ_INIT(&swap_pager_object_list[i]);
  491         mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
  492         mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
  493 
  494         /*
  495          * Device Stripe, in PAGE_SIZE'd blocks
  496          */
  497         dmmax = SWB_NPAGES * 2;
  498 }
  499 
  500 /*
  501  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
  502  *
  503  *      Expected to be started from pageout process once, prior to entering
  504  *      its main loop.
  505  */
  506 void
  507 swap_pager_swap_init(void)
  508 {
  509         int n, n2;
  510 
  511         /*
  512          * Number of in-transit swap bp operations.  Don't
  513          * exhaust the pbufs completely.  Make sure we
  514          * initialize workable values (0 will work for hysteresis
  515          * but it isn't very efficient).
  516          *
  517          * The nsw_cluster_max is constrained by the bp->b_pages[]
  518          * array (MAXPHYS/PAGE_SIZE) and our locally defined
  519          * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
  520          * constrained by the swap device interleave stripe size.
  521          *
  522          * Currently we hardwire nsw_wcount_async to 4.  This limit is 
  523          * designed to prevent other I/O from having high latencies due to
  524          * our pageout I/O.  The value 4 works well for one or two active swap
  525          * devices but is probably a little low if you have more.  Even so,
  526          * a higher value would probably generate only a limited improvement
  527          * with three or four active swap devices since the system does not
  528          * typically have to pageout at extreme bandwidths.   We will want
  529          * at least 2 per swap devices, and 4 is a pretty good value if you
  530          * have one NFS swap device due to the command/ack latency over NFS.
  531          * So it all works out pretty well.
  532          */
  533         nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
  534 
  535         mtx_lock(&pbuf_mtx);
  536         nsw_rcount = (nswbuf + 1) / 2;
  537         nsw_wcount_sync = (nswbuf + 3) / 4;
  538         nsw_wcount_async = 4;
  539         nsw_wcount_async_max = nsw_wcount_async;
  540         mtx_unlock(&pbuf_mtx);
  541 
  542         /*
  543          * Initialize our zone.  Right now I'm just guessing on the number
  544          * we need based on the number of pages in the system.  Each swblock
  545          * can hold 16 pages, so this is probably overkill.  This reservation
  546          * is typically limited to around 32MB by default.
  547          */
  548         n = cnt.v_page_count / 2;
  549         if (maxswzone && n > maxswzone / sizeof(struct swblock))
  550                 n = maxswzone / sizeof(struct swblock);
  551         n2 = n;
  552         swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
  553             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
  554         if (swap_zone == NULL)
  555                 panic("failed to create swap_zone.");
  556         do {
  557                 if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
  558                         break;
  559                 /*
  560                  * if the allocation failed, try a zone two thirds the
  561                  * size of the previous attempt.
  562                  */
  563                 n -= ((n + 2) / 3);
  564         } while (n > 0);
  565         if (n2 != n)
  566                 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
  567         n2 = n;
  568 
  569         /*
  570          * Initialize our meta-data hash table.  The swapper does not need to
  571          * be quite as efficient as the VM system, so we do not use an 
  572          * oversized hash table.
  573          *
  574          *      n:              size of hash table, must be power of 2
  575          *      swhash_mask:    hash table index mask
  576          */
  577         for (n = 1; n < n2 / 8; n *= 2)
  578                 ;
  579         swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
  580         swhash_mask = n - 1;
  581         mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
  582 }
  583 
  584 /*
  585  * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
  586  *                      its metadata structures.
  587  *
  588  *      This routine is called from the mmap and fork code to create a new
  589  *      OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
  590  *      and then converting it with swp_pager_meta_build().
  591  *
  592  *      This routine may block in vm_object_allocate() and create a named
  593  *      object lookup race, so we must interlock.
  594  *
  595  * MPSAFE
  596  */
  597 static vm_object_t
  598 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
  599     vm_ooffset_t offset, struct ucred *cred)
  600 {
  601         vm_object_t object;
  602         vm_pindex_t pindex;
  603 
  604         pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
  605         if (handle) {
  606                 mtx_lock(&Giant);
  607                 /*
  608                  * Reference existing named region or allocate new one.  There
  609                  * should not be a race here against swp_pager_meta_build()
  610                  * as called from vm_page_remove() in regards to the lookup
  611                  * of the handle.
  612                  */
  613                 sx_xlock(&sw_alloc_sx);
  614                 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
  615                 if (object == NULL) {
  616                         if (cred != NULL) {
  617                                 if (!swap_reserve_by_cred(size, cred)) {
  618                                         sx_xunlock(&sw_alloc_sx);
  619                                         mtx_unlock(&Giant);
  620                                         return (NULL);
  621                                 }
  622                                 crhold(cred);
  623                         }
  624                         object = vm_object_allocate(OBJT_DEFAULT, pindex);
  625                         VM_OBJECT_LOCK(object);
  626                         object->handle = handle;
  627                         if (cred != NULL) {
  628                                 object->cred = cred;
  629                                 object->charge = size;
  630                         }
  631                         swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  632                         VM_OBJECT_UNLOCK(object);
  633                 }
  634                 sx_xunlock(&sw_alloc_sx);
  635                 mtx_unlock(&Giant);
  636         } else {
  637                 if (cred != NULL) {
  638                         if (!swap_reserve_by_cred(size, cred))
  639                                 return (NULL);
  640                         crhold(cred);
  641                 }
  642                 object = vm_object_allocate(OBJT_DEFAULT, pindex);
  643                 VM_OBJECT_LOCK(object);
  644                 if (cred != NULL) {
  645                         object->cred = cred;
  646                         object->charge = size;
  647                 }
  648                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  649                 VM_OBJECT_UNLOCK(object);
  650         }
  651         return (object);
  652 }
  653 
  654 /*
  655  * SWAP_PAGER_DEALLOC() -       remove swap metadata from object
  656  *
  657  *      The swap backing for the object is destroyed.  The code is 
  658  *      designed such that we can reinstantiate it later, but this
  659  *      routine is typically called only when the entire object is
  660  *      about to be destroyed.
  661  *
  662  *      The object must be locked.
  663  */
  664 static void
  665 swap_pager_dealloc(vm_object_t object)
  666 {
  667 
  668         /*
  669          * Remove from list right away so lookups will fail if we block for
  670          * pageout completion.
  671          */
  672         if (object->handle != NULL) {
  673                 mtx_lock(&sw_alloc_mtx);
  674                 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
  675                 mtx_unlock(&sw_alloc_mtx);
  676         }
  677 
  678         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  679         vm_object_pip_wait(object, "swpdea");
  680 
  681         /*
  682          * Free all remaining metadata.  We only bother to free it from 
  683          * the swap meta data.  We do not attempt to free swapblk's still
  684          * associated with vm_page_t's for this object.  We do not care
  685          * if paging is still in progress on some objects.
  686          */
  687         swp_pager_meta_free_all(object);
  688 }
  689 
  690 /************************************************************************
  691  *                      SWAP PAGER BITMAP ROUTINES                      *
  692  ************************************************************************/
  693 
  694 /*
  695  * SWP_PAGER_GETSWAPSPACE() -   allocate raw swap space
  696  *
  697  *      Allocate swap for the requested number of pages.  The starting
  698  *      swap block number (a page index) is returned or SWAPBLK_NONE
  699  *      if the allocation failed.
  700  *
  701  *      Also has the side effect of advising that somebody made a mistake
  702  *      when they configured swap and didn't configure enough.
  703  *
  704  *      This routine may not sleep.
  705  *
  706  *      We allocate in round-robin fashion from the configured devices.
  707  */
  708 static daddr_t
  709 swp_pager_getswapspace(int npages)
  710 {
  711         daddr_t blk;
  712         struct swdevt *sp;
  713         int i;
  714 
  715         blk = SWAPBLK_NONE;
  716         mtx_lock(&sw_dev_mtx);
  717         sp = swdevhd;
  718         for (i = 0; i < nswapdev; i++) {
  719                 if (sp == NULL)
  720                         sp = TAILQ_FIRST(&swtailq);
  721                 if (!(sp->sw_flags & SW_CLOSING)) {
  722                         blk = blist_alloc(sp->sw_blist, npages);
  723                         if (blk != SWAPBLK_NONE) {
  724                                 blk += sp->sw_first;
  725                                 sp->sw_used += npages;
  726                                 swap_pager_avail -= npages;
  727                                 swp_sizecheck();
  728                                 swdevhd = TAILQ_NEXT(sp, sw_list);
  729                                 goto done;
  730                         }
  731                 }
  732                 sp = TAILQ_NEXT(sp, sw_list);
  733         }
  734         if (swap_pager_full != 2) {
  735                 printf("swap_pager_getswapspace(%d): failed\n", npages);
  736                 swap_pager_full = 2;
  737                 swap_pager_almost_full = 1;
  738         }
  739         swdevhd = NULL;
  740 done:
  741         mtx_unlock(&sw_dev_mtx);
  742         return (blk);
  743 }
  744 
  745 static int
  746 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
  747 {
  748 
  749         return (blk >= sp->sw_first && blk < sp->sw_end);
  750 }
  751         
  752 static void
  753 swp_pager_strategy(struct buf *bp)
  754 {
  755         struct swdevt *sp;
  756 
  757         mtx_lock(&sw_dev_mtx);
  758         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  759                 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
  760                         mtx_unlock(&sw_dev_mtx);
  761                         sp->sw_strategy(bp, sp);
  762                         return;
  763                 }
  764         }
  765         panic("Swapdev not found");
  766 }
  767         
  768 
  769 /*
  770  * SWP_PAGER_FREESWAPSPACE() -  free raw swap space 
  771  *
  772  *      This routine returns the specified swap blocks back to the bitmap.
  773  *
  774  *      This routine may not sleep.
  775  */
  776 static void
  777 swp_pager_freeswapspace(daddr_t blk, int npages)
  778 {
  779         struct swdevt *sp;
  780 
  781         mtx_lock(&sw_dev_mtx);
  782         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  783                 if (blk >= sp->sw_first && blk < sp->sw_end) {
  784                         sp->sw_used -= npages;
  785                         /*
  786                          * If we are attempting to stop swapping on
  787                          * this device, we don't want to mark any
  788                          * blocks free lest they be reused.  
  789                          */
  790                         if ((sp->sw_flags & SW_CLOSING) == 0) {
  791                                 blist_free(sp->sw_blist, blk - sp->sw_first,
  792                                     npages);
  793                                 swap_pager_avail += npages;
  794                                 swp_sizecheck();
  795                         }
  796                         mtx_unlock(&sw_dev_mtx);
  797                         return;
  798                 }
  799         }
  800         panic("Swapdev not found");
  801 }
  802 
  803 /*
  804  * SWAP_PAGER_FREESPACE() -     frees swap blocks associated with a page
  805  *                              range within an object.
  806  *
  807  *      This is a globally accessible routine.
  808  *
  809  *      This routine removes swapblk assignments from swap metadata.
  810  *
  811  *      The external callers of this routine typically have already destroyed 
  812  *      or renamed vm_page_t's associated with this range in the object so 
  813  *      we should be ok.
  814  */
  815 void
  816 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
  817 {
  818 
  819         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  820         swp_pager_meta_free(object, start, size);
  821 }
  822 
  823 /*
  824  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
  825  *
  826  *      Assigns swap blocks to the specified range within the object.  The 
  827  *      swap blocks are not zerod.  Any previous swap assignment is destroyed.
  828  *
  829  *      Returns 0 on success, -1 on failure.
  830  */
  831 int
  832 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
  833 {
  834         int n = 0;
  835         daddr_t blk = SWAPBLK_NONE;
  836         vm_pindex_t beg = start;        /* save start index */
  837 
  838         VM_OBJECT_LOCK(object);
  839         while (size) {
  840                 if (n == 0) {
  841                         n = BLIST_MAX_ALLOC;
  842                         while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
  843                                 n >>= 1;
  844                                 if (n == 0) {
  845                                         swp_pager_meta_free(object, beg, start - beg);
  846                                         VM_OBJECT_UNLOCK(object);
  847                                         return (-1);
  848                                 }
  849                         }
  850                 }
  851                 swp_pager_meta_build(object, start, blk);
  852                 --size;
  853                 ++start;
  854                 ++blk;
  855                 --n;
  856         }
  857         swp_pager_meta_free(object, start, n);
  858         VM_OBJECT_UNLOCK(object);
  859         return (0);
  860 }
  861 
  862 /*
  863  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
  864  *                      and destroy the source.
  865  *
  866  *      Copy any valid swapblks from the source to the destination.  In
  867  *      cases where both the source and destination have a valid swapblk,
  868  *      we keep the destination's.
  869  *
  870  *      This routine is allowed to sleep.  It may sleep allocating metadata
  871  *      indirectly through swp_pager_meta_build() or if paging is still in
  872  *      progress on the source. 
  873  *
  874  *      The source object contains no vm_page_t's (which is just as well)
  875  *
  876  *      The source object is of type OBJT_SWAP.
  877  *
  878  *      The source and destination objects must be locked.
  879  *      Both object locks may temporarily be released.
  880  */
  881 void
  882 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
  883     vm_pindex_t offset, int destroysource)
  884 {
  885         vm_pindex_t i;
  886 
  887         VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
  888         VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
  889 
  890         /*
  891          * If destroysource is set, we remove the source object from the 
  892          * swap_pager internal queue now. 
  893          */
  894         if (destroysource) {
  895                 if (srcobject->handle != NULL) {
  896                         mtx_lock(&sw_alloc_mtx);
  897                         TAILQ_REMOVE(
  898                             NOBJLIST(srcobject->handle),
  899                             srcobject,
  900                             pager_object_list
  901                         );
  902                         mtx_unlock(&sw_alloc_mtx);
  903                 }
  904         }
  905 
  906         /*
  907          * transfer source to destination.
  908          */
  909         for (i = 0; i < dstobject->size; ++i) {
  910                 daddr_t dstaddr;
  911 
  912                 /*
  913                  * Locate (without changing) the swapblk on the destination,
  914                  * unless it is invalid in which case free it silently, or
  915                  * if the destination is a resident page, in which case the
  916                  * source is thrown away.
  917                  */
  918                 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
  919 
  920                 if (dstaddr == SWAPBLK_NONE) {
  921                         /*
  922                          * Destination has no swapblk and is not resident,
  923                          * copy source.
  924                          */
  925                         daddr_t srcaddr;
  926 
  927                         srcaddr = swp_pager_meta_ctl(
  928                             srcobject, 
  929                             i + offset,
  930                             SWM_POP
  931                         );
  932 
  933                         if (srcaddr != SWAPBLK_NONE) {
  934                                 /*
  935                                  * swp_pager_meta_build() can sleep.
  936                                  */
  937                                 vm_object_pip_add(srcobject, 1);
  938                                 VM_OBJECT_UNLOCK(srcobject);
  939                                 vm_object_pip_add(dstobject, 1);
  940                                 swp_pager_meta_build(dstobject, i, srcaddr);
  941                                 vm_object_pip_wakeup(dstobject);
  942                                 VM_OBJECT_LOCK(srcobject);
  943                                 vm_object_pip_wakeup(srcobject);
  944                         }
  945                 } else {
  946                         /*
  947                          * Destination has valid swapblk or it is represented
  948                          * by a resident page.  We destroy the sourceblock.
  949                          */
  950                         
  951                         swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
  952                 }
  953         }
  954 
  955         /*
  956          * Free left over swap blocks in source.
  957          *
  958          * We have to revert the type to OBJT_DEFAULT so we do not accidently
  959          * double-remove the object from the swap queues.
  960          */
  961         if (destroysource) {
  962                 swp_pager_meta_free_all(srcobject);
  963                 /*
  964                  * Reverting the type is not necessary, the caller is going
  965                  * to destroy srcobject directly, but I'm doing it here
  966                  * for consistency since we've removed the object from its
  967                  * queues.
  968                  */
  969                 srcobject->type = OBJT_DEFAULT;
  970         }
  971 }
  972 
  973 /*
  974  * SWAP_PAGER_HASPAGE() -       determine if we have good backing store for
  975  *                              the requested page.
  976  *
  977  *      We determine whether good backing store exists for the requested
  978  *      page and return TRUE if it does, FALSE if it doesn't.
  979  *
  980  *      If TRUE, we also try to determine how much valid, contiguous backing
  981  *      store exists before and after the requested page within a reasonable
  982  *      distance.  We do not try to restrict it to the swap device stripe
  983  *      (that is handled in getpages/putpages).  It probably isn't worth
  984  *      doing here.
  985  */
  986 static boolean_t
  987 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
  988 {
  989         daddr_t blk0;
  990 
  991         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  992         /*
  993          * do we have good backing store at the requested index ?
  994          */
  995         blk0 = swp_pager_meta_ctl(object, pindex, 0);
  996 
  997         if (blk0 == SWAPBLK_NONE) {
  998                 if (before)
  999                         *before = 0;
 1000                 if (after)
 1001                         *after = 0;
 1002                 return (FALSE);
 1003         }
 1004 
 1005         /*
 1006          * find backwards-looking contiguous good backing store
 1007          */
 1008         if (before != NULL) {
 1009                 int i;
 1010 
 1011                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
 1012                         daddr_t blk;
 1013 
 1014                         if (i > pindex)
 1015                                 break;
 1016                         blk = swp_pager_meta_ctl(object, pindex - i, 0);
 1017                         if (blk != blk0 - i)
 1018                                 break;
 1019                 }
 1020                 *before = (i - 1);
 1021         }
 1022 
 1023         /*
 1024          * find forward-looking contiguous good backing store
 1025          */
 1026         if (after != NULL) {
 1027                 int i;
 1028 
 1029                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
 1030                         daddr_t blk;
 1031 
 1032                         blk = swp_pager_meta_ctl(object, pindex + i, 0);
 1033                         if (blk != blk0 + i)
 1034                                 break;
 1035                 }
 1036                 *after = (i - 1);
 1037         }
 1038         return (TRUE);
 1039 }
 1040 
 1041 /*
 1042  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
 1043  *
 1044  *      This removes any associated swap backing store, whether valid or
 1045  *      not, from the page.  
 1046  *
 1047  *      This routine is typically called when a page is made dirty, at
 1048  *      which point any associated swap can be freed.  MADV_FREE also
 1049  *      calls us in a special-case situation
 1050  *
 1051  *      NOTE!!!  If the page is clean and the swap was valid, the caller
 1052  *      should make the page dirty before calling this routine.  This routine
 1053  *      does NOT change the m->dirty status of the page.  Also: MADV_FREE
 1054  *      depends on it.
 1055  *
 1056  *      This routine may not sleep.
 1057  */
 1058 static void
 1059 swap_pager_unswapped(vm_page_t m)
 1060 {
 1061 
 1062         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1063         swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
 1064 }
 1065 
 1066 /*
 1067  * SWAP_PAGER_GETPAGES() - bring pages in from swap
 1068  *
 1069  *      Attempt to retrieve (m, count) pages from backing store, but make
 1070  *      sure we retrieve at least m[reqpage].  We try to load in as large
 1071  *      a chunk surrounding m[reqpage] as is contiguous in swap and which
 1072  *      belongs to the same object.
 1073  *
 1074  *      The code is designed for asynchronous operation and 
 1075  *      immediate-notification of 'reqpage' but tends not to be
 1076  *      used that way.  Please do not optimize-out this algorithmic
 1077  *      feature, I intend to improve on it in the future.
 1078  *
 1079  *      The parent has a single vm_object_pip_add() reference prior to
 1080  *      calling us and we should return with the same.
 1081  *
 1082  *      The parent has BUSY'd the pages.  We should return with 'm'
 1083  *      left busy, but the others adjusted.
 1084  */
 1085 static int
 1086 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
 1087 {
 1088         struct buf *bp;
 1089         vm_page_t mreq;
 1090         int i;
 1091         int j;
 1092         daddr_t blk;
 1093 
 1094         mreq = m[reqpage];
 1095 
 1096         KASSERT(mreq->object == object,
 1097             ("swap_pager_getpages: object mismatch %p/%p",
 1098             object, mreq->object));
 1099 
 1100         /*
 1101          * Calculate range to retrieve.  The pages have already been assigned
 1102          * their swapblks.  We require a *contiguous* range but we know it to
 1103          * not span devices.   If we do not supply it, bad things
 1104          * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the 
 1105          * loops are set up such that the case(s) are handled implicitly.
 1106          *
 1107          * The swp_*() calls must be made with the object locked.
 1108          */
 1109         blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
 1110 
 1111         for (i = reqpage - 1; i >= 0; --i) {
 1112                 daddr_t iblk;
 1113 
 1114                 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
 1115                 if (blk != iblk + (reqpage - i))
 1116                         break;
 1117         }
 1118         ++i;
 1119 
 1120         for (j = reqpage + 1; j < count; ++j) {
 1121                 daddr_t jblk;
 1122 
 1123                 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
 1124                 if (blk != jblk - (j - reqpage))
 1125                         break;
 1126         }
 1127 
 1128         /*
 1129          * free pages outside our collection range.   Note: we never free
 1130          * mreq, it must remain busy throughout.
 1131          */
 1132         if (0 < i || j < count) {
 1133                 int k;
 1134 
 1135                 for (k = 0; k < i; ++k)
 1136                         swp_pager_free_nrpage(m[k]);
 1137                 for (k = j; k < count; ++k)
 1138                         swp_pager_free_nrpage(m[k]);
 1139         }
 1140 
 1141         /*
 1142          * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq 
 1143          * still busy, but the others unbusied.
 1144          */
 1145         if (blk == SWAPBLK_NONE)
 1146                 return (VM_PAGER_FAIL);
 1147 
 1148         /*
 1149          * Getpbuf() can sleep.
 1150          */
 1151         VM_OBJECT_UNLOCK(object);
 1152         /*
 1153          * Get a swap buffer header to perform the IO
 1154          */
 1155         bp = getpbuf(&nsw_rcount);
 1156         bp->b_flags |= B_PAGING;
 1157 
 1158         /*
 1159          * map our page(s) into kva for input
 1160          */
 1161         pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
 1162 
 1163         bp->b_iocmd = BIO_READ;
 1164         bp->b_iodone = swp_pager_async_iodone;
 1165         bp->b_rcred = crhold(thread0.td_ucred);
 1166         bp->b_wcred = crhold(thread0.td_ucred);
 1167         bp->b_blkno = blk - (reqpage - i);
 1168         bp->b_bcount = PAGE_SIZE * (j - i);
 1169         bp->b_bufsize = PAGE_SIZE * (j - i);
 1170         bp->b_pager.pg_reqpage = reqpage - i;
 1171 
 1172         VM_OBJECT_LOCK(object);
 1173         {
 1174                 int k;
 1175 
 1176                 for (k = i; k < j; ++k) {
 1177                         bp->b_pages[k - i] = m[k];
 1178                         m[k]->oflags |= VPO_SWAPINPROG;
 1179                 }
 1180         }
 1181         bp->b_npages = j - i;
 1182 
 1183         PCPU_INC(cnt.v_swapin);
 1184         PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
 1185 
 1186         /*
 1187          * We still hold the lock on mreq, and our automatic completion routine
 1188          * does not remove it.
 1189          */
 1190         vm_object_pip_add(object, bp->b_npages);
 1191         VM_OBJECT_UNLOCK(object);
 1192 
 1193         /*
 1194          * perform the I/O.  NOTE!!!  bp cannot be considered valid after
 1195          * this point because we automatically release it on completion.
 1196          * Instead, we look at the one page we are interested in which we
 1197          * still hold a lock on even through the I/O completion.
 1198          *
 1199          * The other pages in our m[] array are also released on completion,
 1200          * so we cannot assume they are valid anymore either.
 1201          *
 1202          * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1203          */
 1204         BUF_KERNPROC(bp);
 1205         swp_pager_strategy(bp);
 1206 
 1207         /*
 1208          * wait for the page we want to complete.  VPO_SWAPINPROG is always
 1209          * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
 1210          * is set in the meta-data.
 1211          */
 1212         VM_OBJECT_LOCK(object);
 1213         while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
 1214                 mreq->oflags |= VPO_WANTED;
 1215                 PCPU_INC(cnt.v_intrans);
 1216                 if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) {
 1217                         printf(
 1218 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
 1219                             bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
 1220                 }
 1221         }
 1222 
 1223         /*
 1224          * mreq is left busied after completion, but all the other pages
 1225          * are freed.  If we had an unrecoverable read error the page will
 1226          * not be valid.
 1227          */
 1228         if (mreq->valid != VM_PAGE_BITS_ALL) {
 1229                 return (VM_PAGER_ERROR);
 1230         } else {
 1231                 return (VM_PAGER_OK);
 1232         }
 1233 
 1234         /*
 1235          * A final note: in a low swap situation, we cannot deallocate swap
 1236          * and mark a page dirty here because the caller is likely to mark
 1237          * the page clean when we return, causing the page to possibly revert 
 1238          * to all-zero's later.
 1239          */
 1240 }
 1241 
 1242 /*
 1243  *      swap_pager_putpages: 
 1244  *
 1245  *      Assign swap (if necessary) and initiate I/O on the specified pages.
 1246  *
 1247  *      We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
 1248  *      are automatically converted to SWAP objects.
 1249  *
 1250  *      In a low memory situation we may block in VOP_STRATEGY(), but the new 
 1251  *      vm_page reservation system coupled with properly written VFS devices 
 1252  *      should ensure that no low-memory deadlock occurs.  This is an area
 1253  *      which needs work.
 1254  *
 1255  *      The parent has N vm_object_pip_add() references prior to
 1256  *      calling us and will remove references for rtvals[] that are
 1257  *      not set to VM_PAGER_PEND.  We need to remove the rest on I/O
 1258  *      completion.
 1259  *
 1260  *      The parent has soft-busy'd the pages it passes us and will unbusy
 1261  *      those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
 1262  *      We need to unbusy the rest on I/O completion.
 1263  */
 1264 void
 1265 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 1266     boolean_t sync, int *rtvals)
 1267 {
 1268         int i;
 1269         int n = 0;
 1270 
 1271         if (count && m[0]->object != object) {
 1272                 panic("swap_pager_putpages: object mismatch %p/%p", 
 1273                     object, 
 1274                     m[0]->object
 1275                 );
 1276         }
 1277 
 1278         /*
 1279          * Step 1
 1280          *
 1281          * Turn object into OBJT_SWAP
 1282          * check for bogus sysops
 1283          * force sync if not pageout process
 1284          */
 1285         if (object->type != OBJT_SWAP)
 1286                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
 1287         VM_OBJECT_UNLOCK(object);
 1288 
 1289         if (curproc != pageproc)
 1290                 sync = TRUE;
 1291 
 1292         /*
 1293          * Step 2
 1294          *
 1295          * Update nsw parameters from swap_async_max sysctl values.  
 1296          * Do not let the sysop crash the machine with bogus numbers.
 1297          */
 1298         mtx_lock(&pbuf_mtx);
 1299         if (swap_async_max != nsw_wcount_async_max) {
 1300                 int n;
 1301 
 1302                 /*
 1303                  * limit range
 1304                  */
 1305                 if ((n = swap_async_max) > nswbuf / 2)
 1306                         n = nswbuf / 2;
 1307                 if (n < 1)
 1308                         n = 1;
 1309                 swap_async_max = n;
 1310 
 1311                 /*
 1312                  * Adjust difference ( if possible ).  If the current async
 1313                  * count is too low, we may not be able to make the adjustment
 1314                  * at this time.
 1315                  */
 1316                 n -= nsw_wcount_async_max;
 1317                 if (nsw_wcount_async + n >= 0) {
 1318                         nsw_wcount_async += n;
 1319                         nsw_wcount_async_max += n;
 1320                         wakeup(&nsw_wcount_async);
 1321                 }
 1322         }
 1323         mtx_unlock(&pbuf_mtx);
 1324 
 1325         /*
 1326          * Step 3
 1327          *
 1328          * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
 1329          * The page is left dirty until the pageout operation completes
 1330          * successfully.
 1331          */
 1332         for (i = 0; i < count; i += n) {
 1333                 int j;
 1334                 struct buf *bp;
 1335                 daddr_t blk;
 1336 
 1337                 /*
 1338                  * Maximum I/O size is limited by a number of factors.
 1339                  */
 1340                 n = min(BLIST_MAX_ALLOC, count - i);
 1341                 n = min(n, nsw_cluster_max);
 1342 
 1343                 /*
 1344                  * Get biggest block of swap we can.  If we fail, fall
 1345                  * back and try to allocate a smaller block.  Don't go
 1346                  * overboard trying to allocate space if it would overly
 1347                  * fragment swap.
 1348                  */
 1349                 while (
 1350                     (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
 1351                     n > 4
 1352                 ) {
 1353                         n >>= 1;
 1354                 }
 1355                 if (blk == SWAPBLK_NONE) {
 1356                         for (j = 0; j < n; ++j)
 1357                                 rtvals[i+j] = VM_PAGER_FAIL;
 1358                         continue;
 1359                 }
 1360 
 1361                 /*
 1362                  * All I/O parameters have been satisfied, build the I/O
 1363                  * request and assign the swap space.
 1364                  */
 1365                 if (sync == TRUE) {
 1366                         bp = getpbuf(&nsw_wcount_sync);
 1367                 } else {
 1368                         bp = getpbuf(&nsw_wcount_async);
 1369                         bp->b_flags = B_ASYNC;
 1370                 }
 1371                 bp->b_flags |= B_PAGING;
 1372                 bp->b_iocmd = BIO_WRITE;
 1373 
 1374                 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
 1375 
 1376                 bp->b_rcred = crhold(thread0.td_ucred);
 1377                 bp->b_wcred = crhold(thread0.td_ucred);
 1378                 bp->b_bcount = PAGE_SIZE * n;
 1379                 bp->b_bufsize = PAGE_SIZE * n;
 1380                 bp->b_blkno = blk;
 1381 
 1382                 VM_OBJECT_LOCK(object);
 1383                 for (j = 0; j < n; ++j) {
 1384                         vm_page_t mreq = m[i+j];
 1385 
 1386                         swp_pager_meta_build(
 1387                             mreq->object, 
 1388                             mreq->pindex,
 1389                             blk + j
 1390                         );
 1391                         vm_page_dirty(mreq);
 1392                         rtvals[i+j] = VM_PAGER_OK;
 1393 
 1394                         mreq->oflags |= VPO_SWAPINPROG;
 1395                         bp->b_pages[j] = mreq;
 1396                 }
 1397                 VM_OBJECT_UNLOCK(object);
 1398                 bp->b_npages = n;
 1399                 /*
 1400                  * Must set dirty range for NFS to work.
 1401                  */
 1402                 bp->b_dirtyoff = 0;
 1403                 bp->b_dirtyend = bp->b_bcount;
 1404 
 1405                 PCPU_INC(cnt.v_swapout);
 1406                 PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
 1407 
 1408                 /*
 1409                  * asynchronous
 1410                  *
 1411                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1412                  */
 1413                 if (sync == FALSE) {
 1414                         bp->b_iodone = swp_pager_async_iodone;
 1415                         BUF_KERNPROC(bp);
 1416                         swp_pager_strategy(bp);
 1417 
 1418                         for (j = 0; j < n; ++j)
 1419                                 rtvals[i+j] = VM_PAGER_PEND;
 1420                         /* restart outter loop */
 1421                         continue;
 1422                 }
 1423 
 1424                 /*
 1425                  * synchronous
 1426                  *
 1427                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1428                  */
 1429                 bp->b_iodone = bdone;
 1430                 swp_pager_strategy(bp);
 1431 
 1432                 /*
 1433                  * Wait for the sync I/O to complete, then update rtvals.
 1434                  * We just set the rtvals[] to VM_PAGER_PEND so we can call
 1435                  * our async completion routine at the end, thus avoiding a
 1436                  * double-free.
 1437                  */
 1438                 bwait(bp, PVM, "swwrt");
 1439                 for (j = 0; j < n; ++j)
 1440                         rtvals[i+j] = VM_PAGER_PEND;
 1441                 /*
 1442                  * Now that we are through with the bp, we can call the
 1443                  * normal async completion, which frees everything up.
 1444                  */
 1445                 swp_pager_async_iodone(bp);
 1446         }
 1447         VM_OBJECT_LOCK(object);
 1448 }
 1449 
 1450 /*
 1451  *      swp_pager_async_iodone:
 1452  *
 1453  *      Completion routine for asynchronous reads and writes from/to swap.
 1454  *      Also called manually by synchronous code to finish up a bp.
 1455  *
 1456  *      For READ operations, the pages are VPO_BUSY'd.  For WRITE operations, 
 1457  *      the pages are vm_page_t->busy'd.  For READ operations, we VPO_BUSY 
 1458  *      unbusy all pages except the 'main' request page.  For WRITE 
 1459  *      operations, we vm_page_t->busy'd unbusy all pages ( we can do this 
 1460  *      because we marked them all VM_PAGER_PEND on return from putpages ).
 1461  *
 1462  *      This routine may not sleep.
 1463  */
 1464 static void
 1465 swp_pager_async_iodone(struct buf *bp)
 1466 {
 1467         int i;
 1468         vm_object_t object = NULL;
 1469 
 1470         /*
 1471          * report error
 1472          */
 1473         if (bp->b_ioflags & BIO_ERROR) {
 1474                 printf(
 1475                     "swap_pager: I/O error - %s failed; blkno %ld,"
 1476                         "size %ld, error %d\n",
 1477                     ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
 1478                     (long)bp->b_blkno, 
 1479                     (long)bp->b_bcount,
 1480                     bp->b_error
 1481                 );
 1482         }
 1483 
 1484         /*
 1485          * remove the mapping for kernel virtual
 1486          */
 1487         pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
 1488 
 1489         if (bp->b_npages) {
 1490                 object = bp->b_pages[0]->object;
 1491                 VM_OBJECT_LOCK(object);
 1492         }
 1493 
 1494         /*
 1495          * cleanup pages.  If an error occurs writing to swap, we are in
 1496          * very serious trouble.  If it happens to be a disk error, though,
 1497          * we may be able to recover by reassigning the swap later on.  So
 1498          * in this case we remove the m->swapblk assignment for the page 
 1499          * but do not free it in the rlist.  The errornous block(s) are thus
 1500          * never reallocated as swap.  Redirty the page and continue.
 1501          */
 1502         for (i = 0; i < bp->b_npages; ++i) {
 1503                 vm_page_t m = bp->b_pages[i];
 1504 
 1505                 m->oflags &= ~VPO_SWAPINPROG;
 1506 
 1507                 if (bp->b_ioflags & BIO_ERROR) {
 1508                         /*
 1509                          * If an error occurs I'd love to throw the swapblk
 1510                          * away without freeing it back to swapspace, so it
 1511                          * can never be used again.  But I can't from an 
 1512                          * interrupt.
 1513                          */
 1514                         if (bp->b_iocmd == BIO_READ) {
 1515                                 /*
 1516                                  * When reading, reqpage needs to stay
 1517                                  * locked for the parent, but all other
 1518                                  * pages can be freed.  We still want to
 1519                                  * wakeup the parent waiting on the page,
 1520                                  * though.  ( also: pg_reqpage can be -1 and 
 1521                                  * not match anything ).
 1522                                  *
 1523                                  * We have to wake specifically requested pages
 1524                                  * up too because we cleared VPO_SWAPINPROG and
 1525                                  * someone may be waiting for that.
 1526                                  *
 1527                                  * NOTE: for reads, m->dirty will probably
 1528                                  * be overridden by the original caller of
 1529                                  * getpages so don't play cute tricks here.
 1530                                  */
 1531                                 m->valid = 0;
 1532                                 if (i != bp->b_pager.pg_reqpage)
 1533                                         swp_pager_free_nrpage(m);
 1534                                 else
 1535                                         vm_page_flash(m);
 1536                                 /*
 1537                                  * If i == bp->b_pager.pg_reqpage, do not wake 
 1538                                  * the page up.  The caller needs to.
 1539                                  */
 1540                         } else {
 1541                                 /*
 1542                                  * If a write error occurs, reactivate page
 1543                                  * so it doesn't clog the inactive list,
 1544                                  * then finish the I/O.
 1545                                  */
 1546                                 vm_page_dirty(m);
 1547                                 vm_page_lock(m);
 1548                                 vm_page_activate(m);
 1549                                 vm_page_unlock(m);
 1550                                 vm_page_io_finish(m);
 1551                         }
 1552                 } else if (bp->b_iocmd == BIO_READ) {
 1553                         /*
 1554                          * NOTE: for reads, m->dirty will probably be 
 1555                          * overridden by the original caller of getpages so
 1556                          * we cannot set them in order to free the underlying
 1557                          * swap in a low-swap situation.  I don't think we'd
 1558                          * want to do that anyway, but it was an optimization
 1559                          * that existed in the old swapper for a time before
 1560                          * it got ripped out due to precisely this problem.
 1561                          *
 1562                          * If not the requested page then deactivate it.
 1563                          *
 1564                          * Note that the requested page, reqpage, is left
 1565                          * busied, but we still have to wake it up.  The
 1566                          * other pages are released (unbusied) by 
 1567                          * vm_page_wakeup().
 1568                          */
 1569                         KASSERT(!pmap_page_is_mapped(m),
 1570                             ("swp_pager_async_iodone: page %p is mapped", m));
 1571                         m->valid = VM_PAGE_BITS_ALL;
 1572                         KASSERT(m->dirty == 0,
 1573                             ("swp_pager_async_iodone: page %p is dirty", m));
 1574 
 1575                         /*
 1576                          * We have to wake specifically requested pages
 1577                          * up too because we cleared VPO_SWAPINPROG and
 1578                          * could be waiting for it in getpages.  However,
 1579                          * be sure to not unbusy getpages specifically
 1580                          * requested page - getpages expects it to be 
 1581                          * left busy.
 1582                          */
 1583                         if (i != bp->b_pager.pg_reqpage) {
 1584                                 vm_page_lock(m);
 1585                                 vm_page_deactivate(m);
 1586                                 vm_page_unlock(m);
 1587                                 vm_page_wakeup(m);
 1588                         } else
 1589                                 vm_page_flash(m);
 1590                 } else {
 1591                         /*
 1592                          * For write success, clear the dirty
 1593                          * status, then finish the I/O ( which decrements the 
 1594                          * busy count and possibly wakes waiter's up ).
 1595                          */
 1596                         KASSERT((m->aflags & PGA_WRITEABLE) == 0,
 1597                             ("swp_pager_async_iodone: page %p is not write"
 1598                             " protected", m));
 1599                         vm_page_undirty(m);
 1600                         vm_page_io_finish(m);
 1601                         if (vm_page_count_severe()) {
 1602                                 vm_page_lock(m);
 1603                                 vm_page_try_to_cache(m);
 1604                                 vm_page_unlock(m);
 1605                         }
 1606                 }
 1607         }
 1608 
 1609         /*
 1610          * adjust pip.  NOTE: the original parent may still have its own
 1611          * pip refs on the object.
 1612          */
 1613         if (object != NULL) {
 1614                 vm_object_pip_wakeupn(object, bp->b_npages);
 1615                 VM_OBJECT_UNLOCK(object);
 1616         }
 1617 
 1618         /* 
 1619          * swapdev_strategy() manually sets b_vp and b_bufobj before calling 
 1620          * bstrategy(). Set them back to NULL now we're done with it, or we'll
 1621          * trigger a KASSERT in relpbuf().
 1622          */
 1623         if (bp->b_vp) {
 1624                     bp->b_vp = NULL;
 1625                     bp->b_bufobj = NULL;
 1626         }
 1627         /*
 1628          * release the physical I/O buffer
 1629          */
 1630         relpbuf(
 1631             bp, 
 1632             ((bp->b_iocmd == BIO_READ) ? &nsw_rcount : 
 1633                 ((bp->b_flags & B_ASYNC) ? 
 1634                     &nsw_wcount_async : 
 1635                     &nsw_wcount_sync
 1636                 )
 1637             )
 1638         );
 1639 }
 1640 
 1641 /*
 1642  *      swap_pager_isswapped:
 1643  *
 1644  *      Return 1 if at least one page in the given object is paged
 1645  *      out to the given swap device.
 1646  *
 1647  *      This routine may not sleep.
 1648  */
 1649 int
 1650 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
 1651 {
 1652         daddr_t index = 0;
 1653         int bcount;
 1654         int i;
 1655 
 1656         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1657         if (object->type != OBJT_SWAP)
 1658                 return (0);
 1659 
 1660         mtx_lock(&swhash_mtx);
 1661         for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
 1662                 struct swblock *swap;
 1663 
 1664                 if ((swap = *swp_pager_hash(object, index)) != NULL) {
 1665                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1666                                 if (swp_pager_isondev(swap->swb_pages[i], sp)) {
 1667                                         mtx_unlock(&swhash_mtx);
 1668                                         return (1);
 1669                                 }
 1670                         }
 1671                 }
 1672                 index += SWAP_META_PAGES;
 1673         }
 1674         mtx_unlock(&swhash_mtx);
 1675         return (0);
 1676 }
 1677 
 1678 /*
 1679  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
 1680  *
 1681  *      This routine dissociates the page at the given index within a
 1682  *      swap block from its backing store, paging it in if necessary.
 1683  *      If the page is paged in, it is placed in the inactive queue,
 1684  *      since it had its backing store ripped out from under it.
 1685  *      We also attempt to swap in all other pages in the swap block,
 1686  *      we only guarantee that the one at the specified index is
 1687  *      paged in.
 1688  *
 1689  *      XXX - The code to page the whole block in doesn't work, so we
 1690  *            revert to the one-by-one behavior for now.  Sigh.
 1691  */
 1692 static inline void
 1693 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
 1694 {
 1695         vm_page_t m;
 1696 
 1697         vm_object_pip_add(object, 1);
 1698         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
 1699         if (m->valid == VM_PAGE_BITS_ALL) {
 1700                 vm_object_pip_subtract(object, 1);
 1701                 vm_page_dirty(m);
 1702                 vm_page_lock(m);
 1703                 vm_page_activate(m);
 1704                 vm_page_unlock(m);
 1705                 vm_page_wakeup(m);
 1706                 vm_pager_page_unswapped(m);
 1707                 return;
 1708         }
 1709 
 1710         if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
 1711                 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
 1712         vm_object_pip_subtract(object, 1);
 1713         vm_page_dirty(m);
 1714         vm_page_lock(m);
 1715         vm_page_deactivate(m);
 1716         vm_page_unlock(m);
 1717         vm_page_wakeup(m);
 1718         vm_pager_page_unswapped(m);
 1719 }
 1720 
 1721 /*
 1722  *      swap_pager_swapoff:
 1723  *
 1724  *      Page in all of the pages that have been paged out to the
 1725  *      given device.  The corresponding blocks in the bitmap must be
 1726  *      marked as allocated and the device must be flagged SW_CLOSING.
 1727  *      There may be no processes swapped out to the device.
 1728  *
 1729  *      This routine may block.
 1730  */
 1731 static void
 1732 swap_pager_swapoff(struct swdevt *sp)
 1733 {
 1734         struct swblock *swap;
 1735         int i, j, retries;
 1736 
 1737         GIANT_REQUIRED;
 1738 
 1739         retries = 0;
 1740 full_rescan:
 1741         mtx_lock(&swhash_mtx);
 1742         for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
 1743 restart:
 1744                 for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
 1745                         vm_object_t object = swap->swb_object;
 1746                         vm_pindex_t pindex = swap->swb_index;
 1747                         for (j = 0; j < SWAP_META_PAGES; ++j) {
 1748                                 if (swp_pager_isondev(swap->swb_pages[j], sp)) {
 1749                                         /* avoid deadlock */
 1750                                         if (!VM_OBJECT_TRYLOCK(object)) {
 1751                                                 break;
 1752                                         } else {
 1753                                                 mtx_unlock(&swhash_mtx);
 1754                                                 swp_pager_force_pagein(object,
 1755                                                     pindex + j);
 1756                                                 VM_OBJECT_UNLOCK(object);
 1757                                                 mtx_lock(&swhash_mtx);
 1758                                                 goto restart;
 1759                                         }
 1760                                 }
 1761                         }
 1762                 }
 1763         }
 1764         mtx_unlock(&swhash_mtx);
 1765         if (sp->sw_used) {
 1766                 /*
 1767                  * Objects may be locked or paging to the device being
 1768                  * removed, so we will miss their pages and need to
 1769                  * make another pass.  We have marked this device as
 1770                  * SW_CLOSING, so the activity should finish soon.
 1771                  */
 1772                 retries++;
 1773                 if (retries > 100) {
 1774                         panic("swapoff: failed to locate %d swap blocks",
 1775                             sp->sw_used);
 1776                 }
 1777                 pause("swpoff", hz / 20);
 1778                 goto full_rescan;
 1779         }
 1780 }
 1781 
 1782 /************************************************************************
 1783  *                              SWAP META DATA                          *
 1784  ************************************************************************
 1785  *
 1786  *      These routines manipulate the swap metadata stored in the 
 1787  *      OBJT_SWAP object.
 1788  *
 1789  *      Swap metadata is implemented with a global hash and not directly
 1790  *      linked into the object.  Instead the object simply contains
 1791  *      appropriate tracking counters.
 1792  */
 1793 
 1794 /*
 1795  * SWP_PAGER_META_BUILD() -     add swap block to swap meta data for object
 1796  *
 1797  *      We first convert the object to a swap object if it is a default
 1798  *      object.
 1799  *
 1800  *      The specified swapblk is added to the object's swap metadata.  If
 1801  *      the swapblk is not valid, it is freed instead.  Any previously
 1802  *      assigned swapblk is freed.
 1803  */
 1804 static void
 1805 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
 1806 {
 1807         struct swblock *swap;
 1808         struct swblock **pswap;
 1809         int idx;
 1810 
 1811         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1812         /*
 1813          * Convert default object to swap object if necessary
 1814          */
 1815         if (object->type != OBJT_SWAP) {
 1816                 object->type = OBJT_SWAP;
 1817                 object->un_pager.swp.swp_bcount = 0;
 1818 
 1819                 if (object->handle != NULL) {
 1820                         mtx_lock(&sw_alloc_mtx);
 1821                         TAILQ_INSERT_TAIL(
 1822                             NOBJLIST(object->handle),
 1823                             object, 
 1824                             pager_object_list
 1825                         );
 1826                         mtx_unlock(&sw_alloc_mtx);
 1827                 }
 1828         }
 1829         
 1830         /*
 1831          * Locate hash entry.  If not found create, but if we aren't adding
 1832          * anything just return.  If we run out of space in the map we wait
 1833          * and, since the hash table may have changed, retry.
 1834          */
 1835 retry:
 1836         mtx_lock(&swhash_mtx);
 1837         pswap = swp_pager_hash(object, pindex);
 1838 
 1839         if ((swap = *pswap) == NULL) {
 1840                 int i;
 1841 
 1842                 if (swapblk == SWAPBLK_NONE)
 1843                         goto done;
 1844 
 1845                 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
 1846                 if (swap == NULL) {
 1847                         mtx_unlock(&swhash_mtx);
 1848                         VM_OBJECT_UNLOCK(object);
 1849                         if (uma_zone_exhausted(swap_zone)) {
 1850                                 printf("swap zone exhausted, increase kern.maxswzone\n");
 1851                                 vm_pageout_oom(VM_OOM_SWAPZ);
 1852                                 pause("swzonex", 10);
 1853                         } else
 1854                                 VM_WAIT;
 1855                         VM_OBJECT_LOCK(object);
 1856                         goto retry;
 1857                 }
 1858 
 1859                 swap->swb_hnext = NULL;
 1860                 swap->swb_object = object;
 1861                 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
 1862                 swap->swb_count = 0;
 1863 
 1864                 ++object->un_pager.swp.swp_bcount;
 1865 
 1866                 for (i = 0; i < SWAP_META_PAGES; ++i)
 1867                         swap->swb_pages[i] = SWAPBLK_NONE;
 1868         }
 1869 
 1870         /*
 1871          * Delete prior contents of metadata
 1872          */
 1873         idx = pindex & SWAP_META_MASK;
 1874 
 1875         if (swap->swb_pages[idx] != SWAPBLK_NONE) {
 1876                 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
 1877                 --swap->swb_count;
 1878         }
 1879 
 1880         /*
 1881          * Enter block into metadata
 1882          */
 1883         swap->swb_pages[idx] = swapblk;
 1884         if (swapblk != SWAPBLK_NONE)
 1885                 ++swap->swb_count;
 1886 done:
 1887         mtx_unlock(&swhash_mtx);
 1888 }
 1889 
 1890 /*
 1891  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
 1892  *
 1893  *      The requested range of blocks is freed, with any associated swap 
 1894  *      returned to the swap bitmap.
 1895  *
 1896  *      This routine will free swap metadata structures as they are cleaned 
 1897  *      out.  This routine does *NOT* operate on swap metadata associated
 1898  *      with resident pages.
 1899  */
 1900 static void
 1901 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
 1902 {
 1903 
 1904         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1905         if (object->type != OBJT_SWAP)
 1906                 return;
 1907 
 1908         while (count > 0) {
 1909                 struct swblock **pswap;
 1910                 struct swblock *swap;
 1911 
 1912                 mtx_lock(&swhash_mtx);
 1913                 pswap = swp_pager_hash(object, index);
 1914 
 1915                 if ((swap = *pswap) != NULL) {
 1916                         daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
 1917 
 1918                         if (v != SWAPBLK_NONE) {
 1919                                 swp_pager_freeswapspace(v, 1);
 1920                                 swap->swb_pages[index & SWAP_META_MASK] =
 1921                                         SWAPBLK_NONE;
 1922                                 if (--swap->swb_count == 0) {
 1923                                         *pswap = swap->swb_hnext;
 1924                                         uma_zfree(swap_zone, swap);
 1925                                         --object->un_pager.swp.swp_bcount;
 1926                                 }
 1927                         }
 1928                         --count;
 1929                         ++index;
 1930                 } else {
 1931                         int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
 1932                         count -= n;
 1933                         index += n;
 1934                 }
 1935                 mtx_unlock(&swhash_mtx);
 1936         }
 1937 }
 1938 
 1939 /*
 1940  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
 1941  *
 1942  *      This routine locates and destroys all swap metadata associated with
 1943  *      an object.
 1944  */
 1945 static void
 1946 swp_pager_meta_free_all(vm_object_t object)
 1947 {
 1948         daddr_t index = 0;
 1949 
 1950         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1951         if (object->type != OBJT_SWAP)
 1952                 return;
 1953 
 1954         while (object->un_pager.swp.swp_bcount) {
 1955                 struct swblock **pswap;
 1956                 struct swblock *swap;
 1957 
 1958                 mtx_lock(&swhash_mtx);
 1959                 pswap = swp_pager_hash(object, index);
 1960                 if ((swap = *pswap) != NULL) {
 1961                         int i;
 1962 
 1963                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1964                                 daddr_t v = swap->swb_pages[i];
 1965                                 if (v != SWAPBLK_NONE) {
 1966                                         --swap->swb_count;
 1967                                         swp_pager_freeswapspace(v, 1);
 1968                                 }
 1969                         }
 1970                         if (swap->swb_count != 0)
 1971                                 panic("swap_pager_meta_free_all: swb_count != 0");
 1972                         *pswap = swap->swb_hnext;
 1973                         uma_zfree(swap_zone, swap);
 1974                         --object->un_pager.swp.swp_bcount;
 1975                 }
 1976                 mtx_unlock(&swhash_mtx);
 1977                 index += SWAP_META_PAGES;
 1978         }
 1979 }
 1980 
 1981 /*
 1982  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
 1983  *
 1984  *      This routine is capable of looking up, popping, or freeing
 1985  *      swapblk assignments in the swap meta data or in the vm_page_t.
 1986  *      The routine typically returns the swapblk being looked-up, or popped,
 1987  *      or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
 1988  *      was invalid.  This routine will automatically free any invalid 
 1989  *      meta-data swapblks.
 1990  *
 1991  *      It is not possible to store invalid swapblks in the swap meta data
 1992  *      (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
 1993  *
 1994  *      When acting on a busy resident page and paging is in progress, we 
 1995  *      have to wait until paging is complete but otherwise can act on the 
 1996  *      busy page.
 1997  *
 1998  *      SWM_FREE        remove and free swap block from metadata
 1999  *      SWM_POP         remove from meta data but do not free.. pop it out
 2000  */
 2001 static daddr_t
 2002 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
 2003 {
 2004         struct swblock **pswap;
 2005         struct swblock *swap;
 2006         daddr_t r1;
 2007         int idx;
 2008 
 2009         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 2010         /*
 2011          * The meta data only exists of the object is OBJT_SWAP 
 2012          * and even then might not be allocated yet.
 2013          */
 2014         if (object->type != OBJT_SWAP)
 2015                 return (SWAPBLK_NONE);
 2016 
 2017         r1 = SWAPBLK_NONE;
 2018         mtx_lock(&swhash_mtx);
 2019         pswap = swp_pager_hash(object, pindex);
 2020 
 2021         if ((swap = *pswap) != NULL) {
 2022                 idx = pindex & SWAP_META_MASK;
 2023                 r1 = swap->swb_pages[idx];
 2024 
 2025                 if (r1 != SWAPBLK_NONE) {
 2026                         if (flags & SWM_FREE) {
 2027                                 swp_pager_freeswapspace(r1, 1);
 2028                                 r1 = SWAPBLK_NONE;
 2029                         }
 2030                         if (flags & (SWM_FREE|SWM_POP)) {
 2031                                 swap->swb_pages[idx] = SWAPBLK_NONE;
 2032                                 if (--swap->swb_count == 0) {
 2033                                         *pswap = swap->swb_hnext;
 2034                                         uma_zfree(swap_zone, swap);
 2035                                         --object->un_pager.swp.swp_bcount;
 2036                                 }
 2037                         } 
 2038                 }
 2039         }
 2040         mtx_unlock(&swhash_mtx);
 2041         return (r1);
 2042 }
 2043 
 2044 /*
 2045  * System call swapon(name) enables swapping on device name,
 2046  * which must be in the swdevsw.  Return EBUSY
 2047  * if already swapping on this device.
 2048  */
 2049 #ifndef _SYS_SYSPROTO_H_
 2050 struct swapon_args {
 2051         char *name;
 2052 };
 2053 #endif
 2054 
 2055 /* 
 2056  * MPSAFE
 2057  */
 2058 /* ARGSUSED */
 2059 int
 2060 sys_swapon(struct thread *td, struct swapon_args *uap)
 2061 {
 2062         struct vattr attr;
 2063         struct vnode *vp;
 2064         struct nameidata nd;
 2065         int error;
 2066 
 2067         error = priv_check(td, PRIV_SWAPON);
 2068         if (error)
 2069                 return (error);
 2070 
 2071         mtx_lock(&Giant);
 2072         while (swdev_syscall_active)
 2073             tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
 2074         swdev_syscall_active = 1;
 2075 
 2076         /*
 2077          * Swap metadata may not fit in the KVM if we have physical
 2078          * memory of >1GB.
 2079          */
 2080         if (swap_zone == NULL) {
 2081                 error = ENOMEM;
 2082                 goto done;
 2083         }
 2084 
 2085         NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
 2086             uap->name, td);
 2087         error = namei(&nd);
 2088         if (error)
 2089                 goto done;
 2090 
 2091         NDFREE(&nd, NDF_ONLY_PNBUF);
 2092         vp = nd.ni_vp;
 2093 
 2094         if (vn_isdisk(vp, &error)) {
 2095                 error = swapongeom(td, vp);
 2096         } else if (vp->v_type == VREG &&
 2097             (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
 2098             (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
 2099                 /*
 2100                  * Allow direct swapping to NFS regular files in the same
 2101                  * way that nfs_mountroot() sets up diskless swapping.
 2102                  */
 2103                 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
 2104         }
 2105 
 2106         if (error)
 2107                 vrele(vp);
 2108 done:
 2109         swdev_syscall_active = 0;
 2110         wakeup_one(&swdev_syscall_active);
 2111         mtx_unlock(&Giant);
 2112         return (error);
 2113 }
 2114 
 2115 static void
 2116 swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
 2117 {
 2118         struct swdevt *sp, *tsp;
 2119         swblk_t dvbase;
 2120         u_long mblocks;
 2121 
 2122         /*
 2123          * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
 2124          * First chop nblks off to page-align it, then convert.
 2125          * 
 2126          * sw->sw_nblks is in page-sized chunks now too.
 2127          */
 2128         nblks &= ~(ctodb(1) - 1);
 2129         nblks = dbtoc(nblks);
 2130 
 2131         /*
 2132          * If we go beyond this, we get overflows in the radix
 2133          * tree bitmap code.
 2134          */
 2135         mblocks = 0x40000000 / BLIST_META_RADIX;
 2136         if (nblks > mblocks) {
 2137                 printf(
 2138     "WARNING: reducing swap size to maximum of %luMB per unit\n",
 2139                     mblocks / 1024 / 1024 * PAGE_SIZE);
 2140                 nblks = mblocks;
 2141         }
 2142 
 2143         sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
 2144         sp->sw_vp = vp;
 2145         sp->sw_id = id;
 2146         sp->sw_dev = dev;
 2147         sp->sw_flags = 0;
 2148         sp->sw_nblks = nblks;
 2149         sp->sw_used = 0;
 2150         sp->sw_strategy = strategy;
 2151         sp->sw_close = close;
 2152 
 2153         sp->sw_blist = blist_create(nblks, M_WAITOK);
 2154         /*
 2155          * Do not free the first two block in order to avoid overwriting
 2156          * any bsd label at the front of the partition
 2157          */
 2158         blist_free(sp->sw_blist, 2, nblks - 2);
 2159 
 2160         dvbase = 0;
 2161         mtx_lock(&sw_dev_mtx);
 2162         TAILQ_FOREACH(tsp, &swtailq, sw_list) {
 2163                 if (tsp->sw_end >= dvbase) {
 2164                         /*
 2165                          * We put one uncovered page between the devices
 2166                          * in order to definitively prevent any cross-device
 2167                          * I/O requests
 2168                          */
 2169                         dvbase = tsp->sw_end + 1;
 2170                 }
 2171         }
 2172         sp->sw_first = dvbase;
 2173         sp->sw_end = dvbase + nblks;
 2174         TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
 2175         nswapdev++;
 2176         swap_pager_avail += nblks;
 2177         swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
 2178         swp_sizecheck();
 2179         mtx_unlock(&sw_dev_mtx);
 2180 }
 2181 
 2182 /*
 2183  * SYSCALL: swapoff(devname)
 2184  *
 2185  * Disable swapping on the given device.
 2186  *
 2187  * XXX: Badly designed system call: it should use a device index
 2188  * rather than filename as specification.  We keep sw_vp around
 2189  * only to make this work.
 2190  */
 2191 #ifndef _SYS_SYSPROTO_H_
 2192 struct swapoff_args {
 2193         char *name;
 2194 };
 2195 #endif
 2196 
 2197 /*
 2198  * MPSAFE
 2199  */
 2200 /* ARGSUSED */
 2201 int
 2202 sys_swapoff(struct thread *td, struct swapoff_args *uap)
 2203 {
 2204         struct vnode *vp;
 2205         struct nameidata nd;
 2206         struct swdevt *sp;
 2207         int error;
 2208 
 2209         error = priv_check(td, PRIV_SWAPOFF);
 2210         if (error)
 2211                 return (error);
 2212 
 2213         mtx_lock(&Giant);
 2214         while (swdev_syscall_active)
 2215             tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2216         swdev_syscall_active = 1;
 2217 
 2218         NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
 2219             td);
 2220         error = namei(&nd);
 2221         if (error)
 2222                 goto done;
 2223         NDFREE(&nd, NDF_ONLY_PNBUF);
 2224         vp = nd.ni_vp;
 2225 
 2226         mtx_lock(&sw_dev_mtx);
 2227         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2228                 if (sp->sw_vp == vp)
 2229                         break;
 2230         }
 2231         mtx_unlock(&sw_dev_mtx);
 2232         if (sp == NULL) {
 2233                 error = EINVAL;
 2234                 goto done;
 2235         }
 2236         error = swapoff_one(sp, td->td_ucred);
 2237 done:
 2238         swdev_syscall_active = 0;
 2239         wakeup_one(&swdev_syscall_active);
 2240         mtx_unlock(&Giant);
 2241         return (error);
 2242 }
 2243 
 2244 static int
 2245 swapoff_one(struct swdevt *sp, struct ucred *cred)
 2246 {
 2247         u_long nblks, dvbase;
 2248 #ifdef MAC
 2249         int error;
 2250 #endif
 2251 
 2252         mtx_assert(&Giant, MA_OWNED);
 2253 #ifdef MAC
 2254         (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
 2255         error = mac_system_check_swapoff(cred, sp->sw_vp);
 2256         (void) VOP_UNLOCK(sp->sw_vp, 0);
 2257         if (error != 0)
 2258                 return (error);
 2259 #endif
 2260         nblks = sp->sw_nblks;
 2261 
 2262         /*
 2263          * We can turn off this swap device safely only if the
 2264          * available virtual memory in the system will fit the amount
 2265          * of data we will have to page back in, plus an epsilon so
 2266          * the system doesn't become critically low on swap space.
 2267          */
 2268         if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
 2269             nblks + nswap_lowat) {
 2270                 return (ENOMEM);
 2271         }
 2272 
 2273         /*
 2274          * Prevent further allocations on this device.
 2275          */
 2276         mtx_lock(&sw_dev_mtx);
 2277         sp->sw_flags |= SW_CLOSING;
 2278         for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
 2279                 swap_pager_avail -= blist_fill(sp->sw_blist,
 2280                      dvbase, dmmax);
 2281         }
 2282         swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
 2283         mtx_unlock(&sw_dev_mtx);
 2284 
 2285         /*
 2286          * Page in the contents of the device and close it.
 2287          */
 2288         swap_pager_swapoff(sp);
 2289 
 2290         sp->sw_close(curthread, sp);
 2291         sp->sw_id = NULL;
 2292         mtx_lock(&sw_dev_mtx);
 2293         TAILQ_REMOVE(&swtailq, sp, sw_list);
 2294         nswapdev--;
 2295         if (nswapdev == 0) {
 2296                 swap_pager_full = 2;
 2297                 swap_pager_almost_full = 1;
 2298         }
 2299         if (swdevhd == sp)
 2300                 swdevhd = NULL;
 2301         mtx_unlock(&sw_dev_mtx);
 2302         blist_destroy(sp->sw_blist);
 2303         free(sp, M_VMPGDATA);
 2304         return (0);
 2305 }
 2306 
 2307 void
 2308 swapoff_all(void)
 2309 {
 2310         struct swdevt *sp, *spt;
 2311         const char *devname;
 2312         int error;
 2313  
 2314         mtx_lock(&Giant);
 2315         while (swdev_syscall_active)
 2316                 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2317         swdev_syscall_active = 1;
 2318  
 2319         mtx_lock(&sw_dev_mtx);
 2320         TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
 2321                 mtx_unlock(&sw_dev_mtx);
 2322                 if (vn_isdisk(sp->sw_vp, NULL))
 2323                         devname = sp->sw_vp->v_rdev->si_name;
 2324                 else
 2325                         devname = "[file]";
 2326                 error = swapoff_one(sp, thread0.td_ucred);
 2327                 if (error != 0) {
 2328                         printf("Cannot remove swap device %s (error=%d), "
 2329                             "skipping.\n", devname, error);
 2330                 } else if (bootverbose) {
 2331                         printf("Swap device %s removed.\n", devname);
 2332                 }
 2333                 mtx_lock(&sw_dev_mtx);
 2334         }
 2335         mtx_unlock(&sw_dev_mtx);
 2336  
 2337         swdev_syscall_active = 0;
 2338         wakeup_one(&swdev_syscall_active);
 2339         mtx_unlock(&Giant);
 2340 }
 2341 
 2342 void
 2343 swap_pager_status(int *total, int *used)
 2344 {
 2345         struct swdevt *sp;
 2346 
 2347         *total = 0;
 2348         *used = 0;
 2349         mtx_lock(&sw_dev_mtx);
 2350         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2351                 *total += sp->sw_nblks;
 2352                 *used += sp->sw_used;
 2353         }
 2354         mtx_unlock(&sw_dev_mtx);
 2355 }
 2356 
 2357 int
 2358 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
 2359 {
 2360         struct swdevt *sp;
 2361         char *tmp_devname;
 2362         int error, n;
 2363 
 2364         n = 0;
 2365         error = ENOENT;
 2366         mtx_lock(&sw_dev_mtx);
 2367         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2368                 if (n != name) {
 2369                         n++;
 2370                         continue;
 2371                 }
 2372                 xs->xsw_version = XSWDEV_VERSION;
 2373                 xs->xsw_dev = sp->sw_dev;
 2374                 xs->xsw_flags = sp->sw_flags;
 2375                 xs->xsw_nblks = sp->sw_nblks;
 2376                 xs->xsw_used = sp->sw_used;
 2377                 if (devname != NULL) {
 2378                         if (vn_isdisk(sp->sw_vp, NULL))
 2379                                 tmp_devname = sp->sw_vp->v_rdev->si_name;
 2380                         else
 2381                                 tmp_devname = "[file]";
 2382                         strncpy(devname, tmp_devname, len);
 2383                 }
 2384                 error = 0;
 2385                 break;
 2386         }
 2387         mtx_unlock(&sw_dev_mtx);
 2388         return (error);
 2389 }
 2390 
 2391 static int
 2392 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
 2393 {
 2394         struct xswdev xs;
 2395         int error;
 2396 
 2397         if (arg2 != 1)                  /* name length */
 2398                 return (EINVAL);
 2399         error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
 2400         if (error != 0)
 2401                 return (error);
 2402         error = SYSCTL_OUT(req, &xs, sizeof(xs));
 2403         return (error);
 2404 }
 2405 
 2406 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
 2407     "Number of swap devices");
 2408 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
 2409     "Swap statistics by device");
 2410 
 2411 /*
 2412  * vmspace_swap_count() - count the approximate swap usage in pages for a
 2413  *                        vmspace.
 2414  *
 2415  *      The map must be locked.
 2416  *
 2417  *      Swap usage is determined by taking the proportional swap used by
 2418  *      VM objects backing the VM map.  To make up for fractional losses,
 2419  *      if the VM object has any swap use at all the associated map entries
 2420  *      count for at least 1 swap page.
 2421  */
 2422 long
 2423 vmspace_swap_count(struct vmspace *vmspace)
 2424 {
 2425         vm_map_t map;
 2426         vm_map_entry_t cur;
 2427         vm_object_t object;
 2428         long count, n;
 2429 
 2430         map = &vmspace->vm_map;
 2431         count = 0;
 2432 
 2433         for (cur = map->header.next; cur != &map->header; cur = cur->next) {
 2434                 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2435                     (object = cur->object.vm_object) != NULL) {
 2436                         VM_OBJECT_LOCK(object);
 2437                         if (object->type == OBJT_SWAP &&
 2438                             object->un_pager.swp.swp_bcount != 0) {
 2439                                 n = (cur->end - cur->start) / PAGE_SIZE;
 2440                                 count += object->un_pager.swp.swp_bcount *
 2441                                     SWAP_META_PAGES * n / object->size + 1;
 2442                         }
 2443                         VM_OBJECT_UNLOCK(object);
 2444                 }
 2445         }
 2446         return (count);
 2447 }
 2448 
 2449 /*
 2450  * GEOM backend
 2451  *
 2452  * Swapping onto disk devices.
 2453  *
 2454  */
 2455 
 2456 static g_orphan_t swapgeom_orphan;
 2457 
 2458 static struct g_class g_swap_class = {
 2459         .name = "SWAP",
 2460         .version = G_VERSION,
 2461         .orphan = swapgeom_orphan,
 2462 };
 2463 
 2464 DECLARE_GEOM_CLASS(g_swap_class, g_class);
 2465 
 2466 
 2467 static void
 2468 swapgeom_done(struct bio *bp2)
 2469 {
 2470         struct buf *bp;
 2471 
 2472         bp = bp2->bio_caller2;
 2473         bp->b_ioflags = bp2->bio_flags;
 2474         if (bp2->bio_error)
 2475                 bp->b_ioflags |= BIO_ERROR;
 2476         bp->b_resid = bp->b_bcount - bp2->bio_completed;
 2477         bp->b_error = bp2->bio_error;
 2478         bufdone(bp);
 2479         g_destroy_bio(bp2);
 2480 }
 2481 
 2482 static void
 2483 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
 2484 {
 2485         struct bio *bio;
 2486         struct g_consumer *cp;
 2487 
 2488         cp = sp->sw_id;
 2489         if (cp == NULL) {
 2490                 bp->b_error = ENXIO;
 2491                 bp->b_ioflags |= BIO_ERROR;
 2492                 bufdone(bp);
 2493                 return;
 2494         }
 2495         if (bp->b_iocmd == BIO_WRITE)
 2496                 bio = g_new_bio();
 2497         else
 2498                 bio = g_alloc_bio();
 2499         if (bio == NULL) {
 2500                 bp->b_error = ENOMEM;
 2501                 bp->b_ioflags |= BIO_ERROR;
 2502                 bufdone(bp);
 2503                 return;
 2504         }
 2505 
 2506         bio->bio_caller2 = bp;
 2507         bio->bio_cmd = bp->b_iocmd;
 2508         bio->bio_data = bp->b_data;
 2509         bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
 2510         bio->bio_length = bp->b_bcount;
 2511         bio->bio_done = swapgeom_done;
 2512         g_io_request(bio, cp);
 2513         return;
 2514 }
 2515 
 2516 static void
 2517 swapgeom_orphan(struct g_consumer *cp)
 2518 {
 2519         struct swdevt *sp;
 2520 
 2521         mtx_lock(&sw_dev_mtx);
 2522         TAILQ_FOREACH(sp, &swtailq, sw_list)
 2523                 if (sp->sw_id == cp)
 2524                         sp->sw_id = NULL;
 2525         mtx_unlock(&sw_dev_mtx);
 2526 }
 2527 
 2528 static void
 2529 swapgeom_close_ev(void *arg, int flags)
 2530 {
 2531         struct g_consumer *cp;
 2532 
 2533         cp = arg;
 2534         g_access(cp, -1, -1, 0);
 2535         g_detach(cp);
 2536         g_destroy_consumer(cp);
 2537 }
 2538 
 2539 static void
 2540 swapgeom_close(struct thread *td, struct swdevt *sw)
 2541 {
 2542 
 2543         /* XXX: direct call when Giant untangled */
 2544         g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
 2545 }
 2546 
 2547 
 2548 struct swh0h0 {
 2549         struct cdev *dev;
 2550         struct vnode *vp;
 2551         int     error;
 2552 };
 2553 
 2554 static void
 2555 swapongeom_ev(void *arg, int flags)
 2556 {
 2557         struct swh0h0 *swh;
 2558         struct g_provider *pp;
 2559         struct g_consumer *cp;
 2560         static struct g_geom *gp;
 2561         struct swdevt *sp;
 2562         u_long nblks;
 2563         int error;
 2564 
 2565         swh = arg;
 2566         swh->error = 0;
 2567         pp = g_dev_getprovider(swh->dev);
 2568         if (pp == NULL) {
 2569                 swh->error = ENODEV;
 2570                 return;
 2571         }
 2572         mtx_lock(&sw_dev_mtx);
 2573         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2574                 cp = sp->sw_id;
 2575                 if (cp != NULL && cp->provider == pp) {
 2576                         mtx_unlock(&sw_dev_mtx);
 2577                         swh->error = EBUSY;
 2578                         return;
 2579                 }
 2580         }
 2581         mtx_unlock(&sw_dev_mtx);
 2582         if (gp == NULL)
 2583                 gp = g_new_geomf(&g_swap_class, "swap", NULL);
 2584         cp = g_new_consumer(gp);
 2585         g_attach(cp, pp);
 2586         /*
 2587          * XXX: Everytime you think you can improve the margin for
 2588          * footshooting, somebody depends on the ability to do so:
 2589          * savecore(8) wants to write to our swapdev so we cannot
 2590          * set an exclusive count :-(
 2591          */
 2592         error = g_access(cp, 1, 1, 0);
 2593         if (error) {
 2594                 g_detach(cp);
 2595                 g_destroy_consumer(cp);
 2596                 swh->error = error;
 2597                 return;
 2598         }
 2599         nblks = pp->mediasize / DEV_BSIZE;
 2600         swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
 2601             swapgeom_close, dev2udev(swh->dev));
 2602         swh->error = 0;
 2603         return;
 2604 }
 2605 
 2606 static int
 2607 swapongeom(struct thread *td, struct vnode *vp)
 2608 {
 2609         int error;
 2610         struct swh0h0 swh;
 2611 
 2612         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2613 
 2614         swh.dev = vp->v_rdev;
 2615         swh.vp = vp;
 2616         swh.error = 0;
 2617         /* XXX: direct call when Giant untangled */
 2618         error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
 2619         if (!error)
 2620                 error = swh.error;
 2621         VOP_UNLOCK(vp, 0);
 2622         return (error);
 2623 }
 2624 
 2625 /*
 2626  * VNODE backend
 2627  *
 2628  * This is used mainly for network filesystem (read: probably only tested
 2629  * with NFS) swapfiles.
 2630  *
 2631  */
 2632 
 2633 static void
 2634 swapdev_strategy(struct buf *bp, struct swdevt *sp)
 2635 {
 2636         struct vnode *vp2;
 2637 
 2638         bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
 2639 
 2640         vp2 = sp->sw_id;
 2641         vhold(vp2);
 2642         if (bp->b_iocmd == BIO_WRITE) {
 2643                 if (bp->b_bufobj)
 2644                         bufobj_wdrop(bp->b_bufobj);
 2645                 bufobj_wref(&vp2->v_bufobj);
 2646         }
 2647         if (bp->b_bufobj != &vp2->v_bufobj)
 2648                 bp->b_bufobj = &vp2->v_bufobj;
 2649         bp->b_vp = vp2;
 2650         bp->b_iooffset = dbtob(bp->b_blkno);
 2651         bstrategy(bp);
 2652         return;
 2653 }
 2654 
 2655 static void
 2656 swapdev_close(struct thread *td, struct swdevt *sp)
 2657 {
 2658 
 2659         VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
 2660         vrele(sp->sw_vp);
 2661 }
 2662 
 2663 
 2664 static int
 2665 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
 2666 {
 2667         struct swdevt *sp;
 2668         int error;
 2669 
 2670         if (nblks == 0)
 2671                 return (ENXIO);
 2672         mtx_lock(&sw_dev_mtx);
 2673         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2674                 if (sp->sw_id == vp) {
 2675                         mtx_unlock(&sw_dev_mtx);
 2676                         return (EBUSY);
 2677                 }
 2678         }
 2679         mtx_unlock(&sw_dev_mtx);
 2680     
 2681         (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2682 #ifdef MAC
 2683         error = mac_system_check_swapon(td->td_ucred, vp);
 2684         if (error == 0)
 2685 #endif
 2686                 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
 2687         (void) VOP_UNLOCK(vp, 0);
 2688         if (error)
 2689                 return (error);
 2690 
 2691         swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
 2692             NODEV);
 2693         return (0);
 2694 }

Cache object: d6b7e0a005cd0360ce2eb70992f38ea5


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.