The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/swap_pager.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Matthew Dillon,
    3  * Copyright (c) 1994 John S. Dyson
    4  * Copyright (c) 1990 University of Utah.
    5  * Copyright (c) 1982, 1986, 1989, 1993
    6  *      The Regents of the University of California.  All rights reserved.
    7  *
    8  * This code is derived from software contributed to Berkeley by
    9  * the Systems Programming Group of the University of Utah Computer
   10  * Science Department.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. All advertising materials mentioning features or use of this software
   21  *    must display the following acknowledgement:
   22  *      This product includes software developed by the University of
   23  *      California, Berkeley and its contributors.
   24  * 4. Neither the name of the University nor the names of its contributors
   25  *    may be used to endorse or promote products derived from this software
   26  *    without specific prior written permission.
   27  *
   28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   38  * SUCH DAMAGE.
   39  *
   40  *                              New Swap System
   41  *                              Matthew Dillon
   42  *
   43  * Radix Bitmap 'blists'.
   44  *
   45  *      - The new swapper uses the new radix bitmap code.  This should scale
   46  *        to arbitrarily small or arbitrarily large swap spaces and an almost
   47  *        arbitrary degree of fragmentation.
   48  *
   49  * Features:
   50  *
   51  *      - on the fly reallocation of swap during putpages.  The new system
   52  *        does not try to keep previously allocated swap blocks for dirty
   53  *        pages.
   54  *
   55  *      - on the fly deallocation of swap
   56  *
   57  *      - No more garbage collection required.  Unnecessarily allocated swap
   58  *        blocks only exist for dirty vm_page_t's now and these are already
   59  *        cycled (in a high-load system) by the pager.  We also do on-the-fly
   60  *        removal of invalidated swap blocks when a page is destroyed
   61  *        or renamed.
   62  *
   63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
   64  *
   65  *      @(#)swap_pager.c        8.9 (Berkeley) 3/21/94
   66  *      @(#)vm_swap.c   8.5 (Berkeley) 2/17/94
   67  */
   68 
   69 #include <sys/cdefs.h>
   70 __FBSDID("$FreeBSD$");
   71 
   72 #include "opt_swap.h"
   73 #include "opt_vm.h"
   74 
   75 #include <sys/param.h>
   76 #include <sys/systm.h>
   77 #include <sys/conf.h>
   78 #include <sys/kernel.h>
   79 #include <sys/priv.h>
   80 #include <sys/proc.h>
   81 #include <sys/bio.h>
   82 #include <sys/buf.h>
   83 #include <sys/disk.h>
   84 #include <sys/fcntl.h>
   85 #include <sys/mount.h>
   86 #include <sys/namei.h>
   87 #include <sys/vnode.h>
   88 #include <sys/malloc.h>
   89 #include <sys/racct.h>
   90 #include <sys/resource.h>
   91 #include <sys/resourcevar.h>
   92 #include <sys/rwlock.h>
   93 #include <sys/sysctl.h>
   94 #include <sys/sysproto.h>
   95 #include <sys/blist.h>
   96 #include <sys/lock.h>
   97 #include <sys/sx.h>
   98 #include <sys/vmmeter.h>
   99 
  100 #include <security/mac/mac_framework.h>
  101 
  102 #include <vm/vm.h>
  103 #include <vm/pmap.h>
  104 #include <vm/vm_map.h>
  105 #include <vm/vm_kern.h>
  106 #include <vm/vm_object.h>
  107 #include <vm/vm_page.h>
  108 #include <vm/vm_pager.h>
  109 #include <vm/vm_pageout.h>
  110 #include <vm/vm_param.h>
  111 #include <vm/swap_pager.h>
  112 #include <vm/vm_extern.h>
  113 #include <vm/uma.h>
  114 
  115 #include <geom/geom.h>
  116 
  117 /*
  118  * MAX_PAGEOUT_CLUSTER must be a power of 2 between 1 and 64.
  119  * The 64-page limit is due to the radix code (kern/subr_blist.c).
  120  */
  121 #ifndef MAX_PAGEOUT_CLUSTER
  122 #define MAX_PAGEOUT_CLUSTER 16
  123 #endif
  124 
  125 #if !defined(SWB_NPAGES)
  126 #define SWB_NPAGES      MAX_PAGEOUT_CLUSTER
  127 #endif
  128 
  129 /*
  130  * The swblock structure maps an object and a small, fixed-size range
  131  * of page indices to disk addresses within a swap area.
  132  * The collection of these mappings is implemented as a hash table.
  133  * Unused disk addresses within a swap area are allocated and managed
  134  * using a blist.
  135  */
  136 #define SWAP_META_PAGES         (SWB_NPAGES * 2)
  137 #define SWAP_META_MASK          (SWAP_META_PAGES - 1)
  138 
  139 struct swblock {
  140         struct swblock  *swb_hnext;
  141         vm_object_t     swb_object;
  142         vm_pindex_t     swb_index;
  143         int             swb_count;
  144         daddr_t         swb_pages[SWAP_META_PAGES];
  145 };
  146 
  147 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
  148 static struct mtx sw_dev_mtx;
  149 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
  150 static struct swdevt *swdevhd;  /* Allocate from here next */
  151 static int nswapdev;            /* Number of swap devices */
  152 int swap_pager_avail;
  153 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
  154 
  155 static vm_ooffset_t swap_total;
  156 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
  157     "Total amount of available swap storage.");
  158 static vm_ooffset_t swap_reserved;
  159 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
  160     "Amount of swap storage needed to back all allocated anonymous memory.");
  161 static int overcommit = 0;
  162 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
  163     "Configure virtual memory overcommit behavior. See tuning(7) "
  164     "for details.");
  165 static unsigned long swzone;
  166 SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0,
  167     "Actual size of swap metadata zone");
  168 static unsigned long swap_maxpages;
  169 SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0,
  170     "Maximum amount of swap supported");
  171 
  172 /* bits from overcommit */
  173 #define SWAP_RESERVE_FORCE_ON           (1 << 0)
  174 #define SWAP_RESERVE_RLIMIT_ON          (1 << 1)
  175 #define SWAP_RESERVE_ALLOW_NONWIRED     (1 << 2)
  176 
  177 int
  178 swap_reserve(vm_ooffset_t incr)
  179 {
  180 
  181         return (swap_reserve_by_cred(incr, curthread->td_ucred));
  182 }
  183 
  184 int
  185 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
  186 {
  187         vm_ooffset_t r, s;
  188         int res, error;
  189         static int curfail;
  190         static struct timeval lastfail;
  191         struct uidinfo *uip;
  192 
  193         uip = cred->cr_ruidinfo;
  194 
  195         if (incr & PAGE_MASK)
  196                 panic("swap_reserve: & PAGE_MASK");
  197 
  198 #ifdef RACCT
  199         if (racct_enable) {
  200                 PROC_LOCK(curproc);
  201                 error = racct_add(curproc, RACCT_SWAP, incr);
  202                 PROC_UNLOCK(curproc);
  203                 if (error != 0)
  204                         return (0);
  205         }
  206 #endif
  207 
  208         res = 0;
  209         mtx_lock(&sw_dev_mtx);
  210         r = swap_reserved + incr;
  211         if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
  212                 s = cnt.v_page_count - cnt.v_free_reserved - cnt.v_wire_count;
  213                 s *= PAGE_SIZE;
  214         } else
  215                 s = 0;
  216         s += swap_total;
  217         if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
  218             (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
  219                 res = 1;
  220                 swap_reserved = r;
  221         }
  222         mtx_unlock(&sw_dev_mtx);
  223 
  224         if (res) {
  225                 PROC_LOCK(curproc);
  226                 UIDINFO_VMSIZE_LOCK(uip);
  227                 if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
  228                     uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
  229                     priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
  230                         res = 0;
  231                 else
  232                         uip->ui_vmsize += incr;
  233                 UIDINFO_VMSIZE_UNLOCK(uip);
  234                 PROC_UNLOCK(curproc);
  235                 if (!res) {
  236                         mtx_lock(&sw_dev_mtx);
  237                         swap_reserved -= incr;
  238                         mtx_unlock(&sw_dev_mtx);
  239                 }
  240         }
  241         if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
  242                 printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
  243                     uip->ui_uid, curproc->p_pid, incr);
  244         }
  245 
  246 #ifdef RACCT
  247         if (!res) {
  248                 PROC_LOCK(curproc);
  249                 racct_sub(curproc, RACCT_SWAP, incr);
  250                 PROC_UNLOCK(curproc);
  251         }
  252 #endif
  253 
  254         return (res);
  255 }
  256 
  257 void
  258 swap_reserve_force(vm_ooffset_t incr)
  259 {
  260         struct uidinfo *uip;
  261 
  262         mtx_lock(&sw_dev_mtx);
  263         swap_reserved += incr;
  264         mtx_unlock(&sw_dev_mtx);
  265 
  266 #ifdef RACCT
  267         PROC_LOCK(curproc);
  268         racct_add_force(curproc, RACCT_SWAP, incr);
  269         PROC_UNLOCK(curproc);
  270 #endif
  271 
  272         uip = curthread->td_ucred->cr_ruidinfo;
  273         PROC_LOCK(curproc);
  274         UIDINFO_VMSIZE_LOCK(uip);
  275         uip->ui_vmsize += incr;
  276         UIDINFO_VMSIZE_UNLOCK(uip);
  277         PROC_UNLOCK(curproc);
  278 }
  279 
  280 void
  281 swap_release(vm_ooffset_t decr)
  282 {
  283         struct ucred *cred;
  284 
  285         PROC_LOCK(curproc);
  286         cred = curthread->td_ucred;
  287         swap_release_by_cred(decr, cred);
  288         PROC_UNLOCK(curproc);
  289 }
  290 
  291 void
  292 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
  293 {
  294         struct uidinfo *uip;
  295 
  296         uip = cred->cr_ruidinfo;
  297 
  298         if (decr & PAGE_MASK)
  299                 panic("swap_release: & PAGE_MASK");
  300 
  301         mtx_lock(&sw_dev_mtx);
  302         if (swap_reserved < decr)
  303                 panic("swap_reserved < decr");
  304         swap_reserved -= decr;
  305         mtx_unlock(&sw_dev_mtx);
  306 
  307         UIDINFO_VMSIZE_LOCK(uip);
  308         if (uip->ui_vmsize < decr)
  309                 printf("negative vmsize for uid = %d\n", uip->ui_uid);
  310         uip->ui_vmsize -= decr;
  311         UIDINFO_VMSIZE_UNLOCK(uip);
  312 
  313         racct_sub_cred(cred, RACCT_SWAP, decr);
  314 }
  315 
  316 static void swapdev_strategy(struct buf *, struct swdevt *sw);
  317 
  318 #define SWM_FREE        0x02    /* free, period                 */
  319 #define SWM_POP         0x04    /* pop out                      */
  320 
  321 int swap_pager_full = 2;        /* swap space exhaustion (task killing) */
  322 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
  323 static int nsw_rcount;          /* free read buffers                    */
  324 static int nsw_wcount_sync;     /* limit write buffers / synchronous    */
  325 static int nsw_wcount_async;    /* limit write buffers / asynchronous   */
  326 static int nsw_wcount_async_max;/* assigned maximum                     */
  327 static int nsw_cluster_max;     /* maximum VOP I/O allowed              */
  328 
  329 static struct swblock **swhash;
  330 static int swhash_mask;
  331 static struct mtx swhash_mtx;
  332 
  333 static int swap_async_max = 4;  /* maximum in-progress async I/O's      */
  334 static struct sx sw_alloc_sx;
  335 
  336 
  337 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
  338         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
  339 
  340 /*
  341  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
  342  * of searching a named list by hashing it just a little.
  343  */
  344 
  345 #define NOBJLISTS               8
  346 
  347 #define NOBJLIST(handle)        \
  348         (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
  349 
  350 static struct mtx sw_alloc_mtx; /* protect list manipulation */
  351 static struct pagerlst  swap_pager_object_list[NOBJLISTS];
  352 static uma_zone_t       swap_zone;
  353 
  354 /*
  355  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
  356  * calls hooked from other parts of the VM system and do not appear here.
  357  * (see vm/swap_pager.h).
  358  */
  359 static vm_object_t
  360                 swap_pager_alloc(void *handle, vm_ooffset_t size,
  361                     vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
  362 static void     swap_pager_dealloc(vm_object_t object);
  363 static int      swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
  364 static void     swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
  365 static boolean_t
  366                 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
  367 static void     swap_pager_init(void);
  368 static void     swap_pager_unswapped(vm_page_t);
  369 static void     swap_pager_swapoff(struct swdevt *sp);
  370 
  371 struct pagerops swappagerops = {
  372         .pgo_init =     swap_pager_init,        /* early system initialization of pager */
  373         .pgo_alloc =    swap_pager_alloc,       /* allocate an OBJT_SWAP object         */
  374         .pgo_dealloc =  swap_pager_dealloc,     /* deallocate an OBJT_SWAP object       */
  375         .pgo_getpages = swap_pager_getpages,    /* pagein                               */
  376         .pgo_putpages = swap_pager_putpages,    /* pageout                              */
  377         .pgo_haspage =  swap_pager_haspage,     /* get backing store status for page    */
  378         .pgo_pageunswapped = swap_pager_unswapped,      /* remove swap related to page          */
  379 };
  380 
  381 /*
  382  * swap_*() routines are externally accessible.  swp_*() routines are
  383  * internal.
  384  */
  385 static int nswap_lowat = 128;   /* in pages, swap_pager_almost_full warn */
  386 static int nswap_hiwat = 512;   /* in pages, swap_pager_almost_full warn */
  387 
  388 SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_cluster_max, 0,
  389     "Maximum size of a swap block in pages");
  390 
  391 static void     swp_sizecheck(void);
  392 static void     swp_pager_async_iodone(struct buf *bp);
  393 static int      swapongeom(struct thread *, struct vnode *);
  394 static int      swaponvp(struct thread *, struct vnode *, u_long);
  395 static int      swapoff_one(struct swdevt *sp, struct ucred *cred);
  396 
  397 /*
  398  * Swap bitmap functions
  399  */
  400 static void     swp_pager_freeswapspace(daddr_t blk, int npages);
  401 static daddr_t  swp_pager_getswapspace(int npages);
  402 
  403 /*
  404  * Metadata functions
  405  */
  406 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
  407 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
  408 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
  409 static void swp_pager_meta_free_all(vm_object_t);
  410 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
  411 
  412 static void
  413 swp_pager_free_nrpage(vm_page_t m)
  414 {
  415 
  416         vm_page_lock(m);
  417         if (m->wire_count == 0)
  418                 vm_page_free(m);
  419         vm_page_unlock(m);
  420 }
  421 
  422 /*
  423  * SWP_SIZECHECK() -    update swap_pager_full indication
  424  *
  425  *      update the swap_pager_almost_full indication and warn when we are
  426  *      about to run out of swap space, using lowat/hiwat hysteresis.
  427  *
  428  *      Clear swap_pager_full ( task killing ) indication when lowat is met.
  429  *
  430  *      No restrictions on call
  431  *      This routine may not block.
  432  */
  433 static void
  434 swp_sizecheck(void)
  435 {
  436 
  437         if (swap_pager_avail < nswap_lowat) {
  438                 if (swap_pager_almost_full == 0) {
  439                         printf("swap_pager: out of swap space\n");
  440                         swap_pager_almost_full = 1;
  441                 }
  442         } else {
  443                 swap_pager_full = 0;
  444                 if (swap_pager_avail > nswap_hiwat)
  445                         swap_pager_almost_full = 0;
  446         }
  447 }
  448 
  449 /*
  450  * SWP_PAGER_HASH() -   hash swap meta data
  451  *
  452  *      This is an helper function which hashes the swapblk given
  453  *      the object and page index.  It returns a pointer to a pointer
  454  *      to the object, or a pointer to a NULL pointer if it could not
  455  *      find a swapblk.
  456  */
  457 static struct swblock **
  458 swp_pager_hash(vm_object_t object, vm_pindex_t index)
  459 {
  460         struct swblock **pswap;
  461         struct swblock *swap;
  462 
  463         index &= ~(vm_pindex_t)SWAP_META_MASK;
  464         pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
  465         while ((swap = *pswap) != NULL) {
  466                 if (swap->swb_object == object &&
  467                     swap->swb_index == index
  468                 ) {
  469                         break;
  470                 }
  471                 pswap = &swap->swb_hnext;
  472         }
  473         return (pswap);
  474 }
  475 
  476 /*
  477  * SWAP_PAGER_INIT() -  initialize the swap pager!
  478  *
  479  *      Expected to be started from system init.  NOTE:  This code is run
  480  *      before much else so be careful what you depend on.  Most of the VM
  481  *      system has yet to be initialized at this point.
  482  */
  483 static void
  484 swap_pager_init(void)
  485 {
  486         /*
  487          * Initialize object lists
  488          */
  489         int i;
  490 
  491         for (i = 0; i < NOBJLISTS; ++i)
  492                 TAILQ_INIT(&swap_pager_object_list[i]);
  493         mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
  494         mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
  495         sx_init(&sw_alloc_sx, "swspsx");
  496 }
  497 
  498 /*
  499  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
  500  *
  501  *      Expected to be started from pageout process once, prior to entering
  502  *      its main loop.
  503  */
  504 void
  505 swap_pager_swap_init(void)
  506 {
  507         unsigned long n, n2;
  508 
  509         /*
  510          * Number of in-transit swap bp operations.  Don't
  511          * exhaust the pbufs completely.  Make sure we
  512          * initialize workable values (0 will work for hysteresis
  513          * but it isn't very efficient).
  514          *
  515          * The nsw_cluster_max is constrained by the bp->b_pages[]
  516          * array (MAXPHYS/PAGE_SIZE) and our locally defined
  517          * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
  518          * constrained by the swap device interleave stripe size.
  519          *
  520          * Currently we hardwire nsw_wcount_async to 4.  This limit is
  521          * designed to prevent other I/O from having high latencies due to
  522          * our pageout I/O.  The value 4 works well for one or two active swap
  523          * devices but is probably a little low if you have more.  Even so,
  524          * a higher value would probably generate only a limited improvement
  525          * with three or four active swap devices since the system does not
  526          * typically have to pageout at extreme bandwidths.   We will want
  527          * at least 2 per swap devices, and 4 is a pretty good value if you
  528          * have one NFS swap device due to the command/ack latency over NFS.
  529          * So it all works out pretty well.
  530          */
  531         nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
  532 
  533         mtx_lock(&pbuf_mtx);
  534         nsw_rcount = (nswbuf + 1) / 2;
  535         nsw_wcount_sync = (nswbuf + 3) / 4;
  536         nsw_wcount_async = 4;
  537         nsw_wcount_async_max = nsw_wcount_async;
  538         mtx_unlock(&pbuf_mtx);
  539 
  540         /*
  541          * Initialize our zone.  Right now I'm just guessing on the number
  542          * we need based on the number of pages in the system.  Each swblock
  543          * can hold 32 pages, so this is probably overkill.  This reservation
  544          * is typically limited to around 32MB by default.
  545          */
  546         n = cnt.v_page_count / 2;
  547         if (maxswzone && n > maxswzone / sizeof(struct swblock))
  548                 n = maxswzone / sizeof(struct swblock);
  549         n2 = n;
  550         swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
  551             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
  552         if (swap_zone == NULL)
  553                 panic("failed to create swap_zone.");
  554         do {
  555                 if (uma_zone_reserve_kva(swap_zone, n))
  556                         break;
  557                 /*
  558                  * if the allocation failed, try a zone two thirds the
  559                  * size of the previous attempt.
  560                  */
  561                 n -= ((n + 2) / 3);
  562         } while (n > 0);
  563         if (n2 != n)
  564                 printf("Swap zone entries reduced from %lu to %lu.\n", n2, n);
  565         swap_maxpages = n * SWAP_META_PAGES;
  566         swzone = n * sizeof(struct swblock);
  567         n2 = n;
  568 
  569         /*
  570          * Initialize our meta-data hash table.  The swapper does not need to
  571          * be quite as efficient as the VM system, so we do not use an
  572          * oversized hash table.
  573          *
  574          *      n:              size of hash table, must be power of 2
  575          *      swhash_mask:    hash table index mask
  576          */
  577         for (n = 1; n < n2 / 8; n *= 2)
  578                 ;
  579         swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
  580         swhash_mask = n - 1;
  581         mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
  582 }
  583 
  584 /*
  585  * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
  586  *                      its metadata structures.
  587  *
  588  *      This routine is called from the mmap and fork code to create a new
  589  *      OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
  590  *      and then converting it with swp_pager_meta_build().
  591  *
  592  *      This routine may block in vm_object_allocate() and create a named
  593  *      object lookup race, so we must interlock.
  594  *
  595  * MPSAFE
  596  */
  597 static vm_object_t
  598 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
  599     vm_ooffset_t offset, struct ucred *cred)
  600 {
  601         vm_object_t object;
  602         vm_pindex_t pindex;
  603 
  604         pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
  605         if (handle) {
  606                 mtx_lock(&Giant);
  607                 /*
  608                  * Reference existing named region or allocate new one.  There
  609                  * should not be a race here against swp_pager_meta_build()
  610                  * as called from vm_page_remove() in regards to the lookup
  611                  * of the handle.
  612                  */
  613                 sx_xlock(&sw_alloc_sx);
  614                 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
  615                 if (object == NULL) {
  616                         if (cred != NULL) {
  617                                 if (!swap_reserve_by_cred(size, cred)) {
  618                                         sx_xunlock(&sw_alloc_sx);
  619                                         mtx_unlock(&Giant);
  620                                         return (NULL);
  621                                 }
  622                                 crhold(cred);
  623                         }
  624                         object = vm_object_allocate(OBJT_DEFAULT, pindex);
  625                         VM_OBJECT_WLOCK(object);
  626                         object->handle = handle;
  627                         if (cred != NULL) {
  628                                 object->cred = cred;
  629                                 object->charge = size;
  630                         }
  631                         swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  632                         VM_OBJECT_WUNLOCK(object);
  633                 }
  634                 sx_xunlock(&sw_alloc_sx);
  635                 mtx_unlock(&Giant);
  636         } else {
  637                 if (cred != NULL) {
  638                         if (!swap_reserve_by_cred(size, cred))
  639                                 return (NULL);
  640                         crhold(cred);
  641                 }
  642                 object = vm_object_allocate(OBJT_DEFAULT, pindex);
  643                 VM_OBJECT_WLOCK(object);
  644                 if (cred != NULL) {
  645                         object->cred = cred;
  646                         object->charge = size;
  647                 }
  648                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  649                 VM_OBJECT_WUNLOCK(object);
  650         }
  651         return (object);
  652 }
  653 
  654 /*
  655  * SWAP_PAGER_DEALLOC() -       remove swap metadata from object
  656  *
  657  *      The swap backing for the object is destroyed.  The code is
  658  *      designed such that we can reinstantiate it later, but this
  659  *      routine is typically called only when the entire object is
  660  *      about to be destroyed.
  661  *
  662  *      The object must be locked.
  663  */
  664 static void
  665 swap_pager_dealloc(vm_object_t object)
  666 {
  667 
  668         /*
  669          * Remove from list right away so lookups will fail if we block for
  670          * pageout completion.
  671          */
  672         if (object->handle != NULL) {
  673                 mtx_lock(&sw_alloc_mtx);
  674                 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
  675                 mtx_unlock(&sw_alloc_mtx);
  676         }
  677 
  678         VM_OBJECT_ASSERT_WLOCKED(object);
  679         vm_object_pip_wait(object, "swpdea");
  680 
  681         /*
  682          * Free all remaining metadata.  We only bother to free it from
  683          * the swap meta data.  We do not attempt to free swapblk's still
  684          * associated with vm_page_t's for this object.  We do not care
  685          * if paging is still in progress on some objects.
  686          */
  687         swp_pager_meta_free_all(object);
  688         object->handle = NULL;
  689         object->type = OBJT_DEAD;
  690 }
  691 
  692 /************************************************************************
  693  *                      SWAP PAGER BITMAP ROUTINES                      *
  694  ************************************************************************/
  695 
  696 /*
  697  * SWP_PAGER_GETSWAPSPACE() -   allocate raw swap space
  698  *
  699  *      Allocate swap for the requested number of pages.  The starting
  700  *      swap block number (a page index) is returned or SWAPBLK_NONE
  701  *      if the allocation failed.
  702  *
  703  *      Also has the side effect of advising that somebody made a mistake
  704  *      when they configured swap and didn't configure enough.
  705  *
  706  *      This routine may not sleep.
  707  *
  708  *      We allocate in round-robin fashion from the configured devices.
  709  */
  710 static daddr_t
  711 swp_pager_getswapspace(int npages)
  712 {
  713         daddr_t blk;
  714         struct swdevt *sp;
  715         int i;
  716 
  717         blk = SWAPBLK_NONE;
  718         mtx_lock(&sw_dev_mtx);
  719         sp = swdevhd;
  720         for (i = 0; i < nswapdev; i++) {
  721                 if (sp == NULL)
  722                         sp = TAILQ_FIRST(&swtailq);
  723                 if (!(sp->sw_flags & SW_CLOSING)) {
  724                         blk = blist_alloc(sp->sw_blist, npages);
  725                         if (blk != SWAPBLK_NONE) {
  726                                 blk += sp->sw_first;
  727                                 sp->sw_used += npages;
  728                                 swap_pager_avail -= npages;
  729                                 swp_sizecheck();
  730                                 swdevhd = TAILQ_NEXT(sp, sw_list);
  731                                 goto done;
  732                         }
  733                 }
  734                 sp = TAILQ_NEXT(sp, sw_list);
  735         }
  736         if (swap_pager_full != 2) {
  737                 printf("swap_pager_getswapspace(%d): failed\n", npages);
  738                 swap_pager_full = 2;
  739                 swap_pager_almost_full = 1;
  740         }
  741         swdevhd = NULL;
  742 done:
  743         mtx_unlock(&sw_dev_mtx);
  744         return (blk);
  745 }
  746 
  747 static int
  748 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
  749 {
  750 
  751         return (blk >= sp->sw_first && blk < sp->sw_end);
  752 }
  753 
  754 static void
  755 swp_pager_strategy(struct buf *bp)
  756 {
  757         struct swdevt *sp;
  758 
  759         mtx_lock(&sw_dev_mtx);
  760         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  761                 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
  762                         mtx_unlock(&sw_dev_mtx);
  763                         if ((sp->sw_flags & SW_UNMAPPED) != 0 &&
  764                             unmapped_buf_allowed) {
  765                                 bp->b_kvaalloc = bp->b_data;
  766                                 bp->b_data = unmapped_buf;
  767                                 bp->b_kvabase = unmapped_buf;
  768                                 bp->b_offset = 0;
  769                                 bp->b_flags |= B_UNMAPPED;
  770                         } else {
  771                                 pmap_qenter((vm_offset_t)bp->b_data,
  772                                     &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
  773                         }
  774                         sp->sw_strategy(bp, sp);
  775                         return;
  776                 }
  777         }
  778         panic("Swapdev not found");
  779 }
  780 
  781 
  782 /*
  783  * SWP_PAGER_FREESWAPSPACE() -  free raw swap space
  784  *
  785  *      This routine returns the specified swap blocks back to the bitmap.
  786  *
  787  *      This routine may not sleep.
  788  */
  789 static void
  790 swp_pager_freeswapspace(daddr_t blk, int npages)
  791 {
  792         struct swdevt *sp;
  793 
  794         mtx_lock(&sw_dev_mtx);
  795         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  796                 if (blk >= sp->sw_first && blk < sp->sw_end) {
  797                         sp->sw_used -= npages;
  798                         /*
  799                          * If we are attempting to stop swapping on
  800                          * this device, we don't want to mark any
  801                          * blocks free lest they be reused.
  802                          */
  803                         if ((sp->sw_flags & SW_CLOSING) == 0) {
  804                                 blist_free(sp->sw_blist, blk - sp->sw_first,
  805                                     npages);
  806                                 swap_pager_avail += npages;
  807                                 swp_sizecheck();
  808                         }
  809                         mtx_unlock(&sw_dev_mtx);
  810                         return;
  811                 }
  812         }
  813         panic("Swapdev not found");
  814 }
  815 
  816 /*
  817  * SWAP_PAGER_FREESPACE() -     frees swap blocks associated with a page
  818  *                              range within an object.
  819  *
  820  *      This is a globally accessible routine.
  821  *
  822  *      This routine removes swapblk assignments from swap metadata.
  823  *
  824  *      The external callers of this routine typically have already destroyed
  825  *      or renamed vm_page_t's associated with this range in the object so
  826  *      we should be ok.
  827  *
  828  *      The object must be locked.
  829  */
  830 void
  831 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
  832 {
  833 
  834         swp_pager_meta_free(object, start, size);
  835 }
  836 
  837 /*
  838  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
  839  *
  840  *      Assigns swap blocks to the specified range within the object.  The
  841  *      swap blocks are not zeroed.  Any previous swap assignment is destroyed.
  842  *
  843  *      Returns 0 on success, -1 on failure.
  844  */
  845 int
  846 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
  847 {
  848         int n = 0;
  849         daddr_t blk = SWAPBLK_NONE;
  850         vm_pindex_t beg = start;        /* save start index */
  851 
  852         VM_OBJECT_WLOCK(object);
  853         while (size) {
  854                 if (n == 0) {
  855                         n = BLIST_MAX_ALLOC;
  856                         while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
  857                                 n >>= 1;
  858                                 if (n == 0) {
  859                                         swp_pager_meta_free(object, beg, start - beg);
  860                                         VM_OBJECT_WUNLOCK(object);
  861                                         return (-1);
  862                                 }
  863                         }
  864                 }
  865                 swp_pager_meta_build(object, start, blk);
  866                 --size;
  867                 ++start;
  868                 ++blk;
  869                 --n;
  870         }
  871         swp_pager_meta_free(object, start, n);
  872         VM_OBJECT_WUNLOCK(object);
  873         return (0);
  874 }
  875 
  876 /*
  877  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
  878  *                      and destroy the source.
  879  *
  880  *      Copy any valid swapblks from the source to the destination.  In
  881  *      cases where both the source and destination have a valid swapblk,
  882  *      we keep the destination's.
  883  *
  884  *      This routine is allowed to sleep.  It may sleep allocating metadata
  885  *      indirectly through swp_pager_meta_build() or if paging is still in
  886  *      progress on the source.
  887  *
  888  *      The source object contains no vm_page_t's (which is just as well)
  889  *
  890  *      The source object is of type OBJT_SWAP.
  891  *
  892  *      The source and destination objects must be locked.
  893  *      Both object locks may temporarily be released.
  894  */
  895 void
  896 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
  897     vm_pindex_t offset, int destroysource)
  898 {
  899         vm_pindex_t i;
  900 
  901         VM_OBJECT_ASSERT_WLOCKED(srcobject);
  902         VM_OBJECT_ASSERT_WLOCKED(dstobject);
  903 
  904         /*
  905          * If destroysource is set, we remove the source object from the
  906          * swap_pager internal queue now.
  907          */
  908         if (destroysource) {
  909                 if (srcobject->handle != NULL) {
  910                         mtx_lock(&sw_alloc_mtx);
  911                         TAILQ_REMOVE(
  912                             NOBJLIST(srcobject->handle),
  913                             srcobject,
  914                             pager_object_list
  915                         );
  916                         mtx_unlock(&sw_alloc_mtx);
  917                 }
  918         }
  919 
  920         /*
  921          * transfer source to destination.
  922          */
  923         for (i = 0; i < dstobject->size; ++i) {
  924                 daddr_t dstaddr;
  925 
  926                 /*
  927                  * Locate (without changing) the swapblk on the destination,
  928                  * unless it is invalid in which case free it silently, or
  929                  * if the destination is a resident page, in which case the
  930                  * source is thrown away.
  931                  */
  932                 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
  933 
  934                 if (dstaddr == SWAPBLK_NONE) {
  935                         /*
  936                          * Destination has no swapblk and is not resident,
  937                          * copy source.
  938                          */
  939                         daddr_t srcaddr;
  940 
  941                         srcaddr = swp_pager_meta_ctl(
  942                             srcobject,
  943                             i + offset,
  944                             SWM_POP
  945                         );
  946 
  947                         if (srcaddr != SWAPBLK_NONE) {
  948                                 /*
  949                                  * swp_pager_meta_build() can sleep.
  950                                  */
  951                                 vm_object_pip_add(srcobject, 1);
  952                                 VM_OBJECT_WUNLOCK(srcobject);
  953                                 vm_object_pip_add(dstobject, 1);
  954                                 swp_pager_meta_build(dstobject, i, srcaddr);
  955                                 vm_object_pip_wakeup(dstobject);
  956                                 VM_OBJECT_WLOCK(srcobject);
  957                                 vm_object_pip_wakeup(srcobject);
  958                         }
  959                 } else {
  960                         /*
  961                          * Destination has valid swapblk or it is represented
  962                          * by a resident page.  We destroy the sourceblock.
  963                          */
  964 
  965                         swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
  966                 }
  967         }
  968 
  969         /*
  970          * Free left over swap blocks in source.
  971          *
  972          * We have to revert the type to OBJT_DEFAULT so we do not accidently
  973          * double-remove the object from the swap queues.
  974          */
  975         if (destroysource) {
  976                 swp_pager_meta_free_all(srcobject);
  977                 /*
  978                  * Reverting the type is not necessary, the caller is going
  979                  * to destroy srcobject directly, but I'm doing it here
  980                  * for consistency since we've removed the object from its
  981                  * queues.
  982                  */
  983                 srcobject->type = OBJT_DEFAULT;
  984         }
  985 }
  986 
  987 /*
  988  * SWAP_PAGER_HASPAGE() -       determine if we have good backing store for
  989  *                              the requested page.
  990  *
  991  *      We determine whether good backing store exists for the requested
  992  *      page and return TRUE if it does, FALSE if it doesn't.
  993  *
  994  *      If TRUE, we also try to determine how much valid, contiguous backing
  995  *      store exists before and after the requested page within a reasonable
  996  *      distance.  We do not try to restrict it to the swap device stripe
  997  *      (that is handled in getpages/putpages).  It probably isn't worth
  998  *      doing here.
  999  */
 1000 static boolean_t
 1001 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
 1002 {
 1003         daddr_t blk0;
 1004 
 1005         VM_OBJECT_ASSERT_LOCKED(object);
 1006         /*
 1007          * do we have good backing store at the requested index ?
 1008          */
 1009         blk0 = swp_pager_meta_ctl(object, pindex, 0);
 1010 
 1011         if (blk0 == SWAPBLK_NONE) {
 1012                 if (before)
 1013                         *before = 0;
 1014                 if (after)
 1015                         *after = 0;
 1016                 return (FALSE);
 1017         }
 1018 
 1019         /*
 1020          * find backwards-looking contiguous good backing store
 1021          */
 1022         if (before != NULL) {
 1023                 int i;
 1024 
 1025                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
 1026                         daddr_t blk;
 1027 
 1028                         if (i > pindex)
 1029                                 break;
 1030                         blk = swp_pager_meta_ctl(object, pindex - i, 0);
 1031                         if (blk != blk0 - i)
 1032                                 break;
 1033                 }
 1034                 *before = (i - 1);
 1035         }
 1036 
 1037         /*
 1038          * find forward-looking contiguous good backing store
 1039          */
 1040         if (after != NULL) {
 1041                 int i;
 1042 
 1043                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
 1044                         daddr_t blk;
 1045 
 1046                         blk = swp_pager_meta_ctl(object, pindex + i, 0);
 1047                         if (blk != blk0 + i)
 1048                                 break;
 1049                 }
 1050                 *after = (i - 1);
 1051         }
 1052         return (TRUE);
 1053 }
 1054 
 1055 /*
 1056  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
 1057  *
 1058  *      This removes any associated swap backing store, whether valid or
 1059  *      not, from the page.
 1060  *
 1061  *      This routine is typically called when a page is made dirty, at
 1062  *      which point any associated swap can be freed.  MADV_FREE also
 1063  *      calls us in a special-case situation
 1064  *
 1065  *      NOTE!!!  If the page is clean and the swap was valid, the caller
 1066  *      should make the page dirty before calling this routine.  This routine
 1067  *      does NOT change the m->dirty status of the page.  Also: MADV_FREE
 1068  *      depends on it.
 1069  *
 1070  *      This routine may not sleep.
 1071  *
 1072  *      The object containing the page must be locked.
 1073  */
 1074 static void
 1075 swap_pager_unswapped(vm_page_t m)
 1076 {
 1077 
 1078         swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
 1079 }
 1080 
 1081 /*
 1082  * SWAP_PAGER_GETPAGES() - bring pages in from swap
 1083  *
 1084  *      Attempt to retrieve (m, count) pages from backing store, but make
 1085  *      sure we retrieve at least m[reqpage].  We try to load in as large
 1086  *      a chunk surrounding m[reqpage] as is contiguous in swap and which
 1087  *      belongs to the same object.
 1088  *
 1089  *      The code is designed for asynchronous operation and
 1090  *      immediate-notification of 'reqpage' but tends not to be
 1091  *      used that way.  Please do not optimize-out this algorithmic
 1092  *      feature, I intend to improve on it in the future.
 1093  *
 1094  *      The parent has a single vm_object_pip_add() reference prior to
 1095  *      calling us and we should return with the same.
 1096  *
 1097  *      The parent has BUSY'd the pages.  We should return with 'm'
 1098  *      left busy, but the others adjusted.
 1099  */
 1100 static int
 1101 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
 1102 {
 1103         struct buf *bp;
 1104         vm_page_t mreq;
 1105         int i;
 1106         int j;
 1107         daddr_t blk;
 1108 
 1109         mreq = m[reqpage];
 1110 
 1111         KASSERT(mreq->object == object,
 1112             ("swap_pager_getpages: object mismatch %p/%p",
 1113             object, mreq->object));
 1114 
 1115         /*
 1116          * Calculate range to retrieve.  The pages have already been assigned
 1117          * their swapblks.  We require a *contiguous* range but we know it to
 1118          * not span devices.   If we do not supply it, bad things
 1119          * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
 1120          * loops are set up such that the case(s) are handled implicitly.
 1121          *
 1122          * The swp_*() calls must be made with the object locked.
 1123          */
 1124         blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
 1125 
 1126         for (i = reqpage - 1; i >= 0; --i) {
 1127                 daddr_t iblk;
 1128 
 1129                 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
 1130                 if (blk != iblk + (reqpage - i))
 1131                         break;
 1132         }
 1133         ++i;
 1134 
 1135         for (j = reqpage + 1; j < count; ++j) {
 1136                 daddr_t jblk;
 1137 
 1138                 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
 1139                 if (blk != jblk - (j - reqpage))
 1140                         break;
 1141         }
 1142 
 1143         /*
 1144          * free pages outside our collection range.   Note: we never free
 1145          * mreq, it must remain busy throughout.
 1146          */
 1147         if (0 < i || j < count) {
 1148                 int k;
 1149 
 1150                 for (k = 0; k < i; ++k)
 1151                         swp_pager_free_nrpage(m[k]);
 1152                 for (k = j; k < count; ++k)
 1153                         swp_pager_free_nrpage(m[k]);
 1154         }
 1155 
 1156         /*
 1157          * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
 1158          * still busy, but the others unbusied.
 1159          */
 1160         if (blk == SWAPBLK_NONE)
 1161                 return (VM_PAGER_FAIL);
 1162 
 1163         /*
 1164          * Getpbuf() can sleep.
 1165          */
 1166         VM_OBJECT_WUNLOCK(object);
 1167         /*
 1168          * Get a swap buffer header to perform the IO
 1169          */
 1170         bp = getpbuf(&nsw_rcount);
 1171         bp->b_flags |= B_PAGING;
 1172 
 1173         bp->b_iocmd = BIO_READ;
 1174         bp->b_iodone = swp_pager_async_iodone;
 1175         bp->b_rcred = crhold(thread0.td_ucred);
 1176         bp->b_wcred = crhold(thread0.td_ucred);
 1177         bp->b_blkno = blk - (reqpage - i);
 1178         bp->b_bcount = PAGE_SIZE * (j - i);
 1179         bp->b_bufsize = PAGE_SIZE * (j - i);
 1180         bp->b_pager.pg_reqpage = reqpage - i;
 1181 
 1182         VM_OBJECT_WLOCK(object);
 1183         {
 1184                 int k;
 1185 
 1186                 for (k = i; k < j; ++k) {
 1187                         bp->b_pages[k - i] = m[k];
 1188                         m[k]->oflags |= VPO_SWAPINPROG;
 1189                 }
 1190         }
 1191         bp->b_npages = j - i;
 1192 
 1193         PCPU_INC(cnt.v_swapin);
 1194         PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
 1195 
 1196         /*
 1197          * We still hold the lock on mreq, and our automatic completion routine
 1198          * does not remove it.
 1199          */
 1200         vm_object_pip_add(object, bp->b_npages);
 1201         VM_OBJECT_WUNLOCK(object);
 1202 
 1203         /*
 1204          * perform the I/O.  NOTE!!!  bp cannot be considered valid after
 1205          * this point because we automatically release it on completion.
 1206          * Instead, we look at the one page we are interested in which we
 1207          * still hold a lock on even through the I/O completion.
 1208          *
 1209          * The other pages in our m[] array are also released on completion,
 1210          * so we cannot assume they are valid anymore either.
 1211          *
 1212          * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1213          */
 1214         BUF_KERNPROC(bp);
 1215         swp_pager_strategy(bp);
 1216 
 1217         /*
 1218          * wait for the page we want to complete.  VPO_SWAPINPROG is always
 1219          * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
 1220          * is set in the meta-data.
 1221          */
 1222         VM_OBJECT_WLOCK(object);
 1223         while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
 1224                 mreq->oflags |= VPO_SWAPSLEEP;
 1225                 PCPU_INC(cnt.v_intrans);
 1226                 if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
 1227                     "swread", hz * 20)) {
 1228                         printf(
 1229 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
 1230                             bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
 1231                 }
 1232         }
 1233 
 1234         /*
 1235          * mreq is left busied after completion, but all the other pages
 1236          * are freed.  If we had an unrecoverable read error the page will
 1237          * not be valid.
 1238          */
 1239         if (mreq->valid != VM_PAGE_BITS_ALL) {
 1240                 return (VM_PAGER_ERROR);
 1241         } else {
 1242                 return (VM_PAGER_OK);
 1243         }
 1244 
 1245         /*
 1246          * A final note: in a low swap situation, we cannot deallocate swap
 1247          * and mark a page dirty here because the caller is likely to mark
 1248          * the page clean when we return, causing the page to possibly revert
 1249          * to all-zero's later.
 1250          */
 1251 }
 1252 
 1253 /*
 1254  *      swap_pager_putpages:
 1255  *
 1256  *      Assign swap (if necessary) and initiate I/O on the specified pages.
 1257  *
 1258  *      We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
 1259  *      are automatically converted to SWAP objects.
 1260  *
 1261  *      In a low memory situation we may block in VOP_STRATEGY(), but the new
 1262  *      vm_page reservation system coupled with properly written VFS devices
 1263  *      should ensure that no low-memory deadlock occurs.  This is an area
 1264  *      which needs work.
 1265  *
 1266  *      The parent has N vm_object_pip_add() references prior to
 1267  *      calling us and will remove references for rtvals[] that are
 1268  *      not set to VM_PAGER_PEND.  We need to remove the rest on I/O
 1269  *      completion.
 1270  *
 1271  *      The parent has soft-busy'd the pages it passes us and will unbusy
 1272  *      those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
 1273  *      We need to unbusy the rest on I/O completion.
 1274  */
 1275 void
 1276 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 1277     int flags, int *rtvals)
 1278 {
 1279         int i, n;
 1280         boolean_t sync;
 1281 
 1282         if (count && m[0]->object != object) {
 1283                 panic("swap_pager_putpages: object mismatch %p/%p",
 1284                     object,
 1285                     m[0]->object
 1286                 );
 1287         }
 1288 
 1289         /*
 1290          * Step 1
 1291          *
 1292          * Turn object into OBJT_SWAP
 1293          * check for bogus sysops
 1294          * force sync if not pageout process
 1295          */
 1296         if (object->type != OBJT_SWAP)
 1297                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
 1298         VM_OBJECT_WUNLOCK(object);
 1299 
 1300         n = 0;
 1301         if (curproc != pageproc)
 1302                 sync = TRUE;
 1303         else
 1304                 sync = (flags & VM_PAGER_PUT_SYNC) != 0;
 1305 
 1306         /*
 1307          * Step 2
 1308          *
 1309          * Update nsw parameters from swap_async_max sysctl values.
 1310          * Do not let the sysop crash the machine with bogus numbers.
 1311          */
 1312         mtx_lock(&pbuf_mtx);
 1313         if (swap_async_max != nsw_wcount_async_max) {
 1314                 int n;
 1315 
 1316                 /*
 1317                  * limit range
 1318                  */
 1319                 if ((n = swap_async_max) > nswbuf / 2)
 1320                         n = nswbuf / 2;
 1321                 if (n < 1)
 1322                         n = 1;
 1323                 swap_async_max = n;
 1324 
 1325                 /*
 1326                  * Adjust difference ( if possible ).  If the current async
 1327                  * count is too low, we may not be able to make the adjustment
 1328                  * at this time.
 1329                  */
 1330                 n -= nsw_wcount_async_max;
 1331                 if (nsw_wcount_async + n >= 0) {
 1332                         nsw_wcount_async += n;
 1333                         nsw_wcount_async_max += n;
 1334                         wakeup(&nsw_wcount_async);
 1335                 }
 1336         }
 1337         mtx_unlock(&pbuf_mtx);
 1338 
 1339         /*
 1340          * Step 3
 1341          *
 1342          * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
 1343          * The page is left dirty until the pageout operation completes
 1344          * successfully.
 1345          */
 1346         for (i = 0; i < count; i += n) {
 1347                 int j;
 1348                 struct buf *bp;
 1349                 daddr_t blk;
 1350 
 1351                 /*
 1352                  * Maximum I/O size is limited by a number of factors.
 1353                  */
 1354                 n = min(BLIST_MAX_ALLOC, count - i);
 1355                 n = min(n, nsw_cluster_max);
 1356 
 1357                 /*
 1358                  * Get biggest block of swap we can.  If we fail, fall
 1359                  * back and try to allocate a smaller block.  Don't go
 1360                  * overboard trying to allocate space if it would overly
 1361                  * fragment swap.
 1362                  */
 1363                 while (
 1364                     (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
 1365                     n > 4
 1366                 ) {
 1367                         n >>= 1;
 1368                 }
 1369                 if (blk == SWAPBLK_NONE) {
 1370                         for (j = 0; j < n; ++j)
 1371                                 rtvals[i+j] = VM_PAGER_FAIL;
 1372                         continue;
 1373                 }
 1374 
 1375                 /*
 1376                  * All I/O parameters have been satisfied, build the I/O
 1377                  * request and assign the swap space.
 1378                  */
 1379                 if (sync == TRUE) {
 1380                         bp = getpbuf(&nsw_wcount_sync);
 1381                 } else {
 1382                         bp = getpbuf(&nsw_wcount_async);
 1383                         bp->b_flags = B_ASYNC;
 1384                 }
 1385                 bp->b_flags |= B_PAGING;
 1386                 bp->b_iocmd = BIO_WRITE;
 1387 
 1388                 bp->b_rcred = crhold(thread0.td_ucred);
 1389                 bp->b_wcred = crhold(thread0.td_ucred);
 1390                 bp->b_bcount = PAGE_SIZE * n;
 1391                 bp->b_bufsize = PAGE_SIZE * n;
 1392                 bp->b_blkno = blk;
 1393 
 1394                 VM_OBJECT_WLOCK(object);
 1395                 for (j = 0; j < n; ++j) {
 1396                         vm_page_t mreq = m[i+j];
 1397 
 1398                         swp_pager_meta_build(
 1399                             mreq->object,
 1400                             mreq->pindex,
 1401                             blk + j
 1402                         );
 1403                         MPASS(mreq->dirty == VM_PAGE_BITS_ALL);
 1404                         rtvals[i+j] = VM_PAGER_OK;
 1405 
 1406                         mreq->oflags |= VPO_SWAPINPROG;
 1407                         bp->b_pages[j] = mreq;
 1408                 }
 1409                 VM_OBJECT_WUNLOCK(object);
 1410                 bp->b_npages = n;
 1411                 /*
 1412                  * Must set dirty range for NFS to work.
 1413                  */
 1414                 bp->b_dirtyoff = 0;
 1415                 bp->b_dirtyend = bp->b_bcount;
 1416 
 1417                 PCPU_INC(cnt.v_swapout);
 1418                 PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
 1419 
 1420                 /*
 1421                  * asynchronous
 1422                  *
 1423                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1424                  */
 1425                 if (sync == FALSE) {
 1426                         bp->b_iodone = swp_pager_async_iodone;
 1427                         BUF_KERNPROC(bp);
 1428                         swp_pager_strategy(bp);
 1429 
 1430                         for (j = 0; j < n; ++j)
 1431                                 rtvals[i+j] = VM_PAGER_PEND;
 1432                         /* restart outter loop */
 1433                         continue;
 1434                 }
 1435 
 1436                 /*
 1437                  * synchronous
 1438                  *
 1439                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1440                  */
 1441                 bp->b_iodone = bdone;
 1442                 swp_pager_strategy(bp);
 1443 
 1444                 /*
 1445                  * Wait for the sync I/O to complete, then update rtvals.
 1446                  * We just set the rtvals[] to VM_PAGER_PEND so we can call
 1447                  * our async completion routine at the end, thus avoiding a
 1448                  * double-free.
 1449                  */
 1450                 bwait(bp, PVM, "swwrt");
 1451                 for (j = 0; j < n; ++j)
 1452                         rtvals[i+j] = VM_PAGER_PEND;
 1453                 /*
 1454                  * Now that we are through with the bp, we can call the
 1455                  * normal async completion, which frees everything up.
 1456                  */
 1457                 swp_pager_async_iodone(bp);
 1458         }
 1459         VM_OBJECT_WLOCK(object);
 1460 }
 1461 
 1462 /*
 1463  *      swp_pager_async_iodone:
 1464  *
 1465  *      Completion routine for asynchronous reads and writes from/to swap.
 1466  *      Also called manually by synchronous code to finish up a bp.
 1467  *
 1468  *      This routine may not sleep.
 1469  */
 1470 static void
 1471 swp_pager_async_iodone(struct buf *bp)
 1472 {
 1473         int i;
 1474         vm_object_t object = NULL;
 1475 
 1476         /*
 1477          * report error
 1478          */
 1479         if (bp->b_ioflags & BIO_ERROR) {
 1480                 printf(
 1481                     "swap_pager: I/O error - %s failed; blkno %ld,"
 1482                         "size %ld, error %d\n",
 1483                     ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
 1484                     (long)bp->b_blkno,
 1485                     (long)bp->b_bcount,
 1486                     bp->b_error
 1487                 );
 1488         }
 1489 
 1490         /*
 1491          * remove the mapping for kernel virtual
 1492          */
 1493         if ((bp->b_flags & B_UNMAPPED) != 0) {
 1494                 bp->b_data = bp->b_kvaalloc;
 1495                 bp->b_kvabase = bp->b_kvaalloc;
 1496                 bp->b_flags &= ~B_UNMAPPED;
 1497         } else
 1498                 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
 1499 
 1500         if (bp->b_npages) {
 1501                 object = bp->b_pages[0]->object;
 1502                 VM_OBJECT_WLOCK(object);
 1503         }
 1504 
 1505         /*
 1506          * cleanup pages.  If an error occurs writing to swap, we are in
 1507          * very serious trouble.  If it happens to be a disk error, though,
 1508          * we may be able to recover by reassigning the swap later on.  So
 1509          * in this case we remove the m->swapblk assignment for the page
 1510          * but do not free it in the rlist.  The errornous block(s) are thus
 1511          * never reallocated as swap.  Redirty the page and continue.
 1512          */
 1513         for (i = 0; i < bp->b_npages; ++i) {
 1514                 vm_page_t m = bp->b_pages[i];
 1515 
 1516                 m->oflags &= ~VPO_SWAPINPROG;
 1517                 if (m->oflags & VPO_SWAPSLEEP) {
 1518                         m->oflags &= ~VPO_SWAPSLEEP;
 1519                         wakeup(&object->paging_in_progress);
 1520                 }
 1521 
 1522                 if (bp->b_ioflags & BIO_ERROR) {
 1523                         /*
 1524                          * If an error occurs I'd love to throw the swapblk
 1525                          * away without freeing it back to swapspace, so it
 1526                          * can never be used again.  But I can't from an
 1527                          * interrupt.
 1528                          */
 1529                         if (bp->b_iocmd == BIO_READ) {
 1530                                 /*
 1531                                  * When reading, reqpage needs to stay
 1532                                  * locked for the parent, but all other
 1533                                  * pages can be freed.  We still want to
 1534                                  * wakeup the parent waiting on the page,
 1535                                  * though.  ( also: pg_reqpage can be -1 and
 1536                                  * not match anything ).
 1537                                  *
 1538                                  * We have to wake specifically requested pages
 1539                                  * up too because we cleared VPO_SWAPINPROG and
 1540                                  * someone may be waiting for that.
 1541                                  *
 1542                                  * NOTE: for reads, m->dirty will probably
 1543                                  * be overridden by the original caller of
 1544                                  * getpages so don't play cute tricks here.
 1545                                  */
 1546                                 m->valid = 0;
 1547                                 if (i != bp->b_pager.pg_reqpage)
 1548                                         swp_pager_free_nrpage(m);
 1549                                 else {
 1550                                         vm_page_lock(m);
 1551                                         vm_page_flash(m);
 1552                                         vm_page_unlock(m);
 1553                                 }
 1554                                 /*
 1555                                  * If i == bp->b_pager.pg_reqpage, do not wake
 1556                                  * the page up.  The caller needs to.
 1557                                  */
 1558                         } else {
 1559                                 /*
 1560                                  * If a write error occurs, reactivate page
 1561                                  * so it doesn't clog the inactive list,
 1562                                  * then finish the I/O.
 1563                                  */
 1564                                 vm_page_dirty(m);
 1565                                 vm_page_lock(m);
 1566                                 vm_page_activate(m);
 1567                                 vm_page_unlock(m);
 1568                                 vm_page_sunbusy(m);
 1569                         }
 1570                 } else if (bp->b_iocmd == BIO_READ) {
 1571                         /*
 1572                          * NOTE: for reads, m->dirty will probably be
 1573                          * overridden by the original caller of getpages so
 1574                          * we cannot set them in order to free the underlying
 1575                          * swap in a low-swap situation.  I don't think we'd
 1576                          * want to do that anyway, but it was an optimization
 1577                          * that existed in the old swapper for a time before
 1578                          * it got ripped out due to precisely this problem.
 1579                          *
 1580                          * If not the requested page then deactivate it.
 1581                          *
 1582                          * Note that the requested page, reqpage, is left
 1583                          * busied, but we still have to wake it up.  The
 1584                          * other pages are released (unbusied) by
 1585                          * vm_page_xunbusy().
 1586                          */
 1587                         KASSERT(!pmap_page_is_mapped(m),
 1588                             ("swp_pager_async_iodone: page %p is mapped", m));
 1589                         m->valid = VM_PAGE_BITS_ALL;
 1590                         KASSERT(m->dirty == 0,
 1591                             ("swp_pager_async_iodone: page %p is dirty", m));
 1592 
 1593                         /*
 1594                          * We have to wake specifically requested pages
 1595                          * up too because we cleared VPO_SWAPINPROG and
 1596                          * could be waiting for it in getpages.  However,
 1597                          * be sure to not unbusy getpages specifically
 1598                          * requested page - getpages expects it to be
 1599                          * left busy.
 1600                          */
 1601                         if (i != bp->b_pager.pg_reqpage) {
 1602                                 vm_page_lock(m);
 1603                                 vm_page_deactivate(m);
 1604                                 vm_page_unlock(m);
 1605                                 vm_page_xunbusy(m);
 1606                         } else {
 1607                                 vm_page_lock(m);
 1608                                 vm_page_flash(m);
 1609                                 vm_page_unlock(m);
 1610                         }
 1611                 } else {
 1612                         /*
 1613                          * For write success, clear the dirty
 1614                          * status, then finish the I/O ( which decrements the
 1615                          * busy count and possibly wakes waiter's up ).
 1616                          */
 1617                         KASSERT(!pmap_page_is_write_mapped(m),
 1618                             ("swp_pager_async_iodone: page %p is not write"
 1619                             " protected", m));
 1620                         vm_page_undirty(m);
 1621                         vm_page_sunbusy(m);
 1622                         if (vm_page_count_severe()) {
 1623                                 vm_page_lock(m);
 1624                                 vm_page_try_to_cache(m);
 1625                                 vm_page_unlock(m);
 1626                         }
 1627                 }
 1628         }
 1629 
 1630         /*
 1631          * adjust pip.  NOTE: the original parent may still have its own
 1632          * pip refs on the object.
 1633          */
 1634         if (object != NULL) {
 1635                 vm_object_pip_wakeupn(object, bp->b_npages);
 1636                 VM_OBJECT_WUNLOCK(object);
 1637         }
 1638 
 1639         /*
 1640          * swapdev_strategy() manually sets b_vp and b_bufobj before calling
 1641          * bstrategy(). Set them back to NULL now we're done with it, or we'll
 1642          * trigger a KASSERT in relpbuf().
 1643          */
 1644         if (bp->b_vp) {
 1645                     bp->b_vp = NULL;
 1646                     bp->b_bufobj = NULL;
 1647         }
 1648         /*
 1649          * release the physical I/O buffer
 1650          */
 1651         relpbuf(
 1652             bp,
 1653             ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
 1654                 ((bp->b_flags & B_ASYNC) ?
 1655                     &nsw_wcount_async :
 1656                     &nsw_wcount_sync
 1657                 )
 1658             )
 1659         );
 1660 }
 1661 
 1662 /*
 1663  *      swap_pager_isswapped:
 1664  *
 1665  *      Return 1 if at least one page in the given object is paged
 1666  *      out to the given swap device.
 1667  *
 1668  *      This routine may not sleep.
 1669  */
 1670 int
 1671 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
 1672 {
 1673         daddr_t index = 0;
 1674         int bcount;
 1675         int i;
 1676 
 1677         VM_OBJECT_ASSERT_WLOCKED(object);
 1678         if (object->type != OBJT_SWAP)
 1679                 return (0);
 1680 
 1681         mtx_lock(&swhash_mtx);
 1682         for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
 1683                 struct swblock *swap;
 1684 
 1685                 if ((swap = *swp_pager_hash(object, index)) != NULL) {
 1686                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1687                                 if (swp_pager_isondev(swap->swb_pages[i], sp)) {
 1688                                         mtx_unlock(&swhash_mtx);
 1689                                         return (1);
 1690                                 }
 1691                         }
 1692                 }
 1693                 index += SWAP_META_PAGES;
 1694         }
 1695         mtx_unlock(&swhash_mtx);
 1696         return (0);
 1697 }
 1698 
 1699 /*
 1700  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
 1701  *
 1702  *      This routine dissociates the page at the given index within a
 1703  *      swap block from its backing store, paging it in if necessary.
 1704  *      If the page is paged in, it is placed in the inactive queue,
 1705  *      since it had its backing store ripped out from under it.
 1706  *      We also attempt to swap in all other pages in the swap block,
 1707  *      we only guarantee that the one at the specified index is
 1708  *      paged in.
 1709  *
 1710  *      XXX - The code to page the whole block in doesn't work, so we
 1711  *            revert to the one-by-one behavior for now.  Sigh.
 1712  */
 1713 static inline void
 1714 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
 1715 {
 1716         vm_page_t m;
 1717 
 1718         vm_object_pip_add(object, 1);
 1719         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
 1720         if (m->valid == VM_PAGE_BITS_ALL) {
 1721                 vm_object_pip_subtract(object, 1);
 1722                 vm_page_dirty(m);
 1723                 vm_page_lock(m);
 1724                 vm_page_activate(m);
 1725                 vm_page_unlock(m);
 1726                 vm_page_xunbusy(m);
 1727                 vm_pager_page_unswapped(m);
 1728                 return;
 1729         }
 1730 
 1731         if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
 1732                 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
 1733         vm_object_pip_subtract(object, 1);
 1734         vm_page_dirty(m);
 1735         vm_page_lock(m);
 1736         vm_page_deactivate(m);
 1737         vm_page_unlock(m);
 1738         vm_page_xunbusy(m);
 1739         vm_pager_page_unswapped(m);
 1740 }
 1741 
 1742 /*
 1743  *      swap_pager_swapoff:
 1744  *
 1745  *      Page in all of the pages that have been paged out to the
 1746  *      given device.  The corresponding blocks in the bitmap must be
 1747  *      marked as allocated and the device must be flagged SW_CLOSING.
 1748  *      There may be no processes swapped out to the device.
 1749  *
 1750  *      This routine may block.
 1751  */
 1752 static void
 1753 swap_pager_swapoff(struct swdevt *sp)
 1754 {
 1755         struct swblock *swap;
 1756         vm_object_t locked_obj, object;
 1757         vm_pindex_t pindex;
 1758         int i, j, retries;
 1759 
 1760         GIANT_REQUIRED;
 1761 
 1762         retries = 0;
 1763         locked_obj = NULL;
 1764 full_rescan:
 1765         mtx_lock(&swhash_mtx);
 1766         for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
 1767 restart:
 1768                 for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
 1769                         object = swap->swb_object;
 1770                         pindex = swap->swb_index;
 1771                         for (j = 0; j < SWAP_META_PAGES; ++j) {
 1772                                 if (!swp_pager_isondev(swap->swb_pages[j], sp))
 1773                                         continue;
 1774                                 if (locked_obj != object) {
 1775                                         if (locked_obj != NULL)
 1776                                                 VM_OBJECT_WUNLOCK(locked_obj);
 1777                                         locked_obj = object;
 1778                                         if (!VM_OBJECT_TRYWLOCK(object)) {
 1779                                                 mtx_unlock(&swhash_mtx);
 1780                                                 /* Depends on type-stability. */
 1781                                                 VM_OBJECT_WLOCK(object);
 1782                                                 mtx_lock(&swhash_mtx);
 1783                                                 goto restart;
 1784                                         }
 1785                                 }
 1786                                 MPASS(locked_obj == object);
 1787                                 mtx_unlock(&swhash_mtx);
 1788                                 swp_pager_force_pagein(object, pindex + j);
 1789                                 mtx_lock(&swhash_mtx);
 1790                                 goto restart;
 1791                         }
 1792                 }
 1793         }
 1794         mtx_unlock(&swhash_mtx);
 1795         if (locked_obj != NULL) {
 1796                 VM_OBJECT_WUNLOCK(locked_obj);
 1797                 locked_obj = NULL;
 1798         }
 1799         if (sp->sw_used) {
 1800                 /*
 1801                  * Objects may be locked or paging to the device being
 1802                  * removed, so we will miss their pages and need to
 1803                  * make another pass.  We have marked this device as
 1804                  * SW_CLOSING, so the activity should finish soon.
 1805                  */
 1806                 retries++;
 1807                 if (retries > 100) {
 1808                         panic("swapoff: failed to locate %d swap blocks",
 1809                             sp->sw_used);
 1810                 }
 1811                 pause("swpoff", hz / 20);
 1812                 goto full_rescan;
 1813         }
 1814 }
 1815 
 1816 /************************************************************************
 1817  *                              SWAP META DATA                          *
 1818  ************************************************************************
 1819  *
 1820  *      These routines manipulate the swap metadata stored in the
 1821  *      OBJT_SWAP object.
 1822  *
 1823  *      Swap metadata is implemented with a global hash and not directly
 1824  *      linked into the object.  Instead the object simply contains
 1825  *      appropriate tracking counters.
 1826  */
 1827 
 1828 /*
 1829  * SWP_PAGER_META_BUILD() -     add swap block to swap meta data for object
 1830  *
 1831  *      We first convert the object to a swap object if it is a default
 1832  *      object.
 1833  *
 1834  *      The specified swapblk is added to the object's swap metadata.  If
 1835  *      the swapblk is not valid, it is freed instead.  Any previously
 1836  *      assigned swapblk is freed.
 1837  */
 1838 static void
 1839 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
 1840 {
 1841         static volatile int exhausted;
 1842         struct swblock *swap;
 1843         struct swblock **pswap;
 1844         int idx;
 1845 
 1846         VM_OBJECT_ASSERT_WLOCKED(object);
 1847         /*
 1848          * Convert default object to swap object if necessary
 1849          */
 1850         if (object->type != OBJT_SWAP) {
 1851                 object->type = OBJT_SWAP;
 1852                 object->un_pager.swp.swp_bcount = 0;
 1853 
 1854                 if (object->handle != NULL) {
 1855                         mtx_lock(&sw_alloc_mtx);
 1856                         TAILQ_INSERT_TAIL(
 1857                             NOBJLIST(object->handle),
 1858                             object,
 1859                             pager_object_list
 1860                         );
 1861                         mtx_unlock(&sw_alloc_mtx);
 1862                 }
 1863         }
 1864 
 1865         /*
 1866          * Locate hash entry.  If not found create, but if we aren't adding
 1867          * anything just return.  If we run out of space in the map we wait
 1868          * and, since the hash table may have changed, retry.
 1869          */
 1870 retry:
 1871         mtx_lock(&swhash_mtx);
 1872         pswap = swp_pager_hash(object, pindex);
 1873 
 1874         if ((swap = *pswap) == NULL) {
 1875                 int i;
 1876 
 1877                 if (swapblk == SWAPBLK_NONE)
 1878                         goto done;
 1879 
 1880                 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT |
 1881                     (curproc == pageproc ? M_USE_RESERVE : 0));
 1882                 if (swap == NULL) {
 1883                         mtx_unlock(&swhash_mtx);
 1884                         VM_OBJECT_WUNLOCK(object);
 1885                         if (uma_zone_exhausted(swap_zone)) {
 1886                                 if (atomic_cmpset_int(&exhausted, 0, 1))
 1887                                         printf("swap zone exhausted, "
 1888                                             "increase kern.maxswzone\n");
 1889                                 vm_pageout_oom(VM_OOM_SWAPZ);
 1890                                 pause("swzonex", 10);
 1891                         } else
 1892                                 VM_WAIT;
 1893                         VM_OBJECT_WLOCK(object);
 1894                         goto retry;
 1895                 }
 1896 
 1897                 if (atomic_cmpset_int(&exhausted, 1, 0))
 1898                         printf("swap zone ok\n");
 1899 
 1900                 swap->swb_hnext = NULL;
 1901                 swap->swb_object = object;
 1902                 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
 1903                 swap->swb_count = 0;
 1904 
 1905                 ++object->un_pager.swp.swp_bcount;
 1906 
 1907                 for (i = 0; i < SWAP_META_PAGES; ++i)
 1908                         swap->swb_pages[i] = SWAPBLK_NONE;
 1909         }
 1910 
 1911         /*
 1912          * Delete prior contents of metadata
 1913          */
 1914         idx = pindex & SWAP_META_MASK;
 1915 
 1916         if (swap->swb_pages[idx] != SWAPBLK_NONE) {
 1917                 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
 1918                 --swap->swb_count;
 1919         }
 1920 
 1921         /*
 1922          * Enter block into metadata
 1923          */
 1924         swap->swb_pages[idx] = swapblk;
 1925         if (swapblk != SWAPBLK_NONE)
 1926                 ++swap->swb_count;
 1927 done:
 1928         mtx_unlock(&swhash_mtx);
 1929 }
 1930 
 1931 /*
 1932  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
 1933  *
 1934  *      The requested range of blocks is freed, with any associated swap
 1935  *      returned to the swap bitmap.
 1936  *
 1937  *      This routine will free swap metadata structures as they are cleaned
 1938  *      out.  This routine does *NOT* operate on swap metadata associated
 1939  *      with resident pages.
 1940  */
 1941 static void
 1942 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
 1943 {
 1944 
 1945         VM_OBJECT_ASSERT_LOCKED(object);
 1946         if (object->type != OBJT_SWAP)
 1947                 return;
 1948 
 1949         while (count > 0) {
 1950                 struct swblock **pswap;
 1951                 struct swblock *swap;
 1952 
 1953                 mtx_lock(&swhash_mtx);
 1954                 pswap = swp_pager_hash(object, index);
 1955 
 1956                 if ((swap = *pswap) != NULL) {
 1957                         daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
 1958 
 1959                         if (v != SWAPBLK_NONE) {
 1960                                 swp_pager_freeswapspace(v, 1);
 1961                                 swap->swb_pages[index & SWAP_META_MASK] =
 1962                                         SWAPBLK_NONE;
 1963                                 if (--swap->swb_count == 0) {
 1964                                         *pswap = swap->swb_hnext;
 1965                                         uma_zfree(swap_zone, swap);
 1966                                         --object->un_pager.swp.swp_bcount;
 1967                                 }
 1968                         }
 1969                         --count;
 1970                         ++index;
 1971                 } else {
 1972                         int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
 1973                         count -= n;
 1974                         index += n;
 1975                 }
 1976                 mtx_unlock(&swhash_mtx);
 1977         }
 1978 }
 1979 
 1980 /*
 1981  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
 1982  *
 1983  *      This routine locates and destroys all swap metadata associated with
 1984  *      an object.
 1985  */
 1986 static void
 1987 swp_pager_meta_free_all(vm_object_t object)
 1988 {
 1989         struct swblock **pswap, *swap;
 1990         vm_pindex_t index;
 1991         daddr_t v;
 1992         int i;
 1993 
 1994         VM_OBJECT_ASSERT_WLOCKED(object);
 1995         if (object->type != OBJT_SWAP)
 1996                 return;
 1997 
 1998         index = 0;
 1999         while (object->un_pager.swp.swp_bcount != 0) {
 2000                 mtx_lock(&swhash_mtx);
 2001                 pswap = swp_pager_hash(object, index);
 2002                 if ((swap = *pswap) != NULL) {
 2003                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 2004                                 v = swap->swb_pages[i];
 2005                                 if (v != SWAPBLK_NONE) {
 2006                                         --swap->swb_count;
 2007                                         swp_pager_freeswapspace(v, 1);
 2008                                 }
 2009                         }
 2010                         if (swap->swb_count != 0)
 2011                                 panic(
 2012                                     "swap_pager_meta_free_all: swb_count != 0");
 2013                         *pswap = swap->swb_hnext;
 2014                         uma_zfree(swap_zone, swap);
 2015                         --object->un_pager.swp.swp_bcount;
 2016                 }
 2017                 mtx_unlock(&swhash_mtx);
 2018                 index += SWAP_META_PAGES;
 2019         }
 2020 }
 2021 
 2022 /*
 2023  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
 2024  *
 2025  *      This routine is capable of looking up, popping, or freeing
 2026  *      swapblk assignments in the swap meta data or in the vm_page_t.
 2027  *      The routine typically returns the swapblk being looked-up, or popped,
 2028  *      or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
 2029  *      was invalid.  This routine will automatically free any invalid
 2030  *      meta-data swapblks.
 2031  *
 2032  *      It is not possible to store invalid swapblks in the swap meta data
 2033  *      (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
 2034  *
 2035  *      When acting on a busy resident page and paging is in progress, we
 2036  *      have to wait until paging is complete but otherwise can act on the
 2037  *      busy page.
 2038  *
 2039  *      SWM_FREE        remove and free swap block from metadata
 2040  *      SWM_POP         remove from meta data but do not free.. pop it out
 2041  */
 2042 static daddr_t
 2043 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
 2044 {
 2045         struct swblock **pswap;
 2046         struct swblock *swap;
 2047         daddr_t r1;
 2048         int idx;
 2049 
 2050         VM_OBJECT_ASSERT_LOCKED(object);
 2051         /*
 2052          * The meta data only exists of the object is OBJT_SWAP
 2053          * and even then might not be allocated yet.
 2054          */
 2055         if (object->type != OBJT_SWAP)
 2056                 return (SWAPBLK_NONE);
 2057 
 2058         r1 = SWAPBLK_NONE;
 2059         mtx_lock(&swhash_mtx);
 2060         pswap = swp_pager_hash(object, pindex);
 2061 
 2062         if ((swap = *pswap) != NULL) {
 2063                 idx = pindex & SWAP_META_MASK;
 2064                 r1 = swap->swb_pages[idx];
 2065 
 2066                 if (r1 != SWAPBLK_NONE) {
 2067                         if (flags & SWM_FREE) {
 2068                                 swp_pager_freeswapspace(r1, 1);
 2069                                 r1 = SWAPBLK_NONE;
 2070                         }
 2071                         if (flags & (SWM_FREE|SWM_POP)) {
 2072                                 swap->swb_pages[idx] = SWAPBLK_NONE;
 2073                                 if (--swap->swb_count == 0) {
 2074                                         *pswap = swap->swb_hnext;
 2075                                         uma_zfree(swap_zone, swap);
 2076                                         --object->un_pager.swp.swp_bcount;
 2077                                 }
 2078                         }
 2079                 }
 2080         }
 2081         mtx_unlock(&swhash_mtx);
 2082         return (r1);
 2083 }
 2084 
 2085 /*
 2086  * System call swapon(name) enables swapping on device name,
 2087  * which must be in the swdevsw.  Return EBUSY
 2088  * if already swapping on this device.
 2089  */
 2090 #ifndef _SYS_SYSPROTO_H_
 2091 struct swapon_args {
 2092         char *name;
 2093 };
 2094 #endif
 2095 
 2096 /*
 2097  * MPSAFE
 2098  */
 2099 /* ARGSUSED */
 2100 int
 2101 sys_swapon(struct thread *td, struct swapon_args *uap)
 2102 {
 2103         struct vattr attr;
 2104         struct vnode *vp;
 2105         struct nameidata nd;
 2106         int error;
 2107 
 2108         error = priv_check(td, PRIV_SWAPON);
 2109         if (error)
 2110                 return (error);
 2111 
 2112         mtx_lock(&Giant);
 2113         while (swdev_syscall_active)
 2114             tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
 2115         swdev_syscall_active = 1;
 2116 
 2117         /*
 2118          * Swap metadata may not fit in the KVM if we have physical
 2119          * memory of >1GB.
 2120          */
 2121         if (swap_zone == NULL) {
 2122                 error = ENOMEM;
 2123                 goto done;
 2124         }
 2125 
 2126         NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
 2127             uap->name, td);
 2128         error = namei(&nd);
 2129         if (error)
 2130                 goto done;
 2131 
 2132         NDFREE(&nd, NDF_ONLY_PNBUF);
 2133         vp = nd.ni_vp;
 2134 
 2135         if (vn_isdisk(vp, &error)) {
 2136                 error = swapongeom(td, vp);
 2137         } else if (vp->v_type == VREG &&
 2138             (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
 2139             (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
 2140                 /*
 2141                  * Allow direct swapping to NFS regular files in the same
 2142                  * way that nfs_mountroot() sets up diskless swapping.
 2143                  */
 2144                 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
 2145         }
 2146 
 2147         if (error)
 2148                 vrele(vp);
 2149 done:
 2150         swdev_syscall_active = 0;
 2151         wakeup_one(&swdev_syscall_active);
 2152         mtx_unlock(&Giant);
 2153         return (error);
 2154 }
 2155 
 2156 /*
 2157  * Check that the total amount of swap currently configured does not
 2158  * exceed half the theoretical maximum.  If it does, print a warning
 2159  * message and return -1; otherwise, return 0.
 2160  */
 2161 static int
 2162 swapon_check_swzone(unsigned long npages)
 2163 {
 2164         unsigned long maxpages;
 2165 
 2166         /* absolute maximum we can handle assuming 100% efficiency */
 2167         maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES;
 2168 
 2169         /* recommend using no more than half that amount */
 2170         if (npages > maxpages / 2) {
 2171                 printf("warning: total configured swap (%lu pages) "
 2172                     "exceeds maximum recommended amount (%lu pages).\n",
 2173                     npages, maxpages / 2);
 2174                 printf("warning: increase kern.maxswzone "
 2175                     "or reduce amount of swap.\n");
 2176                 return (-1);
 2177         }
 2178         return (0);
 2179 }
 2180 
 2181 static void
 2182 swaponsomething(struct vnode *vp, void *id, u_long nblks,
 2183     sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags)
 2184 {
 2185         struct swdevt *sp, *tsp;
 2186         swblk_t dvbase;
 2187         u_long mblocks;
 2188 
 2189         /*
 2190          * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
 2191          * First chop nblks off to page-align it, then convert.
 2192          *
 2193          * sw->sw_nblks is in page-sized chunks now too.
 2194          */
 2195         nblks &= ~(ctodb(1) - 1);
 2196         nblks = dbtoc(nblks);
 2197 
 2198         /*
 2199          * If we go beyond this, we get overflows in the radix
 2200          * tree bitmap code.
 2201          */
 2202         mblocks = 0x40000000 / BLIST_META_RADIX;
 2203         if (nblks > mblocks) {
 2204                 printf(
 2205     "WARNING: reducing swap size to maximum of %luMB per unit\n",
 2206                     mblocks / 1024 / 1024 * PAGE_SIZE);
 2207                 nblks = mblocks;
 2208         }
 2209 
 2210         sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
 2211         sp->sw_vp = vp;
 2212         sp->sw_id = id;
 2213         sp->sw_dev = dev;
 2214         sp->sw_flags = 0;
 2215         sp->sw_nblks = nblks;
 2216         sp->sw_used = 0;
 2217         sp->sw_strategy = strategy;
 2218         sp->sw_close = close;
 2219         sp->sw_flags = flags;
 2220 
 2221         sp->sw_blist = blist_create(nblks, M_WAITOK);
 2222         /*
 2223          * Do not free the first two block in order to avoid overwriting
 2224          * any bsd label at the front of the partition
 2225          */
 2226         blist_free(sp->sw_blist, 2, nblks - 2);
 2227 
 2228         dvbase = 0;
 2229         mtx_lock(&sw_dev_mtx);
 2230         TAILQ_FOREACH(tsp, &swtailq, sw_list) {
 2231                 if (tsp->sw_end >= dvbase) {
 2232                         /*
 2233                          * We put one uncovered page between the devices
 2234                          * in order to definitively prevent any cross-device
 2235                          * I/O requests
 2236                          */
 2237                         dvbase = tsp->sw_end + 1;
 2238                 }
 2239         }
 2240         sp->sw_first = dvbase;
 2241         sp->sw_end = dvbase + nblks;
 2242         TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
 2243         nswapdev++;
 2244         swap_pager_avail += nblks - 2;
 2245         swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
 2246         swapon_check_swzone(swap_total / PAGE_SIZE);
 2247         swp_sizecheck();
 2248         mtx_unlock(&sw_dev_mtx);
 2249 }
 2250 
 2251 /*
 2252  * SYSCALL: swapoff(devname)
 2253  *
 2254  * Disable swapping on the given device.
 2255  *
 2256  * XXX: Badly designed system call: it should use a device index
 2257  * rather than filename as specification.  We keep sw_vp around
 2258  * only to make this work.
 2259  */
 2260 #ifndef _SYS_SYSPROTO_H_
 2261 struct swapoff_args {
 2262         char *name;
 2263 };
 2264 #endif
 2265 
 2266 /*
 2267  * MPSAFE
 2268  */
 2269 /* ARGSUSED */
 2270 int
 2271 sys_swapoff(struct thread *td, struct swapoff_args *uap)
 2272 {
 2273         struct vnode *vp;
 2274         struct nameidata nd;
 2275         struct swdevt *sp;
 2276         int error;
 2277 
 2278         error = priv_check(td, PRIV_SWAPOFF);
 2279         if (error)
 2280                 return (error);
 2281 
 2282         mtx_lock(&Giant);
 2283         while (swdev_syscall_active)
 2284             tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2285         swdev_syscall_active = 1;
 2286 
 2287         NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
 2288             td);
 2289         error = namei(&nd);
 2290         if (error)
 2291                 goto done;
 2292         NDFREE(&nd, NDF_ONLY_PNBUF);
 2293         vp = nd.ni_vp;
 2294 
 2295         mtx_lock(&sw_dev_mtx);
 2296         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2297                 if (sp->sw_vp == vp)
 2298                         break;
 2299         }
 2300         mtx_unlock(&sw_dev_mtx);
 2301         if (sp == NULL) {
 2302                 error = EINVAL;
 2303                 goto done;
 2304         }
 2305         error = swapoff_one(sp, td->td_ucred);
 2306 done:
 2307         swdev_syscall_active = 0;
 2308         wakeup_one(&swdev_syscall_active);
 2309         mtx_unlock(&Giant);
 2310         return (error);
 2311 }
 2312 
 2313 static int
 2314 swapoff_one(struct swdevt *sp, struct ucred *cred)
 2315 {
 2316         u_long nblks;
 2317 #ifdef MAC
 2318         int error;
 2319 #endif
 2320 
 2321         mtx_assert(&Giant, MA_OWNED);
 2322 #ifdef MAC
 2323         (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
 2324         error = mac_system_check_swapoff(cred, sp->sw_vp);
 2325         (void) VOP_UNLOCK(sp->sw_vp, 0);
 2326         if (error != 0)
 2327                 return (error);
 2328 #endif
 2329         nblks = sp->sw_nblks;
 2330 
 2331         /*
 2332          * We can turn off this swap device safely only if the
 2333          * available virtual memory in the system will fit the amount
 2334          * of data we will have to page back in, plus an epsilon so
 2335          * the system doesn't become critically low on swap space.
 2336          */
 2337         if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
 2338             nblks + nswap_lowat) {
 2339                 return (ENOMEM);
 2340         }
 2341 
 2342         /*
 2343          * Prevent further allocations on this device.
 2344          */
 2345         mtx_lock(&sw_dev_mtx);
 2346         sp->sw_flags |= SW_CLOSING;
 2347         swap_pager_avail -= blist_fill(sp->sw_blist, 0, nblks);
 2348         swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
 2349         mtx_unlock(&sw_dev_mtx);
 2350 
 2351         /*
 2352          * Page in the contents of the device and close it.
 2353          */
 2354         swap_pager_swapoff(sp);
 2355 
 2356         sp->sw_close(curthread, sp);
 2357         mtx_lock(&sw_dev_mtx);
 2358         sp->sw_id = NULL;
 2359         TAILQ_REMOVE(&swtailq, sp, sw_list);
 2360         nswapdev--;
 2361         if (nswapdev == 0) {
 2362                 swap_pager_full = 2;
 2363                 swap_pager_almost_full = 1;
 2364         }
 2365         if (swdevhd == sp)
 2366                 swdevhd = NULL;
 2367         mtx_unlock(&sw_dev_mtx);
 2368         blist_destroy(sp->sw_blist);
 2369         free(sp, M_VMPGDATA);
 2370         return (0);
 2371 }
 2372 
 2373 void
 2374 swapoff_all(void)
 2375 {
 2376         struct swdevt *sp, *spt;
 2377         const char *devname;
 2378         int error;
 2379 
 2380         mtx_lock(&Giant);
 2381         while (swdev_syscall_active)
 2382                 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2383         swdev_syscall_active = 1;
 2384 
 2385         mtx_lock(&sw_dev_mtx);
 2386         TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
 2387                 mtx_unlock(&sw_dev_mtx);
 2388                 if (vn_isdisk(sp->sw_vp, NULL))
 2389                         devname = devtoname(sp->sw_vp->v_rdev);
 2390                 else
 2391                         devname = "[file]";
 2392                 error = swapoff_one(sp, thread0.td_ucred);
 2393                 if (error != 0) {
 2394                         printf("Cannot remove swap device %s (error=%d), "
 2395                             "skipping.\n", devname, error);
 2396                 } else if (bootverbose) {
 2397                         printf("Swap device %s removed.\n", devname);
 2398                 }
 2399                 mtx_lock(&sw_dev_mtx);
 2400         }
 2401         mtx_unlock(&sw_dev_mtx);
 2402 
 2403         swdev_syscall_active = 0;
 2404         wakeup_one(&swdev_syscall_active);
 2405         mtx_unlock(&Giant);
 2406 }
 2407 
 2408 void
 2409 swap_pager_status(int *total, int *used)
 2410 {
 2411         struct swdevt *sp;
 2412 
 2413         *total = 0;
 2414         *used = 0;
 2415         mtx_lock(&sw_dev_mtx);
 2416         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2417                 *total += sp->sw_nblks;
 2418                 *used += sp->sw_used;
 2419         }
 2420         mtx_unlock(&sw_dev_mtx);
 2421 }
 2422 
 2423 int
 2424 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
 2425 {
 2426         struct swdevt *sp;
 2427         const char *tmp_devname;
 2428         int error, n;
 2429 
 2430         n = 0;
 2431         error = ENOENT;
 2432         mtx_lock(&sw_dev_mtx);
 2433         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2434                 if (n != name) {
 2435                         n++;
 2436                         continue;
 2437                 }
 2438                 xs->xsw_version = XSWDEV_VERSION;
 2439                 xs->xsw_dev = sp->sw_dev;
 2440                 xs->xsw_flags = sp->sw_flags;
 2441                 xs->xsw_nblks = sp->sw_nblks;
 2442                 xs->xsw_used = sp->sw_used;
 2443                 if (devname != NULL) {
 2444                         if (vn_isdisk(sp->sw_vp, NULL))
 2445                                 tmp_devname = devtoname(sp->sw_vp->v_rdev);
 2446                         else
 2447                                 tmp_devname = "[file]";
 2448                         strncpy(devname, tmp_devname, len);
 2449                 }
 2450                 error = 0;
 2451                 break;
 2452         }
 2453         mtx_unlock(&sw_dev_mtx);
 2454         return (error);
 2455 }
 2456 
 2457 static int
 2458 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
 2459 {
 2460         struct xswdev xs;
 2461         int error;
 2462 
 2463         if (arg2 != 1)                  /* name length */
 2464                 return (EINVAL);
 2465         error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
 2466         if (error != 0)
 2467                 return (error);
 2468         error = SYSCTL_OUT(req, &xs, sizeof(xs));
 2469         return (error);
 2470 }
 2471 
 2472 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
 2473     "Number of swap devices");
 2474 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
 2475     "Swap statistics by device");
 2476 
 2477 /*
 2478  * vmspace_swap_count() - count the approximate swap usage in pages for a
 2479  *                        vmspace.
 2480  *
 2481  *      The map must be locked.
 2482  *
 2483  *      Swap usage is determined by taking the proportional swap used by
 2484  *      VM objects backing the VM map.  To make up for fractional losses,
 2485  *      if the VM object has any swap use at all the associated map entries
 2486  *      count for at least 1 swap page.
 2487  */
 2488 long
 2489 vmspace_swap_count(struct vmspace *vmspace)
 2490 {
 2491         vm_map_t map;
 2492         vm_map_entry_t cur;
 2493         vm_object_t object;
 2494         long count, n;
 2495 
 2496         map = &vmspace->vm_map;
 2497         count = 0;
 2498 
 2499         for (cur = map->header.next; cur != &map->header; cur = cur->next) {
 2500                 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2501                     (object = cur->object.vm_object) != NULL) {
 2502                         VM_OBJECT_WLOCK(object);
 2503                         if (object->type == OBJT_SWAP &&
 2504                             object->un_pager.swp.swp_bcount != 0) {
 2505                                 n = (cur->end - cur->start) / PAGE_SIZE;
 2506                                 count += object->un_pager.swp.swp_bcount *
 2507                                     SWAP_META_PAGES * n / object->size + 1;
 2508                         }
 2509                         VM_OBJECT_WUNLOCK(object);
 2510                 }
 2511         }
 2512         return (count);
 2513 }
 2514 
 2515 /*
 2516  * GEOM backend
 2517  *
 2518  * Swapping onto disk devices.
 2519  *
 2520  */
 2521 
 2522 static g_orphan_t swapgeom_orphan;
 2523 
 2524 static struct g_class g_swap_class = {
 2525         .name = "SWAP",
 2526         .version = G_VERSION,
 2527         .orphan = swapgeom_orphan,
 2528 };
 2529 
 2530 DECLARE_GEOM_CLASS(g_swap_class, g_class);
 2531 
 2532 
 2533 static void
 2534 swapgeom_close_ev(void *arg, int flags)
 2535 {
 2536         struct g_consumer *cp;
 2537 
 2538         cp = arg;
 2539         g_access(cp, -1, -1, 0);
 2540         g_detach(cp);
 2541         g_destroy_consumer(cp);
 2542 }
 2543 
 2544 /*
 2545  * Add a reference to the g_consumer for an inflight transaction.
 2546  */
 2547 static void
 2548 swapgeom_acquire(struct g_consumer *cp)
 2549 {
 2550 
 2551         mtx_assert(&sw_dev_mtx, MA_OWNED);
 2552         cp->index++;
 2553 }
 2554 
 2555 /*
 2556  * Remove a reference from the g_consumer. Post a close event if
 2557  * all referneces go away.
 2558  */
 2559 static void
 2560 swapgeom_release(struct g_consumer *cp, struct swdevt *sp)
 2561 {
 2562 
 2563         mtx_assert(&sw_dev_mtx, MA_OWNED);
 2564         cp->index--;
 2565         if (cp->index == 0) {
 2566                 if (g_post_event(swapgeom_close_ev, cp, M_NOWAIT, NULL) == 0)
 2567                         sp->sw_id = NULL;
 2568         }
 2569 }
 2570 
 2571 static void
 2572 swapgeom_done(struct bio *bp2)
 2573 {
 2574         struct swdevt *sp;
 2575         struct buf *bp;
 2576         struct g_consumer *cp;
 2577 
 2578         bp = bp2->bio_caller2;
 2579         cp = bp2->bio_from;
 2580         bp->b_ioflags = bp2->bio_flags;
 2581         if (bp2->bio_error)
 2582                 bp->b_ioflags |= BIO_ERROR;
 2583         bp->b_resid = bp->b_bcount - bp2->bio_completed;
 2584         bp->b_error = bp2->bio_error;
 2585         bufdone(bp);
 2586         sp = bp2->bio_caller1;
 2587         mtx_lock(&sw_dev_mtx);
 2588         swapgeom_release(cp, sp);
 2589         mtx_unlock(&sw_dev_mtx);
 2590         g_destroy_bio(bp2);
 2591 }
 2592 
 2593 static void
 2594 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
 2595 {
 2596         struct bio *bio;
 2597         struct g_consumer *cp;
 2598 
 2599         mtx_lock(&sw_dev_mtx);
 2600         cp = sp->sw_id;
 2601         if (cp == NULL) {
 2602                 mtx_unlock(&sw_dev_mtx);
 2603                 bp->b_error = ENXIO;
 2604                 bp->b_ioflags |= BIO_ERROR;
 2605                 bufdone(bp);
 2606                 return;
 2607         }
 2608         swapgeom_acquire(cp);
 2609         mtx_unlock(&sw_dev_mtx);
 2610         if (bp->b_iocmd == BIO_WRITE)
 2611                 bio = g_new_bio();
 2612         else
 2613                 bio = g_alloc_bio();
 2614         if (bio == NULL) {
 2615                 mtx_lock(&sw_dev_mtx);
 2616                 swapgeom_release(cp, sp);
 2617                 mtx_unlock(&sw_dev_mtx);
 2618                 bp->b_error = ENOMEM;
 2619                 bp->b_ioflags |= BIO_ERROR;
 2620                 bufdone(bp);
 2621                 return;
 2622         }
 2623 
 2624         bio->bio_caller1 = sp;
 2625         bio->bio_caller2 = bp;
 2626         bio->bio_cmd = bp->b_iocmd;
 2627         bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
 2628         bio->bio_length = bp->b_bcount;
 2629         bio->bio_done = swapgeom_done;
 2630         if ((bp->b_flags & B_UNMAPPED) != 0) {
 2631                 bio->bio_ma = bp->b_pages;
 2632                 bio->bio_data = unmapped_buf;
 2633                 bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
 2634                 bio->bio_ma_n = bp->b_npages;
 2635                 bio->bio_flags |= BIO_UNMAPPED;
 2636         } else {
 2637                 bio->bio_data = bp->b_data;
 2638                 bio->bio_ma = NULL;
 2639         }
 2640         g_io_request(bio, cp);
 2641         return;
 2642 }
 2643 
 2644 static void
 2645 swapgeom_orphan(struct g_consumer *cp)
 2646 {
 2647         struct swdevt *sp;
 2648         int destroy;
 2649 
 2650         mtx_lock(&sw_dev_mtx);
 2651         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2652                 if (sp->sw_id == cp) {
 2653                         sp->sw_flags |= SW_CLOSING;
 2654                         break;
 2655                 }
 2656         }
 2657         /*
 2658          * Drop reference we were created with. Do directly since we're in a
 2659          * special context where we don't have to queue the call to
 2660          * swapgeom_close_ev().
 2661          */
 2662         cp->index--;
 2663         destroy = ((sp != NULL) && (cp->index == 0));
 2664         if (destroy)
 2665                 sp->sw_id = NULL;
 2666         mtx_unlock(&sw_dev_mtx);
 2667         if (destroy)
 2668                 swapgeom_close_ev(cp, 0);
 2669 }
 2670 
 2671 static void
 2672 swapgeom_close(struct thread *td, struct swdevt *sw)
 2673 {
 2674         struct g_consumer *cp;
 2675 
 2676         mtx_lock(&sw_dev_mtx);
 2677         cp = sw->sw_id;
 2678         sw->sw_id = NULL;
 2679         mtx_unlock(&sw_dev_mtx);
 2680         /* XXX: direct call when Giant untangled */
 2681         if (cp != NULL)
 2682                 g_waitfor_event(swapgeom_close_ev, cp, M_WAITOK, NULL);
 2683 }
 2684 
 2685 
 2686 struct swh0h0 {
 2687         struct cdev *dev;
 2688         struct vnode *vp;
 2689         int     error;
 2690 };
 2691 
 2692 static void
 2693 swapongeom_ev(void *arg, int flags)
 2694 {
 2695         struct swh0h0 *swh;
 2696         struct g_provider *pp;
 2697         struct g_consumer *cp;
 2698         static struct g_geom *gp;
 2699         struct swdevt *sp;
 2700         u_long nblks;
 2701         int error;
 2702 
 2703         swh = arg;
 2704         swh->error = 0;
 2705         pp = g_dev_getprovider(swh->dev);
 2706         if (pp == NULL) {
 2707                 swh->error = ENODEV;
 2708                 return;
 2709         }
 2710         mtx_lock(&sw_dev_mtx);
 2711         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2712                 cp = sp->sw_id;
 2713                 if (cp != NULL && cp->provider == pp) {
 2714                         mtx_unlock(&sw_dev_mtx);
 2715                         swh->error = EBUSY;
 2716                         return;
 2717                 }
 2718         }
 2719         mtx_unlock(&sw_dev_mtx);
 2720         if (gp == NULL)
 2721                 gp = g_new_geomf(&g_swap_class, "swap");
 2722         cp = g_new_consumer(gp);
 2723         cp->index = 1;          /* Number of active I/Os, plus one for being active. */
 2724         cp->flags |=  G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
 2725         g_attach(cp, pp);
 2726         /*
 2727          * XXX: Everytime you think you can improve the margin for
 2728          * footshooting, somebody depends on the ability to do so:
 2729          * savecore(8) wants to write to our swapdev so we cannot
 2730          * set an exclusive count :-(
 2731          */
 2732         error = g_access(cp, 1, 1, 0);
 2733         if (error) {
 2734                 g_detach(cp);
 2735                 g_destroy_consumer(cp);
 2736                 swh->error = error;
 2737                 return;
 2738         }
 2739         nblks = pp->mediasize / DEV_BSIZE;
 2740         swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
 2741             swapgeom_close, dev2udev(swh->dev),
 2742             (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0);
 2743         swh->error = 0;
 2744 }
 2745 
 2746 static int
 2747 swapongeom(struct thread *td, struct vnode *vp)
 2748 {
 2749         int error;
 2750         struct swh0h0 swh;
 2751 
 2752         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2753 
 2754         swh.dev = vp->v_rdev;
 2755         swh.vp = vp;
 2756         swh.error = 0;
 2757         /* XXX: direct call when Giant untangled */
 2758         error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
 2759         if (!error)
 2760                 error = swh.error;
 2761         VOP_UNLOCK(vp, 0);
 2762         return (error);
 2763 }
 2764 
 2765 /*
 2766  * VNODE backend
 2767  *
 2768  * This is used mainly for network filesystem (read: probably only tested
 2769  * with NFS) swapfiles.
 2770  *
 2771  */
 2772 
 2773 static void
 2774 swapdev_strategy(struct buf *bp, struct swdevt *sp)
 2775 {
 2776         struct vnode *vp2;
 2777 
 2778         bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
 2779 
 2780         vp2 = sp->sw_id;
 2781         vhold(vp2);
 2782         if (bp->b_iocmd == BIO_WRITE) {
 2783                 if (bp->b_bufobj)
 2784                         bufobj_wdrop(bp->b_bufobj);
 2785                 bufobj_wref(&vp2->v_bufobj);
 2786         }
 2787         if (bp->b_bufobj != &vp2->v_bufobj)
 2788                 bp->b_bufobj = &vp2->v_bufobj;
 2789         bp->b_vp = vp2;
 2790         bp->b_iooffset = dbtob(bp->b_blkno);
 2791         bstrategy(bp);
 2792         return;
 2793 }
 2794 
 2795 static void
 2796 swapdev_close(struct thread *td, struct swdevt *sp)
 2797 {
 2798 
 2799         VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
 2800         vrele(sp->sw_vp);
 2801 }
 2802 
 2803 
 2804 static int
 2805 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
 2806 {
 2807         struct swdevt *sp;
 2808         int error;
 2809 
 2810         if (nblks == 0)
 2811                 return (ENXIO);
 2812         mtx_lock(&sw_dev_mtx);
 2813         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2814                 if (sp->sw_id == vp) {
 2815                         mtx_unlock(&sw_dev_mtx);
 2816                         return (EBUSY);
 2817                 }
 2818         }
 2819         mtx_unlock(&sw_dev_mtx);
 2820 
 2821         (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2822 #ifdef MAC
 2823         error = mac_system_check_swapon(td->td_ucred, vp);
 2824         if (error == 0)
 2825 #endif
 2826                 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
 2827         (void) VOP_UNLOCK(vp, 0);
 2828         if (error)
 2829                 return (error);
 2830 
 2831         swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
 2832             NODEV, 0);
 2833         return (0);
 2834 }

Cache object: f40d696adea4b583cfa44d3932cdad20


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.