[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/swap_pager.c

Version: -  FREEBSD  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  cheribsd  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1  -  FREEBSD-LIBC  -  FREEBSD8-LIBC  -  FREEBSD7-LIBC  -  FREEBSD6-LIBC  -  GLIBC27 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Matthew Dillon,
    3  * Copyright (c) 1994 John S. Dyson
    4  * Copyright (c) 1990 University of Utah.
    5  * Copyright (c) 1982, 1986, 1989, 1993
    6  *      The Regents of the University of California.  All rights reserved.
    7  *
    8  * This code is derived from software contributed to Berkeley by
    9  * the Systems Programming Group of the University of Utah Computer
   10  * Science Department.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. All advertising materials mentioning features or use of this software
   21  *    must display the following acknowledgement:
   22  *      This product includes software developed by the University of
   23  *      California, Berkeley and its contributors.
   24  * 4. Neither the name of the University nor the names of its contributors
   25  *    may be used to endorse or promote products derived from this software
   26  *    without specific prior written permission.
   27  *
   28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   38  * SUCH DAMAGE.
   39  *
   40  *                              New Swap System
   41  *                              Matthew Dillon
   42  *
   43  * Radix Bitmap 'blists'.
   44  *
   45  *      - The new swapper uses the new radix bitmap code.  This should scale
   46  *        to arbitrarily small or arbitrarily large swap spaces and an almost
   47  *        arbitrary degree of fragmentation.
   48  *
   49  * Features:
   50  *
   51  *      - on the fly reallocation of swap during putpages.  The new system
   52  *        does not try to keep previously allocated swap blocks for dirty
   53  *        pages.
   54  *
   55  *      - on the fly deallocation of swap
   56  *
   57  *      - No more garbage collection required.  Unnecessarily allocated swap
   58  *        blocks only exist for dirty vm_page_t's now and these are already
   59  *        cycled (in a high-load system) by the pager.  We also do on-the-fly
   60  *        removal of invalidated swap blocks when a page is destroyed
   61  *        or renamed.
   62  *
   63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
   64  *
   65  *      @(#)swap_pager.c        8.9 (Berkeley) 3/21/94
   66  *      @(#)vm_swap.c   8.5 (Berkeley) 2/17/94
   67  */
   68 
   69 #include <sys/cdefs.h>
   70 __FBSDID("$FreeBSD: head/sys/vm/swap_pager.c 254649 2013-08-22 07:39:53Z kib $");
   71 
   72 #include "opt_swap.h"
   73 #include "opt_vm.h"
   74 
   75 #include <sys/param.h>
   76 #include <sys/systm.h>
   77 #include <sys/conf.h>
   78 #include <sys/kernel.h>
   79 #include <sys/priv.h>
   80 #include <sys/proc.h>
   81 #include <sys/bio.h>
   82 #include <sys/buf.h>
   83 #include <sys/disk.h>
   84 #include <sys/fcntl.h>
   85 #include <sys/mount.h>
   86 #include <sys/namei.h>
   87 #include <sys/vnode.h>
   88 #include <sys/malloc.h>
   89 #include <sys/racct.h>
   90 #include <sys/resource.h>
   91 #include <sys/resourcevar.h>
   92 #include <sys/rwlock.h>
   93 #include <sys/sysctl.h>
   94 #include <sys/sysproto.h>
   95 #include <sys/blist.h>
   96 #include <sys/lock.h>
   97 #include <sys/sx.h>
   98 #include <sys/vmmeter.h>
   99 
  100 #include <security/mac/mac_framework.h>
  101 
  102 #include <vm/vm.h>
  103 #include <vm/pmap.h>
  104 #include <vm/vm_map.h>
  105 #include <vm/vm_kern.h>
  106 #include <vm/vm_object.h>
  107 #include <vm/vm_page.h>
  108 #include <vm/vm_pager.h>
  109 #include <vm/vm_pageout.h>
  110 #include <vm/vm_param.h>
  111 #include <vm/swap_pager.h>
  112 #include <vm/vm_extern.h>
  113 #include <vm/uma.h>
  114 
  115 #include <geom/geom.h>
  116 
  117 /*
  118  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
  119  * or 32 pages per allocation.
  120  * The 32-page limit is due to the radix code (kern/subr_blist.c).
  121  */
  122 #ifndef MAX_PAGEOUT_CLUSTER
  123 #define MAX_PAGEOUT_CLUSTER 16
  124 #endif
  125 
  126 #if !defined(SWB_NPAGES)
  127 #define SWB_NPAGES      MAX_PAGEOUT_CLUSTER
  128 #endif
  129 
  130 /*
  131  * The swblock structure maps an object and a small, fixed-size range
  132  * of page indices to disk addresses within a swap area.
  133  * The collection of these mappings is implemented as a hash table.
  134  * Unused disk addresses within a swap area are allocated and managed
  135  * using a blist.
  136  */
  137 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
  138 #define SWAP_META_PAGES         (SWB_NPAGES * 2)
  139 #define SWAP_META_MASK          (SWAP_META_PAGES - 1)
  140 
  141 struct swblock {
  142         struct swblock  *swb_hnext;
  143         vm_object_t     swb_object;
  144         vm_pindex_t     swb_index;
  145         int             swb_count;
  146         daddr_t         swb_pages[SWAP_META_PAGES];
  147 };
  148 
  149 static MALLOC_DEFINE(M_VMPGDATA, "vm_pgdata", "swap pager private data");
  150 static struct mtx sw_dev_mtx;
  151 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
  152 static struct swdevt *swdevhd;  /* Allocate from here next */
  153 static int nswapdev;            /* Number of swap devices */
  154 int swap_pager_avail;
  155 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
  156 
  157 static vm_ooffset_t swap_total;
  158 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0,
  159     "Total amount of available swap storage.");
  160 static vm_ooffset_t swap_reserved;
  161 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0,
  162     "Amount of swap storage needed to back all allocated anonymous memory.");
  163 static int overcommit = 0;
  164 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0,
  165     "Configure virtual memory overcommit behavior. See tuning(7) "
  166     "for details.");
  167 
  168 /* bits from overcommit */
  169 #define SWAP_RESERVE_FORCE_ON           (1 << 0)
  170 #define SWAP_RESERVE_RLIMIT_ON          (1 << 1)
  171 #define SWAP_RESERVE_ALLOW_NONWIRED     (1 << 2)
  172 
  173 int
  174 swap_reserve(vm_ooffset_t incr)
  175 {
  176 
  177         return (swap_reserve_by_cred(incr, curthread->td_ucred));
  178 }
  179 
  180 int
  181 swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
  182 {
  183         vm_ooffset_t r, s;
  184         int res, error;
  185         static int curfail;
  186         static struct timeval lastfail;
  187         struct uidinfo *uip;
  188 
  189         uip = cred->cr_ruidinfo;
  190 
  191         if (incr & PAGE_MASK)
  192                 panic("swap_reserve: & PAGE_MASK");
  193 
  194 #ifdef RACCT
  195         PROC_LOCK(curproc);
  196         error = racct_add(curproc, RACCT_SWAP, incr);
  197         PROC_UNLOCK(curproc);
  198         if (error != 0)
  199                 return (0);
  200 #endif
  201 
  202         res = 0;
  203         mtx_lock(&sw_dev_mtx);
  204         r = swap_reserved + incr;
  205         if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
  206                 s = cnt.v_page_count - cnt.v_free_reserved - cnt.v_wire_count;
  207                 s *= PAGE_SIZE;
  208         } else
  209                 s = 0;
  210         s += swap_total;
  211         if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
  212             (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
  213                 res = 1;
  214                 swap_reserved = r;
  215         }
  216         mtx_unlock(&sw_dev_mtx);
  217 
  218         if (res) {
  219                 PROC_LOCK(curproc);
  220                 UIDINFO_VMSIZE_LOCK(uip);
  221                 if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
  222                     uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
  223                     priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
  224                         res = 0;
  225                 else
  226                         uip->ui_vmsize += incr;
  227                 UIDINFO_VMSIZE_UNLOCK(uip);
  228                 PROC_UNLOCK(curproc);
  229                 if (!res) {
  230                         mtx_lock(&sw_dev_mtx);
  231                         swap_reserved -= incr;
  232                         mtx_unlock(&sw_dev_mtx);
  233                 }
  234         }
  235         if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
  236                 printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
  237                     uip->ui_uid, curproc->p_pid, incr);
  238         }
  239 
  240 #ifdef RACCT
  241         if (!res) {
  242                 PROC_LOCK(curproc);
  243                 racct_sub(curproc, RACCT_SWAP, incr);
  244                 PROC_UNLOCK(curproc);
  245         }
  246 #endif
  247 
  248         return (res);
  249 }
  250 
  251 void
  252 swap_reserve_force(vm_ooffset_t incr)
  253 {
  254         struct uidinfo *uip;
  255 
  256         mtx_lock(&sw_dev_mtx);
  257         swap_reserved += incr;
  258         mtx_unlock(&sw_dev_mtx);
  259 
  260 #ifdef RACCT
  261         PROC_LOCK(curproc);
  262         racct_add_force(curproc, RACCT_SWAP, incr);
  263         PROC_UNLOCK(curproc);
  264 #endif
  265 
  266         uip = curthread->td_ucred->cr_ruidinfo;
  267         PROC_LOCK(curproc);
  268         UIDINFO_VMSIZE_LOCK(uip);
  269         uip->ui_vmsize += incr;
  270         UIDINFO_VMSIZE_UNLOCK(uip);
  271         PROC_UNLOCK(curproc);
  272 }
  273 
  274 void
  275 swap_release(vm_ooffset_t decr)
  276 {
  277         struct ucred *cred;
  278 
  279         PROC_LOCK(curproc);
  280         cred = curthread->td_ucred;
  281         swap_release_by_cred(decr, cred);
  282         PROC_UNLOCK(curproc);
  283 }
  284 
  285 void
  286 swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
  287 {
  288         struct uidinfo *uip;
  289 
  290         uip = cred->cr_ruidinfo;
  291 
  292         if (decr & PAGE_MASK)
  293                 panic("swap_release: & PAGE_MASK");
  294 
  295         mtx_lock(&sw_dev_mtx);
  296         if (swap_reserved < decr)
  297                 panic("swap_reserved < decr");
  298         swap_reserved -= decr;
  299         mtx_unlock(&sw_dev_mtx);
  300 
  301         UIDINFO_VMSIZE_LOCK(uip);
  302         if (uip->ui_vmsize < decr)
  303                 printf("negative vmsize for uid = %d\n", uip->ui_uid);
  304         uip->ui_vmsize -= decr;
  305         UIDINFO_VMSIZE_UNLOCK(uip);
  306 
  307         racct_sub_cred(cred, RACCT_SWAP, decr);
  308 }
  309 
  310 static void swapdev_strategy(struct buf *, struct swdevt *sw);
  311 
  312 #define SWM_FREE        0x02    /* free, period                 */
  313 #define SWM_POP         0x04    /* pop out                      */
  314 
  315 int swap_pager_full = 2;        /* swap space exhaustion (task killing) */
  316 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
  317 static int nsw_rcount;          /* free read buffers                    */
  318 static int nsw_wcount_sync;     /* limit write buffers / synchronous    */
  319 static int nsw_wcount_async;    /* limit write buffers / asynchronous   */
  320 static int nsw_wcount_async_max;/* assigned maximum                     */
  321 static int nsw_cluster_max;     /* maximum VOP I/O allowed              */
  322 
  323 static struct swblock **swhash;
  324 static int swhash_mask;
  325 static struct mtx swhash_mtx;
  326 
  327 static int swap_async_max = 4;  /* maximum in-progress async I/O's      */
  328 static struct sx sw_alloc_sx;
  329 
  330 
  331 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
  332         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
  333 
  334 /*
  335  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
  336  * of searching a named list by hashing it just a little.
  337  */
  338 
  339 #define NOBJLISTS               8
  340 
  341 #define NOBJLIST(handle)        \
  342         (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
  343 
  344 static struct mtx sw_alloc_mtx; /* protect list manipulation */
  345 static struct pagerlst  swap_pager_object_list[NOBJLISTS];
  346 static uma_zone_t       swap_zone;
  347 
  348 /*
  349  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
  350  * calls hooked from other parts of the VM system and do not appear here.
  351  * (see vm/swap_pager.h).
  352  */
  353 static vm_object_t
  354                 swap_pager_alloc(void *handle, vm_ooffset_t size,
  355                     vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
  356 static void     swap_pager_dealloc(vm_object_t object);
  357 static int      swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
  358 static void     swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
  359 static boolean_t
  360                 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
  361 static void     swap_pager_init(void);
  362 static void     swap_pager_unswapped(vm_page_t);
  363 static void     swap_pager_swapoff(struct swdevt *sp);
  364 
  365 struct pagerops swappagerops = {
  366         .pgo_init =     swap_pager_init,        /* early system initialization of pager */
  367         .pgo_alloc =    swap_pager_alloc,       /* allocate an OBJT_SWAP object         */
  368         .pgo_dealloc =  swap_pager_dealloc,     /* deallocate an OBJT_SWAP object       */
  369         .pgo_getpages = swap_pager_getpages,    /* pagein                               */
  370         .pgo_putpages = swap_pager_putpages,    /* pageout                              */
  371         .pgo_haspage =  swap_pager_haspage,     /* get backing store status for page    */
  372         .pgo_pageunswapped = swap_pager_unswapped,      /* remove swap related to page          */
  373 };
  374 
  375 /*
  376  * dmmax is in page-sized chunks with the new swap system.  It was
  377  * dev-bsized chunks in the old.  dmmax is always a power of 2.
  378  *
  379  * swap_*() routines are externally accessible.  swp_*() routines are
  380  * internal.
  381  */
  382 static int dmmax;
  383 static int nswap_lowat = 128;   /* in pages, swap_pager_almost_full warn */
  384 static int nswap_hiwat = 512;   /* in pages, swap_pager_almost_full warn */
  385 
  386 SYSCTL_INT(_vm, OID_AUTO, dmmax,
  387         CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
  388 
  389 static void     swp_sizecheck(void);
  390 static void     swp_pager_async_iodone(struct buf *bp);
  391 static int      swapongeom(struct thread *, struct vnode *);
  392 static int      swaponvp(struct thread *, struct vnode *, u_long);
  393 static int      swapoff_one(struct swdevt *sp, struct ucred *cred);
  394 
  395 /*
  396  * Swap bitmap functions
  397  */
  398 static void     swp_pager_freeswapspace(daddr_t blk, int npages);
  399 static daddr_t  swp_pager_getswapspace(int npages);
  400 
  401 /*
  402  * Metadata functions
  403  */
  404 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
  405 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
  406 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
  407 static void swp_pager_meta_free_all(vm_object_t);
  408 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
  409 
  410 static void
  411 swp_pager_free_nrpage(vm_page_t m)
  412 {
  413 
  414         vm_page_lock(m);
  415         if (m->wire_count == 0)
  416                 vm_page_free(m);
  417         vm_page_unlock(m);
  418 }
  419 
  420 /*
  421  * SWP_SIZECHECK() -    update swap_pager_full indication
  422  *
  423  *      update the swap_pager_almost_full indication and warn when we are
  424  *      about to run out of swap space, using lowat/hiwat hysteresis.
  425  *
  426  *      Clear swap_pager_full ( task killing ) indication when lowat is met.
  427  *
  428  *      No restrictions on call
  429  *      This routine may not block.
  430  */
  431 static void
  432 swp_sizecheck(void)
  433 {
  434 
  435         if (swap_pager_avail < nswap_lowat) {
  436                 if (swap_pager_almost_full == 0) {
  437                         printf("swap_pager: out of swap space\n");
  438                         swap_pager_almost_full = 1;
  439                 }
  440         } else {
  441                 swap_pager_full = 0;
  442                 if (swap_pager_avail > nswap_hiwat)
  443                         swap_pager_almost_full = 0;
  444         }
  445 }
  446 
  447 /*
  448  * SWP_PAGER_HASH() -   hash swap meta data
  449  *
  450  *      This is an helper function which hashes the swapblk given
  451  *      the object and page index.  It returns a pointer to a pointer
  452  *      to the object, or a pointer to a NULL pointer if it could not
  453  *      find a swapblk.
  454  */
  455 static struct swblock **
  456 swp_pager_hash(vm_object_t object, vm_pindex_t index)
  457 {
  458         struct swblock **pswap;
  459         struct swblock *swap;
  460 
  461         index &= ~(vm_pindex_t)SWAP_META_MASK;
  462         pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
  463         while ((swap = *pswap) != NULL) {
  464                 if (swap->swb_object == object &&
  465                     swap->swb_index == index
  466                 ) {
  467                         break;
  468                 }
  469                 pswap = &swap->swb_hnext;
  470         }
  471         return (pswap);
  472 }
  473 
  474 /*
  475  * SWAP_PAGER_INIT() -  initialize the swap pager!
  476  *
  477  *      Expected to be started from system init.  NOTE:  This code is run
  478  *      before much else so be careful what you depend on.  Most of the VM
  479  *      system has yet to be initialized at this point.
  480  */
  481 static void
  482 swap_pager_init(void)
  483 {
  484         /*
  485          * Initialize object lists
  486          */
  487         int i;
  488 
  489         for (i = 0; i < NOBJLISTS; ++i)
  490                 TAILQ_INIT(&swap_pager_object_list[i]);
  491         mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
  492         mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
  493 
  494         /*
  495          * Device Stripe, in PAGE_SIZE'd blocks
  496          */
  497         dmmax = SWB_NPAGES * 2;
  498 }
  499 
  500 /*
  501  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
  502  *
  503  *      Expected to be started from pageout process once, prior to entering
  504  *      its main loop.
  505  */
  506 void
  507 swap_pager_swap_init(void)
  508 {
  509         int n, n2;
  510 
  511         /*
  512          * Number of in-transit swap bp operations.  Don't
  513          * exhaust the pbufs completely.  Make sure we
  514          * initialize workable values (0 will work for hysteresis
  515          * but it isn't very efficient).
  516          *
  517          * The nsw_cluster_max is constrained by the bp->b_pages[]
  518          * array (MAXPHYS/PAGE_SIZE) and our locally defined
  519          * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
  520          * constrained by the swap device interleave stripe size.
  521          *
  522          * Currently we hardwire nsw_wcount_async to 4.  This limit is
  523          * designed to prevent other I/O from having high latencies due to
  524          * our pageout I/O.  The value 4 works well for one or two active swap
  525          * devices but is probably a little low if you have more.  Even so,
  526          * a higher value would probably generate only a limited improvement
  527          * with three or four active swap devices since the system does not
  528          * typically have to pageout at extreme bandwidths.   We will want
  529          * at least 2 per swap devices, and 4 is a pretty good value if you
  530          * have one NFS swap device due to the command/ack latency over NFS.
  531          * So it all works out pretty well.
  532          */
  533         nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
  534 
  535         mtx_lock(&pbuf_mtx);
  536         nsw_rcount = (nswbuf + 1) / 2;
  537         nsw_wcount_sync = (nswbuf + 3) / 4;
  538         nsw_wcount_async = 4;
  539         nsw_wcount_async_max = nsw_wcount_async;
  540         mtx_unlock(&pbuf_mtx);
  541 
  542         /*
  543          * Initialize our zone.  Right now I'm just guessing on the number
  544          * we need based on the number of pages in the system.  Each swblock
  545          * can hold 16 pages, so this is probably overkill.  This reservation
  546          * is typically limited to around 32MB by default.
  547          */
  548         n = cnt.v_page_count / 2;
  549         if (maxswzone && n > maxswzone / sizeof(struct swblock))
  550                 n = maxswzone / sizeof(struct swblock);
  551         n2 = n;
  552         swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
  553             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
  554         if (swap_zone == NULL)
  555                 panic("failed to create swap_zone.");
  556         do {
  557                 if (uma_zone_reserve_kva(swap_zone, n))
  558                         break;
  559                 /*
  560                  * if the allocation failed, try a zone two thirds the
  561                  * size of the previous attempt.
  562                  */
  563                 n -= ((n + 2) / 3);
  564         } while (n > 0);
  565         if (n2 != n)
  566                 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
  567         n2 = n;
  568 
  569         /*
  570          * Initialize our meta-data hash table.  The swapper does not need to
  571          * be quite as efficient as the VM system, so we do not use an
  572          * oversized hash table.
  573          *
  574          *      n:              size of hash table, must be power of 2
  575          *      swhash_mask:    hash table index mask
  576          */
  577         for (n = 1; n < n2 / 8; n *= 2)
  578                 ;
  579         swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
  580         swhash_mask = n - 1;
  581         mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
  582 }
  583 
  584 /*
  585  * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
  586  *                      its metadata structures.
  587  *
  588  *      This routine is called from the mmap and fork code to create a new
  589  *      OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
  590  *      and then converting it with swp_pager_meta_build().
  591  *
  592  *      This routine may block in vm_object_allocate() and create a named
  593  *      object lookup race, so we must interlock.
  594  *
  595  * MPSAFE
  596  */
  597 static vm_object_t
  598 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
  599     vm_ooffset_t offset, struct ucred *cred)
  600 {
  601         vm_object_t object;
  602         vm_pindex_t pindex;
  603 
  604         pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
  605         if (handle) {
  606                 mtx_lock(&Giant);
  607                 /*
  608                  * Reference existing named region or allocate new one.  There
  609                  * should not be a race here against swp_pager_meta_build()
  610                  * as called from vm_page_remove() in regards to the lookup
  611                  * of the handle.
  612                  */
  613                 sx_xlock(&sw_alloc_sx);
  614                 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
  615                 if (object == NULL) {
  616                         if (cred != NULL) {
  617                                 if (!swap_reserve_by_cred(size, cred)) {
  618                                         sx_xunlock(&sw_alloc_sx);
  619                                         mtx_unlock(&Giant);
  620                                         return (NULL);
  621                                 }
  622                                 crhold(cred);
  623                         }
  624                         object = vm_object_allocate(OBJT_DEFAULT, pindex);
  625                         VM_OBJECT_WLOCK(object);
  626                         object->handle = handle;
  627                         if (cred != NULL) {
  628                                 object->cred = cred;
  629                                 object->charge = size;
  630                         }
  631                         swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  632                         VM_OBJECT_WUNLOCK(object);
  633                 }
  634                 sx_xunlock(&sw_alloc_sx);
  635                 mtx_unlock(&Giant);
  636         } else {
  637                 if (cred != NULL) {
  638                         if (!swap_reserve_by_cred(size, cred))
  639                                 return (NULL);
  640                         crhold(cred);
  641                 }
  642                 object = vm_object_allocate(OBJT_DEFAULT, pindex);
  643                 VM_OBJECT_WLOCK(object);
  644                 if (cred != NULL) {
  645                         object->cred = cred;
  646                         object->charge = size;
  647                 }
  648                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  649                 VM_OBJECT_WUNLOCK(object);
  650         }
  651         return (object);
  652 }
  653 
  654 /*
  655  * SWAP_PAGER_DEALLOC() -       remove swap metadata from object
  656  *
  657  *      The swap backing for the object is destroyed.  The code is
  658  *      designed such that we can reinstantiate it later, but this
  659  *      routine is typically called only when the entire object is
  660  *      about to be destroyed.
  661  *
  662  *      The object must be locked.
  663  */
  664 static void
  665 swap_pager_dealloc(vm_object_t object)
  666 {
  667 
  668         /*
  669          * Remove from list right away so lookups will fail if we block for
  670          * pageout completion.
  671          */
  672         if (object->handle != NULL) {
  673                 mtx_lock(&sw_alloc_mtx);
  674                 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
  675                 mtx_unlock(&sw_alloc_mtx);
  676         }
  677 
  678         VM_OBJECT_ASSERT_WLOCKED(object);
  679         vm_object_pip_wait(object, "swpdea");
  680 
  681         /*
  682          * Free all remaining metadata.  We only bother to free it from
  683          * the swap meta data.  We do not attempt to free swapblk's still
  684          * associated with vm_page_t's for this object.  We do not care
  685          * if paging is still in progress on some objects.
  686          */
  687         swp_pager_meta_free_all(object);
  688 }
  689 
  690 /************************************************************************
  691  *                      SWAP PAGER BITMAP ROUTINES                      *
  692  ************************************************************************/
  693 
  694 /*
  695  * SWP_PAGER_GETSWAPSPACE() -   allocate raw swap space
  696  *
  697  *      Allocate swap for the requested number of pages.  The starting
  698  *      swap block number (a page index) is returned or SWAPBLK_NONE
  699  *      if the allocation failed.
  700  *
  701  *      Also has the side effect of advising that somebody made a mistake
  702  *      when they configured swap and didn't configure enough.
  703  *
  704  *      This routine may not sleep.
  705  *
  706  *      We allocate in round-robin fashion from the configured devices.
  707  */
  708 static daddr_t
  709 swp_pager_getswapspace(int npages)
  710 {
  711         daddr_t blk;
  712         struct swdevt *sp;
  713         int i;
  714 
  715         blk = SWAPBLK_NONE;
  716         mtx_lock(&sw_dev_mtx);
  717         sp = swdevhd;
  718         for (i = 0; i < nswapdev; i++) {
  719                 if (sp == NULL)
  720                         sp = TAILQ_FIRST(&swtailq);
  721                 if (!(sp->sw_flags & SW_CLOSING)) {
  722                         blk = blist_alloc(sp->sw_blist, npages);
  723                         if (blk != SWAPBLK_NONE) {
  724                                 blk += sp->sw_first;
  725                                 sp->sw_used += npages;
  726                                 swap_pager_avail -= npages;
  727                                 swp_sizecheck();
  728                                 swdevhd = TAILQ_NEXT(sp, sw_list);
  729                                 goto done;
  730                         }
  731                 }
  732                 sp = TAILQ_NEXT(sp, sw_list);
  733         }
  734         if (swap_pager_full != 2) {
  735                 printf("swap_pager_getswapspace(%d): failed\n", npages);
  736                 swap_pager_full = 2;
  737                 swap_pager_almost_full = 1;
  738         }
  739         swdevhd = NULL;
  740 done:
  741         mtx_unlock(&sw_dev_mtx);
  742         return (blk);
  743 }
  744 
  745 static int
  746 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
  747 {
  748 
  749         return (blk >= sp->sw_first && blk < sp->sw_end);
  750 }
  751 
  752 static void
  753 swp_pager_strategy(struct buf *bp)
  754 {
  755         struct swdevt *sp;
  756 
  757         mtx_lock(&sw_dev_mtx);
  758         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  759                 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
  760                         mtx_unlock(&sw_dev_mtx);
  761                         if ((sp->sw_flags & SW_UNMAPPED) != 0 &&
  762                             unmapped_buf_allowed) {
  763                                 bp->b_kvaalloc = bp->b_data;
  764                                 bp->b_data = unmapped_buf;
  765                                 bp->b_kvabase = unmapped_buf;
  766                                 bp->b_offset = 0;
  767                                 bp->b_flags |= B_UNMAPPED;
  768                         } else {
  769                                 pmap_qenter((vm_offset_t)bp->b_data,
  770                                     &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
  771                         }
  772                         sp->sw_strategy(bp, sp);
  773                         return;
  774                 }
  775         }
  776         panic("Swapdev not found");
  777 }
  778 
  779 
  780 /*
  781  * SWP_PAGER_FREESWAPSPACE() -  free raw swap space
  782  *
  783  *      This routine returns the specified swap blocks back to the bitmap.
  784  *
  785  *      This routine may not sleep.
  786  */
  787 static void
  788 swp_pager_freeswapspace(daddr_t blk, int npages)
  789 {
  790         struct swdevt *sp;
  791 
  792         mtx_lock(&sw_dev_mtx);
  793         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  794                 if (blk >= sp->sw_first && blk < sp->sw_end) {
  795                         sp->sw_used -= npages;
  796                         /*
  797                          * If we are attempting to stop swapping on
  798                          * this device, we don't want to mark any
  799                          * blocks free lest they be reused.
  800                          */
  801                         if ((sp->sw_flags & SW_CLOSING) == 0) {
  802                                 blist_free(sp->sw_blist, blk - sp->sw_first,
  803                                     npages);
  804                                 swap_pager_avail += npages;
  805                                 swp_sizecheck();
  806                         }
  807                         mtx_unlock(&sw_dev_mtx);
  808                         return;
  809                 }
  810         }
  811         panic("Swapdev not found");
  812 }
  813 
  814 /*
  815  * SWAP_PAGER_FREESPACE() -     frees swap blocks associated with a page
  816  *                              range within an object.
  817  *
  818  *      This is a globally accessible routine.
  819  *
  820  *      This routine removes swapblk assignments from swap metadata.
  821  *
  822  *      The external callers of this routine typically have already destroyed
  823  *      or renamed vm_page_t's associated with this range in the object so
  824  *      we should be ok.
  825  *
  826  *      The object must be locked.
  827  */
  828 void
  829 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
  830 {
  831 
  832         swp_pager_meta_free(object, start, size);
  833 }
  834 
  835 /*
  836  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
  837  *
  838  *      Assigns swap blocks to the specified range within the object.  The
  839  *      swap blocks are not zeroed.  Any previous swap assignment is destroyed.
  840  *
  841  *      Returns 0 on success, -1 on failure.
  842  */
  843 int
  844 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
  845 {
  846         int n = 0;
  847         daddr_t blk = SWAPBLK_NONE;
  848         vm_pindex_t beg = start;        /* save start index */
  849 
  850         VM_OBJECT_WLOCK(object);
  851         while (size) {
  852                 if (n == 0) {
  853                         n = BLIST_MAX_ALLOC;
  854                         while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
  855                                 n >>= 1;
  856                                 if (n == 0) {
  857                                         swp_pager_meta_free(object, beg, start - beg);
  858                                         VM_OBJECT_WUNLOCK(object);
  859                                         return (-1);
  860                                 }
  861                         }
  862                 }
  863                 swp_pager_meta_build(object, start, blk);
  864                 --size;
  865                 ++start;
  866                 ++blk;
  867                 --n;
  868         }
  869         swp_pager_meta_free(object, start, n);
  870         VM_OBJECT_WUNLOCK(object);
  871         return (0);
  872 }
  873 
  874 /*
  875  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
  876  *                      and destroy the source.
  877  *
  878  *      Copy any valid swapblks from the source to the destination.  In
  879  *      cases where both the source and destination have a valid swapblk,
  880  *      we keep the destination's.
  881  *
  882  *      This routine is allowed to sleep.  It may sleep allocating metadata
  883  *      indirectly through swp_pager_meta_build() or if paging is still in
  884  *      progress on the source.
  885  *
  886  *      The source object contains no vm_page_t's (which is just as well)
  887  *
  888  *      The source object is of type OBJT_SWAP.
  889  *
  890  *      The source and destination objects must be locked.
  891  *      Both object locks may temporarily be released.
  892  */
  893 void
  894 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
  895     vm_pindex_t offset, int destroysource)
  896 {
  897         vm_pindex_t i;
  898 
  899         VM_OBJECT_ASSERT_WLOCKED(srcobject);
  900         VM_OBJECT_ASSERT_WLOCKED(dstobject);
  901 
  902         /*
  903          * If destroysource is set, we remove the source object from the
  904          * swap_pager internal queue now.
  905          */
  906         if (destroysource) {
  907                 if (srcobject->handle != NULL) {
  908                         mtx_lock(&sw_alloc_mtx);
  909                         TAILQ_REMOVE(
  910                             NOBJLIST(srcobject->handle),
  911                             srcobject,
  912                             pager_object_list
  913                         );
  914                         mtx_unlock(&sw_alloc_mtx);
  915                 }
  916         }
  917 
  918         /*
  919          * transfer source to destination.
  920          */
  921         for (i = 0; i < dstobject->size; ++i) {
  922                 daddr_t dstaddr;
  923 
  924                 /*
  925                  * Locate (without changing) the swapblk on the destination,
  926                  * unless it is invalid in which case free it silently, or
  927                  * if the destination is a resident page, in which case the
  928                  * source is thrown away.
  929                  */
  930                 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
  931 
  932                 if (dstaddr == SWAPBLK_NONE) {
  933                         /*
  934                          * Destination has no swapblk and is not resident,
  935                          * copy source.
  936                          */
  937                         daddr_t srcaddr;
  938 
  939                         srcaddr = swp_pager_meta_ctl(
  940                             srcobject,
  941                             i + offset,
  942                             SWM_POP
  943                         );
  944 
  945                         if (srcaddr != SWAPBLK_NONE) {
  946                                 /*
  947                                  * swp_pager_meta_build() can sleep.
  948                                  */
  949                                 vm_object_pip_add(srcobject, 1);
  950                                 VM_OBJECT_WUNLOCK(srcobject);
  951                                 vm_object_pip_add(dstobject, 1);
  952                                 swp_pager_meta_build(dstobject, i, srcaddr);
  953                                 vm_object_pip_wakeup(dstobject);
  954                                 VM_OBJECT_WLOCK(srcobject);
  955                                 vm_object_pip_wakeup(srcobject);
  956                         }
  957                 } else {
  958                         /*
  959                          * Destination has valid swapblk or it is represented
  960                          * by a resident page.  We destroy the sourceblock.
  961                          */
  962 
  963                         swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
  964                 }
  965         }
  966 
  967         /*
  968          * Free left over swap blocks in source.
  969          *
  970          * We have to revert the type to OBJT_DEFAULT so we do not accidently
  971          * double-remove the object from the swap queues.
  972          */
  973         if (destroysource) {
  974                 swp_pager_meta_free_all(srcobject);
  975                 /*
  976                  * Reverting the type is not necessary, the caller is going
  977                  * to destroy srcobject directly, but I'm doing it here
  978                  * for consistency since we've removed the object from its
  979                  * queues.
  980                  */
  981                 srcobject->type = OBJT_DEFAULT;
  982         }
  983 }
  984 
  985 /*
  986  * SWAP_PAGER_HASPAGE() -       determine if we have good backing store for
  987  *                              the requested page.
  988  *
  989  *      We determine whether good backing store exists for the requested
  990  *      page and return TRUE if it does, FALSE if it doesn't.
  991  *
  992  *      If TRUE, we also try to determine how much valid, contiguous backing
  993  *      store exists before and after the requested page within a reasonable
  994  *      distance.  We do not try to restrict it to the swap device stripe
  995  *      (that is handled in getpages/putpages).  It probably isn't worth
  996  *      doing here.
  997  */
  998 static boolean_t
  999 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
 1000 {
 1001         daddr_t blk0;
 1002 
 1003         VM_OBJECT_ASSERT_LOCKED(object);
 1004         /*
 1005          * do we have good backing store at the requested index ?
 1006          */
 1007         blk0 = swp_pager_meta_ctl(object, pindex, 0);
 1008 
 1009         if (blk0 == SWAPBLK_NONE) {
 1010                 if (before)
 1011                         *before = 0;
 1012                 if (after)
 1013                         *after = 0;
 1014                 return (FALSE);
 1015         }
 1016 
 1017         /*
 1018          * find backwards-looking contiguous good backing store
 1019          */
 1020         if (before != NULL) {
 1021                 int i;
 1022 
 1023                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
 1024                         daddr_t blk;
 1025 
 1026                         if (i > pindex)
 1027                                 break;
 1028                         blk = swp_pager_meta_ctl(object, pindex - i, 0);
 1029                         if (blk != blk0 - i)
 1030                                 break;
 1031                 }
 1032                 *before = (i - 1);
 1033         }
 1034 
 1035         /*
 1036          * find forward-looking contiguous good backing store
 1037          */
 1038         if (after != NULL) {
 1039                 int i;
 1040 
 1041                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
 1042                         daddr_t blk;
 1043 
 1044                         blk = swp_pager_meta_ctl(object, pindex + i, 0);
 1045                         if (blk != blk0 + i)
 1046                                 break;
 1047                 }
 1048                 *after = (i - 1);
 1049         }
 1050         return (TRUE);
 1051 }
 1052 
 1053 /*
 1054  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
 1055  *
 1056  *      This removes any associated swap backing store, whether valid or
 1057  *      not, from the page.
 1058  *
 1059  *      This routine is typically called when a page is made dirty, at
 1060  *      which point any associated swap can be freed.  MADV_FREE also
 1061  *      calls us in a special-case situation
 1062  *
 1063  *      NOTE!!!  If the page is clean and the swap was valid, the caller
 1064  *      should make the page dirty before calling this routine.  This routine
 1065  *      does NOT change the m->dirty status of the page.  Also: MADV_FREE
 1066  *      depends on it.
 1067  *
 1068  *      This routine may not sleep.
 1069  *
 1070  *      The object containing the page must be locked.
 1071  */
 1072 static void
 1073 swap_pager_unswapped(vm_page_t m)
 1074 {
 1075 
 1076         swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
 1077 }
 1078 
 1079 /*
 1080  * SWAP_PAGER_GETPAGES() - bring pages in from swap
 1081  *
 1082  *      Attempt to retrieve (m, count) pages from backing store, but make
 1083  *      sure we retrieve at least m[reqpage].  We try to load in as large
 1084  *      a chunk surrounding m[reqpage] as is contiguous in swap and which
 1085  *      belongs to the same object.
 1086  *
 1087  *      The code is designed for asynchronous operation and
 1088  *      immediate-notification of 'reqpage' but tends not to be
 1089  *      used that way.  Please do not optimize-out this algorithmic
 1090  *      feature, I intend to improve on it in the future.
 1091  *
 1092  *      The parent has a single vm_object_pip_add() reference prior to
 1093  *      calling us and we should return with the same.
 1094  *
 1095  *      The parent has BUSY'd the pages.  We should return with 'm'
 1096  *      left busy, but the others adjusted.
 1097  */
 1098 static int
 1099 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
 1100 {
 1101         struct buf *bp;
 1102         vm_page_t mreq;
 1103         int i;
 1104         int j;
 1105         daddr_t blk;
 1106 
 1107         mreq = m[reqpage];
 1108 
 1109         KASSERT(mreq->object == object,
 1110             ("swap_pager_getpages: object mismatch %p/%p",
 1111             object, mreq->object));
 1112 
 1113         /*
 1114          * Calculate range to retrieve.  The pages have already been assigned
 1115          * their swapblks.  We require a *contiguous* range but we know it to
 1116          * not span devices.   If we do not supply it, bad things
 1117          * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the
 1118          * loops are set up such that the case(s) are handled implicitly.
 1119          *
 1120          * The swp_*() calls must be made with the object locked.
 1121          */
 1122         blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
 1123 
 1124         for (i = reqpage - 1; i >= 0; --i) {
 1125                 daddr_t iblk;
 1126 
 1127                 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
 1128                 if (blk != iblk + (reqpage - i))
 1129                         break;
 1130         }
 1131         ++i;
 1132 
 1133         for (j = reqpage + 1; j < count; ++j) {
 1134                 daddr_t jblk;
 1135 
 1136                 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
 1137                 if (blk != jblk - (j - reqpage))
 1138                         break;
 1139         }
 1140 
 1141         /*
 1142          * free pages outside our collection range.   Note: we never free
 1143          * mreq, it must remain busy throughout.
 1144          */
 1145         if (0 < i || j < count) {
 1146                 int k;
 1147 
 1148                 for (k = 0; k < i; ++k)
 1149                         swp_pager_free_nrpage(m[k]);
 1150                 for (k = j; k < count; ++k)
 1151                         swp_pager_free_nrpage(m[k]);
 1152         }
 1153 
 1154         /*
 1155          * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq
 1156          * still busy, but the others unbusied.
 1157          */
 1158         if (blk == SWAPBLK_NONE)
 1159                 return (VM_PAGER_FAIL);
 1160 
 1161         /*
 1162          * Getpbuf() can sleep.
 1163          */
 1164         VM_OBJECT_WUNLOCK(object);
 1165         /*
 1166          * Get a swap buffer header to perform the IO
 1167          */
 1168         bp = getpbuf(&nsw_rcount);
 1169         bp->b_flags |= B_PAGING;
 1170 
 1171         bp->b_iocmd = BIO_READ;
 1172         bp->b_iodone = swp_pager_async_iodone;
 1173         bp->b_rcred = crhold(thread0.td_ucred);
 1174         bp->b_wcred = crhold(thread0.td_ucred);
 1175         bp->b_blkno = blk - (reqpage - i);
 1176         bp->b_bcount = PAGE_SIZE * (j - i);
 1177         bp->b_bufsize = PAGE_SIZE * (j - i);
 1178         bp->b_pager.pg_reqpage = reqpage - i;
 1179 
 1180         VM_OBJECT_WLOCK(object);
 1181         {
 1182                 int k;
 1183 
 1184                 for (k = i; k < j; ++k) {
 1185                         bp->b_pages[k - i] = m[k];
 1186                         m[k]->oflags |= VPO_SWAPINPROG;
 1187                 }
 1188         }
 1189         bp->b_npages = j - i;
 1190 
 1191         PCPU_INC(cnt.v_swapin);
 1192         PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
 1193 
 1194         /*
 1195          * We still hold the lock on mreq, and our automatic completion routine
 1196          * does not remove it.
 1197          */
 1198         vm_object_pip_add(object, bp->b_npages);
 1199         VM_OBJECT_WUNLOCK(object);
 1200 
 1201         /*
 1202          * perform the I/O.  NOTE!!!  bp cannot be considered valid after
 1203          * this point because we automatically release it on completion.
 1204          * Instead, we look at the one page we are interested in which we
 1205          * still hold a lock on even through the I/O completion.
 1206          *
 1207          * The other pages in our m[] array are also released on completion,
 1208          * so we cannot assume they are valid anymore either.
 1209          *
 1210          * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1211          */
 1212         BUF_KERNPROC(bp);
 1213         swp_pager_strategy(bp);
 1214 
 1215         /*
 1216          * wait for the page we want to complete.  VPO_SWAPINPROG is always
 1217          * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
 1218          * is set in the meta-data.
 1219          */
 1220         VM_OBJECT_WLOCK(object);
 1221         while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
 1222                 mreq->oflags |= VPO_SWAPSLEEP;
 1223                 PCPU_INC(cnt.v_intrans);
 1224                 if (VM_OBJECT_SLEEP(object, &object->paging_in_progress, PSWP,
 1225                     "swread", hz * 20)) {
 1226                         printf(
 1227 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
 1228                             bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
 1229                 }
 1230         }
 1231 
 1232         /*
 1233          * mreq is left busied after completion, but all the other pages
 1234          * are freed.  If we had an unrecoverable read error the page will
 1235          * not be valid.
 1236          */
 1237         if (mreq->valid != VM_PAGE_BITS_ALL) {
 1238                 return (VM_PAGER_ERROR);
 1239         } else {
 1240                 return (VM_PAGER_OK);
 1241         }
 1242 
 1243         /*
 1244          * A final note: in a low swap situation, we cannot deallocate swap
 1245          * and mark a page dirty here because the caller is likely to mark
 1246          * the page clean when we return, causing the page to possibly revert
 1247          * to all-zero's later.
 1248          */
 1249 }
 1250 
 1251 /*
 1252  *      swap_pager_putpages:
 1253  *
 1254  *      Assign swap (if necessary) and initiate I/O on the specified pages.
 1255  *
 1256  *      We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
 1257  *      are automatically converted to SWAP objects.
 1258  *
 1259  *      In a low memory situation we may block in VOP_STRATEGY(), but the new
 1260  *      vm_page reservation system coupled with properly written VFS devices
 1261  *      should ensure that no low-memory deadlock occurs.  This is an area
 1262  *      which needs work.
 1263  *
 1264  *      The parent has N vm_object_pip_add() references prior to
 1265  *      calling us and will remove references for rtvals[] that are
 1266  *      not set to VM_PAGER_PEND.  We need to remove the rest on I/O
 1267  *      completion.
 1268  *
 1269  *      The parent has soft-busy'd the pages it passes us and will unbusy
 1270  *      those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
 1271  *      We need to unbusy the rest on I/O completion.
 1272  */
 1273 void
 1274 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 1275     boolean_t sync, int *rtvals)
 1276 {
 1277         int i;
 1278         int n = 0;
 1279 
 1280         if (count && m[0]->object != object) {
 1281                 panic("swap_pager_putpages: object mismatch %p/%p",
 1282                     object,
 1283                     m[0]->object
 1284                 );
 1285         }
 1286 
 1287         /*
 1288          * Step 1
 1289          *
 1290          * Turn object into OBJT_SWAP
 1291          * check for bogus sysops
 1292          * force sync if not pageout process
 1293          */
 1294         if (object->type != OBJT_SWAP)
 1295                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
 1296         VM_OBJECT_WUNLOCK(object);
 1297 
 1298         if (curproc != pageproc)
 1299                 sync = TRUE;
 1300 
 1301         /*
 1302          * Step 2
 1303          *
 1304          * Update nsw parameters from swap_async_max sysctl values.
 1305          * Do not let the sysop crash the machine with bogus numbers.
 1306          */
 1307         mtx_lock(&pbuf_mtx);
 1308         if (swap_async_max != nsw_wcount_async_max) {
 1309                 int n;
 1310 
 1311                 /*
 1312                  * limit range
 1313                  */
 1314                 if ((n = swap_async_max) > nswbuf / 2)
 1315                         n = nswbuf / 2;
 1316                 if (n < 1)
 1317                         n = 1;
 1318                 swap_async_max = n;
 1319 
 1320                 /*
 1321                  * Adjust difference ( if possible ).  If the current async
 1322                  * count is too low, we may not be able to make the adjustment
 1323                  * at this time.
 1324                  */
 1325                 n -= nsw_wcount_async_max;
 1326                 if (nsw_wcount_async + n >= 0) {
 1327                         nsw_wcount_async += n;
 1328                         nsw_wcount_async_max += n;
 1329                         wakeup(&nsw_wcount_async);
 1330                 }
 1331         }
 1332         mtx_unlock(&pbuf_mtx);
 1333 
 1334         /*
 1335          * Step 3
 1336          *
 1337          * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
 1338          * The page is left dirty until the pageout operation completes
 1339          * successfully.
 1340          */
 1341         for (i = 0; i < count; i += n) {
 1342                 int j;
 1343                 struct buf *bp;
 1344                 daddr_t blk;
 1345 
 1346                 /*
 1347                  * Maximum I/O size is limited by a number of factors.
 1348                  */
 1349                 n = min(BLIST_MAX_ALLOC, count - i);
 1350                 n = min(n, nsw_cluster_max);
 1351 
 1352                 /*
 1353                  * Get biggest block of swap we can.  If we fail, fall
 1354                  * back and try to allocate a smaller block.  Don't go
 1355                  * overboard trying to allocate space if it would overly
 1356                  * fragment swap.
 1357                  */
 1358                 while (
 1359                     (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
 1360                     n > 4
 1361                 ) {
 1362                         n >>= 1;
 1363                 }
 1364                 if (blk == SWAPBLK_NONE) {
 1365                         for (j = 0; j < n; ++j)
 1366                                 rtvals[i+j] = VM_PAGER_FAIL;
 1367                         continue;
 1368                 }
 1369 
 1370                 /*
 1371                  * All I/O parameters have been satisfied, build the I/O
 1372                  * request and assign the swap space.
 1373                  */
 1374                 if (sync == TRUE) {
 1375                         bp = getpbuf(&nsw_wcount_sync);
 1376                 } else {
 1377                         bp = getpbuf(&nsw_wcount_async);
 1378                         bp->b_flags = B_ASYNC;
 1379                 }
 1380                 bp->b_flags |= B_PAGING;
 1381                 bp->b_iocmd = BIO_WRITE;
 1382 
 1383                 bp->b_rcred = crhold(thread0.td_ucred);
 1384                 bp->b_wcred = crhold(thread0.td_ucred);
 1385                 bp->b_bcount = PAGE_SIZE * n;
 1386                 bp->b_bufsize = PAGE_SIZE * n;
 1387                 bp->b_blkno = blk;
 1388 
 1389                 VM_OBJECT_WLOCK(object);
 1390                 for (j = 0; j < n; ++j) {
 1391                         vm_page_t mreq = m[i+j];
 1392 
 1393                         swp_pager_meta_build(
 1394                             mreq->object,
 1395                             mreq->pindex,
 1396                             blk + j
 1397                         );
 1398                         vm_page_dirty(mreq);
 1399                         rtvals[i+j] = VM_PAGER_OK;
 1400 
 1401                         mreq->oflags |= VPO_SWAPINPROG;
 1402                         bp->b_pages[j] = mreq;
 1403                 }
 1404                 VM_OBJECT_WUNLOCK(object);
 1405                 bp->b_npages = n;
 1406                 /*
 1407                  * Must set dirty range for NFS to work.
 1408                  */
 1409                 bp->b_dirtyoff = 0;
 1410                 bp->b_dirtyend = bp->b_bcount;
 1411 
 1412                 PCPU_INC(cnt.v_swapout);
 1413                 PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
 1414 
 1415                 /*
 1416                  * asynchronous
 1417                  *
 1418                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1419                  */
 1420                 if (sync == FALSE) {
 1421                         bp->b_iodone = swp_pager_async_iodone;
 1422                         BUF_KERNPROC(bp);
 1423                         swp_pager_strategy(bp);
 1424 
 1425                         for (j = 0; j < n; ++j)
 1426                                 rtvals[i+j] = VM_PAGER_PEND;
 1427                         /* restart outter loop */
 1428                         continue;
 1429                 }
 1430 
 1431                 /*
 1432                  * synchronous
 1433                  *
 1434                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1435                  */
 1436                 bp->b_iodone = bdone;
 1437                 swp_pager_strategy(bp);
 1438 
 1439                 /*
 1440                  * Wait for the sync I/O to complete, then update rtvals.
 1441                  * We just set the rtvals[] to VM_PAGER_PEND so we can call
 1442                  * our async completion routine at the end, thus avoiding a
 1443                  * double-free.
 1444                  */
 1445                 bwait(bp, PVM, "swwrt");
 1446                 for (j = 0; j < n; ++j)
 1447                         rtvals[i+j] = VM_PAGER_PEND;
 1448                 /*
 1449                  * Now that we are through with the bp, we can call the
 1450                  * normal async completion, which frees everything up.
 1451                  */
 1452                 swp_pager_async_iodone(bp);
 1453         }
 1454         VM_OBJECT_WLOCK(object);
 1455 }
 1456 
 1457 /*
 1458  *      swp_pager_async_iodone:
 1459  *
 1460  *      Completion routine for asynchronous reads and writes from/to swap.
 1461  *      Also called manually by synchronous code to finish up a bp.
 1462  *
 1463  *      This routine may not sleep.
 1464  */
 1465 static void
 1466 swp_pager_async_iodone(struct buf *bp)
 1467 {
 1468         int i;
 1469         vm_object_t object = NULL;
 1470 
 1471         /*
 1472          * report error
 1473          */
 1474         if (bp->b_ioflags & BIO_ERROR) {
 1475                 printf(
 1476                     "swap_pager: I/O error - %s failed; blkno %ld,"
 1477                         "size %ld, error %d\n",
 1478                     ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
 1479                     (long)bp->b_blkno,
 1480                     (long)bp->b_bcount,
 1481                     bp->b_error
 1482                 );
 1483         }
 1484 
 1485         /*
 1486          * remove the mapping for kernel virtual
 1487          */
 1488         if ((bp->b_flags & B_UNMAPPED) != 0) {
 1489                 bp->b_data = bp->b_kvaalloc;
 1490                 bp->b_kvabase = bp->b_kvaalloc;
 1491                 bp->b_flags &= ~B_UNMAPPED;
 1492         } else
 1493                 pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
 1494 
 1495         if (bp->b_npages) {
 1496                 object = bp->b_pages[0]->object;
 1497                 VM_OBJECT_WLOCK(object);
 1498         }
 1499 
 1500         /*
 1501          * cleanup pages.  If an error occurs writing to swap, we are in
 1502          * very serious trouble.  If it happens to be a disk error, though,
 1503          * we may be able to recover by reassigning the swap later on.  So
 1504          * in this case we remove the m->swapblk assignment for the page
 1505          * but do not free it in the rlist.  The errornous block(s) are thus
 1506          * never reallocated as swap.  Redirty the page and continue.
 1507          */
 1508         for (i = 0; i < bp->b_npages; ++i) {
 1509                 vm_page_t m = bp->b_pages[i];
 1510 
 1511                 m->oflags &= ~VPO_SWAPINPROG;
 1512                 if (m->oflags & VPO_SWAPSLEEP) {
 1513                         m->oflags &= ~VPO_SWAPSLEEP;
 1514                         wakeup(&object->paging_in_progress);
 1515                 }
 1516 
 1517                 if (bp->b_ioflags & BIO_ERROR) {
 1518                         /*
 1519                          * If an error occurs I'd love to throw the swapblk
 1520                          * away without freeing it back to swapspace, so it
 1521                          * can never be used again.  But I can't from an
 1522                          * interrupt.
 1523                          */
 1524                         if (bp->b_iocmd == BIO_READ) {
 1525                                 /*
 1526                                  * When reading, reqpage needs to stay
 1527                                  * locked for the parent, but all other
 1528                                  * pages can be freed.  We still want to
 1529                                  * wakeup the parent waiting on the page,
 1530                                  * though.  ( also: pg_reqpage can be -1 and
 1531                                  * not match anything ).
 1532                                  *
 1533                                  * We have to wake specifically requested pages
 1534                                  * up too because we cleared VPO_SWAPINPROG and
 1535                                  * someone may be waiting for that.
 1536                                  *
 1537                                  * NOTE: for reads, m->dirty will probably
 1538                                  * be overridden by the original caller of
 1539                                  * getpages so don't play cute tricks here.
 1540                                  */
 1541                                 m->valid = 0;
 1542                                 if (i != bp->b_pager.pg_reqpage)
 1543                                         swp_pager_free_nrpage(m);
 1544                                 else {
 1545                                         vm_page_lock(m);
 1546                                         vm_page_flash(m);
 1547                                         vm_page_unlock(m);
 1548                                 }
 1549                                 /*
 1550                                  * If i == bp->b_pager.pg_reqpage, do not wake
 1551                                  * the page up.  The caller needs to.
 1552                                  */
 1553                         } else {
 1554                                 /*
 1555                                  * If a write error occurs, reactivate page
 1556                                  * so it doesn't clog the inactive list,
 1557                                  * then finish the I/O.
 1558                                  */
 1559                                 vm_page_dirty(m);
 1560                                 vm_page_lock(m);
 1561                                 vm_page_activate(m);
 1562                                 vm_page_unlock(m);
 1563                                 vm_page_sunbusy(m);
 1564                         }
 1565                 } else if (bp->b_iocmd == BIO_READ) {
 1566                         /*
 1567                          * NOTE: for reads, m->dirty will probably be
 1568                          * overridden by the original caller of getpages so
 1569                          * we cannot set them in order to free the underlying
 1570                          * swap in a low-swap situation.  I don't think we'd
 1571                          * want to do that anyway, but it was an optimization
 1572                          * that existed in the old swapper for a time before
 1573                          * it got ripped out due to precisely this problem.
 1574                          *
 1575                          * If not the requested page then deactivate it.
 1576                          *
 1577                          * Note that the requested page, reqpage, is left
 1578                          * busied, but we still have to wake it up.  The
 1579                          * other pages are released (unbusied) by
 1580                          * vm_page_xunbusy().
 1581                          */
 1582                         KASSERT(!pmap_page_is_mapped(m),
 1583                             ("swp_pager_async_iodone: page %p is mapped", m));
 1584                         m->valid = VM_PAGE_BITS_ALL;
 1585                         KASSERT(m->dirty == 0,
 1586                             ("swp_pager_async_iodone: page %p is dirty", m));
 1587 
 1588                         /*
 1589                          * We have to wake specifically requested pages
 1590                          * up too because we cleared VPO_SWAPINPROG and
 1591                          * could be waiting for it in getpages.  However,
 1592                          * be sure to not unbusy getpages specifically
 1593                          * requested page - getpages expects it to be
 1594                          * left busy.
 1595                          */
 1596                         if (i != bp->b_pager.pg_reqpage) {
 1597                                 vm_page_lock(m);
 1598                                 vm_page_deactivate(m);
 1599                                 vm_page_unlock(m);
 1600                                 vm_page_xunbusy(m);
 1601                         } else {
 1602                                 vm_page_lock(m);
 1603                                 vm_page_flash(m);
 1604                                 vm_page_unlock(m);
 1605                         }
 1606                 } else {
 1607                         /*
 1608                          * For write success, clear the dirty
 1609                          * status, then finish the I/O ( which decrements the
 1610                          * busy count and possibly wakes waiter's up ).
 1611                          */
 1612                         KASSERT(!pmap_page_is_write_mapped(m),
 1613                             ("swp_pager_async_iodone: page %p is not write"
 1614                             " protected", m));
 1615                         vm_page_undirty(m);
 1616                         vm_page_sunbusy(m);
 1617                         if (vm_page_count_severe()) {
 1618                                 vm_page_lock(m);
 1619                                 vm_page_try_to_cache(m);
 1620                                 vm_page_unlock(m);
 1621                         }
 1622                 }
 1623         }
 1624 
 1625         /*
 1626          * adjust pip.  NOTE: the original parent may still have its own
 1627          * pip refs on the object.
 1628          */
 1629         if (object != NULL) {
 1630                 vm_object_pip_wakeupn(object, bp->b_npages);
 1631                 VM_OBJECT_WUNLOCK(object);
 1632         }
 1633 
 1634         /*
 1635          * swapdev_strategy() manually sets b_vp and b_bufobj before calling
 1636          * bstrategy(). Set them back to NULL now we're done with it, or we'll
 1637          * trigger a KASSERT in relpbuf().
 1638          */
 1639         if (bp->b_vp) {
 1640                     bp->b_vp = NULL;
 1641                     bp->b_bufobj = NULL;
 1642         }
 1643         /*
 1644          * release the physical I/O buffer
 1645          */
 1646         relpbuf(
 1647             bp,
 1648             ((bp->b_iocmd == BIO_READ) ? &nsw_rcount :
 1649                 ((bp->b_flags & B_ASYNC) ?
 1650                     &nsw_wcount_async :
 1651                     &nsw_wcount_sync
 1652                 )
 1653             )
 1654         );
 1655 }
 1656 
 1657 /*
 1658  *      swap_pager_isswapped:
 1659  *
 1660  *      Return 1 if at least one page in the given object is paged
 1661  *      out to the given swap device.
 1662  *
 1663  *      This routine may not sleep.
 1664  */
 1665 int
 1666 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
 1667 {
 1668         daddr_t index = 0;
 1669         int bcount;
 1670         int i;
 1671 
 1672         VM_OBJECT_ASSERT_WLOCKED(object);
 1673         if (object->type != OBJT_SWAP)
 1674                 return (0);
 1675 
 1676         mtx_lock(&swhash_mtx);
 1677         for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
 1678                 struct swblock *swap;
 1679 
 1680                 if ((swap = *swp_pager_hash(object, index)) != NULL) {
 1681                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1682                                 if (swp_pager_isondev(swap->swb_pages[i], sp)) {
 1683                                         mtx_unlock(&swhash_mtx);
 1684                                         return (1);
 1685                                 }
 1686                         }
 1687                 }
 1688                 index += SWAP_META_PAGES;
 1689         }
 1690         mtx_unlock(&swhash_mtx);
 1691         return (0);
 1692 }
 1693 
 1694 /*
 1695  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
 1696  *
 1697  *      This routine dissociates the page at the given index within a
 1698  *      swap block from its backing store, paging it in if necessary.
 1699  *      If the page is paged in, it is placed in the inactive queue,
 1700  *      since it had its backing store ripped out from under it.
 1701  *      We also attempt to swap in all other pages in the swap block,
 1702  *      we only guarantee that the one at the specified index is
 1703  *      paged in.
 1704  *
 1705  *      XXX - The code to page the whole block in doesn't work, so we
 1706  *            revert to the one-by-one behavior for now.  Sigh.
 1707  */
 1708 static inline void
 1709 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
 1710 {
 1711         vm_page_t m;
 1712 
 1713         vm_object_pip_add(object, 1);
 1714         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
 1715         if (m->valid == VM_PAGE_BITS_ALL) {
 1716                 vm_object_pip_subtract(object, 1);
 1717                 vm_page_dirty(m);
 1718                 vm_page_lock(m);
 1719                 vm_page_activate(m);
 1720                 vm_page_unlock(m);
 1721                 vm_page_xunbusy(m);
 1722                 vm_pager_page_unswapped(m);
 1723                 return;
 1724         }
 1725 
 1726         if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
 1727                 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
 1728         vm_object_pip_subtract(object, 1);
 1729         vm_page_dirty(m);
 1730         vm_page_lock(m);
 1731         vm_page_deactivate(m);
 1732         vm_page_unlock(m);
 1733         vm_page_xunbusy(m);
 1734         vm_pager_page_unswapped(m);
 1735 }
 1736 
 1737 /*
 1738  *      swap_pager_swapoff:
 1739  *
 1740  *      Page in all of the pages that have been paged out to the
 1741  *      given device.  The corresponding blocks in the bitmap must be
 1742  *      marked as allocated and the device must be flagged SW_CLOSING.
 1743  *      There may be no processes swapped out to the device.
 1744  *
 1745  *      This routine may block.
 1746  */
 1747 static void
 1748 swap_pager_swapoff(struct swdevt *sp)
 1749 {
 1750         struct swblock *swap;
 1751         int i, j, retries;
 1752 
 1753         GIANT_REQUIRED;
 1754 
 1755         retries = 0;
 1756 full_rescan:
 1757         mtx_lock(&swhash_mtx);
 1758         for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
 1759 restart:
 1760                 for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
 1761                         vm_object_t object = swap->swb_object;
 1762                         vm_pindex_t pindex = swap->swb_index;
 1763                         for (j = 0; j < SWAP_META_PAGES; ++j) {
 1764                                 if (swp_pager_isondev(swap->swb_pages[j], sp)) {
 1765                                         /* avoid deadlock */
 1766                                         if (!VM_OBJECT_TRYWLOCK(object)) {
 1767                                                 break;
 1768                                         } else {
 1769                                                 mtx_unlock(&swhash_mtx);
 1770                                                 swp_pager_force_pagein(object,
 1771                                                     pindex + j);
 1772                                                 VM_OBJECT_WUNLOCK(object);
 1773                                                 mtx_lock(&swhash_mtx);
 1774                                                 goto restart;
 1775                                         }
 1776                                 }
 1777                         }
 1778                 }
 1779         }
 1780         mtx_unlock(&swhash_mtx);
 1781         if (sp->sw_used) {
 1782                 /*
 1783                  * Objects may be locked or paging to the device being
 1784                  * removed, so we will miss their pages and need to
 1785                  * make another pass.  We have marked this device as
 1786                  * SW_CLOSING, so the activity should finish soon.
 1787                  */
 1788                 retries++;
 1789                 if (retries > 100) {
 1790                         panic("swapoff: failed to locate %d swap blocks",
 1791                             sp->sw_used);
 1792                 }
 1793                 pause("swpoff", hz / 20);
 1794                 goto full_rescan;
 1795         }
 1796 }
 1797 
 1798 /************************************************************************
 1799  *                              SWAP META DATA                          *
 1800  ************************************************************************
 1801  *
 1802  *      These routines manipulate the swap metadata stored in the
 1803  *      OBJT_SWAP object.
 1804  *
 1805  *      Swap metadata is implemented with a global hash and not directly
 1806  *      linked into the object.  Instead the object simply contains
 1807  *      appropriate tracking counters.
 1808  */
 1809 
 1810 /*
 1811  * SWP_PAGER_META_BUILD() -     add swap block to swap meta data for object
 1812  *
 1813  *      We first convert the object to a swap object if it is a default
 1814  *      object.
 1815  *
 1816  *      The specified swapblk is added to the object's swap metadata.  If
 1817  *      the swapblk is not valid, it is freed instead.  Any previously
 1818  *      assigned swapblk is freed.
 1819  */
 1820 static void
 1821 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
 1822 {
 1823         static volatile int exhausted;
 1824         struct swblock *swap;
 1825         struct swblock **pswap;
 1826         int idx;
 1827 
 1828         VM_OBJECT_ASSERT_WLOCKED(object);
 1829         /*
 1830          * Convert default object to swap object if necessary
 1831          */
 1832         if (object->type != OBJT_SWAP) {
 1833                 object->type = OBJT_SWAP;
 1834                 object->un_pager.swp.swp_bcount = 0;
 1835 
 1836                 if (object->handle != NULL) {
 1837                         mtx_lock(&sw_alloc_mtx);
 1838                         TAILQ_INSERT_TAIL(
 1839                             NOBJLIST(object->handle),
 1840                             object,
 1841                             pager_object_list
 1842                         );
 1843                         mtx_unlock(&sw_alloc_mtx);
 1844                 }
 1845         }
 1846 
 1847         /*
 1848          * Locate hash entry.  If not found create, but if we aren't adding
 1849          * anything just return.  If we run out of space in the map we wait
 1850          * and, since the hash table may have changed, retry.
 1851          */
 1852 retry:
 1853         mtx_lock(&swhash_mtx);
 1854         pswap = swp_pager_hash(object, pindex);
 1855 
 1856         if ((swap = *pswap) == NULL) {
 1857                 int i;
 1858 
 1859                 if (swapblk == SWAPBLK_NONE)
 1860                         goto done;
 1861 
 1862                 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT |
 1863                     (curproc == pageproc ? M_USE_RESERVE : 0));
 1864                 if (swap == NULL) {
 1865                         mtx_unlock(&swhash_mtx);
 1866                         VM_OBJECT_WUNLOCK(object);
 1867                         if (uma_zone_exhausted(swap_zone)) {
 1868                                 if (atomic_cmpset_int(&exhausted, 0, 1))
 1869                                         printf("swap zone exhausted, "
 1870                                             "increase kern.maxswzone\n");
 1871                                 vm_pageout_oom(VM_OOM_SWAPZ);
 1872                                 pause("swzonex", 10);
 1873                         } else
 1874                                 VM_WAIT;
 1875                         VM_OBJECT_WLOCK(object);
 1876                         goto retry;
 1877                 }
 1878 
 1879                 if (atomic_cmpset_int(&exhausted, 1, 0))
 1880                         printf("swap zone ok\n");
 1881 
 1882                 swap->swb_hnext = NULL;
 1883                 swap->swb_object = object;
 1884                 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
 1885                 swap->swb_count = 0;
 1886 
 1887                 ++object->un_pager.swp.swp_bcount;
 1888 
 1889                 for (i = 0; i < SWAP_META_PAGES; ++i)
 1890                         swap->swb_pages[i] = SWAPBLK_NONE;
 1891         }
 1892 
 1893         /*
 1894          * Delete prior contents of metadata
 1895          */
 1896         idx = pindex & SWAP_META_MASK;
 1897 
 1898         if (swap->swb_pages[idx] != SWAPBLK_NONE) {
 1899                 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
 1900                 --swap->swb_count;
 1901         }
 1902 
 1903         /*
 1904          * Enter block into metadata
 1905          */
 1906         swap->swb_pages[idx] = swapblk;
 1907         if (swapblk != SWAPBLK_NONE)
 1908                 ++swap->swb_count;
 1909 done:
 1910         mtx_unlock(&swhash_mtx);
 1911 }
 1912 
 1913 /*
 1914  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
 1915  *
 1916  *      The requested range of blocks is freed, with any associated swap
 1917  *      returned to the swap bitmap.
 1918  *
 1919  *      This routine will free swap metadata structures as they are cleaned
 1920  *      out.  This routine does *NOT* operate on swap metadata associated
 1921  *      with resident pages.
 1922  */
 1923 static void
 1924 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
 1925 {
 1926 
 1927         VM_OBJECT_ASSERT_LOCKED(object);
 1928         if (object->type != OBJT_SWAP)
 1929                 return;
 1930 
 1931         while (count > 0) {
 1932                 struct swblock **pswap;
 1933                 struct swblock *swap;
 1934 
 1935                 mtx_lock(&swhash_mtx);
 1936                 pswap = swp_pager_hash(object, index);
 1937 
 1938                 if ((swap = *pswap) != NULL) {
 1939                         daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
 1940 
 1941                         if (v != SWAPBLK_NONE) {
 1942                                 swp_pager_freeswapspace(v, 1);
 1943                                 swap->swb_pages[index & SWAP_META_MASK] =
 1944                                         SWAPBLK_NONE;
 1945                                 if (--swap->swb_count == 0) {
 1946                                         *pswap = swap->swb_hnext;
 1947                                         uma_zfree(swap_zone, swap);
 1948                                         --object->un_pager.swp.swp_bcount;
 1949                                 }
 1950                         }
 1951                         --count;
 1952                         ++index;
 1953                 } else {
 1954                         int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
 1955                         count -= n;
 1956                         index += n;
 1957                 }
 1958                 mtx_unlock(&swhash_mtx);
 1959         }
 1960 }
 1961 
 1962 /*
 1963  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
 1964  *
 1965  *      This routine locates and destroys all swap metadata associated with
 1966  *      an object.
 1967  */
 1968 static void
 1969 swp_pager_meta_free_all(vm_object_t object)
 1970 {
 1971         daddr_t index = 0;
 1972 
 1973         VM_OBJECT_ASSERT_WLOCKED(object);
 1974         if (object->type != OBJT_SWAP)
 1975                 return;
 1976 
 1977         while (object->un_pager.swp.swp_bcount) {
 1978                 struct swblock **pswap;
 1979                 struct swblock *swap;
 1980 
 1981                 mtx_lock(&swhash_mtx);
 1982                 pswap = swp_pager_hash(object, index);
 1983                 if ((swap = *pswap) != NULL) {
 1984                         int i;
 1985 
 1986                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1987                                 daddr_t v = swap->swb_pages[i];
 1988                                 if (v != SWAPBLK_NONE) {
 1989                                         --swap->swb_count;
 1990                                         swp_pager_freeswapspace(v, 1);
 1991                                 }
 1992                         }
 1993                         if (swap->swb_count != 0)
 1994                                 panic("swap_pager_meta_free_all: swb_count != 0");
 1995                         *pswap = swap->swb_hnext;
 1996                         uma_zfree(swap_zone, swap);
 1997                         --object->un_pager.swp.swp_bcount;
 1998                 }
 1999                 mtx_unlock(&swhash_mtx);
 2000                 index += SWAP_META_PAGES;
 2001         }
 2002 }
 2003 
 2004 /*
 2005  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
 2006  *
 2007  *      This routine is capable of looking up, popping, or freeing
 2008  *      swapblk assignments in the swap meta data or in the vm_page_t.
 2009  *      The routine typically returns the swapblk being looked-up, or popped,
 2010  *      or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
 2011  *      was invalid.  This routine will automatically free any invalid
 2012  *      meta-data swapblks.
 2013  *
 2014  *      It is not possible to store invalid swapblks in the swap meta data
 2015  *      (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
 2016  *
 2017  *      When acting on a busy resident page and paging is in progress, we
 2018  *      have to wait until paging is complete but otherwise can act on the
 2019  *      busy page.
 2020  *
 2021  *      SWM_FREE        remove and free swap block from metadata
 2022  *      SWM_POP         remove from meta data but do not free.. pop it out
 2023  */
 2024 static daddr_t
 2025 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
 2026 {
 2027         struct swblock **pswap;
 2028         struct swblock *swap;
 2029         daddr_t r1;
 2030         int idx;
 2031 
 2032         VM_OBJECT_ASSERT_LOCKED(object);
 2033         /*
 2034          * The meta data only exists of the object is OBJT_SWAP
 2035          * and even then might not be allocated yet.
 2036          */
 2037         if (object->type != OBJT_SWAP)
 2038                 return (SWAPBLK_NONE);
 2039 
 2040         r1 = SWAPBLK_NONE;
 2041         mtx_lock(&swhash_mtx);
 2042         pswap = swp_pager_hash(object, pindex);
 2043 
 2044         if ((swap = *pswap) != NULL) {
 2045                 idx = pindex & SWAP_META_MASK;
 2046                 r1 = swap->swb_pages[idx];
 2047 
 2048                 if (r1 != SWAPBLK_NONE) {
 2049                         if (flags & SWM_FREE) {
 2050                                 swp_pager_freeswapspace(r1, 1);
 2051                                 r1 = SWAPBLK_NONE;
 2052                         }
 2053                         if (flags & (SWM_FREE|SWM_POP)) {
 2054                                 swap->swb_pages[idx] = SWAPBLK_NONE;
 2055                                 if (--swap->swb_count == 0) {
 2056                                         *pswap = swap->swb_hnext;
 2057                                         uma_zfree(swap_zone, swap);
 2058                                         --object->un_pager.swp.swp_bcount;
 2059                                 }
 2060                         }
 2061                 }
 2062         }
 2063         mtx_unlock(&swhash_mtx);
 2064         return (r1);
 2065 }
 2066 
 2067 /*
 2068  * System call swapon(name) enables swapping on device name,
 2069  * which must be in the swdevsw.  Return EBUSY
 2070  * if already swapping on this device.
 2071  */
 2072 #ifndef _SYS_SYSPROTO_H_
 2073 struct swapon_args {
 2074         char *name;
 2075 };
 2076 #endif
 2077 
 2078 /*
 2079  * MPSAFE
 2080  */
 2081 /* ARGSUSED */
 2082 int
 2083 sys_swapon(struct thread *td, struct swapon_args *uap)
 2084 {
 2085         struct vattr attr;
 2086         struct vnode *vp;
 2087         struct nameidata nd;
 2088         int error;
 2089 
 2090         error = priv_check(td, PRIV_SWAPON);
 2091         if (error)
 2092                 return (error);
 2093 
 2094         mtx_lock(&Giant);
 2095         while (swdev_syscall_active)
 2096             tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
 2097         swdev_syscall_active = 1;
 2098 
 2099         /*
 2100          * Swap metadata may not fit in the KVM if we have physical
 2101          * memory of >1GB.
 2102          */
 2103         if (swap_zone == NULL) {
 2104                 error = ENOMEM;
 2105                 goto done;
 2106         }
 2107 
 2108         NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
 2109             uap->name, td);
 2110         error = namei(&nd);
 2111         if (error)
 2112                 goto done;
 2113 
 2114         NDFREE(&nd, NDF_ONLY_PNBUF);
 2115         vp = nd.ni_vp;
 2116 
 2117         if (vn_isdisk(vp, &error)) {
 2118                 error = swapongeom(td, vp);
 2119         } else if (vp->v_type == VREG &&
 2120             (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
 2121             (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
 2122                 /*
 2123                  * Allow direct swapping to NFS regular files in the same
 2124                  * way that nfs_mountroot() sets up diskless swapping.
 2125                  */
 2126                 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
 2127         }
 2128 
 2129         if (error)
 2130                 vrele(vp);
 2131 done:
 2132         swdev_syscall_active = 0;
 2133         wakeup_one(&swdev_syscall_active);
 2134         mtx_unlock(&Giant);
 2135         return (error);
 2136 }
 2137 
 2138 /*
 2139  * Check that the total amount of swap currently configured does not
 2140  * exceed half the theoretical maximum.  If it does, print a warning
 2141  * message and return -1; otherwise, return 0.
 2142  */
 2143 static int
 2144 swapon_check_swzone(unsigned long npages)
 2145 {
 2146         unsigned long maxpages;
 2147 
 2148         /* absolute maximum we can handle assuming 100% efficiency */
 2149         maxpages = uma_zone_get_max(swap_zone) * SWAP_META_PAGES;
 2150 
 2151         /* recommend using no more than half that amount */
 2152         if (npages > maxpages / 2) {
 2153                 printf("warning: total configured swap (%lu pages) "
 2154                     "exceeds maximum recommended amount (%lu pages).\n",
 2155                     npages, maxpages / 2);
 2156                 printf("warning: increase kern.maxswzone "
 2157                     "or reduce amount of swap.\n");
 2158                 return (-1);
 2159         }
 2160         return (0);
 2161 }
 2162 
 2163 static void
 2164 swaponsomething(struct vnode *vp, void *id, u_long nblks,
 2165     sw_strategy_t *strategy, sw_close_t *close, dev_t dev, int flags)
 2166 {
 2167         struct swdevt *sp, *tsp;
 2168         swblk_t dvbase;
 2169         u_long mblocks;
 2170 
 2171         /*
 2172          * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
 2173          * First chop nblks off to page-align it, then convert.
 2174          *
 2175          * sw->sw_nblks is in page-sized chunks now too.
 2176          */
 2177         nblks &= ~(ctodb(1) - 1);
 2178         nblks = dbtoc(nblks);
 2179 
 2180         /*
 2181          * If we go beyond this, we get overflows in the radix
 2182          * tree bitmap code.
 2183          */
 2184         mblocks = 0x40000000 / BLIST_META_RADIX;
 2185         if (nblks > mblocks) {
 2186                 printf(
 2187     "WARNING: reducing swap size to maximum of %luMB per unit\n",
 2188                     mblocks / 1024 / 1024 * PAGE_SIZE);
 2189                 nblks = mblocks;
 2190         }
 2191 
 2192         sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
 2193         sp->sw_vp = vp;
 2194         sp->sw_id = id;
 2195         sp->sw_dev = dev;
 2196         sp->sw_flags = 0;
 2197         sp->sw_nblks = nblks;
 2198         sp->sw_used = 0;
 2199         sp->sw_strategy = strategy;
 2200         sp->sw_close = close;
 2201         sp->sw_flags = flags;
 2202 
 2203         sp->sw_blist = blist_create(nblks, M_WAITOK);
 2204         /*
 2205          * Do not free the first two block in order to avoid overwriting
 2206          * any bsd label at the front of the partition
 2207          */
 2208         blist_free(sp->sw_blist, 2, nblks - 2);
 2209 
 2210         dvbase = 0;
 2211         mtx_lock(&sw_dev_mtx);
 2212         TAILQ_FOREACH(tsp, &swtailq, sw_list) {
 2213                 if (tsp->sw_end >= dvbase) {
 2214                         /*
 2215                          * We put one uncovered page between the devices
 2216                          * in order to definitively prevent any cross-device
 2217                          * I/O requests
 2218                          */
 2219                         dvbase = tsp->sw_end + 1;
 2220                 }
 2221         }
 2222         sp->sw_first = dvbase;
 2223         sp->sw_end = dvbase + nblks;
 2224         TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
 2225         nswapdev++;
 2226         swap_pager_avail += nblks;
 2227         swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
 2228         swapon_check_swzone(swap_total / PAGE_SIZE);
 2229         swp_sizecheck();
 2230         mtx_unlock(&sw_dev_mtx);
 2231 }
 2232 
 2233 /*
 2234  * SYSCALL: swapoff(devname)
 2235  *
 2236  * Disable swapping on the given device.
 2237  *
 2238  * XXX: Badly designed system call: it should use a device index
 2239  * rather than filename as specification.  We keep sw_vp around
 2240  * only to make this work.
 2241  */
 2242 #ifndef _SYS_SYSPROTO_H_
 2243 struct swapoff_args {
 2244         char *name;
 2245 };
 2246 #endif
 2247 
 2248 /*
 2249  * MPSAFE
 2250  */
 2251 /* ARGSUSED */
 2252 int
 2253 sys_swapoff(struct thread *td, struct swapoff_args *uap)
 2254 {
 2255         struct vnode *vp;
 2256         struct nameidata nd;
 2257         struct swdevt *sp;
 2258         int error;
 2259 
 2260         error = priv_check(td, PRIV_SWAPOFF);
 2261         if (error)
 2262                 return (error);
 2263 
 2264         mtx_lock(&Giant);
 2265         while (swdev_syscall_active)
 2266             tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2267         swdev_syscall_active = 1;
 2268 
 2269         NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
 2270             td);
 2271         error = namei(&nd);
 2272         if (error)
 2273                 goto done;
 2274         NDFREE(&nd, NDF_ONLY_PNBUF);
 2275         vp = nd.ni_vp;
 2276 
 2277         mtx_lock(&sw_dev_mtx);
 2278         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2279                 if (sp->sw_vp == vp)
 2280                         break;
 2281         }
 2282         mtx_unlock(&sw_dev_mtx);
 2283         if (sp == NULL) {
 2284                 error = EINVAL;
 2285                 goto done;
 2286         }
 2287         error = swapoff_one(sp, td->td_ucred);
 2288 done:
 2289         swdev_syscall_active = 0;
 2290         wakeup_one(&swdev_syscall_active);
 2291         mtx_unlock(&Giant);
 2292         return (error);
 2293 }
 2294 
 2295 static int
 2296 swapoff_one(struct swdevt *sp, struct ucred *cred)
 2297 {
 2298         u_long nblks, dvbase;
 2299 #ifdef MAC
 2300         int error;
 2301 #endif
 2302 
 2303         mtx_assert(&Giant, MA_OWNED);
 2304 #ifdef MAC
 2305         (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
 2306         error = mac_system_check_swapoff(cred, sp->sw_vp);
 2307         (void) VOP_UNLOCK(sp->sw_vp, 0);
 2308         if (error != 0)
 2309                 return (error);
 2310 #endif
 2311         nblks = sp->sw_nblks;
 2312 
 2313         /*
 2314          * We can turn off this swap device safely only if the
 2315          * available virtual memory in the system will fit the amount
 2316          * of data we will have to page back in, plus an epsilon so
 2317          * the system doesn't become critically low on swap space.
 2318          */
 2319         if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
 2320             nblks + nswap_lowat) {
 2321                 return (ENOMEM);
 2322         }
 2323 
 2324         /*
 2325          * Prevent further allocations on this device.
 2326          */
 2327         mtx_lock(&sw_dev_mtx);
 2328         sp->sw_flags |= SW_CLOSING;
 2329         for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
 2330                 swap_pager_avail -= blist_fill(sp->sw_blist,
 2331                      dvbase, dmmax);
 2332         }
 2333         swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
 2334         mtx_unlock(&sw_dev_mtx);
 2335 
 2336         /*
 2337          * Page in the contents of the device and close it.
 2338          */
 2339         swap_pager_swapoff(sp);
 2340 
 2341         sp->sw_close(curthread, sp);
 2342         sp->sw_id = NULL;
 2343         mtx_lock(&sw_dev_mtx);
 2344         TAILQ_REMOVE(&swtailq, sp, sw_list);
 2345         nswapdev--;
 2346         if (nswapdev == 0) {
 2347                 swap_pager_full = 2;
 2348                 swap_pager_almost_full = 1;
 2349         }
 2350         if (swdevhd == sp)
 2351                 swdevhd = NULL;
 2352         mtx_unlock(&sw_dev_mtx);
 2353         blist_destroy(sp->sw_blist);
 2354         free(sp, M_VMPGDATA);
 2355         return (0);
 2356 }
 2357 
 2358 void
 2359 swapoff_all(void)
 2360 {
 2361         struct swdevt *sp, *spt;
 2362         const char *devname;
 2363         int error;
 2364 
 2365         mtx_lock(&Giant);
 2366         while (swdev_syscall_active)
 2367                 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2368         swdev_syscall_active = 1;
 2369 
 2370         mtx_lock(&sw_dev_mtx);
 2371         TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
 2372                 mtx_unlock(&sw_dev_mtx);
 2373                 if (vn_isdisk(sp->sw_vp, NULL))
 2374                         devname = devtoname(sp->sw_vp->v_rdev);
 2375                 else
 2376                         devname = "[file]";
 2377                 error = swapoff_one(sp, thread0.td_ucred);
 2378                 if (error != 0) {
 2379                         printf("Cannot remove swap device %s (error=%d), "
 2380                             "skipping.\n", devname, error);
 2381                 } else if (bootverbose) {
 2382                         printf("Swap device %s removed.\n", devname);
 2383                 }
 2384                 mtx_lock(&sw_dev_mtx);
 2385         }
 2386         mtx_unlock(&sw_dev_mtx);
 2387 
 2388         swdev_syscall_active = 0;
 2389         wakeup_one(&swdev_syscall_active);
 2390         mtx_unlock(&Giant);
 2391 }
 2392 
 2393 void
 2394 swap_pager_status(int *total, int *used)
 2395 {
 2396         struct swdevt *sp;
 2397 
 2398         *total = 0;
 2399         *used = 0;
 2400         mtx_lock(&sw_dev_mtx);
 2401         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2402                 *total += sp->sw_nblks;
 2403                 *used += sp->sw_used;
 2404         }
 2405         mtx_unlock(&sw_dev_mtx);
 2406 }
 2407 
 2408 int
 2409 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
 2410 {
 2411         struct swdevt *sp;
 2412         const char *tmp_devname;
 2413         int error, n;
 2414 
 2415         n = 0;
 2416         error = ENOENT;
 2417         mtx_lock(&sw_dev_mtx);
 2418         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2419                 if (n != name) {
 2420                         n++;
 2421                         continue;
 2422                 }
 2423                 xs->xsw_version = XSWDEV_VERSION;
 2424                 xs->xsw_dev = sp->sw_dev;
 2425                 xs->xsw_flags = sp->sw_flags;
 2426                 xs->xsw_nblks = sp->sw_nblks;
 2427                 xs->xsw_used = sp->sw_used;
 2428                 if (devname != NULL) {
 2429                         if (vn_isdisk(sp->sw_vp, NULL))
 2430                                 tmp_devname = devtoname(sp->sw_vp->v_rdev);
 2431                         else
 2432                                 tmp_devname = "[file]";
 2433                         strncpy(devname, tmp_devname, len);
 2434                 }
 2435                 error = 0;
 2436                 break;
 2437         }
 2438         mtx_unlock(&sw_dev_mtx);
 2439         return (error);
 2440 }
 2441 
 2442 static int
 2443 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
 2444 {
 2445         struct xswdev xs;
 2446         int error;
 2447 
 2448         if (arg2 != 1)                  /* name length */
 2449                 return (EINVAL);
 2450         error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
 2451         if (error != 0)
 2452                 return (error);
 2453         error = SYSCTL_OUT(req, &xs, sizeof(xs));
 2454         return (error);
 2455 }
 2456 
 2457 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
 2458     "Number of swap devices");
 2459 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
 2460     "Swap statistics by device");
 2461 
 2462 /*
 2463  * vmspace_swap_count() - count the approximate swap usage in pages for a
 2464  *                        vmspace.
 2465  *
 2466  *      The map must be locked.
 2467  *
 2468  *      Swap usage is determined by taking the proportional swap used by
 2469  *      VM objects backing the VM map.  To make up for fractional losses,
 2470  *      if the VM object has any swap use at all the associated map entries
 2471  *      count for at least 1 swap page.
 2472  */
 2473 long
 2474 vmspace_swap_count(struct vmspace *vmspace)
 2475 {
 2476         vm_map_t map;
 2477         vm_map_entry_t cur;
 2478         vm_object_t object;
 2479         long count, n;
 2480 
 2481         map = &vmspace->vm_map;
 2482         count = 0;
 2483 
 2484         for (cur = map->header.next; cur != &map->header; cur = cur->next) {
 2485                 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2486                     (object = cur->object.vm_object) != NULL) {
 2487                         VM_OBJECT_WLOCK(object);
 2488                         if (object->type == OBJT_SWAP &&
 2489                             object->un_pager.swp.swp_bcount != 0) {
 2490                                 n = (cur->end - cur->start) / PAGE_SIZE;
 2491                                 count += object->un_pager.swp.swp_bcount *
 2492                                     SWAP_META_PAGES * n / object->size + 1;
 2493                         }
 2494                         VM_OBJECT_WUNLOCK(object);
 2495                 }
 2496         }
 2497         return (count);
 2498 }
 2499 
 2500 /*
 2501  * GEOM backend
 2502  *
 2503  * Swapping onto disk devices.
 2504  *
 2505  */
 2506 
 2507 static g_orphan_t swapgeom_orphan;
 2508 
 2509 static struct g_class g_swap_class = {
 2510         .name = "SWAP",
 2511         .version = G_VERSION,
 2512         .orphan = swapgeom_orphan,
 2513 };
 2514 
 2515 DECLARE_GEOM_CLASS(g_swap_class, g_class);
 2516 
 2517 
 2518 static void
 2519 swapgeom_done(struct bio *bp2)
 2520 {
 2521         struct buf *bp;
 2522 
 2523         bp = bp2->bio_caller2;
 2524         bp->b_ioflags = bp2->bio_flags;
 2525         if (bp2->bio_error)
 2526                 bp->b_ioflags |= BIO_ERROR;
 2527         bp->b_resid = bp->b_bcount - bp2->bio_completed;
 2528         bp->b_error = bp2->bio_error;
 2529         bufdone(bp);
 2530         g_destroy_bio(bp2);
 2531 }
 2532 
 2533 static void
 2534 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
 2535 {
 2536         struct bio *bio;
 2537         struct g_consumer *cp;
 2538 
 2539         cp = sp->sw_id;
 2540         if (cp == NULL) {
 2541                 bp->b_error = ENXIO;
 2542                 bp->b_ioflags |= BIO_ERROR;
 2543                 bufdone(bp);
 2544                 return;
 2545         }
 2546         if (bp->b_iocmd == BIO_WRITE)
 2547                 bio = g_new_bio();
 2548         else
 2549                 bio = g_alloc_bio();
 2550         if (bio == NULL) {
 2551                 bp->b_error = ENOMEM;
 2552                 bp->b_ioflags |= BIO_ERROR;
 2553                 bufdone(bp);
 2554                 return;
 2555         }
 2556 
 2557         bio->bio_caller2 = bp;
 2558         bio->bio_cmd = bp->b_iocmd;
 2559         bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
 2560         bio->bio_length = bp->b_bcount;
 2561         bio->bio_done = swapgeom_done;
 2562         if ((bp->b_flags & B_UNMAPPED) != 0) {
 2563                 bio->bio_ma = bp->b_pages;
 2564                 bio->bio_data = unmapped_buf;
 2565                 bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
 2566                 bio->bio_ma_n = bp->b_npages;
 2567                 bio->bio_flags |= BIO_UNMAPPED;
 2568         } else {
 2569                 bio->bio_data = bp->b_data;
 2570                 bio->bio_ma = NULL;
 2571         }
 2572         g_io_request(bio, cp);
 2573         return;
 2574 }
 2575 
 2576 static void
 2577 swapgeom_orphan(struct g_consumer *cp)
 2578 {
 2579         struct swdevt *sp;
 2580 
 2581         mtx_lock(&sw_dev_mtx);
 2582         TAILQ_FOREACH(sp, &swtailq, sw_list)
 2583                 if (sp->sw_id == cp)
 2584                         sp->sw_flags |= SW_CLOSING;
 2585         mtx_unlock(&sw_dev_mtx);
 2586 }
 2587 
 2588 static void
 2589 swapgeom_close_ev(void *arg, int flags)
 2590 {
 2591         struct g_consumer *cp;
 2592 
 2593         cp = arg;
 2594         g_access(cp, -1, -1, 0);
 2595         g_detach(cp);
 2596         g_destroy_consumer(cp);
 2597 }
 2598 
 2599 static void
 2600 swapgeom_close(struct thread *td, struct swdevt *sw)
 2601 {
 2602 
 2603         /* XXX: direct call when Giant untangled */
 2604         g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
 2605 }
 2606 
 2607 
 2608 struct swh0h0 {
 2609         struct cdev *dev;
 2610         struct vnode *vp;
 2611         int     error;
 2612 };
 2613 
 2614 static void
 2615 swapongeom_ev(void *arg, int flags)
 2616 {
 2617         struct swh0h0 *swh;
 2618         struct g_provider *pp;
 2619         struct g_consumer *cp;
 2620         static struct g_geom *gp;
 2621         struct swdevt *sp;
 2622         u_long nblks;
 2623         int error;
 2624 
 2625         swh = arg;
 2626         swh->error = 0;
 2627         pp = g_dev_getprovider(swh->dev);
 2628         if (pp == NULL) {
 2629                 swh->error = ENODEV;
 2630                 return;
 2631         }
 2632         mtx_lock(&sw_dev_mtx);
 2633         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2634                 cp = sp->sw_id;
 2635                 if (cp != NULL && cp->provider == pp) {
 2636                         mtx_unlock(&sw_dev_mtx);
 2637                         swh->error = EBUSY;
 2638                         return;
 2639                 }
 2640         }
 2641         mtx_unlock(&sw_dev_mtx);
 2642         if (gp == NULL)
 2643                 gp = g_new_geomf(&g_swap_class, "swap");
 2644         cp = g_new_consumer(gp);
 2645         g_attach(cp, pp);
 2646         /*
 2647          * XXX: Everytime you think you can improve the margin for
 2648          * footshooting, somebody depends on the ability to do so:
 2649          * savecore(8) wants to write to our swapdev so we cannot
 2650          * set an exclusive count :-(
 2651          */
 2652         error = g_access(cp, 1, 1, 0);
 2653         if (error) {
 2654                 g_detach(cp);
 2655                 g_destroy_consumer(cp);
 2656                 swh->error = error;
 2657                 return;
 2658         }
 2659         nblks = pp->mediasize / DEV_BSIZE;
 2660         swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
 2661             swapgeom_close, dev2udev(swh->dev),
 2662             (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ? SW_UNMAPPED : 0);
 2663         swh->error = 0;
 2664 }
 2665 
 2666 static int
 2667 swapongeom(struct thread *td, struct vnode *vp)
 2668 {
 2669         int error;
 2670         struct swh0h0 swh;
 2671 
 2672         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2673 
 2674         swh.dev = vp->v_rdev;
 2675         swh.vp = vp;
 2676         swh.error = 0;
 2677         /* XXX: direct call when Giant untangled */
 2678         error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
 2679         if (!error)
 2680                 error = swh.error;
 2681         VOP_UNLOCK(vp, 0);
 2682         return (error);
 2683 }
 2684 
 2685 /*
 2686  * VNODE backend
 2687  *
 2688  * This is used mainly for network filesystem (read: probably only tested
 2689  * with NFS) swapfiles.
 2690  *
 2691  */
 2692 
 2693 static void
 2694 swapdev_strategy(struct buf *bp, struct swdevt *sp)
 2695 {
 2696         struct vnode *vp2;
 2697 
 2698         bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
 2699 
 2700         vp2 = sp->sw_id;
 2701         vhold(vp2);
 2702         if (bp->b_iocmd == BIO_WRITE) {
 2703                 if (bp->b_bufobj)
 2704                         bufobj_wdrop(bp->b_bufobj);
 2705                 bufobj_wref(&vp2->v_bufobj);
 2706         }
 2707         if (bp->b_bufobj != &vp2->v_bufobj)
 2708                 bp->b_bufobj = &vp2->v_bufobj;
 2709         bp->b_vp = vp2;
 2710         bp->b_iooffset = dbtob(bp->b_blkno);
 2711         bstrategy(bp);
 2712         return;
 2713 }
 2714 
 2715 static void
 2716 swapdev_close(struct thread *td, struct swdevt *sp)
 2717 {
 2718 
 2719         VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
 2720         vrele(sp->sw_vp);
 2721 }
 2722 
 2723 
 2724 static int
 2725 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
 2726 {
 2727         struct swdevt *sp;
 2728         int error;
 2729 
 2730         if (nblks == 0)
 2731                 return (ENXIO);
 2732         mtx_lock(&sw_dev_mtx);
 2733         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2734                 if (sp->sw_id == vp) {
 2735                         mtx_unlock(&sw_dev_mtx);
 2736                         return (EBUSY);
 2737                 }
 2738         }
 2739         mtx_unlock(&sw_dev_mtx);
 2740 
 2741         (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2742 #ifdef MAC
 2743         error = mac_system_check_swapon(td->td_ucred, vp);
 2744         if (error == 0)
 2745 #endif
 2746                 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
 2747         (void) VOP_UNLOCK(vp, 0);
 2748         if (error)
 2749                 return (error);
 2750 
 2751         swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
 2752             NODEV, 0);
 2753         return (0);
 2754 }

Cache object: ad1e051f6418f3437d74fedc76cfb3fa


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.