[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/swap_pager.c

Version: -  FREEBSD  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  cheribsd  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1  -  FREEBSD-LIBC  -  FREEBSD8-LIBC  -  FREEBSD7-LIBC  -  FREEBSD6-LIBC  -  GLIBC27 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Matthew Dillon,
    3  * Copyright (c) 1994 John S. Dyson
    4  * Copyright (c) 1990 University of Utah.
    5  * Copyright (c) 1982, 1986, 1989, 1993
    6  *      The Regents of the University of California.  All rights reserved.
    7  *
    8  * This code is derived from software contributed to Berkeley by
    9  * the Systems Programming Group of the University of Utah Computer
   10  * Science Department.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. All advertising materials mentioning features or use of this software
   21  *    must display the following acknowledgement:
   22  *      This product includes software developed by the University of
   23  *      California, Berkeley and its contributors.
   24  * 4. Neither the name of the University nor the names of its contributors
   25  *    may be used to endorse or promote products derived from this software
   26  *    without specific prior written permission.
   27  *
   28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   38  * SUCH DAMAGE.
   39  *
   40  *                              New Swap System
   41  *                              Matthew Dillon
   42  *
   43  * Radix Bitmap 'blists'.
   44  *
   45  *      - The new swapper uses the new radix bitmap code.  This should scale
   46  *        to arbitrarily small or arbitrarily large swap spaces and an almost
   47  *        arbitrary degree of fragmentation.
   48  *
   49  * Features:
   50  *
   51  *      - on the fly reallocation of swap during putpages.  The new system
   52  *        does not try to keep previously allocated swap blocks for dirty
   53  *        pages.  
   54  *
   55  *      - on the fly deallocation of swap
   56  *
   57  *      - No more garbage collection required.  Unnecessarily allocated swap
   58  *        blocks only exist for dirty vm_page_t's now and these are already
   59  *        cycled (in a high-load system) by the pager.  We also do on-the-fly
   60  *        removal of invalidated swap blocks when a page is destroyed
   61  *        or renamed.
   62  *
   63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
   64  *
   65  *      @(#)swap_pager.c        8.9 (Berkeley) 3/21/94
   66  *      @(#)vm_swap.c   8.5 (Berkeley) 2/17/94
   67  */
   68 
   69 #include <sys/cdefs.h>
   70 __FBSDID("$FreeBSD: stable/8/sys/vm/swap_pager.c 231189 2012-02-08 07:31:55Z mav $");
   71 
   72 #include "opt_swap.h"
   73 #include "opt_vm.h"
   74 
   75 #include <sys/param.h>
   76 #include <sys/systm.h>
   77 #include <sys/conf.h>
   78 #include <sys/kernel.h>
   79 #include <sys/priv.h>
   80 #include <sys/proc.h>
   81 #include <sys/bio.h>
   82 #include <sys/buf.h>
   83 #include <sys/disk.h>
   84 #include <sys/fcntl.h>
   85 #include <sys/mount.h>
   86 #include <sys/namei.h>
   87 #include <sys/vnode.h>
   88 #include <sys/malloc.h>
   89 #include <sys/resource.h>
   90 #include <sys/resourcevar.h>
   91 #include <sys/sysctl.h>
   92 #include <sys/sysproto.h>
   93 #include <sys/blist.h>
   94 #include <sys/lock.h>
   95 #include <sys/sx.h>
   96 #include <sys/vmmeter.h>
   97 
   98 #include <security/mac/mac_framework.h>
   99 
  100 #include <vm/vm.h>
  101 #include <vm/pmap.h>
  102 #include <vm/vm_map.h>
  103 #include <vm/vm_kern.h>
  104 #include <vm/vm_object.h>
  105 #include <vm/vm_page.h>
  106 #include <vm/vm_pager.h>
  107 #include <vm/vm_pageout.h>
  108 #include <vm/vm_param.h>
  109 #include <vm/swap_pager.h>
  110 #include <vm/vm_extern.h>
  111 #include <vm/uma.h>
  112 
  113 #include <geom/geom.h>
  114 
  115 /*
  116  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, 16
  117  * or 32 pages per allocation.
  118  * The 32-page limit is due to the radix code (kern/subr_blist.c).
  119  */
  120 #ifndef MAX_PAGEOUT_CLUSTER
  121 #define MAX_PAGEOUT_CLUSTER 16
  122 #endif
  123 
  124 #if !defined(SWB_NPAGES)
  125 #define SWB_NPAGES      MAX_PAGEOUT_CLUSTER
  126 #endif
  127 
  128 /*
  129  * The swblock structure maps an object and a small, fixed-size range
  130  * of page indices to disk addresses within a swap area.
  131  * The collection of these mappings is implemented as a hash table.
  132  * Unused disk addresses within a swap area are allocated and managed
  133  * using a blist.
  134  */
  135 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
  136 #define SWAP_META_PAGES         (SWB_NPAGES * 2)
  137 #define SWAP_META_MASK          (SWAP_META_PAGES - 1)
  138 
  139 struct swblock {
  140         struct swblock  *swb_hnext;
  141         vm_object_t     swb_object;
  142         vm_pindex_t     swb_index;
  143         int             swb_count;
  144         daddr_t         swb_pages[SWAP_META_PAGES];
  145 };
  146 
  147 static struct mtx sw_dev_mtx;
  148 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
  149 static struct swdevt *swdevhd;  /* Allocate from here next */
  150 static int nswapdev;            /* Number of swap devices */
  151 int swap_pager_avail;
  152 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
  153 
  154 static vm_ooffset_t swap_total;
  155 SYSCTL_QUAD(_vm, OID_AUTO, swap_total, CTLFLAG_RD, &swap_total, 0, "");
  156 static vm_ooffset_t swap_reserved;
  157 SYSCTL_QUAD(_vm, OID_AUTO, swap_reserved, CTLFLAG_RD, &swap_reserved, 0, "");
  158 static int overcommit = 0;
  159 SYSCTL_INT(_vm, OID_AUTO, overcommit, CTLFLAG_RW, &overcommit, 0, "");
  160 
  161 /* bits from overcommit */
  162 #define SWAP_RESERVE_FORCE_ON           (1 << 0)
  163 #define SWAP_RESERVE_RLIMIT_ON          (1 << 1)
  164 #define SWAP_RESERVE_ALLOW_NONWIRED     (1 << 2)
  165 
  166 int
  167 swap_reserve(vm_ooffset_t incr)
  168 {
  169 
  170         return (swap_reserve_by_uid(incr, curthread->td_ucred->cr_ruidinfo));
  171 }
  172 
  173 int
  174 swap_reserve_by_uid(vm_ooffset_t incr, struct uidinfo *uip)
  175 {
  176         vm_ooffset_t r, s;
  177         int res, error;
  178         static int curfail;
  179         static struct timeval lastfail;
  180 
  181         if (incr & PAGE_MASK)
  182                 panic("swap_reserve: & PAGE_MASK");
  183 
  184         res = 0;
  185         mtx_lock(&sw_dev_mtx);
  186         r = swap_reserved + incr;
  187         if (overcommit & SWAP_RESERVE_ALLOW_NONWIRED) {
  188                 s = cnt.v_page_count - cnt.v_free_reserved - cnt.v_wire_count;
  189                 s *= PAGE_SIZE;
  190         } else
  191                 s = 0;
  192         s += swap_total;
  193         if ((overcommit & SWAP_RESERVE_FORCE_ON) == 0 || r <= s ||
  194             (error = priv_check(curthread, PRIV_VM_SWAP_NOQUOTA)) == 0) {
  195                 res = 1;
  196                 swap_reserved = r;
  197         }
  198         mtx_unlock(&sw_dev_mtx);
  199 
  200         if (res) {
  201                 PROC_LOCK(curproc);
  202                 UIDINFO_VMSIZE_LOCK(uip);
  203                 if ((overcommit & SWAP_RESERVE_RLIMIT_ON) != 0 &&
  204                     uip->ui_vmsize + incr > lim_cur(curproc, RLIMIT_SWAP) &&
  205                     priv_check(curthread, PRIV_VM_SWAP_NORLIMIT))
  206                         res = 0;
  207                 else
  208                         uip->ui_vmsize += incr;
  209                 UIDINFO_VMSIZE_UNLOCK(uip);
  210                 PROC_UNLOCK(curproc);
  211                 if (!res) {
  212                         mtx_lock(&sw_dev_mtx);
  213                         swap_reserved -= incr;
  214                         mtx_unlock(&sw_dev_mtx);
  215                 }
  216         }
  217         if (!res && ppsratecheck(&lastfail, &curfail, 1)) {
  218                 printf("uid %d, pid %d: swap reservation for %jd bytes failed\n",
  219                     uip->ui_uid, curproc->p_pid, incr);
  220         }
  221 
  222         return (res);
  223 }
  224 
  225 void
  226 swap_reserve_force(vm_ooffset_t incr)
  227 {
  228         struct uidinfo *uip;
  229 
  230         mtx_lock(&sw_dev_mtx);
  231         swap_reserved += incr;
  232         mtx_unlock(&sw_dev_mtx);
  233 
  234         uip = curthread->td_ucred->cr_ruidinfo;
  235         PROC_LOCK(curproc);
  236         UIDINFO_VMSIZE_LOCK(uip);
  237         uip->ui_vmsize += incr;
  238         UIDINFO_VMSIZE_UNLOCK(uip);
  239         PROC_UNLOCK(curproc);
  240 }
  241 
  242 void
  243 swap_release(vm_ooffset_t decr)
  244 {
  245         struct uidinfo *uip;
  246 
  247         PROC_LOCK(curproc);
  248         uip = curthread->td_ucred->cr_ruidinfo;
  249         swap_release_by_uid(decr, uip);
  250         PROC_UNLOCK(curproc);
  251 }
  252 
  253 void
  254 swap_release_by_uid(vm_ooffset_t decr, struct uidinfo *uip)
  255 {
  256 
  257         if (decr & PAGE_MASK)
  258                 panic("swap_release: & PAGE_MASK");
  259 
  260         mtx_lock(&sw_dev_mtx);
  261         if (swap_reserved < decr)
  262                 panic("swap_reserved < decr");
  263         swap_reserved -= decr;
  264         mtx_unlock(&sw_dev_mtx);
  265 
  266         UIDINFO_VMSIZE_LOCK(uip);
  267         if (uip->ui_vmsize < decr)
  268                 printf("negative vmsize for uid = %d\n", uip->ui_uid);
  269         uip->ui_vmsize -= decr;
  270         UIDINFO_VMSIZE_UNLOCK(uip);
  271 }
  272 
  273 static void swapdev_strategy(struct buf *, struct swdevt *sw);
  274 
  275 #define SWM_FREE        0x02    /* free, period                 */
  276 #define SWM_POP         0x04    /* pop out                      */
  277 
  278 int swap_pager_full = 2;        /* swap space exhaustion (task killing) */
  279 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
  280 static int nsw_rcount;          /* free read buffers                    */
  281 static int nsw_wcount_sync;     /* limit write buffers / synchronous    */
  282 static int nsw_wcount_async;    /* limit write buffers / asynchronous   */
  283 static int nsw_wcount_async_max;/* assigned maximum                     */
  284 static int nsw_cluster_max;     /* maximum VOP I/O allowed              */
  285 
  286 static struct swblock **swhash;
  287 static int swhash_mask;
  288 static struct mtx swhash_mtx;
  289 
  290 static int swap_async_max = 4;  /* maximum in-progress async I/O's      */
  291 static struct sx sw_alloc_sx;
  292 
  293 
  294 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
  295         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
  296 
  297 /*
  298  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
  299  * of searching a named list by hashing it just a little.
  300  */
  301 
  302 #define NOBJLISTS               8
  303 
  304 #define NOBJLIST(handle)        \
  305         (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
  306 
  307 static struct mtx sw_alloc_mtx; /* protect list manipulation */ 
  308 static struct pagerlst  swap_pager_object_list[NOBJLISTS];
  309 static uma_zone_t       swap_zone;
  310 static struct vm_object swap_zone_obj;
  311 
  312 /*
  313  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
  314  * calls hooked from other parts of the VM system and do not appear here.
  315  * (see vm/swap_pager.h).
  316  */
  317 static vm_object_t
  318                 swap_pager_alloc(void *handle, vm_ooffset_t size,
  319                     vm_prot_t prot, vm_ooffset_t offset, struct ucred *);
  320 static void     swap_pager_dealloc(vm_object_t object);
  321 static int      swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
  322 static void     swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
  323 static boolean_t
  324                 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
  325 static void     swap_pager_init(void);
  326 static void     swap_pager_unswapped(vm_page_t);
  327 static void     swap_pager_swapoff(struct swdevt *sp);
  328 
  329 struct pagerops swappagerops = {
  330         .pgo_init =     swap_pager_init,        /* early system initialization of pager */
  331         .pgo_alloc =    swap_pager_alloc,       /* allocate an OBJT_SWAP object         */
  332         .pgo_dealloc =  swap_pager_dealloc,     /* deallocate an OBJT_SWAP object       */
  333         .pgo_getpages = swap_pager_getpages,    /* pagein                               */
  334         .pgo_putpages = swap_pager_putpages,    /* pageout                              */
  335         .pgo_haspage =  swap_pager_haspage,     /* get backing store status for page    */
  336         .pgo_pageunswapped = swap_pager_unswapped,      /* remove swap related to page          */
  337 };
  338 
  339 /*
  340  * dmmax is in page-sized chunks with the new swap system.  It was
  341  * dev-bsized chunks in the old.  dmmax is always a power of 2.
  342  *
  343  * swap_*() routines are externally accessible.  swp_*() routines are
  344  * internal.
  345  */
  346 static int dmmax;
  347 static int nswap_lowat = 128;   /* in pages, swap_pager_almost_full warn */
  348 static int nswap_hiwat = 512;   /* in pages, swap_pager_almost_full warn */
  349 
  350 SYSCTL_INT(_vm, OID_AUTO, dmmax,
  351         CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
  352 
  353 static void     swp_sizecheck(void);
  354 static void     swp_pager_async_iodone(struct buf *bp);
  355 static int      swapongeom(struct thread *, struct vnode *);
  356 static int      swaponvp(struct thread *, struct vnode *, u_long);
  357 static int      swapoff_one(struct swdevt *sp, struct ucred *cred);
  358 
  359 /*
  360  * Swap bitmap functions
  361  */
  362 static void     swp_pager_freeswapspace(daddr_t blk, int npages);
  363 static daddr_t  swp_pager_getswapspace(int npages);
  364 
  365 /*
  366  * Metadata functions
  367  */
  368 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
  369 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
  370 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
  371 static void swp_pager_meta_free_all(vm_object_t);
  372 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
  373 
  374 static void
  375 swp_pager_free_nrpage(vm_page_t m)
  376 {
  377 
  378         if (m->wire_count == 0)
  379                 vm_page_free(m);
  380 }
  381 
  382 /*
  383  * SWP_SIZECHECK() -    update swap_pager_full indication
  384  *      
  385  *      update the swap_pager_almost_full indication and warn when we are
  386  *      about to run out of swap space, using lowat/hiwat hysteresis.
  387  *
  388  *      Clear swap_pager_full ( task killing ) indication when lowat is met.
  389  *
  390  *      No restrictions on call
  391  *      This routine may not block.
  392  */
  393 static void
  394 swp_sizecheck(void)
  395 {
  396 
  397         if (swap_pager_avail < nswap_lowat) {
  398                 if (swap_pager_almost_full == 0) {
  399                         printf("swap_pager: out of swap space\n");
  400                         swap_pager_almost_full = 1;
  401                 }
  402         } else {
  403                 swap_pager_full = 0;
  404                 if (swap_pager_avail > nswap_hiwat)
  405                         swap_pager_almost_full = 0;
  406         }
  407 }
  408 
  409 /*
  410  * SWP_PAGER_HASH() -   hash swap meta data
  411  *
  412  *      This is an helper function which hashes the swapblk given
  413  *      the object and page index.  It returns a pointer to a pointer
  414  *      to the object, or a pointer to a NULL pointer if it could not
  415  *      find a swapblk.
  416  */
  417 static struct swblock **
  418 swp_pager_hash(vm_object_t object, vm_pindex_t index)
  419 {
  420         struct swblock **pswap;
  421         struct swblock *swap;
  422 
  423         index &= ~(vm_pindex_t)SWAP_META_MASK;
  424         pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
  425         while ((swap = *pswap) != NULL) {
  426                 if (swap->swb_object == object &&
  427                     swap->swb_index == index
  428                 ) {
  429                         break;
  430                 }
  431                 pswap = &swap->swb_hnext;
  432         }
  433         return (pswap);
  434 }
  435 
  436 /*
  437  * SWAP_PAGER_INIT() -  initialize the swap pager!
  438  *
  439  *      Expected to be started from system init.  NOTE:  This code is run 
  440  *      before much else so be careful what you depend on.  Most of the VM
  441  *      system has yet to be initialized at this point.
  442  */
  443 static void
  444 swap_pager_init(void)
  445 {
  446         /*
  447          * Initialize object lists
  448          */
  449         int i;
  450 
  451         for (i = 0; i < NOBJLISTS; ++i)
  452                 TAILQ_INIT(&swap_pager_object_list[i]);
  453         mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
  454         mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
  455 
  456         /*
  457          * Device Stripe, in PAGE_SIZE'd blocks
  458          */
  459         dmmax = SWB_NPAGES * 2;
  460 }
  461 
  462 /*
  463  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
  464  *
  465  *      Expected to be started from pageout process once, prior to entering
  466  *      its main loop.
  467  */
  468 void
  469 swap_pager_swap_init(void)
  470 {
  471         int n, n2;
  472 
  473         /*
  474          * Number of in-transit swap bp operations.  Don't
  475          * exhaust the pbufs completely.  Make sure we
  476          * initialize workable values (0 will work for hysteresis
  477          * but it isn't very efficient).
  478          *
  479          * The nsw_cluster_max is constrained by the bp->b_pages[]
  480          * array (MAXPHYS/PAGE_SIZE) and our locally defined
  481          * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
  482          * constrained by the swap device interleave stripe size.
  483          *
  484          * Currently we hardwire nsw_wcount_async to 4.  This limit is 
  485          * designed to prevent other I/O from having high latencies due to
  486          * our pageout I/O.  The value 4 works well for one or two active swap
  487          * devices but is probably a little low if you have more.  Even so,
  488          * a higher value would probably generate only a limited improvement
  489          * with three or four active swap devices since the system does not
  490          * typically have to pageout at extreme bandwidths.   We will want
  491          * at least 2 per swap devices, and 4 is a pretty good value if you
  492          * have one NFS swap device due to the command/ack latency over NFS.
  493          * So it all works out pretty well.
  494          */
  495         nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
  496 
  497         mtx_lock(&pbuf_mtx);
  498         nsw_rcount = (nswbuf + 1) / 2;
  499         nsw_wcount_sync = (nswbuf + 3) / 4;
  500         nsw_wcount_async = 4;
  501         nsw_wcount_async_max = nsw_wcount_async;
  502         mtx_unlock(&pbuf_mtx);
  503 
  504         /*
  505          * Initialize our zone.  Right now I'm just guessing on the number
  506          * we need based on the number of pages in the system.  Each swblock
  507          * can hold 16 pages, so this is probably overkill.  This reservation
  508          * is typically limited to around 32MB by default.
  509          */
  510         n = cnt.v_page_count / 2;
  511         if (maxswzone && n > maxswzone / sizeof(struct swblock))
  512                 n = maxswzone / sizeof(struct swblock);
  513         n2 = n;
  514         swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
  515             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
  516         if (swap_zone == NULL)
  517                 panic("failed to create swap_zone.");
  518         do {
  519                 if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
  520                         break;
  521                 /*
  522                  * if the allocation failed, try a zone two thirds the
  523                  * size of the previous attempt.
  524                  */
  525                 n -= ((n + 2) / 3);
  526         } while (n > 0);
  527         if (n2 != n)
  528                 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
  529         n2 = n;
  530 
  531         /*
  532          * Initialize our meta-data hash table.  The swapper does not need to
  533          * be quite as efficient as the VM system, so we do not use an 
  534          * oversized hash table.
  535          *
  536          *      n:              size of hash table, must be power of 2
  537          *      swhash_mask:    hash table index mask
  538          */
  539         for (n = 1; n < n2 / 8; n *= 2)
  540                 ;
  541         swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
  542         swhash_mask = n - 1;
  543         mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
  544 }
  545 
  546 /*
  547  * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
  548  *                      its metadata structures.
  549  *
  550  *      This routine is called from the mmap and fork code to create a new
  551  *      OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
  552  *      and then converting it with swp_pager_meta_build().
  553  *
  554  *      This routine may block in vm_object_allocate() and create a named
  555  *      object lookup race, so we must interlock.
  556  *
  557  * MPSAFE
  558  */
  559 static vm_object_t
  560 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
  561     vm_ooffset_t offset, struct ucred *cred)
  562 {
  563         vm_object_t object;
  564         vm_pindex_t pindex;
  565         struct uidinfo *uip;
  566 
  567         uip = NULL;
  568         pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
  569         if (handle) {
  570                 mtx_lock(&Giant);
  571                 /*
  572                  * Reference existing named region or allocate new one.  There
  573                  * should not be a race here against swp_pager_meta_build()
  574                  * as called from vm_page_remove() in regards to the lookup
  575                  * of the handle.
  576                  */
  577                 sx_xlock(&sw_alloc_sx);
  578                 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
  579                 if (object == NULL) {
  580                         if (cred != NULL) {
  581                                 uip = cred->cr_ruidinfo;
  582                                 if (!swap_reserve_by_uid(size, uip)) {
  583                                         sx_xunlock(&sw_alloc_sx);
  584                                         mtx_unlock(&Giant);
  585                                         return (NULL);
  586                                 }
  587                                 uihold(uip);
  588                         }
  589                         object = vm_object_allocate(OBJT_DEFAULT, pindex);
  590                         VM_OBJECT_LOCK(object);
  591                         object->handle = handle;
  592                         if (cred != NULL) {
  593                                 object->uip = uip;
  594                                 object->charge = size;
  595                         }
  596                         swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  597                         VM_OBJECT_UNLOCK(object);
  598                 }
  599                 sx_xunlock(&sw_alloc_sx);
  600                 mtx_unlock(&Giant);
  601         } else {
  602                 if (cred != NULL) {
  603                         uip = cred->cr_ruidinfo;
  604                         if (!swap_reserve_by_uid(size, uip))
  605                                 return (NULL);
  606                         uihold(uip);
  607                 }
  608                 object = vm_object_allocate(OBJT_DEFAULT, pindex);
  609                 VM_OBJECT_LOCK(object);
  610                 if (cred != NULL) {
  611                         object->uip = uip;
  612                         object->charge = size;
  613                 }
  614                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  615                 VM_OBJECT_UNLOCK(object);
  616         }
  617         return (object);
  618 }
  619 
  620 /*
  621  * SWAP_PAGER_DEALLOC() -       remove swap metadata from object
  622  *
  623  *      The swap backing for the object is destroyed.  The code is 
  624  *      designed such that we can reinstantiate it later, but this
  625  *      routine is typically called only when the entire object is
  626  *      about to be destroyed.
  627  *
  628  *      The object must be locked.
  629  */
  630 static void
  631 swap_pager_dealloc(vm_object_t object)
  632 {
  633 
  634         /*
  635          * Remove from list right away so lookups will fail if we block for
  636          * pageout completion.
  637          */
  638         if (object->handle != NULL) {
  639                 mtx_lock(&sw_alloc_mtx);
  640                 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
  641                 mtx_unlock(&sw_alloc_mtx);
  642         }
  643 
  644         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  645         vm_object_pip_wait(object, "swpdea");
  646 
  647         /*
  648          * Free all remaining metadata.  We only bother to free it from 
  649          * the swap meta data.  We do not attempt to free swapblk's still
  650          * associated with vm_page_t's for this object.  We do not care
  651          * if paging is still in progress on some objects.
  652          */
  653         swp_pager_meta_free_all(object);
  654 }
  655 
  656 /************************************************************************
  657  *                      SWAP PAGER BITMAP ROUTINES                      *
  658  ************************************************************************/
  659 
  660 /*
  661  * SWP_PAGER_GETSWAPSPACE() -   allocate raw swap space
  662  *
  663  *      Allocate swap for the requested number of pages.  The starting
  664  *      swap block number (a page index) is returned or SWAPBLK_NONE
  665  *      if the allocation failed.
  666  *
  667  *      Also has the side effect of advising that somebody made a mistake
  668  *      when they configured swap and didn't configure enough.
  669  *
  670  *      This routine may not sleep.
  671  *
  672  *      We allocate in round-robin fashion from the configured devices.
  673  */
  674 static daddr_t
  675 swp_pager_getswapspace(int npages)
  676 {
  677         daddr_t blk;
  678         struct swdevt *sp;
  679         int i;
  680 
  681         blk = SWAPBLK_NONE;
  682         mtx_lock(&sw_dev_mtx);
  683         sp = swdevhd;
  684         for (i = 0; i < nswapdev; i++) {
  685                 if (sp == NULL)
  686                         sp = TAILQ_FIRST(&swtailq);
  687                 if (!(sp->sw_flags & SW_CLOSING)) {
  688                         blk = blist_alloc(sp->sw_blist, npages);
  689                         if (blk != SWAPBLK_NONE) {
  690                                 blk += sp->sw_first;
  691                                 sp->sw_used += npages;
  692                                 swap_pager_avail -= npages;
  693                                 swp_sizecheck();
  694                                 swdevhd = TAILQ_NEXT(sp, sw_list);
  695                                 goto done;
  696                         }
  697                 }
  698                 sp = TAILQ_NEXT(sp, sw_list);
  699         }
  700         if (swap_pager_full != 2) {
  701                 printf("swap_pager_getswapspace(%d): failed\n", npages);
  702                 swap_pager_full = 2;
  703                 swap_pager_almost_full = 1;
  704         }
  705         swdevhd = NULL;
  706 done:
  707         mtx_unlock(&sw_dev_mtx);
  708         return (blk);
  709 }
  710 
  711 static int
  712 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
  713 {
  714 
  715         return (blk >= sp->sw_first && blk < sp->sw_end);
  716 }
  717         
  718 static void
  719 swp_pager_strategy(struct buf *bp)
  720 {
  721         struct swdevt *sp;
  722 
  723         mtx_lock(&sw_dev_mtx);
  724         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  725                 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
  726                         mtx_unlock(&sw_dev_mtx);
  727                         sp->sw_strategy(bp, sp);
  728                         return;
  729                 }
  730         }
  731         panic("Swapdev not found");
  732 }
  733         
  734 
  735 /*
  736  * SWP_PAGER_FREESWAPSPACE() -  free raw swap space 
  737  *
  738  *      This routine returns the specified swap blocks back to the bitmap.
  739  *
  740  *      This routine may not sleep.
  741  */
  742 static void
  743 swp_pager_freeswapspace(daddr_t blk, int npages)
  744 {
  745         struct swdevt *sp;
  746 
  747         mtx_lock(&sw_dev_mtx);
  748         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  749                 if (blk >= sp->sw_first && blk < sp->sw_end) {
  750                         sp->sw_used -= npages;
  751                         /*
  752                          * If we are attempting to stop swapping on
  753                          * this device, we don't want to mark any
  754                          * blocks free lest they be reused.  
  755                          */
  756                         if ((sp->sw_flags & SW_CLOSING) == 0) {
  757                                 blist_free(sp->sw_blist, blk - sp->sw_first,
  758                                     npages);
  759                                 swap_pager_avail += npages;
  760                                 swp_sizecheck();
  761                         }
  762                         mtx_unlock(&sw_dev_mtx);
  763                         return;
  764                 }
  765         }
  766         panic("Swapdev not found");
  767 }
  768 
  769 /*
  770  * SWAP_PAGER_FREESPACE() -     frees swap blocks associated with a page
  771  *                              range within an object.
  772  *
  773  *      This is a globally accessible routine.
  774  *
  775  *      This routine removes swapblk assignments from swap metadata.
  776  *
  777  *      The external callers of this routine typically have already destroyed 
  778  *      or renamed vm_page_t's associated with this range in the object so 
  779  *      we should be ok.
  780  */
  781 void
  782 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
  783 {
  784 
  785         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  786         swp_pager_meta_free(object, start, size);
  787 }
  788 
  789 /*
  790  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
  791  *
  792  *      Assigns swap blocks to the specified range within the object.  The 
  793  *      swap blocks are not zerod.  Any previous swap assignment is destroyed.
  794  *
  795  *      Returns 0 on success, -1 on failure.
  796  */
  797 int
  798 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
  799 {
  800         int n = 0;
  801         daddr_t blk = SWAPBLK_NONE;
  802         vm_pindex_t beg = start;        /* save start index */
  803 
  804         VM_OBJECT_LOCK(object);
  805         while (size) {
  806                 if (n == 0) {
  807                         n = BLIST_MAX_ALLOC;
  808                         while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
  809                                 n >>= 1;
  810                                 if (n == 0) {
  811                                         swp_pager_meta_free(object, beg, start - beg);
  812                                         VM_OBJECT_UNLOCK(object);
  813                                         return (-1);
  814                                 }
  815                         }
  816                 }
  817                 swp_pager_meta_build(object, start, blk);
  818                 --size;
  819                 ++start;
  820                 ++blk;
  821                 --n;
  822         }
  823         swp_pager_meta_free(object, start, n);
  824         VM_OBJECT_UNLOCK(object);
  825         return (0);
  826 }
  827 
  828 /*
  829  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
  830  *                      and destroy the source.
  831  *
  832  *      Copy any valid swapblks from the source to the destination.  In
  833  *      cases where both the source and destination have a valid swapblk,
  834  *      we keep the destination's.
  835  *
  836  *      This routine is allowed to sleep.  It may sleep allocating metadata
  837  *      indirectly through swp_pager_meta_build() or if paging is still in
  838  *      progress on the source. 
  839  *
  840  *      The source object contains no vm_page_t's (which is just as well)
  841  *
  842  *      The source object is of type OBJT_SWAP.
  843  *
  844  *      The source and destination objects must be locked.
  845  *      Both object locks may temporarily be released.
  846  */
  847 void
  848 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
  849     vm_pindex_t offset, int destroysource)
  850 {
  851         vm_pindex_t i;
  852 
  853         VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
  854         VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
  855 
  856         /*
  857          * If destroysource is set, we remove the source object from the 
  858          * swap_pager internal queue now. 
  859          */
  860         if (destroysource) {
  861                 if (srcobject->handle != NULL) {
  862                         mtx_lock(&sw_alloc_mtx);
  863                         TAILQ_REMOVE(
  864                             NOBJLIST(srcobject->handle),
  865                             srcobject,
  866                             pager_object_list
  867                         );
  868                         mtx_unlock(&sw_alloc_mtx);
  869                 }
  870         }
  871 
  872         /*
  873          * transfer source to destination.
  874          */
  875         for (i = 0; i < dstobject->size; ++i) {
  876                 daddr_t dstaddr;
  877 
  878                 /*
  879                  * Locate (without changing) the swapblk on the destination,
  880                  * unless it is invalid in which case free it silently, or
  881                  * if the destination is a resident page, in which case the
  882                  * source is thrown away.
  883                  */
  884                 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
  885 
  886                 if (dstaddr == SWAPBLK_NONE) {
  887                         /*
  888                          * Destination has no swapblk and is not resident,
  889                          * copy source.
  890                          */
  891                         daddr_t srcaddr;
  892 
  893                         srcaddr = swp_pager_meta_ctl(
  894                             srcobject, 
  895                             i + offset,
  896                             SWM_POP
  897                         );
  898 
  899                         if (srcaddr != SWAPBLK_NONE) {
  900                                 /*
  901                                  * swp_pager_meta_build() can sleep.
  902                                  */
  903                                 vm_object_pip_add(srcobject, 1);
  904                                 VM_OBJECT_UNLOCK(srcobject);
  905                                 vm_object_pip_add(dstobject, 1);
  906                                 swp_pager_meta_build(dstobject, i, srcaddr);
  907                                 vm_object_pip_wakeup(dstobject);
  908                                 VM_OBJECT_LOCK(srcobject);
  909                                 vm_object_pip_wakeup(srcobject);
  910                         }
  911                 } else {
  912                         /*
  913                          * Destination has valid swapblk or it is represented
  914                          * by a resident page.  We destroy the sourceblock.
  915                          */
  916                         
  917                         swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
  918                 }
  919         }
  920 
  921         /*
  922          * Free left over swap blocks in source.
  923          *
  924          * We have to revert the type to OBJT_DEFAULT so we do not accidently
  925          * double-remove the object from the swap queues.
  926          */
  927         if (destroysource) {
  928                 swp_pager_meta_free_all(srcobject);
  929                 /*
  930                  * Reverting the type is not necessary, the caller is going
  931                  * to destroy srcobject directly, but I'm doing it here
  932                  * for consistency since we've removed the object from its
  933                  * queues.
  934                  */
  935                 srcobject->type = OBJT_DEFAULT;
  936         }
  937 }
  938 
  939 /*
  940  * SWAP_PAGER_HASPAGE() -       determine if we have good backing store for
  941  *                              the requested page.
  942  *
  943  *      We determine whether good backing store exists for the requested
  944  *      page and return TRUE if it does, FALSE if it doesn't.
  945  *
  946  *      If TRUE, we also try to determine how much valid, contiguous backing
  947  *      store exists before and after the requested page within a reasonable
  948  *      distance.  We do not try to restrict it to the swap device stripe
  949  *      (that is handled in getpages/putpages).  It probably isn't worth
  950  *      doing here.
  951  */
  952 static boolean_t
  953 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
  954 {
  955         daddr_t blk0;
  956 
  957         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  958         /*
  959          * do we have good backing store at the requested index ?
  960          */
  961         blk0 = swp_pager_meta_ctl(object, pindex, 0);
  962 
  963         if (blk0 == SWAPBLK_NONE) {
  964                 if (before)
  965                         *before = 0;
  966                 if (after)
  967                         *after = 0;
  968                 return (FALSE);
  969         }
  970 
  971         /*
  972          * find backwards-looking contiguous good backing store
  973          */
  974         if (before != NULL) {
  975                 int i;
  976 
  977                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
  978                         daddr_t blk;
  979 
  980                         if (i > pindex)
  981                                 break;
  982                         blk = swp_pager_meta_ctl(object, pindex - i, 0);
  983                         if (blk != blk0 - i)
  984                                 break;
  985                 }
  986                 *before = (i - 1);
  987         }
  988 
  989         /*
  990          * find forward-looking contiguous good backing store
  991          */
  992         if (after != NULL) {
  993                 int i;
  994 
  995                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
  996                         daddr_t blk;
  997 
  998                         blk = swp_pager_meta_ctl(object, pindex + i, 0);
  999                         if (blk != blk0 + i)
 1000                                 break;
 1001                 }
 1002                 *after = (i - 1);
 1003         }
 1004         return (TRUE);
 1005 }
 1006 
 1007 /*
 1008  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
 1009  *
 1010  *      This removes any associated swap backing store, whether valid or
 1011  *      not, from the page.  
 1012  *
 1013  *      This routine is typically called when a page is made dirty, at
 1014  *      which point any associated swap can be freed.  MADV_FREE also
 1015  *      calls us in a special-case situation
 1016  *
 1017  *      NOTE!!!  If the page is clean and the swap was valid, the caller
 1018  *      should make the page dirty before calling this routine.  This routine
 1019  *      does NOT change the m->dirty status of the page.  Also: MADV_FREE
 1020  *      depends on it.
 1021  *
 1022  *      This routine may not sleep.
 1023  */
 1024 static void
 1025 swap_pager_unswapped(vm_page_t m)
 1026 {
 1027 
 1028         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 1029         swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
 1030 }
 1031 
 1032 /*
 1033  * SWAP_PAGER_GETPAGES() - bring pages in from swap
 1034  *
 1035  *      Attempt to retrieve (m, count) pages from backing store, but make
 1036  *      sure we retrieve at least m[reqpage].  We try to load in as large
 1037  *      a chunk surrounding m[reqpage] as is contiguous in swap and which
 1038  *      belongs to the same object.
 1039  *
 1040  *      The code is designed for asynchronous operation and 
 1041  *      immediate-notification of 'reqpage' but tends not to be
 1042  *      used that way.  Please do not optimize-out this algorithmic
 1043  *      feature, I intend to improve on it in the future.
 1044  *
 1045  *      The parent has a single vm_object_pip_add() reference prior to
 1046  *      calling us and we should return with the same.
 1047  *
 1048  *      The parent has BUSY'd the pages.  We should return with 'm'
 1049  *      left busy, but the others adjusted.
 1050  */
 1051 static int
 1052 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
 1053 {
 1054         struct buf *bp;
 1055         vm_page_t mreq;
 1056         int i;
 1057         int j;
 1058         daddr_t blk;
 1059 
 1060         mreq = m[reqpage];
 1061 
 1062         KASSERT(mreq->object == object,
 1063             ("swap_pager_getpages: object mismatch %p/%p",
 1064             object, mreq->object));
 1065 
 1066         /*
 1067          * Calculate range to retrieve.  The pages have already been assigned
 1068          * their swapblks.  We require a *contiguous* range but we know it to
 1069          * not span devices.   If we do not supply it, bad things
 1070          * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the 
 1071          * loops are set up such that the case(s) are handled implicitly.
 1072          *
 1073          * The swp_*() calls must be made with the object locked.
 1074          */
 1075         blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
 1076 
 1077         for (i = reqpage - 1; i >= 0; --i) {
 1078                 daddr_t iblk;
 1079 
 1080                 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
 1081                 if (blk != iblk + (reqpage - i))
 1082                         break;
 1083         }
 1084         ++i;
 1085 
 1086         for (j = reqpage + 1; j < count; ++j) {
 1087                 daddr_t jblk;
 1088 
 1089                 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
 1090                 if (blk != jblk - (j - reqpage))
 1091                         break;
 1092         }
 1093 
 1094         /*
 1095          * free pages outside our collection range.   Note: we never free
 1096          * mreq, it must remain busy throughout.
 1097          */
 1098         if (0 < i || j < count) {
 1099                 int k;
 1100 
 1101                 vm_page_lock_queues();
 1102                 for (k = 0; k < i; ++k)
 1103                         swp_pager_free_nrpage(m[k]);
 1104                 for (k = j; k < count; ++k)
 1105                         swp_pager_free_nrpage(m[k]);
 1106                 vm_page_unlock_queues();
 1107         }
 1108 
 1109         /*
 1110          * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq 
 1111          * still busy, but the others unbusied.
 1112          */
 1113         if (blk == SWAPBLK_NONE)
 1114                 return (VM_PAGER_FAIL);
 1115 
 1116         /*
 1117          * Getpbuf() can sleep.
 1118          */
 1119         VM_OBJECT_UNLOCK(object);
 1120         /*
 1121          * Get a swap buffer header to perform the IO
 1122          */
 1123         bp = getpbuf(&nsw_rcount);
 1124         bp->b_flags |= B_PAGING;
 1125 
 1126         /*
 1127          * map our page(s) into kva for input
 1128          */
 1129         pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
 1130 
 1131         bp->b_iocmd = BIO_READ;
 1132         bp->b_iodone = swp_pager_async_iodone;
 1133         bp->b_rcred = crhold(thread0.td_ucred);
 1134         bp->b_wcred = crhold(thread0.td_ucred);
 1135         bp->b_blkno = blk - (reqpage - i);
 1136         bp->b_bcount = PAGE_SIZE * (j - i);
 1137         bp->b_bufsize = PAGE_SIZE * (j - i);
 1138         bp->b_pager.pg_reqpage = reqpage - i;
 1139 
 1140         VM_OBJECT_LOCK(object);
 1141         {
 1142                 int k;
 1143 
 1144                 for (k = i; k < j; ++k) {
 1145                         bp->b_pages[k - i] = m[k];
 1146                         m[k]->oflags |= VPO_SWAPINPROG;
 1147                 }
 1148         }
 1149         bp->b_npages = j - i;
 1150 
 1151         PCPU_INC(cnt.v_swapin);
 1152         PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
 1153 
 1154         /*
 1155          * We still hold the lock on mreq, and our automatic completion routine
 1156          * does not remove it.
 1157          */
 1158         vm_object_pip_add(object, bp->b_npages);
 1159         VM_OBJECT_UNLOCK(object);
 1160 
 1161         /*
 1162          * perform the I/O.  NOTE!!!  bp cannot be considered valid after
 1163          * this point because we automatically release it on completion.
 1164          * Instead, we look at the one page we are interested in which we
 1165          * still hold a lock on even through the I/O completion.
 1166          *
 1167          * The other pages in our m[] array are also released on completion,
 1168          * so we cannot assume they are valid anymore either.
 1169          *
 1170          * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1171          */
 1172         BUF_KERNPROC(bp);
 1173         swp_pager_strategy(bp);
 1174 
 1175         /*
 1176          * wait for the page we want to complete.  VPO_SWAPINPROG is always
 1177          * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
 1178          * is set in the meta-data.
 1179          */
 1180         VM_OBJECT_LOCK(object);
 1181         while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
 1182                 mreq->oflags |= VPO_WANTED;
 1183                 PCPU_INC(cnt.v_intrans);
 1184                 if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) {
 1185                         printf(
 1186 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
 1187                             bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
 1188                 }
 1189         }
 1190 
 1191         /*
 1192          * mreq is left busied after completion, but all the other pages
 1193          * are freed.  If we had an unrecoverable read error the page will
 1194          * not be valid.
 1195          */
 1196         if (mreq->valid != VM_PAGE_BITS_ALL) {
 1197                 return (VM_PAGER_ERROR);
 1198         } else {
 1199                 return (VM_PAGER_OK);
 1200         }
 1201 
 1202         /*
 1203          * A final note: in a low swap situation, we cannot deallocate swap
 1204          * and mark a page dirty here because the caller is likely to mark
 1205          * the page clean when we return, causing the page to possibly revert 
 1206          * to all-zero's later.
 1207          */
 1208 }
 1209 
 1210 /*
 1211  *      swap_pager_putpages: 
 1212  *
 1213  *      Assign swap (if necessary) and initiate I/O on the specified pages.
 1214  *
 1215  *      We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
 1216  *      are automatically converted to SWAP objects.
 1217  *
 1218  *      In a low memory situation we may block in VOP_STRATEGY(), but the new 
 1219  *      vm_page reservation system coupled with properly written VFS devices 
 1220  *      should ensure that no low-memory deadlock occurs.  This is an area
 1221  *      which needs work.
 1222  *
 1223  *      The parent has N vm_object_pip_add() references prior to
 1224  *      calling us and will remove references for rtvals[] that are
 1225  *      not set to VM_PAGER_PEND.  We need to remove the rest on I/O
 1226  *      completion.
 1227  *
 1228  *      The parent has soft-busy'd the pages it passes us and will unbusy
 1229  *      those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
 1230  *      We need to unbusy the rest on I/O completion.
 1231  */
 1232 void
 1233 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 1234     boolean_t sync, int *rtvals)
 1235 {
 1236         int i;
 1237         int n = 0;
 1238 
 1239         if (count && m[0]->object != object) {
 1240                 panic("swap_pager_putpages: object mismatch %p/%p", 
 1241                     object, 
 1242                     m[0]->object
 1243                 );
 1244         }
 1245 
 1246         /*
 1247          * Step 1
 1248          *
 1249          * Turn object into OBJT_SWAP
 1250          * check for bogus sysops
 1251          * force sync if not pageout process
 1252          */
 1253         if (object->type != OBJT_SWAP)
 1254                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
 1255         VM_OBJECT_UNLOCK(object);
 1256 
 1257         if (curproc != pageproc)
 1258                 sync = TRUE;
 1259 
 1260         /*
 1261          * Step 2
 1262          *
 1263          * Update nsw parameters from swap_async_max sysctl values.  
 1264          * Do not let the sysop crash the machine with bogus numbers.
 1265          */
 1266         mtx_lock(&pbuf_mtx);
 1267         if (swap_async_max != nsw_wcount_async_max) {
 1268                 int n;
 1269 
 1270                 /*
 1271                  * limit range
 1272                  */
 1273                 if ((n = swap_async_max) > nswbuf / 2)
 1274                         n = nswbuf / 2;
 1275                 if (n < 1)
 1276                         n = 1;
 1277                 swap_async_max = n;
 1278 
 1279                 /*
 1280                  * Adjust difference ( if possible ).  If the current async
 1281                  * count is too low, we may not be able to make the adjustment
 1282                  * at this time.
 1283                  */
 1284                 n -= nsw_wcount_async_max;
 1285                 if (nsw_wcount_async + n >= 0) {
 1286                         nsw_wcount_async += n;
 1287                         nsw_wcount_async_max += n;
 1288                         wakeup(&nsw_wcount_async);
 1289                 }
 1290         }
 1291         mtx_unlock(&pbuf_mtx);
 1292 
 1293         /*
 1294          * Step 3
 1295          *
 1296          * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
 1297          * The page is left dirty until the pageout operation completes
 1298          * successfully.
 1299          */
 1300         for (i = 0; i < count; i += n) {
 1301                 int j;
 1302                 struct buf *bp;
 1303                 daddr_t blk;
 1304 
 1305                 /*
 1306                  * Maximum I/O size is limited by a number of factors.
 1307                  */
 1308                 n = min(BLIST_MAX_ALLOC, count - i);
 1309                 n = min(n, nsw_cluster_max);
 1310 
 1311                 /*
 1312                  * Get biggest block of swap we can.  If we fail, fall
 1313                  * back and try to allocate a smaller block.  Don't go
 1314                  * overboard trying to allocate space if it would overly
 1315                  * fragment swap.
 1316                  */
 1317                 while (
 1318                     (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
 1319                     n > 4
 1320                 ) {
 1321                         n >>= 1;
 1322                 }
 1323                 if (blk == SWAPBLK_NONE) {
 1324                         for (j = 0; j < n; ++j)
 1325                                 rtvals[i+j] = VM_PAGER_FAIL;
 1326                         continue;
 1327                 }
 1328 
 1329                 /*
 1330                  * All I/O parameters have been satisfied, build the I/O
 1331                  * request and assign the swap space.
 1332                  */
 1333                 if (sync == TRUE) {
 1334                         bp = getpbuf(&nsw_wcount_sync);
 1335                 } else {
 1336                         bp = getpbuf(&nsw_wcount_async);
 1337                         bp->b_flags = B_ASYNC;
 1338                 }
 1339                 bp->b_flags |= B_PAGING;
 1340                 bp->b_iocmd = BIO_WRITE;
 1341 
 1342                 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
 1343 
 1344                 bp->b_rcred = crhold(thread0.td_ucred);
 1345                 bp->b_wcred = crhold(thread0.td_ucred);
 1346                 bp->b_bcount = PAGE_SIZE * n;
 1347                 bp->b_bufsize = PAGE_SIZE * n;
 1348                 bp->b_blkno = blk;
 1349 
 1350                 VM_OBJECT_LOCK(object);
 1351                 for (j = 0; j < n; ++j) {
 1352                         vm_page_t mreq = m[i+j];
 1353 
 1354                         swp_pager_meta_build(
 1355                             mreq->object, 
 1356                             mreq->pindex,
 1357                             blk + j
 1358                         );
 1359                         vm_page_dirty(mreq);
 1360                         rtvals[i+j] = VM_PAGER_OK;
 1361 
 1362                         mreq->oflags |= VPO_SWAPINPROG;
 1363                         bp->b_pages[j] = mreq;
 1364                 }
 1365                 VM_OBJECT_UNLOCK(object);
 1366                 bp->b_npages = n;
 1367                 /*
 1368                  * Must set dirty range for NFS to work.
 1369                  */
 1370                 bp->b_dirtyoff = 0;
 1371                 bp->b_dirtyend = bp->b_bcount;
 1372 
 1373                 PCPU_INC(cnt.v_swapout);
 1374                 PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
 1375 
 1376                 /*
 1377                  * asynchronous
 1378                  *
 1379                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1380                  */
 1381                 if (sync == FALSE) {
 1382                         bp->b_iodone = swp_pager_async_iodone;
 1383                         BUF_KERNPROC(bp);
 1384                         swp_pager_strategy(bp);
 1385 
 1386                         for (j = 0; j < n; ++j)
 1387                                 rtvals[i+j] = VM_PAGER_PEND;
 1388                         /* restart outter loop */
 1389                         continue;
 1390                 }
 1391 
 1392                 /*
 1393                  * synchronous
 1394                  *
 1395                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1396                  */
 1397                 bp->b_iodone = bdone;
 1398                 swp_pager_strategy(bp);
 1399 
 1400                 /*
 1401                  * Wait for the sync I/O to complete, then update rtvals.
 1402                  * We just set the rtvals[] to VM_PAGER_PEND so we can call
 1403                  * our async completion routine at the end, thus avoiding a
 1404                  * double-free.
 1405                  */
 1406                 bwait(bp, PVM, "swwrt");
 1407                 for (j = 0; j < n; ++j)
 1408                         rtvals[i+j] = VM_PAGER_PEND;
 1409                 /*
 1410                  * Now that we are through with the bp, we can call the
 1411                  * normal async completion, which frees everything up.
 1412                  */
 1413                 swp_pager_async_iodone(bp);
 1414         }
 1415         VM_OBJECT_LOCK(object);
 1416 }
 1417 
 1418 /*
 1419  *      swp_pager_async_iodone:
 1420  *
 1421  *      Completion routine for asynchronous reads and writes from/to swap.
 1422  *      Also called manually by synchronous code to finish up a bp.
 1423  *
 1424  *      For READ operations, the pages are VPO_BUSY'd.  For WRITE operations, 
 1425  *      the pages are vm_page_t->busy'd.  For READ operations, we VPO_BUSY 
 1426  *      unbusy all pages except the 'main' request page.  For WRITE 
 1427  *      operations, we vm_page_t->busy'd unbusy all pages ( we can do this 
 1428  *      because we marked them all VM_PAGER_PEND on return from putpages ).
 1429  *
 1430  *      This routine may not sleep.
 1431  */
 1432 static void
 1433 swp_pager_async_iodone(struct buf *bp)
 1434 {
 1435         int i;
 1436         vm_object_t object = NULL;
 1437 
 1438         /*
 1439          * report error
 1440          */
 1441         if (bp->b_ioflags & BIO_ERROR) {
 1442                 printf(
 1443                     "swap_pager: I/O error - %s failed; blkno %ld,"
 1444                         "size %ld, error %d\n",
 1445                     ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
 1446                     (long)bp->b_blkno, 
 1447                     (long)bp->b_bcount,
 1448                     bp->b_error
 1449                 );
 1450         }
 1451 
 1452         /*
 1453          * remove the mapping for kernel virtual
 1454          */
 1455         pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
 1456 
 1457         if (bp->b_npages) {
 1458                 object = bp->b_pages[0]->object;
 1459                 VM_OBJECT_LOCK(object);
 1460         }
 1461         vm_page_lock_queues();
 1462         /*
 1463          * cleanup pages.  If an error occurs writing to swap, we are in
 1464          * very serious trouble.  If it happens to be a disk error, though,
 1465          * we may be able to recover by reassigning the swap later on.  So
 1466          * in this case we remove the m->swapblk assignment for the page 
 1467          * but do not free it in the rlist.  The errornous block(s) are thus
 1468          * never reallocated as swap.  Redirty the page and continue.
 1469          */
 1470         for (i = 0; i < bp->b_npages; ++i) {
 1471                 vm_page_t m = bp->b_pages[i];
 1472 
 1473                 m->oflags &= ~VPO_SWAPINPROG;
 1474 
 1475                 if (bp->b_ioflags & BIO_ERROR) {
 1476                         /*
 1477                          * If an error occurs I'd love to throw the swapblk
 1478                          * away without freeing it back to swapspace, so it
 1479                          * can never be used again.  But I can't from an 
 1480                          * interrupt.
 1481                          */
 1482                         if (bp->b_iocmd == BIO_READ) {
 1483                                 /*
 1484                                  * When reading, reqpage needs to stay
 1485                                  * locked for the parent, but all other
 1486                                  * pages can be freed.  We still want to
 1487                                  * wakeup the parent waiting on the page,
 1488                                  * though.  ( also: pg_reqpage can be -1 and 
 1489                                  * not match anything ).
 1490                                  *
 1491                                  * We have to wake specifically requested pages
 1492                                  * up too because we cleared VPO_SWAPINPROG and
 1493                                  * someone may be waiting for that.
 1494                                  *
 1495                                  * NOTE: for reads, m->dirty will probably
 1496                                  * be overridden by the original caller of
 1497                                  * getpages so don't play cute tricks here.
 1498                                  */
 1499                                 m->valid = 0;
 1500                                 if (i != bp->b_pager.pg_reqpage)
 1501                                         swp_pager_free_nrpage(m);
 1502                                 else
 1503                                         vm_page_flash(m);
 1504                                 /*
 1505                                  * If i == bp->b_pager.pg_reqpage, do not wake 
 1506                                  * the page up.  The caller needs to.
 1507                                  */
 1508                         } else {
 1509                                 /*
 1510                                  * If a write error occurs, reactivate page
 1511                                  * so it doesn't clog the inactive list,
 1512                                  * then finish the I/O.
 1513                                  */
 1514                                 vm_page_dirty(m);
 1515                                 vm_page_activate(m);
 1516                                 vm_page_io_finish(m);
 1517                         }
 1518                 } else if (bp->b_iocmd == BIO_READ) {
 1519                         /*
 1520                          * NOTE: for reads, m->dirty will probably be 
 1521                          * overridden by the original caller of getpages so
 1522                          * we cannot set them in order to free the underlying
 1523                          * swap in a low-swap situation.  I don't think we'd
 1524                          * want to do that anyway, but it was an optimization
 1525                          * that existed in the old swapper for a time before
 1526                          * it got ripped out due to precisely this problem.
 1527                          *
 1528                          * If not the requested page then deactivate it.
 1529                          *
 1530                          * Note that the requested page, reqpage, is left
 1531                          * busied, but we still have to wake it up.  The
 1532                          * other pages are released (unbusied) by 
 1533                          * vm_page_wakeup().
 1534                          */
 1535                         KASSERT(!pmap_page_is_mapped(m),
 1536                             ("swp_pager_async_iodone: page %p is mapped", m));
 1537                         m->valid = VM_PAGE_BITS_ALL;
 1538                         KASSERT(m->dirty == 0,
 1539                             ("swp_pager_async_iodone: page %p is dirty", m));
 1540 
 1541                         /*
 1542                          * We have to wake specifically requested pages
 1543                          * up too because we cleared VPO_SWAPINPROG and
 1544                          * could be waiting for it in getpages.  However,
 1545                          * be sure to not unbusy getpages specifically
 1546                          * requested page - getpages expects it to be 
 1547                          * left busy.
 1548                          */
 1549                         if (i != bp->b_pager.pg_reqpage) {
 1550                                 vm_page_deactivate(m);
 1551                                 vm_page_wakeup(m);
 1552                         } else {
 1553                                 vm_page_flash(m);
 1554                         }
 1555                 } else {
 1556                         /*
 1557                          * For write success, clear the dirty
 1558                          * status, then finish the I/O ( which decrements the 
 1559                          * busy count and possibly wakes waiter's up ).
 1560                          */
 1561                         KASSERT((m->flags & PG_WRITEABLE) == 0,
 1562                             ("swp_pager_async_iodone: page %p is not write"
 1563                             " protected", m));
 1564                         vm_page_undirty(m);
 1565                         vm_page_io_finish(m);
 1566                         if (vm_page_count_severe())
 1567                                 vm_page_try_to_cache(m);
 1568                 }
 1569         }
 1570         vm_page_unlock_queues();
 1571 
 1572         /*
 1573          * adjust pip.  NOTE: the original parent may still have its own
 1574          * pip refs on the object.
 1575          */
 1576         if (object != NULL) {
 1577                 vm_object_pip_wakeupn(object, bp->b_npages);
 1578                 VM_OBJECT_UNLOCK(object);
 1579         }
 1580 
 1581         /* 
 1582          * swapdev_strategy() manually sets b_vp and b_bufobj before calling 
 1583          * bstrategy(). Set them back to NULL now we're done with it, or we'll
 1584          * trigger a KASSERT in relpbuf().
 1585          */
 1586         if (bp->b_vp) {
 1587                     bp->b_vp = NULL;
 1588                     bp->b_bufobj = NULL;
 1589         }
 1590         /*
 1591          * release the physical I/O buffer
 1592          */
 1593         relpbuf(
 1594             bp, 
 1595             ((bp->b_iocmd == BIO_READ) ? &nsw_rcount : 
 1596                 ((bp->b_flags & B_ASYNC) ? 
 1597                     &nsw_wcount_async : 
 1598                     &nsw_wcount_sync
 1599                 )
 1600             )
 1601         );
 1602 }
 1603 
 1604 /*
 1605  *      swap_pager_isswapped:
 1606  *
 1607  *      Return 1 if at least one page in the given object is paged
 1608  *      out to the given swap device.
 1609  *
 1610  *      This routine may not sleep.
 1611  */
 1612 int
 1613 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
 1614 {
 1615         daddr_t index = 0;
 1616         int bcount;
 1617         int i;
 1618 
 1619         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1620         if (object->type != OBJT_SWAP)
 1621                 return (0);
 1622 
 1623         mtx_lock(&swhash_mtx);
 1624         for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
 1625                 struct swblock *swap;
 1626 
 1627                 if ((swap = *swp_pager_hash(object, index)) != NULL) {
 1628                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1629                                 if (swp_pager_isondev(swap->swb_pages[i], sp)) {
 1630                                         mtx_unlock(&swhash_mtx);
 1631                                         return (1);
 1632                                 }
 1633                         }
 1634                 }
 1635                 index += SWAP_META_PAGES;
 1636         }
 1637         mtx_unlock(&swhash_mtx);
 1638         return (0);
 1639 }
 1640 
 1641 /*
 1642  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
 1643  *
 1644  *      This routine dissociates the page at the given index within a
 1645  *      swap block from its backing store, paging it in if necessary.
 1646  *      If the page is paged in, it is placed in the inactive queue,
 1647  *      since it had its backing store ripped out from under it.
 1648  *      We also attempt to swap in all other pages in the swap block,
 1649  *      we only guarantee that the one at the specified index is
 1650  *      paged in.
 1651  *
 1652  *      XXX - The code to page the whole block in doesn't work, so we
 1653  *            revert to the one-by-one behavior for now.  Sigh.
 1654  */
 1655 static inline void
 1656 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
 1657 {
 1658         vm_page_t m;
 1659 
 1660         vm_object_pip_add(object, 1);
 1661         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
 1662         if (m->valid == VM_PAGE_BITS_ALL) {
 1663                 vm_object_pip_subtract(object, 1);
 1664                 vm_page_lock_queues();
 1665                 vm_page_activate(m);
 1666                 vm_page_dirty(m);
 1667                 vm_page_unlock_queues();
 1668                 vm_page_wakeup(m);
 1669                 vm_pager_page_unswapped(m);
 1670                 return;
 1671         }
 1672 
 1673         if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
 1674                 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
 1675         vm_object_pip_subtract(object, 1);
 1676         vm_page_lock_queues();
 1677         vm_page_dirty(m);
 1678         vm_page_dontneed(m);
 1679         vm_page_unlock_queues();
 1680         vm_page_wakeup(m);
 1681         vm_pager_page_unswapped(m);
 1682 }
 1683 
 1684 /*
 1685  *      swap_pager_swapoff:
 1686  *
 1687  *      Page in all of the pages that have been paged out to the
 1688  *      given device.  The corresponding blocks in the bitmap must be
 1689  *      marked as allocated and the device must be flagged SW_CLOSING.
 1690  *      There may be no processes swapped out to the device.
 1691  *
 1692  *      This routine may block.
 1693  */
 1694 static void
 1695 swap_pager_swapoff(struct swdevt *sp)
 1696 {
 1697         struct swblock *swap;
 1698         int i, j, retries;
 1699 
 1700         GIANT_REQUIRED;
 1701 
 1702         retries = 0;
 1703 full_rescan:
 1704         mtx_lock(&swhash_mtx);
 1705         for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
 1706 restart:
 1707                 for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
 1708                         vm_object_t object = swap->swb_object;
 1709                         vm_pindex_t pindex = swap->swb_index;
 1710                         for (j = 0; j < SWAP_META_PAGES; ++j) {
 1711                                 if (swp_pager_isondev(swap->swb_pages[j], sp)) {
 1712                                         /* avoid deadlock */
 1713                                         if (!VM_OBJECT_TRYLOCK(object)) {
 1714                                                 break;
 1715                                         } else {
 1716                                                 mtx_unlock(&swhash_mtx);
 1717                                                 swp_pager_force_pagein(object,
 1718                                                     pindex + j);
 1719                                                 VM_OBJECT_UNLOCK(object);
 1720                                                 mtx_lock(&swhash_mtx);
 1721                                                 goto restart;
 1722                                         }
 1723                                 }
 1724                         }
 1725                 }
 1726         }
 1727         mtx_unlock(&swhash_mtx);
 1728         if (sp->sw_used) {
 1729                 /*
 1730                  * Objects may be locked or paging to the device being
 1731                  * removed, so we will miss their pages and need to
 1732                  * make another pass.  We have marked this device as
 1733                  * SW_CLOSING, so the activity should finish soon.
 1734                  */
 1735                 retries++;
 1736                 if (retries > 100) {
 1737                         panic("swapoff: failed to locate %d swap blocks",
 1738                             sp->sw_used);
 1739                 }
 1740                 pause("swpoff", hz / 20);
 1741                 goto full_rescan;
 1742         }
 1743 }
 1744 
 1745 /************************************************************************
 1746  *                              SWAP META DATA                          *
 1747  ************************************************************************
 1748  *
 1749  *      These routines manipulate the swap metadata stored in the 
 1750  *      OBJT_SWAP object.
 1751  *
 1752  *      Swap metadata is implemented with a global hash and not directly
 1753  *      linked into the object.  Instead the object simply contains
 1754  *      appropriate tracking counters.
 1755  */
 1756 
 1757 /*
 1758  * SWP_PAGER_META_BUILD() -     add swap block to swap meta data for object
 1759  *
 1760  *      We first convert the object to a swap object if it is a default
 1761  *      object.
 1762  *
 1763  *      The specified swapblk is added to the object's swap metadata.  If
 1764  *      the swapblk is not valid, it is freed instead.  Any previously
 1765  *      assigned swapblk is freed.
 1766  */
 1767 static void
 1768 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
 1769 {
 1770         struct swblock *swap;
 1771         struct swblock **pswap;
 1772         int idx;
 1773 
 1774         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1775         /*
 1776          * Convert default object to swap object if necessary
 1777          */
 1778         if (object->type != OBJT_SWAP) {
 1779                 object->type = OBJT_SWAP;
 1780                 object->un_pager.swp.swp_bcount = 0;
 1781 
 1782                 if (object->handle != NULL) {
 1783                         mtx_lock(&sw_alloc_mtx);
 1784                         TAILQ_INSERT_TAIL(
 1785                             NOBJLIST(object->handle),
 1786                             object, 
 1787                             pager_object_list
 1788                         );
 1789                         mtx_unlock(&sw_alloc_mtx);
 1790                 }
 1791         }
 1792         
 1793         /*
 1794          * Locate hash entry.  If not found create, but if we aren't adding
 1795          * anything just return.  If we run out of space in the map we wait
 1796          * and, since the hash table may have changed, retry.
 1797          */
 1798 retry:
 1799         mtx_lock(&swhash_mtx);
 1800         pswap = swp_pager_hash(object, pindex);
 1801 
 1802         if ((swap = *pswap) == NULL) {
 1803                 int i;
 1804 
 1805                 if (swapblk == SWAPBLK_NONE)
 1806                         goto done;
 1807 
 1808                 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
 1809                 if (swap == NULL) {
 1810                         mtx_unlock(&swhash_mtx);
 1811                         VM_OBJECT_UNLOCK(object);
 1812                         if (uma_zone_exhausted(swap_zone)) {
 1813                                 printf("swap zone exhausted, increase kern.maxswzone\n");
 1814                                 vm_pageout_oom(VM_OOM_SWAPZ);
 1815                                 pause("swzonex", 10);
 1816                         } else
 1817                                 VM_WAIT;
 1818                         VM_OBJECT_LOCK(object);
 1819                         goto retry;
 1820                 }
 1821 
 1822                 swap->swb_hnext = NULL;
 1823                 swap->swb_object = object;
 1824                 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
 1825                 swap->swb_count = 0;
 1826 
 1827                 ++object->un_pager.swp.swp_bcount;
 1828 
 1829                 for (i = 0; i < SWAP_META_PAGES; ++i)
 1830                         swap->swb_pages[i] = SWAPBLK_NONE;
 1831         }
 1832 
 1833         /*
 1834          * Delete prior contents of metadata
 1835          */
 1836         idx = pindex & SWAP_META_MASK;
 1837 
 1838         if (swap->swb_pages[idx] != SWAPBLK_NONE) {
 1839                 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
 1840                 --swap->swb_count;
 1841         }
 1842 
 1843         /*
 1844          * Enter block into metadata
 1845          */
 1846         swap->swb_pages[idx] = swapblk;
 1847         if (swapblk != SWAPBLK_NONE)
 1848                 ++swap->swb_count;
 1849 done:
 1850         mtx_unlock(&swhash_mtx);
 1851 }
 1852 
 1853 /*
 1854  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
 1855  *
 1856  *      The requested range of blocks is freed, with any associated swap 
 1857  *      returned to the swap bitmap.
 1858  *
 1859  *      This routine will free swap metadata structures as they are cleaned 
 1860  *      out.  This routine does *NOT* operate on swap metadata associated
 1861  *      with resident pages.
 1862  */
 1863 static void
 1864 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
 1865 {
 1866 
 1867         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1868         if (object->type != OBJT_SWAP)
 1869                 return;
 1870 
 1871         while (count > 0) {
 1872                 struct swblock **pswap;
 1873                 struct swblock *swap;
 1874 
 1875                 mtx_lock(&swhash_mtx);
 1876                 pswap = swp_pager_hash(object, index);
 1877 
 1878                 if ((swap = *pswap) != NULL) {
 1879                         daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
 1880 
 1881                         if (v != SWAPBLK_NONE) {
 1882                                 swp_pager_freeswapspace(v, 1);
 1883                                 swap->swb_pages[index & SWAP_META_MASK] =
 1884                                         SWAPBLK_NONE;
 1885                                 if (--swap->swb_count == 0) {
 1886                                         *pswap = swap->swb_hnext;
 1887                                         uma_zfree(swap_zone, swap);
 1888                                         --object->un_pager.swp.swp_bcount;
 1889                                 }
 1890                         }
 1891                         --count;
 1892                         ++index;
 1893                 } else {
 1894                         int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
 1895                         count -= n;
 1896                         index += n;
 1897                 }
 1898                 mtx_unlock(&swhash_mtx);
 1899         }
 1900 }
 1901 
 1902 /*
 1903  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
 1904  *
 1905  *      This routine locates and destroys all swap metadata associated with
 1906  *      an object.
 1907  */
 1908 static void
 1909 swp_pager_meta_free_all(vm_object_t object)
 1910 {
 1911         daddr_t index = 0;
 1912 
 1913         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1914         if (object->type != OBJT_SWAP)
 1915                 return;
 1916 
 1917         while (object->un_pager.swp.swp_bcount) {
 1918                 struct swblock **pswap;
 1919                 struct swblock *swap;
 1920 
 1921                 mtx_lock(&swhash_mtx);
 1922                 pswap = swp_pager_hash(object, index);
 1923                 if ((swap = *pswap) != NULL) {
 1924                         int i;
 1925 
 1926                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1927                                 daddr_t v = swap->swb_pages[i];
 1928                                 if (v != SWAPBLK_NONE) {
 1929                                         --swap->swb_count;
 1930                                         swp_pager_freeswapspace(v, 1);
 1931                                 }
 1932                         }
 1933                         if (swap->swb_count != 0)
 1934                                 panic("swap_pager_meta_free_all: swb_count != 0");
 1935                         *pswap = swap->swb_hnext;
 1936                         uma_zfree(swap_zone, swap);
 1937                         --object->un_pager.swp.swp_bcount;
 1938                 }
 1939                 mtx_unlock(&swhash_mtx);
 1940                 index += SWAP_META_PAGES;
 1941         }
 1942 }
 1943 
 1944 /*
 1945  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
 1946  *
 1947  *      This routine is capable of looking up, popping, or freeing
 1948  *      swapblk assignments in the swap meta data or in the vm_page_t.
 1949  *      The routine typically returns the swapblk being looked-up, or popped,
 1950  *      or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
 1951  *      was invalid.  This routine will automatically free any invalid 
 1952  *      meta-data swapblks.
 1953  *
 1954  *      It is not possible to store invalid swapblks in the swap meta data
 1955  *      (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
 1956  *
 1957  *      When acting on a busy resident page and paging is in progress, we 
 1958  *      have to wait until paging is complete but otherwise can act on the 
 1959  *      busy page.
 1960  *
 1961  *      SWM_FREE        remove and free swap block from metadata
 1962  *      SWM_POP         remove from meta data but do not free.. pop it out
 1963  */
 1964 static daddr_t
 1965 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
 1966 {
 1967         struct swblock **pswap;
 1968         struct swblock *swap;
 1969         daddr_t r1;
 1970         int idx;
 1971 
 1972         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1973         /*
 1974          * The meta data only exists of the object is OBJT_SWAP 
 1975          * and even then might not be allocated yet.
 1976          */
 1977         if (object->type != OBJT_SWAP)
 1978                 return (SWAPBLK_NONE);
 1979 
 1980         r1 = SWAPBLK_NONE;
 1981         mtx_lock(&swhash_mtx);
 1982         pswap = swp_pager_hash(object, pindex);
 1983 
 1984         if ((swap = *pswap) != NULL) {
 1985                 idx = pindex & SWAP_META_MASK;
 1986                 r1 = swap->swb_pages[idx];
 1987 
 1988                 if (r1 != SWAPBLK_NONE) {
 1989                         if (flags & SWM_FREE) {
 1990                                 swp_pager_freeswapspace(r1, 1);
 1991                                 r1 = SWAPBLK_NONE;
 1992                         }
 1993                         if (flags & (SWM_FREE|SWM_POP)) {
 1994                                 swap->swb_pages[idx] = SWAPBLK_NONE;
 1995                                 if (--swap->swb_count == 0) {
 1996                                         *pswap = swap->swb_hnext;
 1997                                         uma_zfree(swap_zone, swap);
 1998                                         --object->un_pager.swp.swp_bcount;
 1999                                 }
 2000                         } 
 2001                 }
 2002         }
 2003         mtx_unlock(&swhash_mtx);
 2004         return (r1);
 2005 }
 2006 
 2007 /*
 2008  * System call swapon(name) enables swapping on device name,
 2009  * which must be in the swdevsw.  Return EBUSY
 2010  * if already swapping on this device.
 2011  */
 2012 #ifndef _SYS_SYSPROTO_H_
 2013 struct swapon_args {
 2014         char *name;
 2015 };
 2016 #endif
 2017 
 2018 /* 
 2019  * MPSAFE
 2020  */
 2021 /* ARGSUSED */
 2022 int
 2023 swapon(struct thread *td, struct swapon_args *uap)
 2024 {
 2025         struct vattr attr;
 2026         struct vnode *vp;
 2027         struct nameidata nd;
 2028         int error;
 2029 
 2030         error = priv_check(td, PRIV_SWAPON);
 2031         if (error)
 2032                 return (error);
 2033 
 2034         mtx_lock(&Giant);
 2035         while (swdev_syscall_active)
 2036             tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
 2037         swdev_syscall_active = 1;
 2038 
 2039         /*
 2040          * Swap metadata may not fit in the KVM if we have physical
 2041          * memory of >1GB.
 2042          */
 2043         if (swap_zone == NULL) {
 2044                 error = ENOMEM;
 2045                 goto done;
 2046         }
 2047 
 2048         NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
 2049             uap->name, td);
 2050         error = namei(&nd);
 2051         if (error)
 2052                 goto done;
 2053 
 2054         NDFREE(&nd, NDF_ONLY_PNBUF);
 2055         vp = nd.ni_vp;
 2056 
 2057         if (vn_isdisk(vp, &error)) {
 2058                 error = swapongeom(td, vp);
 2059         } else if (vp->v_type == VREG &&
 2060             (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
 2061             (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
 2062                 /*
 2063                  * Allow direct swapping to NFS regular files in the same
 2064                  * way that nfs_mountroot() sets up diskless swapping.
 2065                  */
 2066                 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
 2067         }
 2068 
 2069         if (error)
 2070                 vrele(vp);
 2071 done:
 2072         swdev_syscall_active = 0;
 2073         wakeup_one(&swdev_syscall_active);
 2074         mtx_unlock(&Giant);
 2075         return (error);
 2076 }
 2077 
 2078 static void
 2079 swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
 2080 {
 2081         struct swdevt *sp, *tsp;
 2082         swblk_t dvbase;
 2083         u_long mblocks;
 2084 
 2085         /*
 2086          * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
 2087          * First chop nblks off to page-align it, then convert.
 2088          * 
 2089          * sw->sw_nblks is in page-sized chunks now too.
 2090          */
 2091         nblks &= ~(ctodb(1) - 1);
 2092         nblks = dbtoc(nblks);
 2093 
 2094         /*
 2095          * If we go beyond this, we get overflows in the radix
 2096          * tree bitmap code.
 2097          */
 2098         mblocks = 0x40000000 / BLIST_META_RADIX;
 2099         if (nblks > mblocks) {
 2100                 printf(
 2101     "WARNING: reducing swap size to maximum of %luMB per unit\n",
 2102                     mblocks / 1024 / 1024 * PAGE_SIZE);
 2103                 nblks = mblocks;
 2104         }
 2105 
 2106         sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
 2107         sp->sw_vp = vp;
 2108         sp->sw_id = id;
 2109         sp->sw_dev = dev;
 2110         sp->sw_flags = 0;
 2111         sp->sw_nblks = nblks;
 2112         sp->sw_used = 0;
 2113         sp->sw_strategy = strategy;
 2114         sp->sw_close = close;
 2115 
 2116         sp->sw_blist = blist_create(nblks, M_WAITOK);
 2117         /*
 2118          * Do not free the first two block in order to avoid overwriting
 2119          * any bsd label at the front of the partition
 2120          */
 2121         blist_free(sp->sw_blist, 2, nblks - 2);
 2122 
 2123         dvbase = 0;
 2124         mtx_lock(&sw_dev_mtx);
 2125         TAILQ_FOREACH(tsp, &swtailq, sw_list) {
 2126                 if (tsp->sw_end >= dvbase) {
 2127                         /*
 2128                          * We put one uncovered page between the devices
 2129                          * in order to definitively prevent any cross-device
 2130                          * I/O requests
 2131                          */
 2132                         dvbase = tsp->sw_end + 1;
 2133                 }
 2134         }
 2135         sp->sw_first = dvbase;
 2136         sp->sw_end = dvbase + nblks;
 2137         TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
 2138         nswapdev++;
 2139         swap_pager_avail += nblks;
 2140         swap_total += (vm_ooffset_t)nblks * PAGE_SIZE;
 2141         swp_sizecheck();
 2142         mtx_unlock(&sw_dev_mtx);
 2143 }
 2144 
 2145 /*
 2146  * SYSCALL: swapoff(devname)
 2147  *
 2148  * Disable swapping on the given device.
 2149  *
 2150  * XXX: Badly designed system call: it should use a device index
 2151  * rather than filename as specification.  We keep sw_vp around
 2152  * only to make this work.
 2153  */
 2154 #ifndef _SYS_SYSPROTO_H_
 2155 struct swapoff_args {
 2156         char *name;
 2157 };
 2158 #endif
 2159 
 2160 /*
 2161  * MPSAFE
 2162  */
 2163 /* ARGSUSED */
 2164 int
 2165 swapoff(struct thread *td, struct swapoff_args *uap)
 2166 {
 2167         struct vnode *vp;
 2168         struct nameidata nd;
 2169         struct swdevt *sp;
 2170         int error;
 2171 
 2172         error = priv_check(td, PRIV_SWAPOFF);
 2173         if (error)
 2174                 return (error);
 2175 
 2176         mtx_lock(&Giant);
 2177         while (swdev_syscall_active)
 2178             tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2179         swdev_syscall_active = 1;
 2180 
 2181         NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
 2182             td);
 2183         error = namei(&nd);
 2184         if (error)
 2185                 goto done;
 2186         NDFREE(&nd, NDF_ONLY_PNBUF);
 2187         vp = nd.ni_vp;
 2188 
 2189         mtx_lock(&sw_dev_mtx);
 2190         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2191                 if (sp->sw_vp == vp)
 2192                         break;
 2193         }
 2194         mtx_unlock(&sw_dev_mtx);
 2195         if (sp == NULL) {
 2196                 error = EINVAL;
 2197                 goto done;
 2198         }
 2199         error = swapoff_one(sp, td->td_ucred);
 2200 done:
 2201         swdev_syscall_active = 0;
 2202         wakeup_one(&swdev_syscall_active);
 2203         mtx_unlock(&Giant);
 2204         return (error);
 2205 }
 2206 
 2207 static int
 2208 swapoff_one(struct swdevt *sp, struct ucred *cred)
 2209 {
 2210         u_long nblks, dvbase;
 2211 #ifdef MAC
 2212         int error;
 2213 #endif
 2214 
 2215         mtx_assert(&Giant, MA_OWNED);
 2216 #ifdef MAC
 2217         (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY);
 2218         error = mac_system_check_swapoff(cred, sp->sw_vp);
 2219         (void) VOP_UNLOCK(sp->sw_vp, 0);
 2220         if (error != 0)
 2221                 return (error);
 2222 #endif
 2223         nblks = sp->sw_nblks;
 2224 
 2225         /*
 2226          * We can turn off this swap device safely only if the
 2227          * available virtual memory in the system will fit the amount
 2228          * of data we will have to page back in, plus an epsilon so
 2229          * the system doesn't become critically low on swap space.
 2230          */
 2231         if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
 2232             nblks + nswap_lowat) {
 2233                 return (ENOMEM);
 2234         }
 2235 
 2236         /*
 2237          * Prevent further allocations on this device.
 2238          */
 2239         mtx_lock(&sw_dev_mtx);
 2240         sp->sw_flags |= SW_CLOSING;
 2241         for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
 2242                 swap_pager_avail -= blist_fill(sp->sw_blist,
 2243                      dvbase, dmmax);
 2244         }
 2245         swap_total -= (vm_ooffset_t)nblks * PAGE_SIZE;
 2246         mtx_unlock(&sw_dev_mtx);
 2247 
 2248         /*
 2249          * Page in the contents of the device and close it.
 2250          */
 2251         swap_pager_swapoff(sp);
 2252 
 2253         sp->sw_close(curthread, sp);
 2254         sp->sw_id = NULL;
 2255         mtx_lock(&sw_dev_mtx);
 2256         TAILQ_REMOVE(&swtailq, sp, sw_list);
 2257         nswapdev--;
 2258         if (nswapdev == 0) {
 2259                 swap_pager_full = 2;
 2260                 swap_pager_almost_full = 1;
 2261         }
 2262         if (swdevhd == sp)
 2263                 swdevhd = NULL;
 2264         mtx_unlock(&sw_dev_mtx);
 2265         blist_destroy(sp->sw_blist);
 2266         free(sp, M_VMPGDATA);
 2267         return (0);
 2268 }
 2269 
 2270 void
 2271 swapoff_all(void)
 2272 {
 2273         struct swdevt *sp, *spt;
 2274         const char *devname;
 2275         int error;
 2276  
 2277         mtx_lock(&Giant);
 2278         while (swdev_syscall_active)
 2279                 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2280         swdev_syscall_active = 1;
 2281  
 2282         mtx_lock(&sw_dev_mtx);
 2283         TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
 2284                 mtx_unlock(&sw_dev_mtx);
 2285                 if (vn_isdisk(sp->sw_vp, NULL))
 2286                         devname = sp->sw_vp->v_rdev->si_name;
 2287                 else
 2288                         devname = "[file]";
 2289                 error = swapoff_one(sp, thread0.td_ucred);
 2290                 if (error != 0) {
 2291                         printf("Cannot remove swap device %s (error=%d), "
 2292                             "skipping.\n", devname, error);
 2293                 } else if (bootverbose) {
 2294                         printf("Swap device %s removed.\n", devname);
 2295                 }
 2296                 mtx_lock(&sw_dev_mtx);
 2297         }
 2298         mtx_unlock(&sw_dev_mtx);
 2299  
 2300         swdev_syscall_active = 0;
 2301         wakeup_one(&swdev_syscall_active);
 2302         mtx_unlock(&Giant);
 2303 }
 2304 
 2305 void
 2306 swap_pager_status(int *total, int *used)
 2307 {
 2308         struct swdevt *sp;
 2309 
 2310         *total = 0;
 2311         *used = 0;
 2312         mtx_lock(&sw_dev_mtx);
 2313         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2314                 *total += sp->sw_nblks;
 2315                 *used += sp->sw_used;
 2316         }
 2317         mtx_unlock(&sw_dev_mtx);
 2318 }
 2319 
 2320 int
 2321 swap_dev_info(int name, struct xswdev *xs, char *devname, size_t len)
 2322 {
 2323         struct swdevt *sp;
 2324         char *tmp_devname;
 2325         int error, n;
 2326 
 2327         n = 0;
 2328         error = ENOENT;
 2329         mtx_lock(&sw_dev_mtx);
 2330         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2331                 if (n != name) {
 2332                         n++;
 2333                         continue;
 2334                 }
 2335                 xs->xsw_version = XSWDEV_VERSION;
 2336                 xs->xsw_dev = sp->sw_dev;
 2337                 xs->xsw_flags = sp->sw_flags;
 2338                 xs->xsw_nblks = sp->sw_nblks;
 2339                 xs->xsw_used = sp->sw_used;
 2340                 if (devname != NULL) {
 2341                         if (vn_isdisk(sp->sw_vp, NULL))
 2342                                 tmp_devname = sp->sw_vp->v_rdev->si_name;
 2343                         else
 2344                                 tmp_devname = "[file]";
 2345                         strncpy(devname, tmp_devname, len);
 2346                 }
 2347                 error = 0;
 2348                 break;
 2349         }
 2350         mtx_unlock(&sw_dev_mtx);
 2351         return (error);
 2352 }
 2353 
 2354 static int
 2355 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
 2356 {
 2357         struct xswdev xs;
 2358         int error;
 2359 
 2360         if (arg2 != 1)                  /* name length */
 2361                 return (EINVAL);
 2362         error = swap_dev_info(*(int *)arg1, &xs, NULL, 0);
 2363         if (error != 0)
 2364                 return (error);
 2365         error = SYSCTL_OUT(req, &xs, sizeof(xs));
 2366         return (error);
 2367 }
 2368 
 2369 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
 2370     "Number of swap devices");
 2371 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
 2372     "Swap statistics by device");
 2373 
 2374 /*
 2375  * vmspace_swap_count() - count the approximate swap usage in pages for a
 2376  *                        vmspace.
 2377  *
 2378  *      The map must be locked.
 2379  *
 2380  *      Swap usage is determined by taking the proportional swap used by
 2381  *      VM objects backing the VM map.  To make up for fractional losses,
 2382  *      if the VM object has any swap use at all the associated map entries
 2383  *      count for at least 1 swap page.
 2384  */
 2385 long
 2386 vmspace_swap_count(struct vmspace *vmspace)
 2387 {
 2388         vm_map_t map;
 2389         vm_map_entry_t cur;
 2390         vm_object_t object;
 2391         long count, n;
 2392 
 2393         map = &vmspace->vm_map;
 2394         count = 0;
 2395 
 2396         for (cur = map->header.next; cur != &map->header; cur = cur->next) {
 2397                 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2398                     (object = cur->object.vm_object) != NULL) {
 2399                         VM_OBJECT_LOCK(object);
 2400                         if (object->type == OBJT_SWAP &&
 2401                             object->un_pager.swp.swp_bcount != 0) {
 2402                                 n = (cur->end - cur->start) / PAGE_SIZE;
 2403                                 count += object->un_pager.swp.swp_bcount *
 2404                                     SWAP_META_PAGES * n / object->size + 1;
 2405                         }
 2406                         VM_OBJECT_UNLOCK(object);
 2407                 }
 2408         }
 2409         return (count);
 2410 }
 2411 
 2412 /*
 2413  * GEOM backend
 2414  *
 2415  * Swapping onto disk devices.
 2416  *
 2417  */
 2418 
 2419 static g_orphan_t swapgeom_orphan;
 2420 
 2421 static struct g_class g_swap_class = {
 2422         .name = "SWAP",
 2423         .version = G_VERSION,
 2424         .orphan = swapgeom_orphan,
 2425 };
 2426 
 2427 DECLARE_GEOM_CLASS(g_swap_class, g_class);
 2428 
 2429 
 2430 static void
 2431 swapgeom_done(struct bio *bp2)
 2432 {
 2433         struct buf *bp;
 2434 
 2435         bp = bp2->bio_caller2;
 2436         bp->b_ioflags = bp2->bio_flags;
 2437         if (bp2->bio_error)
 2438                 bp->b_ioflags |= BIO_ERROR;
 2439         bp->b_resid = bp->b_bcount - bp2->bio_completed;
 2440         bp->b_error = bp2->bio_error;
 2441         bufdone(bp);
 2442         g_destroy_bio(bp2);
 2443 }
 2444 
 2445 static void
 2446 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
 2447 {
 2448         struct bio *bio;
 2449         struct g_consumer *cp;
 2450 
 2451         cp = sp->sw_id;
 2452         if (cp == NULL) {
 2453                 bp->b_error = ENXIO;
 2454                 bp->b_ioflags |= BIO_ERROR;
 2455                 bufdone(bp);
 2456                 return;
 2457         }
 2458         if (bp->b_iocmd == BIO_WRITE)
 2459                 bio = g_new_bio();
 2460         else
 2461                 bio = g_alloc_bio();
 2462         if (bio == NULL) {
 2463                 bp->b_error = ENOMEM;
 2464                 bp->b_ioflags |= BIO_ERROR;
 2465                 bufdone(bp);
 2466                 return;
 2467         }
 2468 
 2469         bio->bio_caller2 = bp;
 2470         bio->bio_cmd = bp->b_iocmd;
 2471         bio->bio_data = bp->b_data;
 2472         bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
 2473         bio->bio_length = bp->b_bcount;
 2474         bio->bio_done = swapgeom_done;
 2475         g_io_request(bio, cp);
 2476         return;
 2477 }
 2478 
 2479 static void
 2480 swapgeom_orphan(struct g_consumer *cp)
 2481 {
 2482         struct swdevt *sp;
 2483 
 2484         mtx_lock(&sw_dev_mtx);
 2485         TAILQ_FOREACH(sp, &swtailq, sw_list)
 2486                 if (sp->sw_id == cp)
 2487                         sp->sw_flags |= SW_CLOSING;
 2488         mtx_unlock(&sw_dev_mtx);
 2489 }
 2490 
 2491 static void
 2492 swapgeom_close_ev(void *arg, int flags)
 2493 {
 2494         struct g_consumer *cp;
 2495 
 2496         cp = arg;
 2497         g_access(cp, -1, -1, 0);
 2498         g_detach(cp);
 2499         g_destroy_consumer(cp);
 2500 }
 2501 
 2502 static void
 2503 swapgeom_close(struct thread *td, struct swdevt *sw)
 2504 {
 2505 
 2506         /* XXX: direct call when Giant untangled */
 2507         g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
 2508 }
 2509 
 2510 
 2511 struct swh0h0 {
 2512         struct cdev *dev;
 2513         struct vnode *vp;
 2514         int     error;
 2515 };
 2516 
 2517 static void
 2518 swapongeom_ev(void *arg, int flags)
 2519 {
 2520         struct swh0h0 *swh;
 2521         struct g_provider *pp;
 2522         struct g_consumer *cp;
 2523         static struct g_geom *gp;
 2524         struct swdevt *sp;
 2525         u_long nblks;
 2526         int error;
 2527 
 2528         swh = arg;
 2529         swh->error = 0;
 2530         pp = g_dev_getprovider(swh->dev);
 2531         if (pp == NULL) {
 2532                 swh->error = ENODEV;
 2533                 return;
 2534         }
 2535         mtx_lock(&sw_dev_mtx);
 2536         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2537                 cp = sp->sw_id;
 2538                 if (cp != NULL && cp->provider == pp) {
 2539                         mtx_unlock(&sw_dev_mtx);
 2540                         swh->error = EBUSY;
 2541                         return;
 2542                 }
 2543         }
 2544         mtx_unlock(&sw_dev_mtx);
 2545         if (gp == NULL)
 2546                 gp = g_new_geomf(&g_swap_class, "swap", NULL);
 2547         cp = g_new_consumer(gp);
 2548         g_attach(cp, pp);
 2549         /*
 2550          * XXX: Everytime you think you can improve the margin for
 2551          * footshooting, somebody depends on the ability to do so:
 2552          * savecore(8) wants to write to our swapdev so we cannot
 2553          * set an exclusive count :-(
 2554          */
 2555         error = g_access(cp, 1, 1, 0);
 2556         if (error) {
 2557                 g_detach(cp);
 2558                 g_destroy_consumer(cp);
 2559                 swh->error = error;
 2560                 return;
 2561         }
 2562         nblks = pp->mediasize / DEV_BSIZE;
 2563         swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
 2564             swapgeom_close, dev2udev(swh->dev));
 2565         swh->error = 0;
 2566         return;
 2567 }
 2568 
 2569 static int
 2570 swapongeom(struct thread *td, struct vnode *vp)
 2571 {
 2572         int error;
 2573         struct swh0h0 swh;
 2574 
 2575         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2576 
 2577         swh.dev = vp->v_rdev;
 2578         swh.vp = vp;
 2579         swh.error = 0;
 2580         /* XXX: direct call when Giant untangled */
 2581         error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
 2582         if (!error)
 2583                 error = swh.error;
 2584         VOP_UNLOCK(vp, 0);
 2585         return (error);
 2586 }
 2587 
 2588 /*
 2589  * VNODE backend
 2590  *
 2591  * This is used mainly for network filesystem (read: probably only tested
 2592  * with NFS) swapfiles.
 2593  *
 2594  */
 2595 
 2596 static void
 2597 swapdev_strategy(struct buf *bp, struct swdevt *sp)
 2598 {
 2599         struct vnode *vp2;
 2600 
 2601         bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
 2602 
 2603         vp2 = sp->sw_id;
 2604         vhold(vp2);
 2605         if (bp->b_iocmd == BIO_WRITE) {
 2606                 if (bp->b_bufobj)
 2607                         bufobj_wdrop(bp->b_bufobj);
 2608                 bufobj_wref(&vp2->v_bufobj);
 2609         }
 2610         if (bp->b_bufobj != &vp2->v_bufobj)
 2611                 bp->b_bufobj = &vp2->v_bufobj;
 2612         bp->b_vp = vp2;
 2613         bp->b_iooffset = dbtob(bp->b_blkno);
 2614         bstrategy(bp);
 2615         return;
 2616 }
 2617 
 2618 static void
 2619 swapdev_close(struct thread *td, struct swdevt *sp)
 2620 {
 2621 
 2622         VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
 2623         vrele(sp->sw_vp);
 2624 }
 2625 
 2626 
 2627 static int
 2628 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
 2629 {
 2630         struct swdevt *sp;
 2631         int error;
 2632 
 2633         if (nblks == 0)
 2634                 return (ENXIO);
 2635         mtx_lock(&sw_dev_mtx);
 2636         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2637                 if (sp->sw_id == vp) {
 2638                         mtx_unlock(&sw_dev_mtx);
 2639                         return (EBUSY);
 2640                 }
 2641         }
 2642         mtx_unlock(&sw_dev_mtx);
 2643     
 2644         (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 2645 #ifdef MAC
 2646         error = mac_system_check_swapon(td->td_ucred, vp);
 2647         if (error == 0)
 2648 #endif
 2649                 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
 2650         (void) VOP_UNLOCK(vp, 0);
 2651         if (error)
 2652                 return (error);
 2653 
 2654         swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
 2655             NODEV);
 2656         return (0);
 2657 }

Cache object: e8270b43884595008108e87846d26626


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.