The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/swap_pager.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Matthew Dillon,
    3  * Copyright (c) 1994 John S. Dyson
    4  * Copyright (c) 1990 University of Utah.
    5  * Copyright (c) 1982, 1986, 1989, 1993
    6  *      The Regents of the University of California.  All rights reserved.
    7  *
    8  * This code is derived from software contributed to Berkeley by
    9  * the Systems Programming Group of the University of Utah Computer
   10  * Science Department.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. All advertising materials mentioning features or use of this software
   21  *    must display the following acknowledgement:
   22  *      This product includes software developed by the University of
   23  *      California, Berkeley and its contributors.
   24  * 4. Neither the name of the University nor the names of its contributors
   25  *    may be used to endorse or promote products derived from this software
   26  *    without specific prior written permission.
   27  *
   28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   38  * SUCH DAMAGE.
   39  *
   40  *                              New Swap System
   41  *                              Matthew Dillon
   42  *
   43  * Radix Bitmap 'blists'.
   44  *
   45  *      - The new swapper uses the new radix bitmap code.  This should scale
   46  *        to arbitrarily small or arbitrarily large swap spaces and an almost
   47  *        arbitrary degree of fragmentation.
   48  *
   49  * Features:
   50  *
   51  *      - on the fly reallocation of swap during putpages.  The new system
   52  *        does not try to keep previously allocated swap blocks for dirty
   53  *        pages.  
   54  *
   55  *      - on the fly deallocation of swap
   56  *
   57  *      - No more garbage collection required.  Unnecessarily allocated swap
   58  *        blocks only exist for dirty vm_page_t's now and these are already
   59  *        cycled (in a high-load system) by the pager.  We also do on-the-fly
   60  *        removal of invalidated swap blocks when a page is destroyed
   61  *        or renamed.
   62  *
   63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
   64  *
   65  *      @(#)swap_pager.c        8.9 (Berkeley) 3/21/94
   66  *      @(#)vm_swap.c   8.5 (Berkeley) 2/17/94
   67  */
   68 
   69 #include <sys/cdefs.h>
   70 __FBSDID("$FreeBSD$");
   71 
   72 #include "opt_mac.h"
   73 #include "opt_swap.h"
   74 #include "opt_vm.h"
   75 
   76 #include <sys/param.h>
   77 #include <sys/systm.h>
   78 #include <sys/conf.h>
   79 #include <sys/kernel.h>
   80 #include <sys/proc.h>
   81 #include <sys/bio.h>
   82 #include <sys/buf.h>
   83 #include <sys/disk.h>
   84 #include <sys/fcntl.h>
   85 #include <sys/mount.h>
   86 #include <sys/namei.h>
   87 #include <sys/vnode.h>
   88 #include <sys/mac.h>
   89 #include <sys/malloc.h>
   90 #include <sys/sysctl.h>
   91 #include <sys/sysproto.h>
   92 #include <sys/blist.h>
   93 #include <sys/lock.h>
   94 #include <sys/sx.h>
   95 #include <sys/vmmeter.h>
   96 
   97 #include <vm/vm.h>
   98 #include <vm/pmap.h>
   99 #include <vm/vm_map.h>
  100 #include <vm/vm_kern.h>
  101 #include <vm/vm_object.h>
  102 #include <vm/vm_page.h>
  103 #include <vm/vm_pager.h>
  104 #include <vm/vm_pageout.h>
  105 #include <vm/vm_param.h>
  106 #include <vm/swap_pager.h>
  107 #include <vm/vm_extern.h>
  108 #include <vm/uma.h>
  109 
  110 #include <geom/geom.h>
  111 
  112 /*
  113  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, or 16
  114  * pages per allocation.  We recommend you stick with the default of 8.
  115  * The 16-page limit is due to the radix code (kern/subr_blist.c).
  116  */
  117 #ifndef MAX_PAGEOUT_CLUSTER
  118 #define MAX_PAGEOUT_CLUSTER 16
  119 #endif
  120 
  121 #if !defined(SWB_NPAGES)
  122 #define SWB_NPAGES      MAX_PAGEOUT_CLUSTER
  123 #endif
  124 
  125 /*
  126  * Piecemeal swap metadata structure.  Swap is stored in a radix tree.
  127  *
  128  * If SWB_NPAGES is 8 and sizeof(char *) == sizeof(daddr_t), our radix
  129  * is basically 8.  Assuming PAGE_SIZE == 4096, one tree level represents
  130  * 32K worth of data, two levels represent 256K, three levels represent
  131  * 2 MBytes.   This is acceptable.
  132  *
  133  * Overall memory utilization is about the same as the old swap structure.
  134  */
  135 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
  136 #define SWAP_META_PAGES         (SWB_NPAGES * 2)
  137 #define SWAP_META_MASK          (SWAP_META_PAGES - 1)
  138 
  139 struct swblock {
  140         struct swblock  *swb_hnext;
  141         vm_object_t     swb_object;
  142         vm_pindex_t     swb_index;
  143         int             swb_count;
  144         daddr_t         swb_pages[SWAP_META_PAGES];
  145 };
  146 
  147 static struct mtx sw_dev_mtx;
  148 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
  149 static struct swdevt *swdevhd;  /* Allocate from here next */
  150 static int nswapdev;            /* Number of swap devices */
  151 int swap_pager_avail;
  152 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
  153 
  154 static void swapdev_strategy(struct buf *, struct swdevt *sw);
  155 
  156 #define SWM_FREE        0x02    /* free, period                 */
  157 #define SWM_POP         0x04    /* pop out                      */
  158 
  159 int swap_pager_full = 2;        /* swap space exhaustion (task killing) */
  160 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
  161 static int nsw_rcount;          /* free read buffers                    */
  162 static int nsw_wcount_sync;     /* limit write buffers / synchronous    */
  163 static int nsw_wcount_async;    /* limit write buffers / asynchronous   */
  164 static int nsw_wcount_async_max;/* assigned maximum                     */
  165 static int nsw_cluster_max;     /* maximum VOP I/O allowed              */
  166 
  167 static struct swblock **swhash;
  168 static int swhash_mask;
  169 static struct mtx swhash_mtx;
  170 
  171 static int swap_async_max = 4;  /* maximum in-progress async I/O's      */
  172 static struct sx sw_alloc_sx;
  173 
  174 
  175 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
  176         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
  177 
  178 /*
  179  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
  180  * of searching a named list by hashing it just a little.
  181  */
  182 
  183 #define NOBJLISTS               8
  184 
  185 #define NOBJLIST(handle)        \
  186         (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
  187 
  188 static struct mtx sw_alloc_mtx; /* protect list manipulation */ 
  189 static struct pagerlst  swap_pager_object_list[NOBJLISTS];
  190 static uma_zone_t       swap_zone;
  191 static struct vm_object swap_zone_obj;
  192 
  193 /*
  194  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
  195  * calls hooked from other parts of the VM system and do not appear here.
  196  * (see vm/swap_pager.h).
  197  */
  198 static vm_object_t
  199                 swap_pager_alloc(void *handle, vm_ooffset_t size,
  200                                       vm_prot_t prot, vm_ooffset_t offset);
  201 static void     swap_pager_dealloc(vm_object_t object);
  202 static int      swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
  203 static void     swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
  204 static boolean_t
  205                 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
  206 static void     swap_pager_init(void);
  207 static void     swap_pager_unswapped(vm_page_t);
  208 static void     swap_pager_swapoff(struct swdevt *sp);
  209 
  210 struct pagerops swappagerops = {
  211         .pgo_init =     swap_pager_init,        /* early system initialization of pager */
  212         .pgo_alloc =    swap_pager_alloc,       /* allocate an OBJT_SWAP object         */
  213         .pgo_dealloc =  swap_pager_dealloc,     /* deallocate an OBJT_SWAP object       */
  214         .pgo_getpages = swap_pager_getpages,    /* pagein                               */
  215         .pgo_putpages = swap_pager_putpages,    /* pageout                              */
  216         .pgo_haspage =  swap_pager_haspage,     /* get backing store status for page    */
  217         .pgo_pageunswapped = swap_pager_unswapped,      /* remove swap related to page          */
  218 };
  219 
  220 /*
  221  * dmmax is in page-sized chunks with the new swap system.  It was
  222  * dev-bsized chunks in the old.  dmmax is always a power of 2.
  223  *
  224  * swap_*() routines are externally accessible.  swp_*() routines are
  225  * internal.
  226  */
  227 static int dmmax;
  228 static int nswap_lowat = 128;   /* in pages, swap_pager_almost_full warn */
  229 static int nswap_hiwat = 512;   /* in pages, swap_pager_almost_full warn */
  230 
  231 SYSCTL_INT(_vm, OID_AUTO, dmmax,
  232         CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
  233 
  234 static void     swp_sizecheck(void);
  235 static void     swp_pager_async_iodone(struct buf *bp);
  236 static int      swapongeom(struct thread *, struct vnode *);
  237 static int      swaponvp(struct thread *, struct vnode *, u_long);
  238 static int      swapoff_one(struct swdevt *sp, struct thread *td);
  239 
  240 /*
  241  * Swap bitmap functions
  242  */
  243 static void     swp_pager_freeswapspace(daddr_t blk, int npages);
  244 static daddr_t  swp_pager_getswapspace(int npages);
  245 
  246 /*
  247  * Metadata functions
  248  */
  249 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
  250 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
  251 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
  252 static void swp_pager_meta_free_all(vm_object_t);
  253 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
  254 
  255 /*
  256  * SWP_SIZECHECK() -    update swap_pager_full indication
  257  *      
  258  *      update the swap_pager_almost_full indication and warn when we are
  259  *      about to run out of swap space, using lowat/hiwat hysteresis.
  260  *
  261  *      Clear swap_pager_full ( task killing ) indication when lowat is met.
  262  *
  263  *      No restrictions on call
  264  *      This routine may not block.
  265  *      This routine must be called at splvm()
  266  */
  267 static void
  268 swp_sizecheck(void)
  269 {
  270 
  271         if (swap_pager_avail < nswap_lowat) {
  272                 if (swap_pager_almost_full == 0) {
  273                         printf("swap_pager: out of swap space\n");
  274                         swap_pager_almost_full = 1;
  275                 }
  276         } else {
  277                 swap_pager_full = 0;
  278                 if (swap_pager_avail > nswap_hiwat)
  279                         swap_pager_almost_full = 0;
  280         }
  281 }
  282 
  283 /*
  284  * SWP_PAGER_HASH() -   hash swap meta data
  285  *
  286  *      This is an helper function which hashes the swapblk given
  287  *      the object and page index.  It returns a pointer to a pointer
  288  *      to the object, or a pointer to a NULL pointer if it could not
  289  *      find a swapblk.
  290  *
  291  *      This routine must be called at splvm().
  292  */
  293 static struct swblock **
  294 swp_pager_hash(vm_object_t object, vm_pindex_t index)
  295 {
  296         struct swblock **pswap;
  297         struct swblock *swap;
  298 
  299         index &= ~(vm_pindex_t)SWAP_META_MASK;
  300         pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
  301         while ((swap = *pswap) != NULL) {
  302                 if (swap->swb_object == object &&
  303                     swap->swb_index == index
  304                 ) {
  305                         break;
  306                 }
  307                 pswap = &swap->swb_hnext;
  308         }
  309         return (pswap);
  310 }
  311 
  312 /*
  313  * SWAP_PAGER_INIT() -  initialize the swap pager!
  314  *
  315  *      Expected to be started from system init.  NOTE:  This code is run 
  316  *      before much else so be careful what you depend on.  Most of the VM
  317  *      system has yet to be initialized at this point.
  318  */
  319 static void
  320 swap_pager_init(void)
  321 {
  322         /*
  323          * Initialize object lists
  324          */
  325         int i;
  326 
  327         for (i = 0; i < NOBJLISTS; ++i)
  328                 TAILQ_INIT(&swap_pager_object_list[i]);
  329         mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
  330         mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
  331 
  332         /*
  333          * Device Stripe, in PAGE_SIZE'd blocks
  334          */
  335         dmmax = SWB_NPAGES * 2;
  336 }
  337 
  338 /*
  339  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
  340  *
  341  *      Expected to be started from pageout process once, prior to entering
  342  *      its main loop.
  343  */
  344 void
  345 swap_pager_swap_init(void)
  346 {
  347         int n, n2;
  348 
  349         /*
  350          * Number of in-transit swap bp operations.  Don't
  351          * exhaust the pbufs completely.  Make sure we
  352          * initialize workable values (0 will work for hysteresis
  353          * but it isn't very efficient).
  354          *
  355          * The nsw_cluster_max is constrained by the bp->b_pages[]
  356          * array (MAXPHYS/PAGE_SIZE) and our locally defined
  357          * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
  358          * constrained by the swap device interleave stripe size.
  359          *
  360          * Currently we hardwire nsw_wcount_async to 4.  This limit is 
  361          * designed to prevent other I/O from having high latencies due to
  362          * our pageout I/O.  The value 4 works well for one or two active swap
  363          * devices but is probably a little low if you have more.  Even so,
  364          * a higher value would probably generate only a limited improvement
  365          * with three or four active swap devices since the system does not
  366          * typically have to pageout at extreme bandwidths.   We will want
  367          * at least 2 per swap devices, and 4 is a pretty good value if you
  368          * have one NFS swap device due to the command/ack latency over NFS.
  369          * So it all works out pretty well.
  370          */
  371         nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
  372 
  373         mtx_lock(&pbuf_mtx);
  374         nsw_rcount = (nswbuf + 1) / 2;
  375         nsw_wcount_sync = (nswbuf + 3) / 4;
  376         nsw_wcount_async = 4;
  377         nsw_wcount_async_max = nsw_wcount_async;
  378         mtx_unlock(&pbuf_mtx);
  379 
  380         /*
  381          * Initialize our zone.  Right now I'm just guessing on the number
  382          * we need based on the number of pages in the system.  Each swblock
  383          * can hold 16 pages, so this is probably overkill.  This reservation
  384          * is typically limited to around 32MB by default.
  385          */
  386         n = cnt.v_page_count / 2;
  387         if (maxswzone && n > maxswzone / sizeof(struct swblock))
  388                 n = maxswzone / sizeof(struct swblock);
  389         n2 = n;
  390         swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
  391             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
  392         if (swap_zone == NULL)
  393                 panic("failed to create swap_zone.");
  394         do {
  395                 if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
  396                         break;
  397                 /*
  398                  * if the allocation failed, try a zone two thirds the
  399                  * size of the previous attempt.
  400                  */
  401                 n -= ((n + 2) / 3);
  402         } while (n > 0);
  403         if (n2 != n)
  404                 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
  405         n2 = n;
  406 
  407         /*
  408          * Initialize our meta-data hash table.  The swapper does not need to
  409          * be quite as efficient as the VM system, so we do not use an 
  410          * oversized hash table.
  411          *
  412          *      n:              size of hash table, must be power of 2
  413          *      swhash_mask:    hash table index mask
  414          */
  415         for (n = 1; n < n2 / 8; n *= 2)
  416                 ;
  417         swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
  418         swhash_mask = n - 1;
  419         mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
  420 }
  421 
  422 /*
  423  * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
  424  *                      its metadata structures.
  425  *
  426  *      This routine is called from the mmap and fork code to create a new
  427  *      OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
  428  *      and then converting it with swp_pager_meta_build().
  429  *
  430  *      This routine may block in vm_object_allocate() and create a named
  431  *      object lookup race, so we must interlock.   We must also run at
  432  *      splvm() for the object lookup to handle races with interrupts, but
  433  *      we do not have to maintain splvm() in between the lookup and the
  434  *      add because (I believe) it is not possible to attempt to create
  435  *      a new swap object w/handle when a default object with that handle
  436  *      already exists.
  437  *
  438  * MPSAFE
  439  */
  440 static vm_object_t
  441 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
  442                  vm_ooffset_t offset)
  443 {
  444         vm_object_t object;
  445         vm_pindex_t pindex;
  446 
  447         pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
  448 
  449         if (handle) {
  450                 mtx_lock(&Giant);
  451                 /*
  452                  * Reference existing named region or allocate new one.  There
  453                  * should not be a race here against swp_pager_meta_build()
  454                  * as called from vm_page_remove() in regards to the lookup
  455                  * of the handle.
  456                  */
  457                 sx_xlock(&sw_alloc_sx);
  458                 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
  459 
  460                 if (object != NULL) {
  461                         vm_object_reference(object);
  462                 } else {
  463                         object = vm_object_allocate(OBJT_DEFAULT, pindex);
  464                         object->handle = handle;
  465 
  466                         VM_OBJECT_LOCK(object);
  467                         swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  468                         VM_OBJECT_UNLOCK(object);
  469                 }
  470                 sx_xunlock(&sw_alloc_sx);
  471                 mtx_unlock(&Giant);
  472         } else {
  473                 object = vm_object_allocate(OBJT_DEFAULT, pindex);
  474 
  475                 VM_OBJECT_LOCK(object);
  476                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  477                 VM_OBJECT_UNLOCK(object);
  478         }
  479         return (object);
  480 }
  481 
  482 /*
  483  * SWAP_PAGER_DEALLOC() -       remove swap metadata from object
  484  *
  485  *      The swap backing for the object is destroyed.  The code is 
  486  *      designed such that we can reinstantiate it later, but this
  487  *      routine is typically called only when the entire object is
  488  *      about to be destroyed.
  489  *
  490  *      This routine may block, but no longer does. 
  491  *
  492  *      The object must be locked or unreferenceable.
  493  */
  494 static void
  495 swap_pager_dealloc(vm_object_t object)
  496 {
  497 
  498         /*
  499          * Remove from list right away so lookups will fail if we block for
  500          * pageout completion.
  501          */
  502         if (object->handle != NULL) {
  503                 mtx_lock(&sw_alloc_mtx);
  504                 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
  505                 mtx_unlock(&sw_alloc_mtx);
  506         }
  507 
  508         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  509         vm_object_pip_wait(object, "swpdea");
  510 
  511         /*
  512          * Free all remaining metadata.  We only bother to free it from 
  513          * the swap meta data.  We do not attempt to free swapblk's still
  514          * associated with vm_page_t's for this object.  We do not care
  515          * if paging is still in progress on some objects.
  516          */
  517         swp_pager_meta_free_all(object);
  518 }
  519 
  520 /************************************************************************
  521  *                      SWAP PAGER BITMAP ROUTINES                      *
  522  ************************************************************************/
  523 
  524 /*
  525  * SWP_PAGER_GETSWAPSPACE() -   allocate raw swap space
  526  *
  527  *      Allocate swap for the requested number of pages.  The starting
  528  *      swap block number (a page index) is returned or SWAPBLK_NONE
  529  *      if the allocation failed.
  530  *
  531  *      Also has the side effect of advising that somebody made a mistake
  532  *      when they configured swap and didn't configure enough.
  533  *
  534  *      Must be called at splvm() to avoid races with bitmap frees from
  535  *      vm_page_remove() aka swap_pager_page_removed().
  536  *
  537  *      This routine may not block
  538  *      This routine must be called at splvm().
  539  *
  540  *      We allocate in round-robin fashion from the configured devices.
  541  */
  542 static daddr_t
  543 swp_pager_getswapspace(int npages)
  544 {
  545         daddr_t blk;
  546         struct swdevt *sp;
  547         int i;
  548 
  549         blk = SWAPBLK_NONE;
  550         mtx_lock(&sw_dev_mtx);
  551         sp = swdevhd;
  552         for (i = 0; i < nswapdev; i++) {
  553                 if (sp == NULL)
  554                         sp = TAILQ_FIRST(&swtailq);
  555                 if (!(sp->sw_flags & SW_CLOSING)) {
  556                         blk = blist_alloc(sp->sw_blist, npages);
  557                         if (blk != SWAPBLK_NONE) {
  558                                 blk += sp->sw_first;
  559                                 sp->sw_used += npages;
  560                                 swap_pager_avail -= npages;
  561                                 swp_sizecheck();
  562                                 swdevhd = TAILQ_NEXT(sp, sw_list);
  563                                 goto done;
  564                         }
  565                 }
  566                 sp = TAILQ_NEXT(sp, sw_list);
  567         }
  568         if (swap_pager_full != 2) {
  569                 printf("swap_pager_getswapspace(%d): failed\n", npages);
  570                 swap_pager_full = 2;
  571                 swap_pager_almost_full = 1;
  572         }
  573         swdevhd = NULL;
  574 done:
  575         mtx_unlock(&sw_dev_mtx);
  576         return (blk);
  577 }
  578 
  579 static int
  580 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
  581 {
  582 
  583         return (blk >= sp->sw_first && blk < sp->sw_end);
  584 }
  585         
  586 static void
  587 swp_pager_strategy(struct buf *bp)
  588 {
  589         struct swdevt *sp;
  590 
  591         mtx_lock(&sw_dev_mtx);
  592         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  593                 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
  594                         mtx_unlock(&sw_dev_mtx);
  595                         sp->sw_strategy(bp, sp);
  596                         return;
  597                 }
  598         }
  599         panic("Swapdev not found");
  600 }
  601         
  602 
  603 /*
  604  * SWP_PAGER_FREESWAPSPACE() -  free raw swap space 
  605  *
  606  *      This routine returns the specified swap blocks back to the bitmap.
  607  *
  608  *      Note:  This routine may not block (it could in the old swap code),
  609  *      and through the use of the new blist routines it does not block.
  610  *
  611  *      We must be called at splvm() to avoid races with bitmap frees from
  612  *      vm_page_remove() aka swap_pager_page_removed().
  613  *
  614  *      This routine may not block
  615  *      This routine must be called at splvm().
  616  */
  617 static void
  618 swp_pager_freeswapspace(daddr_t blk, int npages)
  619 {
  620         struct swdevt *sp;
  621 
  622         mtx_lock(&sw_dev_mtx);
  623         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  624                 if (blk >= sp->sw_first && blk < sp->sw_end) {
  625                         sp->sw_used -= npages;
  626                         /*
  627                          * If we are attempting to stop swapping on
  628                          * this device, we don't want to mark any
  629                          * blocks free lest they be reused.  
  630                          */
  631                         if ((sp->sw_flags & SW_CLOSING) == 0) {
  632                                 blist_free(sp->sw_blist, blk - sp->sw_first,
  633                                     npages);
  634                                 swap_pager_avail += npages;
  635                                 swp_sizecheck();
  636                         }
  637                         mtx_unlock(&sw_dev_mtx);
  638                         return;
  639                 }
  640         }
  641         panic("Swapdev not found");
  642 }
  643 
  644 /*
  645  * SWAP_PAGER_FREESPACE() -     frees swap blocks associated with a page
  646  *                              range within an object.
  647  *
  648  *      This is a globally accessible routine.
  649  *
  650  *      This routine removes swapblk assignments from swap metadata.
  651  *
  652  *      The external callers of this routine typically have already destroyed 
  653  *      or renamed vm_page_t's associated with this range in the object so 
  654  *      we should be ok.
  655  *
  656  *      This routine may be called at any spl.  We up our spl to splvm temporarily
  657  *      in order to perform the metadata removal.
  658  */
  659 void
  660 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
  661 {
  662 
  663         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  664         swp_pager_meta_free(object, start, size);
  665 }
  666 
  667 /*
  668  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
  669  *
  670  *      Assigns swap blocks to the specified range within the object.  The 
  671  *      swap blocks are not zerod.  Any previous swap assignment is destroyed.
  672  *
  673  *      Returns 0 on success, -1 on failure.
  674  */
  675 int
  676 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
  677 {
  678         int n = 0;
  679         daddr_t blk = SWAPBLK_NONE;
  680         vm_pindex_t beg = start;        /* save start index */
  681 
  682         VM_OBJECT_LOCK(object);
  683         while (size) {
  684                 if (n == 0) {
  685                         n = BLIST_MAX_ALLOC;
  686                         while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
  687                                 n >>= 1;
  688                                 if (n == 0) {
  689                                         swp_pager_meta_free(object, beg, start - beg);
  690                                         VM_OBJECT_UNLOCK(object);
  691                                         return (-1);
  692                                 }
  693                         }
  694                 }
  695                 swp_pager_meta_build(object, start, blk);
  696                 --size;
  697                 ++start;
  698                 ++blk;
  699                 --n;
  700         }
  701         swp_pager_meta_free(object, start, n);
  702         VM_OBJECT_UNLOCK(object);
  703         return (0);
  704 }
  705 
  706 /*
  707  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
  708  *                      and destroy the source.
  709  *
  710  *      Copy any valid swapblks from the source to the destination.  In
  711  *      cases where both the source and destination have a valid swapblk,
  712  *      we keep the destination's.
  713  *
  714  *      This routine is allowed to block.  It may block allocating metadata
  715  *      indirectly through swp_pager_meta_build() or if paging is still in
  716  *      progress on the source. 
  717  *
  718  *      This routine can be called at any spl
  719  *
  720  *      XXX vm_page_collapse() kinda expects us not to block because we 
  721  *      supposedly do not need to allocate memory, but for the moment we
  722  *      *may* have to get a little memory from the zone allocator, but
  723  *      it is taken from the interrupt memory.  We should be ok. 
  724  *
  725  *      The source object contains no vm_page_t's (which is just as well)
  726  *
  727  *      The source object is of type OBJT_SWAP.
  728  *
  729  *      The source and destination objects must be locked or 
  730  *      inaccessible (XXX are they ?)
  731  */
  732 void
  733 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
  734     vm_pindex_t offset, int destroysource)
  735 {
  736         vm_pindex_t i;
  737 
  738         VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
  739         VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
  740 
  741         /*
  742          * If destroysource is set, we remove the source object from the 
  743          * swap_pager internal queue now. 
  744          */
  745         if (destroysource) {
  746                 if (srcobject->handle != NULL) {
  747                         mtx_lock(&sw_alloc_mtx);
  748                         TAILQ_REMOVE(
  749                             NOBJLIST(srcobject->handle),
  750                             srcobject,
  751                             pager_object_list
  752                         );
  753                         mtx_unlock(&sw_alloc_mtx);
  754                 }
  755         }
  756 
  757         /*
  758          * transfer source to destination.
  759          */
  760         for (i = 0; i < dstobject->size; ++i) {
  761                 daddr_t dstaddr;
  762 
  763                 /*
  764                  * Locate (without changing) the swapblk on the destination,
  765                  * unless it is invalid in which case free it silently, or
  766                  * if the destination is a resident page, in which case the
  767                  * source is thrown away.
  768                  */
  769                 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
  770 
  771                 if (dstaddr == SWAPBLK_NONE) {
  772                         /*
  773                          * Destination has no swapblk and is not resident,
  774                          * copy source.
  775                          */
  776                         daddr_t srcaddr;
  777 
  778                         srcaddr = swp_pager_meta_ctl(
  779                             srcobject, 
  780                             i + offset,
  781                             SWM_POP
  782                         );
  783 
  784                         if (srcaddr != SWAPBLK_NONE) {
  785                                 /*
  786                                  * swp_pager_meta_build() can sleep.
  787                                  */
  788                                 vm_object_pip_add(srcobject, 1);
  789                                 VM_OBJECT_UNLOCK(srcobject);
  790                                 vm_object_pip_add(dstobject, 1);
  791                                 swp_pager_meta_build(dstobject, i, srcaddr);
  792                                 vm_object_pip_wakeup(dstobject);
  793                                 VM_OBJECT_LOCK(srcobject);
  794                                 vm_object_pip_wakeup(srcobject);
  795                         }
  796                 } else {
  797                         /*
  798                          * Destination has valid swapblk or it is represented
  799                          * by a resident page.  We destroy the sourceblock.
  800                          */
  801                         
  802                         swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
  803                 }
  804         }
  805 
  806         /*
  807          * Free left over swap blocks in source.
  808          *
  809          * We have to revert the type to OBJT_DEFAULT so we do not accidently
  810          * double-remove the object from the swap queues.
  811          */
  812         if (destroysource) {
  813                 swp_pager_meta_free_all(srcobject);
  814                 /*
  815                  * Reverting the type is not necessary, the caller is going
  816                  * to destroy srcobject directly, but I'm doing it here
  817                  * for consistency since we've removed the object from its
  818                  * queues.
  819                  */
  820                 srcobject->type = OBJT_DEFAULT;
  821         }
  822 }
  823 
  824 /*
  825  * SWAP_PAGER_HASPAGE() -       determine if we have good backing store for
  826  *                              the requested page.
  827  *
  828  *      We determine whether good backing store exists for the requested
  829  *      page and return TRUE if it does, FALSE if it doesn't.
  830  *
  831  *      If TRUE, we also try to determine how much valid, contiguous backing
  832  *      store exists before and after the requested page within a reasonable
  833  *      distance.  We do not try to restrict it to the swap device stripe
  834  *      (that is handled in getpages/putpages).  It probably isn't worth
  835  *      doing here.
  836  */
  837 static boolean_t
  838 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
  839 {
  840         daddr_t blk0;
  841 
  842         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  843         /*
  844          * do we have good backing store at the requested index ?
  845          */
  846         blk0 = swp_pager_meta_ctl(object, pindex, 0);
  847 
  848         if (blk0 == SWAPBLK_NONE) {
  849                 if (before)
  850                         *before = 0;
  851                 if (after)
  852                         *after = 0;
  853                 return (FALSE);
  854         }
  855 
  856         /*
  857          * find backwards-looking contiguous good backing store
  858          */
  859         if (before != NULL) {
  860                 int i;
  861 
  862                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
  863                         daddr_t blk;
  864 
  865                         if (i > pindex)
  866                                 break;
  867                         blk = swp_pager_meta_ctl(object, pindex - i, 0);
  868                         if (blk != blk0 - i)
  869                                 break;
  870                 }
  871                 *before = (i - 1);
  872         }
  873 
  874         /*
  875          * find forward-looking contiguous good backing store
  876          */
  877         if (after != NULL) {
  878                 int i;
  879 
  880                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
  881                         daddr_t blk;
  882 
  883                         blk = swp_pager_meta_ctl(object, pindex + i, 0);
  884                         if (blk != blk0 + i)
  885                                 break;
  886                 }
  887                 *after = (i - 1);
  888         }
  889         return (TRUE);
  890 }
  891 
  892 /*
  893  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
  894  *
  895  *      This removes any associated swap backing store, whether valid or
  896  *      not, from the page.  
  897  *
  898  *      This routine is typically called when a page is made dirty, at
  899  *      which point any associated swap can be freed.  MADV_FREE also
  900  *      calls us in a special-case situation
  901  *
  902  *      NOTE!!!  If the page is clean and the swap was valid, the caller
  903  *      should make the page dirty before calling this routine.  This routine
  904  *      does NOT change the m->dirty status of the page.  Also: MADV_FREE
  905  *      depends on it.
  906  *
  907  *      This routine may not block
  908  *      This routine must be called at splvm()
  909  */
  910 static void
  911 swap_pager_unswapped(vm_page_t m)
  912 {
  913 
  914         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  915         swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
  916 }
  917 
  918 /*
  919  * SWAP_PAGER_GETPAGES() - bring pages in from swap
  920  *
  921  *      Attempt to retrieve (m, count) pages from backing store, but make
  922  *      sure we retrieve at least m[reqpage].  We try to load in as large
  923  *      a chunk surrounding m[reqpage] as is contiguous in swap and which
  924  *      belongs to the same object.
  925  *
  926  *      The code is designed for asynchronous operation and 
  927  *      immediate-notification of 'reqpage' but tends not to be
  928  *      used that way.  Please do not optimize-out this algorithmic
  929  *      feature, I intend to improve on it in the future.
  930  *
  931  *      The parent has a single vm_object_pip_add() reference prior to
  932  *      calling us and we should return with the same.
  933  *
  934  *      The parent has BUSY'd the pages.  We should return with 'm'
  935  *      left busy, but the others adjusted.
  936  */
  937 static int
  938 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
  939 {
  940         struct buf *bp;
  941         vm_page_t mreq;
  942         int i;
  943         int j;
  944         daddr_t blk;
  945 
  946         mreq = m[reqpage];
  947 
  948         KASSERT(mreq->object == object,
  949             ("swap_pager_getpages: object mismatch %p/%p",
  950             object, mreq->object));
  951 
  952         /*
  953          * Calculate range to retrieve.  The pages have already been assigned
  954          * their swapblks.  We require a *contiguous* range but we know it to
  955          * not span devices.   If we do not supply it, bad things
  956          * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the 
  957          * loops are set up such that the case(s) are handled implicitly.
  958          *
  959          * The swp_*() calls must be made at splvm().  vm_page_free() does
  960          * not need to be, but it will go a little faster if it is.
  961          */
  962         blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
  963 
  964         for (i = reqpage - 1; i >= 0; --i) {
  965                 daddr_t iblk;
  966 
  967                 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
  968                 if (blk != iblk + (reqpage - i))
  969                         break;
  970         }
  971         ++i;
  972 
  973         for (j = reqpage + 1; j < count; ++j) {
  974                 daddr_t jblk;
  975 
  976                 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
  977                 if (blk != jblk - (j - reqpage))
  978                         break;
  979         }
  980 
  981         /*
  982          * free pages outside our collection range.   Note: we never free
  983          * mreq, it must remain busy throughout.
  984          */
  985         if (0 < i || j < count) {
  986                 int k;
  987 
  988                 vm_page_lock_queues();
  989                 for (k = 0; k < i; ++k)
  990                         vm_page_free(m[k]);
  991                 for (k = j; k < count; ++k)
  992                         vm_page_free(m[k]);
  993                 vm_page_unlock_queues();
  994         }
  995 
  996         /*
  997          * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq 
  998          * still busy, but the others unbusied.
  999          */
 1000         if (blk == SWAPBLK_NONE)
 1001                 return (VM_PAGER_FAIL);
 1002 
 1003         /*
 1004          * Getpbuf() can sleep.
 1005          */
 1006         VM_OBJECT_UNLOCK(object);
 1007         /*
 1008          * Get a swap buffer header to perform the IO
 1009          */
 1010         bp = getpbuf(&nsw_rcount);
 1011         bp->b_flags |= B_PAGING;
 1012 
 1013         /*
 1014          * map our page(s) into kva for input
 1015          */
 1016         pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
 1017 
 1018         bp->b_iocmd = BIO_READ;
 1019         bp->b_iodone = swp_pager_async_iodone;
 1020         bp->b_rcred = crhold(thread0.td_ucred);
 1021         bp->b_wcred = crhold(thread0.td_ucred);
 1022         bp->b_blkno = blk - (reqpage - i);
 1023         bp->b_bcount = PAGE_SIZE * (j - i);
 1024         bp->b_bufsize = PAGE_SIZE * (j - i);
 1025         bp->b_pager.pg_reqpage = reqpage - i;
 1026 
 1027         VM_OBJECT_LOCK(object);
 1028         vm_page_lock_queues();
 1029         {
 1030                 int k;
 1031 
 1032                 for (k = i; k < j; ++k) {
 1033                         bp->b_pages[k - i] = m[k];
 1034                         vm_page_flag_set(m[k], PG_SWAPINPROG);
 1035                 }
 1036         }
 1037         vm_page_unlock_queues();
 1038         bp->b_npages = j - i;
 1039 
 1040         cnt.v_swapin++;
 1041         cnt.v_swappgsin += bp->b_npages;
 1042 
 1043         /*
 1044          * We still hold the lock on mreq, and our automatic completion routine
 1045          * does not remove it.
 1046          */
 1047         vm_object_pip_add(object, bp->b_npages);
 1048         VM_OBJECT_UNLOCK(object);
 1049 
 1050         /*
 1051          * perform the I/O.  NOTE!!!  bp cannot be considered valid after
 1052          * this point because we automatically release it on completion.
 1053          * Instead, we look at the one page we are interested in which we
 1054          * still hold a lock on even through the I/O completion.
 1055          *
 1056          * The other pages in our m[] array are also released on completion,
 1057          * so we cannot assume they are valid anymore either.
 1058          *
 1059          * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1060          */
 1061         BUF_KERNPROC(bp);
 1062         swp_pager_strategy(bp);
 1063 
 1064         /*
 1065          * wait for the page we want to complete.  PG_SWAPINPROG is always
 1066          * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
 1067          * is set in the meta-data.
 1068          */
 1069         vm_page_lock_queues();
 1070         while ((mreq->flags & PG_SWAPINPROG) != 0) {
 1071                 vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
 1072                 cnt.v_intrans++;
 1073                 if (msleep(mreq, &vm_page_queue_mtx, PSWP, "swread", hz*20)) {
 1074                         printf(
 1075 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
 1076                             bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
 1077                 }
 1078         }
 1079         vm_page_unlock_queues();
 1080 
 1081         VM_OBJECT_LOCK(object);
 1082         /*
 1083          * mreq is left busied after completion, but all the other pages
 1084          * are freed.  If we had an unrecoverable read error the page will
 1085          * not be valid.
 1086          */
 1087         if (mreq->valid != VM_PAGE_BITS_ALL) {
 1088                 return (VM_PAGER_ERROR);
 1089         } else {
 1090                 return (VM_PAGER_OK);
 1091         }
 1092 
 1093         /*
 1094          * A final note: in a low swap situation, we cannot deallocate swap
 1095          * and mark a page dirty here because the caller is likely to mark
 1096          * the page clean when we return, causing the page to possibly revert 
 1097          * to all-zero's later.
 1098          */
 1099 }
 1100 
 1101 /*
 1102  *      swap_pager_putpages: 
 1103  *
 1104  *      Assign swap (if necessary) and initiate I/O on the specified pages.
 1105  *
 1106  *      We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
 1107  *      are automatically converted to SWAP objects.
 1108  *
 1109  *      In a low memory situation we may block in VOP_STRATEGY(), but the new 
 1110  *      vm_page reservation system coupled with properly written VFS devices 
 1111  *      should ensure that no low-memory deadlock occurs.  This is an area
 1112  *      which needs work.
 1113  *
 1114  *      The parent has N vm_object_pip_add() references prior to
 1115  *      calling us and will remove references for rtvals[] that are
 1116  *      not set to VM_PAGER_PEND.  We need to remove the rest on I/O
 1117  *      completion.
 1118  *
 1119  *      The parent has soft-busy'd the pages it passes us and will unbusy
 1120  *      those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
 1121  *      We need to unbusy the rest on I/O completion.
 1122  */
 1123 void
 1124 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 1125     boolean_t sync, int *rtvals)
 1126 {
 1127         int i;
 1128         int n = 0;
 1129 
 1130         if (count && m[0]->object != object) {
 1131                 panic("swap_pager_getpages: object mismatch %p/%p", 
 1132                     object, 
 1133                     m[0]->object
 1134                 );
 1135         }
 1136 
 1137         /*
 1138          * Step 1
 1139          *
 1140          * Turn object into OBJT_SWAP
 1141          * check for bogus sysops
 1142          * force sync if not pageout process
 1143          */
 1144         if (object->type != OBJT_SWAP)
 1145                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
 1146         VM_OBJECT_UNLOCK(object);
 1147 
 1148         if (curproc != pageproc)
 1149                 sync = TRUE;
 1150 
 1151         /*
 1152          * Step 2
 1153          *
 1154          * Update nsw parameters from swap_async_max sysctl values.  
 1155          * Do not let the sysop crash the machine with bogus numbers.
 1156          */
 1157         mtx_lock(&pbuf_mtx);
 1158         if (swap_async_max != nsw_wcount_async_max) {
 1159                 int n;
 1160 
 1161                 /*
 1162                  * limit range
 1163                  */
 1164                 if ((n = swap_async_max) > nswbuf / 2)
 1165                         n = nswbuf / 2;
 1166                 if (n < 1)
 1167                         n = 1;
 1168                 swap_async_max = n;
 1169 
 1170                 /*
 1171                  * Adjust difference ( if possible ).  If the current async
 1172                  * count is too low, we may not be able to make the adjustment
 1173                  * at this time.
 1174                  */
 1175                 n -= nsw_wcount_async_max;
 1176                 if (nsw_wcount_async + n >= 0) {
 1177                         nsw_wcount_async += n;
 1178                         nsw_wcount_async_max += n;
 1179                         wakeup(&nsw_wcount_async);
 1180                 }
 1181         }
 1182         mtx_unlock(&pbuf_mtx);
 1183 
 1184         /*
 1185          * Step 3
 1186          *
 1187          * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
 1188          * The page is left dirty until the pageout operation completes
 1189          * successfully.
 1190          */
 1191         for (i = 0; i < count; i += n) {
 1192                 int j;
 1193                 struct buf *bp;
 1194                 daddr_t blk;
 1195 
 1196                 /*
 1197                  * Maximum I/O size is limited by a number of factors.
 1198                  */
 1199                 n = min(BLIST_MAX_ALLOC, count - i);
 1200                 n = min(n, nsw_cluster_max);
 1201 
 1202                 /*
 1203                  * Get biggest block of swap we can.  If we fail, fall
 1204                  * back and try to allocate a smaller block.  Don't go
 1205                  * overboard trying to allocate space if it would overly
 1206                  * fragment swap.
 1207                  */
 1208                 while (
 1209                     (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
 1210                     n > 4
 1211                 ) {
 1212                         n >>= 1;
 1213                 }
 1214                 if (blk == SWAPBLK_NONE) {
 1215                         for (j = 0; j < n; ++j)
 1216                                 rtvals[i+j] = VM_PAGER_FAIL;
 1217                         continue;
 1218                 }
 1219 
 1220                 /*
 1221                  * All I/O parameters have been satisfied, build the I/O
 1222                  * request and assign the swap space.
 1223                  */
 1224                 if (sync == TRUE) {
 1225                         bp = getpbuf(&nsw_wcount_sync);
 1226                 } else {
 1227                         bp = getpbuf(&nsw_wcount_async);
 1228                         bp->b_flags = B_ASYNC;
 1229                 }
 1230                 bp->b_flags |= B_PAGING;
 1231                 bp->b_iocmd = BIO_WRITE;
 1232 
 1233                 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
 1234 
 1235                 bp->b_rcred = crhold(thread0.td_ucred);
 1236                 bp->b_wcred = crhold(thread0.td_ucred);
 1237                 bp->b_bcount = PAGE_SIZE * n;
 1238                 bp->b_bufsize = PAGE_SIZE * n;
 1239                 bp->b_blkno = blk;
 1240 
 1241                 VM_OBJECT_LOCK(object);
 1242                 for (j = 0; j < n; ++j) {
 1243                         vm_page_t mreq = m[i+j];
 1244 
 1245                         swp_pager_meta_build(
 1246                             mreq->object, 
 1247                             mreq->pindex,
 1248                             blk + j
 1249                         );
 1250                         vm_page_dirty(mreq);
 1251                         rtvals[i+j] = VM_PAGER_OK;
 1252 
 1253                         vm_page_lock_queues();
 1254                         vm_page_flag_set(mreq, PG_SWAPINPROG);
 1255                         vm_page_unlock_queues();
 1256                         bp->b_pages[j] = mreq;
 1257                 }
 1258                 VM_OBJECT_UNLOCK(object);
 1259                 bp->b_npages = n;
 1260                 /*
 1261                  * Must set dirty range for NFS to work.
 1262                  */
 1263                 bp->b_dirtyoff = 0;
 1264                 bp->b_dirtyend = bp->b_bcount;
 1265 
 1266                 cnt.v_swapout++;
 1267                 cnt.v_swappgsout += bp->b_npages;
 1268 
 1269                 /*
 1270                  * asynchronous
 1271                  *
 1272                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1273                  */
 1274                 if (sync == FALSE) {
 1275                         bp->b_iodone = swp_pager_async_iodone;
 1276                         BUF_KERNPROC(bp);
 1277                         swp_pager_strategy(bp);
 1278 
 1279                         for (j = 0; j < n; ++j)
 1280                                 rtvals[i+j] = VM_PAGER_PEND;
 1281                         /* restart outter loop */
 1282                         continue;
 1283                 }
 1284 
 1285                 /*
 1286                  * synchronous
 1287                  *
 1288                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1289                  */
 1290                 bp->b_iodone = bdone;
 1291                 swp_pager_strategy(bp);
 1292 
 1293                 /*
 1294                  * Wait for the sync I/O to complete, then update rtvals.
 1295                  * We just set the rtvals[] to VM_PAGER_PEND so we can call
 1296                  * our async completion routine at the end, thus avoiding a
 1297                  * double-free.
 1298                  */
 1299                 bwait(bp, PVM, "swwrt");
 1300                 for (j = 0; j < n; ++j)
 1301                         rtvals[i+j] = VM_PAGER_PEND;
 1302                 /*
 1303                  * Now that we are through with the bp, we can call the
 1304                  * normal async completion, which frees everything up.
 1305                  */
 1306                 swp_pager_async_iodone(bp);
 1307         }
 1308         VM_OBJECT_LOCK(object);
 1309 }
 1310 
 1311 /*
 1312  *      swp_pager_async_iodone:
 1313  *
 1314  *      Completion routine for asynchronous reads and writes from/to swap.
 1315  *      Also called manually by synchronous code to finish up a bp.
 1316  *
 1317  *      For READ operations, the pages are PG_BUSY'd.  For WRITE operations, 
 1318  *      the pages are vm_page_t->busy'd.  For READ operations, we PG_BUSY 
 1319  *      unbusy all pages except the 'main' request page.  For WRITE 
 1320  *      operations, we vm_page_t->busy'd unbusy all pages ( we can do this 
 1321  *      because we marked them all VM_PAGER_PEND on return from putpages ).
 1322  *
 1323  *      This routine may not block.
 1324  *      This routine is called at splbio() or better
 1325  *
 1326  *      We up ourselves to splvm() as required for various vm_page related
 1327  *      calls.
 1328  */
 1329 static void
 1330 swp_pager_async_iodone(struct buf *bp)
 1331 {
 1332         int i;
 1333         vm_object_t object = NULL;
 1334 
 1335         /*
 1336          * report error
 1337          */
 1338         if (bp->b_ioflags & BIO_ERROR) {
 1339                 printf(
 1340                     "swap_pager: I/O error - %s failed; blkno %ld,"
 1341                         "size %ld, error %d\n",
 1342                     ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
 1343                     (long)bp->b_blkno, 
 1344                     (long)bp->b_bcount,
 1345                     bp->b_error
 1346                 );
 1347         }
 1348 
 1349         /*
 1350          * remove the mapping for kernel virtual
 1351          */
 1352         pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
 1353 
 1354         if (bp->b_npages) {
 1355                 object = bp->b_pages[0]->object;
 1356                 VM_OBJECT_LOCK(object);
 1357         }
 1358         vm_page_lock_queues();
 1359         /*
 1360          * cleanup pages.  If an error occurs writing to swap, we are in
 1361          * very serious trouble.  If it happens to be a disk error, though,
 1362          * we may be able to recover by reassigning the swap later on.  So
 1363          * in this case we remove the m->swapblk assignment for the page 
 1364          * but do not free it in the rlist.  The errornous block(s) are thus
 1365          * never reallocated as swap.  Redirty the page and continue.
 1366          */
 1367         for (i = 0; i < bp->b_npages; ++i) {
 1368                 vm_page_t m = bp->b_pages[i];
 1369 
 1370                 vm_page_flag_clear(m, PG_SWAPINPROG);
 1371 
 1372                 if (bp->b_ioflags & BIO_ERROR) {
 1373                         /*
 1374                          * If an error occurs I'd love to throw the swapblk
 1375                          * away without freeing it back to swapspace, so it
 1376                          * can never be used again.  But I can't from an 
 1377                          * interrupt.
 1378                          */
 1379                         if (bp->b_iocmd == BIO_READ) {
 1380                                 /*
 1381                                  * When reading, reqpage needs to stay
 1382                                  * locked for the parent, but all other
 1383                                  * pages can be freed.  We still want to
 1384                                  * wakeup the parent waiting on the page,
 1385                                  * though.  ( also: pg_reqpage can be -1 and 
 1386                                  * not match anything ).
 1387                                  *
 1388                                  * We have to wake specifically requested pages
 1389                                  * up too because we cleared PG_SWAPINPROG and
 1390                                  * someone may be waiting for that.
 1391                                  *
 1392                                  * NOTE: for reads, m->dirty will probably
 1393                                  * be overridden by the original caller of
 1394                                  * getpages so don't play cute tricks here.
 1395                                  *
 1396                                  * XXX IT IS NOT LEGAL TO FREE THE PAGE HERE
 1397                                  * AS THIS MESSES WITH object->memq, and it is
 1398                                  * not legal to mess with object->memq from an
 1399                                  * interrupt.
 1400                                  */
 1401                                 m->valid = 0;
 1402                                 if (i != bp->b_pager.pg_reqpage)
 1403                                         vm_page_free(m);
 1404                                 else
 1405                                         vm_page_flash(m);
 1406                                 /*
 1407                                  * If i == bp->b_pager.pg_reqpage, do not wake 
 1408                                  * the page up.  The caller needs to.
 1409                                  */
 1410                         } else {
 1411                                 /*
 1412                                  * If a write error occurs, reactivate page
 1413                                  * so it doesn't clog the inactive list,
 1414                                  * then finish the I/O.
 1415                                  */
 1416                                 vm_page_dirty(m);
 1417                                 vm_page_activate(m);
 1418                                 vm_page_io_finish(m);
 1419                         }
 1420                 } else if (bp->b_iocmd == BIO_READ) {
 1421                         /*
 1422                          * For read success, clear dirty bits.  Nobody should
 1423                          * have this page mapped but don't take any chances,
 1424                          * make sure the pmap modify bits are also cleared.
 1425                          *
 1426                          * NOTE: for reads, m->dirty will probably be 
 1427                          * overridden by the original caller of getpages so
 1428                          * we cannot set them in order to free the underlying
 1429                          * swap in a low-swap situation.  I don't think we'd
 1430                          * want to do that anyway, but it was an optimization
 1431                          * that existed in the old swapper for a time before
 1432                          * it got ripped out due to precisely this problem.
 1433                          *
 1434                          * If not the requested page then deactivate it.
 1435                          *
 1436                          * Note that the requested page, reqpage, is left
 1437                          * busied, but we still have to wake it up.  The
 1438                          * other pages are released (unbusied) by 
 1439                          * vm_page_wakeup().  We do not set reqpage's
 1440                          * valid bits here, it is up to the caller.
 1441                          */
 1442                         pmap_clear_modify(m);
 1443                         m->valid = VM_PAGE_BITS_ALL;
 1444                         vm_page_undirty(m);
 1445 
 1446                         /*
 1447                          * We have to wake specifically requested pages
 1448                          * up too because we cleared PG_SWAPINPROG and
 1449                          * could be waiting for it in getpages.  However,
 1450                          * be sure to not unbusy getpages specifically
 1451                          * requested page - getpages expects it to be 
 1452                          * left busy.
 1453                          */
 1454                         if (i != bp->b_pager.pg_reqpage) {
 1455                                 vm_page_deactivate(m);
 1456                                 vm_page_wakeup(m);
 1457                         } else {
 1458                                 vm_page_flash(m);
 1459                         }
 1460                 } else {
 1461                         /*
 1462                          * For write success, clear the modify and dirty 
 1463                          * status, then finish the I/O ( which decrements the 
 1464                          * busy count and possibly wakes waiter's up ).
 1465                          */
 1466                         pmap_clear_modify(m);
 1467                         vm_page_undirty(m);
 1468                         vm_page_io_finish(m);
 1469                         if (vm_page_count_severe())
 1470                                 vm_page_try_to_cache(m);
 1471                 }
 1472         }
 1473         vm_page_unlock_queues();
 1474 
 1475         /*
 1476          * adjust pip.  NOTE: the original parent may still have its own
 1477          * pip refs on the object.
 1478          */
 1479         if (object != NULL) {
 1480                 vm_object_pip_wakeupn(object, bp->b_npages);
 1481                 VM_OBJECT_UNLOCK(object);
 1482         }
 1483         /* 
 1484          * swapdev_strategy() manually sets b_vp and b_bufobj before calling 
 1485          * bstrategy(). Set them back to NULL now we're done with it, or we'll
 1486          * trigger a KASSERT in relpbuf().
 1487          */
 1488         if (bp->b_vp) {
 1489                     bp->b_vp = NULL;
 1490                     bp->b_bufobj = NULL;
 1491         }
 1492 
 1493         /*
 1494          * release the physical I/O buffer
 1495          */
 1496         relpbuf(
 1497             bp, 
 1498             ((bp->b_iocmd == BIO_READ) ? &nsw_rcount : 
 1499                 ((bp->b_flags & B_ASYNC) ? 
 1500                     &nsw_wcount_async : 
 1501                     &nsw_wcount_sync
 1502                 )
 1503             )
 1504         );
 1505 }
 1506 
 1507 /*
 1508  *      swap_pager_isswapped:
 1509  *
 1510  *      Return 1 if at least one page in the given object is paged
 1511  *      out to the given swap device.
 1512  *
 1513  *      This routine may not block.
 1514  */
 1515 int
 1516 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
 1517 {
 1518         daddr_t index = 0;
 1519         int bcount;
 1520         int i;
 1521 
 1522         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1523         if (object->type != OBJT_SWAP)
 1524                 return (0);
 1525 
 1526         mtx_lock(&swhash_mtx);
 1527         for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
 1528                 struct swblock *swap;
 1529 
 1530                 if ((swap = *swp_pager_hash(object, index)) != NULL) {
 1531                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1532                                 if (swp_pager_isondev(swap->swb_pages[i], sp)) {
 1533                                         mtx_unlock(&swhash_mtx);
 1534                                         return (1);
 1535                                 }
 1536                         }
 1537                 }
 1538                 index += SWAP_META_PAGES;
 1539                 if (index > 0x20000000)
 1540                         panic("swap_pager_isswapped: failed to locate all swap meta blocks");
 1541         }
 1542         mtx_unlock(&swhash_mtx);
 1543         return (0);
 1544 }
 1545 
 1546 /*
 1547  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
 1548  *
 1549  *      This routine dissociates the page at the given index within a
 1550  *      swap block from its backing store, paging it in if necessary.
 1551  *      If the page is paged in, it is placed in the inactive queue,
 1552  *      since it had its backing store ripped out from under it.
 1553  *      We also attempt to swap in all other pages in the swap block,
 1554  *      we only guarantee that the one at the specified index is
 1555  *      paged in.
 1556  *
 1557  *      XXX - The code to page the whole block in doesn't work, so we
 1558  *            revert to the one-by-one behavior for now.  Sigh.
 1559  */
 1560 static __inline void
 1561 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
 1562 {
 1563         vm_page_t m;
 1564 
 1565         vm_object_pip_add(object, 1);
 1566         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
 1567         if (m->valid == VM_PAGE_BITS_ALL) {
 1568                 vm_object_pip_subtract(object, 1);
 1569                 vm_page_lock_queues();
 1570                 vm_page_activate(m);
 1571                 vm_page_dirty(m);
 1572                 vm_page_wakeup(m);
 1573                 vm_page_unlock_queues();
 1574                 vm_pager_page_unswapped(m);
 1575                 return;
 1576         }
 1577 
 1578         if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
 1579                 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
 1580         vm_object_pip_subtract(object, 1);
 1581         vm_page_lock_queues();
 1582         vm_page_dirty(m);
 1583         vm_page_dontneed(m);
 1584         vm_page_wakeup(m);
 1585         vm_page_unlock_queues();
 1586         vm_pager_page_unswapped(m);
 1587 }
 1588 
 1589 /*
 1590  *      swap_pager_swapoff:
 1591  *
 1592  *      Page in all of the pages that have been paged out to the
 1593  *      given device.  The corresponding blocks in the bitmap must be
 1594  *      marked as allocated and the device must be flagged SW_CLOSING.
 1595  *      There may be no processes swapped out to the device.
 1596  *
 1597  *      This routine may block.
 1598  */
 1599 static void
 1600 swap_pager_swapoff(struct swdevt *sp)
 1601 {
 1602         struct swblock *swap;
 1603         int i, j, retries;
 1604 
 1605         GIANT_REQUIRED;
 1606 
 1607         retries = 0;
 1608 full_rescan:
 1609         mtx_lock(&swhash_mtx);
 1610         for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
 1611 restart:
 1612                 for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
 1613                         vm_object_t object = swap->swb_object;
 1614                         vm_pindex_t pindex = swap->swb_index;
 1615                         for (j = 0; j < SWAP_META_PAGES; ++j) {
 1616                                 if (swp_pager_isondev(swap->swb_pages[j], sp)) {
 1617                                         /* avoid deadlock */
 1618                                         if (!VM_OBJECT_TRYLOCK(object)) {
 1619                                                 break;
 1620                                         } else {
 1621                                                 mtx_unlock(&swhash_mtx);
 1622                                                 swp_pager_force_pagein(object,
 1623                                                     pindex + j);
 1624                                                 VM_OBJECT_UNLOCK(object);
 1625                                                 mtx_lock(&swhash_mtx);
 1626                                                 goto restart;
 1627                                         }
 1628                                 }
 1629                         }
 1630                 }
 1631         }
 1632         mtx_unlock(&swhash_mtx);
 1633         if (sp->sw_used) {
 1634                 int dummy;
 1635                 /*
 1636                  * Objects may be locked or paging to the device being
 1637                  * removed, so we will miss their pages and need to
 1638                  * make another pass.  We have marked this device as
 1639                  * SW_CLOSING, so the activity should finish soon.
 1640                  */
 1641                 retries++;
 1642                 if (retries > 100) {
 1643                         panic("swapoff: failed to locate %d swap blocks",
 1644                             sp->sw_used);
 1645                 }
 1646                 tsleep(&dummy, PVM, "swpoff", hz / 20);
 1647                 goto full_rescan;
 1648         }
 1649 }
 1650 
 1651 /************************************************************************
 1652  *                              SWAP META DATA                          *
 1653  ************************************************************************
 1654  *
 1655  *      These routines manipulate the swap metadata stored in the 
 1656  *      OBJT_SWAP object.  All swp_*() routines must be called at
 1657  *      splvm() because swap can be freed up by the low level vm_page
 1658  *      code which might be called from interrupts beyond what splbio() covers.
 1659  *
 1660  *      Swap metadata is implemented with a global hash and not directly
 1661  *      linked into the object.  Instead the object simply contains
 1662  *      appropriate tracking counters.
 1663  */
 1664 
 1665 /*
 1666  * SWP_PAGER_META_BUILD() -     add swap block to swap meta data for object
 1667  *
 1668  *      We first convert the object to a swap object if it is a default
 1669  *      object.
 1670  *
 1671  *      The specified swapblk is added to the object's swap metadata.  If
 1672  *      the swapblk is not valid, it is freed instead.  Any previously
 1673  *      assigned swapblk is freed.
 1674  *
 1675  *      This routine must be called at splvm(), except when used to convert
 1676  *      an OBJT_DEFAULT object into an OBJT_SWAP object.
 1677  */
 1678 static void
 1679 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
 1680 {
 1681         struct swblock *swap;
 1682         struct swblock **pswap;
 1683         int dummy, idx;
 1684 
 1685         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1686         /*
 1687          * Convert default object to swap object if necessary
 1688          */
 1689         if (object->type != OBJT_SWAP) {
 1690                 object->type = OBJT_SWAP;
 1691                 object->un_pager.swp.swp_bcount = 0;
 1692 
 1693                 if (object->handle != NULL) {
 1694                         mtx_lock(&sw_alloc_mtx);
 1695                         TAILQ_INSERT_TAIL(
 1696                             NOBJLIST(object->handle),
 1697                             object, 
 1698                             pager_object_list
 1699                         );
 1700                         mtx_unlock(&sw_alloc_mtx);
 1701                 }
 1702         }
 1703         
 1704         /*
 1705          * Locate hash entry.  If not found create, but if we aren't adding
 1706          * anything just return.  If we run out of space in the map we wait
 1707          * and, since the hash table may have changed, retry.
 1708          */
 1709 retry:
 1710         mtx_lock(&swhash_mtx);
 1711         pswap = swp_pager_hash(object, pindex);
 1712 
 1713         if ((swap = *pswap) == NULL) {
 1714                 int i;
 1715 
 1716                 if (swapblk == SWAPBLK_NONE)
 1717                         goto done;
 1718 
 1719                 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
 1720                 if (swap == NULL) {
 1721                         mtx_unlock(&swhash_mtx);
 1722                         VM_OBJECT_UNLOCK(object);
 1723                         if (uma_zone_exhausted(swap_zone)) {
 1724                                 printf("swap zone exhausted, increase kern.maxswzone\n");
 1725                                 vm_pageout_oom(VM_OOM_SWAPZ);
 1726                                 tsleep(&dummy, PVM, "swzonex", 10);
 1727                         } else
 1728                                 VM_WAIT;
 1729                         VM_OBJECT_LOCK(object);
 1730                         goto retry;
 1731                 }
 1732 
 1733                 swap->swb_hnext = NULL;
 1734                 swap->swb_object = object;
 1735                 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
 1736                 swap->swb_count = 0;
 1737 
 1738                 ++object->un_pager.swp.swp_bcount;
 1739 
 1740                 for (i = 0; i < SWAP_META_PAGES; ++i)
 1741                         swap->swb_pages[i] = SWAPBLK_NONE;
 1742         }
 1743 
 1744         /*
 1745          * Delete prior contents of metadata
 1746          */
 1747         idx = pindex & SWAP_META_MASK;
 1748 
 1749         if (swap->swb_pages[idx] != SWAPBLK_NONE) {
 1750                 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
 1751                 --swap->swb_count;
 1752         }
 1753 
 1754         /*
 1755          * Enter block into metadata
 1756          */
 1757         swap->swb_pages[idx] = swapblk;
 1758         if (swapblk != SWAPBLK_NONE)
 1759                 ++swap->swb_count;
 1760 done:
 1761         mtx_unlock(&swhash_mtx);
 1762 }
 1763 
 1764 /*
 1765  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
 1766  *
 1767  *      The requested range of blocks is freed, with any associated swap 
 1768  *      returned to the swap bitmap.
 1769  *
 1770  *      This routine will free swap metadata structures as they are cleaned 
 1771  *      out.  This routine does *NOT* operate on swap metadata associated
 1772  *      with resident pages.
 1773  *
 1774  *      This routine must be called at splvm()
 1775  */
 1776 static void
 1777 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
 1778 {
 1779 
 1780         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1781         if (object->type != OBJT_SWAP)
 1782                 return;
 1783 
 1784         while (count > 0) {
 1785                 struct swblock **pswap;
 1786                 struct swblock *swap;
 1787 
 1788                 mtx_lock(&swhash_mtx);
 1789                 pswap = swp_pager_hash(object, index);
 1790 
 1791                 if ((swap = *pswap) != NULL) {
 1792                         daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
 1793 
 1794                         if (v != SWAPBLK_NONE) {
 1795                                 swp_pager_freeswapspace(v, 1);
 1796                                 swap->swb_pages[index & SWAP_META_MASK] =
 1797                                         SWAPBLK_NONE;
 1798                                 if (--swap->swb_count == 0) {
 1799                                         *pswap = swap->swb_hnext;
 1800                                         uma_zfree(swap_zone, swap);
 1801                                         --object->un_pager.swp.swp_bcount;
 1802                                 }
 1803                         }
 1804                         --count;
 1805                         ++index;
 1806                 } else {
 1807                         int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
 1808                         count -= n;
 1809                         index += n;
 1810                 }
 1811                 mtx_unlock(&swhash_mtx);
 1812         }
 1813 }
 1814 
 1815 /*
 1816  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
 1817  *
 1818  *      This routine locates and destroys all swap metadata associated with
 1819  *      an object.
 1820  *
 1821  *      This routine must be called at splvm()
 1822  */
 1823 static void
 1824 swp_pager_meta_free_all(vm_object_t object)
 1825 {
 1826         daddr_t index = 0;
 1827 
 1828         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1829         if (object->type != OBJT_SWAP)
 1830                 return;
 1831 
 1832         while (object->un_pager.swp.swp_bcount) {
 1833                 struct swblock **pswap;
 1834                 struct swblock *swap;
 1835 
 1836                 mtx_lock(&swhash_mtx);
 1837                 pswap = swp_pager_hash(object, index);
 1838                 if ((swap = *pswap) != NULL) {
 1839                         int i;
 1840 
 1841                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1842                                 daddr_t v = swap->swb_pages[i];
 1843                                 if (v != SWAPBLK_NONE) {
 1844                                         --swap->swb_count;
 1845                                         swp_pager_freeswapspace(v, 1);
 1846                                 }
 1847                         }
 1848                         if (swap->swb_count != 0)
 1849                                 panic("swap_pager_meta_free_all: swb_count != 0");
 1850                         *pswap = swap->swb_hnext;
 1851                         uma_zfree(swap_zone, swap);
 1852                         --object->un_pager.swp.swp_bcount;
 1853                 }
 1854                 mtx_unlock(&swhash_mtx);
 1855                 index += SWAP_META_PAGES;
 1856                 if (index > 0x20000000)
 1857                         panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
 1858         }
 1859 }
 1860 
 1861 /*
 1862  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
 1863  *
 1864  *      This routine is capable of looking up, popping, or freeing
 1865  *      swapblk assignments in the swap meta data or in the vm_page_t.
 1866  *      The routine typically returns the swapblk being looked-up, or popped,
 1867  *      or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
 1868  *      was invalid.  This routine will automatically free any invalid 
 1869  *      meta-data swapblks.
 1870  *
 1871  *      It is not possible to store invalid swapblks in the swap meta data
 1872  *      (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
 1873  *
 1874  *      When acting on a busy resident page and paging is in progress, we 
 1875  *      have to wait until paging is complete but otherwise can act on the 
 1876  *      busy page.
 1877  *
 1878  *      This routine must be called at splvm().
 1879  *
 1880  *      SWM_FREE        remove and free swap block from metadata
 1881  *      SWM_POP         remove from meta data but do not free.. pop it out
 1882  */
 1883 static daddr_t
 1884 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
 1885 {
 1886         struct swblock **pswap;
 1887         struct swblock *swap;
 1888         daddr_t r1;
 1889         int idx;
 1890 
 1891         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1892         /*
 1893          * The meta data only exists of the object is OBJT_SWAP 
 1894          * and even then might not be allocated yet.
 1895          */
 1896         if (object->type != OBJT_SWAP)
 1897                 return (SWAPBLK_NONE);
 1898 
 1899         r1 = SWAPBLK_NONE;
 1900         mtx_lock(&swhash_mtx);
 1901         pswap = swp_pager_hash(object, pindex);
 1902 
 1903         if ((swap = *pswap) != NULL) {
 1904                 idx = pindex & SWAP_META_MASK;
 1905                 r1 = swap->swb_pages[idx];
 1906 
 1907                 if (r1 != SWAPBLK_NONE) {
 1908                         if (flags & SWM_FREE) {
 1909                                 swp_pager_freeswapspace(r1, 1);
 1910                                 r1 = SWAPBLK_NONE;
 1911                         }
 1912                         if (flags & (SWM_FREE|SWM_POP)) {
 1913                                 swap->swb_pages[idx] = SWAPBLK_NONE;
 1914                                 if (--swap->swb_count == 0) {
 1915                                         *pswap = swap->swb_hnext;
 1916                                         uma_zfree(swap_zone, swap);
 1917                                         --object->un_pager.swp.swp_bcount;
 1918                                 }
 1919                         } 
 1920                 }
 1921         }
 1922         mtx_unlock(&swhash_mtx);
 1923         return (r1);
 1924 }
 1925 
 1926 /*
 1927  * System call swapon(name) enables swapping on device name,
 1928  * which must be in the swdevsw.  Return EBUSY
 1929  * if already swapping on this device.
 1930  */
 1931 #ifndef _SYS_SYSPROTO_H_
 1932 struct swapon_args {
 1933         char *name;
 1934 };
 1935 #endif
 1936 
 1937 /* 
 1938  * MPSAFE
 1939  */
 1940 /* ARGSUSED */
 1941 int
 1942 swapon(struct thread *td, struct swapon_args *uap)
 1943 {
 1944         struct vattr attr;
 1945         struct vnode *vp;
 1946         struct nameidata nd;
 1947         int error;
 1948 
 1949         mtx_lock(&Giant);
 1950         error = suser(td);
 1951         if (error)
 1952                 goto done2;
 1953 
 1954         while (swdev_syscall_active)
 1955             tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
 1956         swdev_syscall_active = 1;
 1957 
 1958         /*
 1959          * Swap metadata may not fit in the KVM if we have physical
 1960          * memory of >1GB.
 1961          */
 1962         if (swap_zone == NULL) {
 1963                 error = ENOMEM;
 1964                 goto done;
 1965         }
 1966 
 1967         NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW, UIO_USERSPACE, uap->name, td);
 1968         error = namei(&nd);
 1969         if (error)
 1970                 goto done;
 1971 
 1972         NDFREE(&nd, NDF_ONLY_PNBUF);
 1973         vp = nd.ni_vp;
 1974 
 1975         if (vn_isdisk(vp, &error)) {
 1976                 error = swapongeom(td, vp);
 1977         } else if (vp->v_type == VREG &&
 1978             (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
 1979             (error = VOP_GETATTR(vp, &attr, td->td_ucred, td)) == 0) {
 1980                 /*
 1981                  * Allow direct swapping to NFS regular files in the same
 1982                  * way that nfs_mountroot() sets up diskless swapping.
 1983                  */
 1984                 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
 1985         }
 1986 
 1987         if (error)
 1988                 vrele(vp);
 1989 done:
 1990         swdev_syscall_active = 0;
 1991         wakeup_one(&swdev_syscall_active);
 1992 done2:
 1993         mtx_unlock(&Giant);
 1994         return (error);
 1995 }
 1996 
 1997 static void
 1998 swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
 1999 {
 2000         struct swdevt *sp, *tsp;
 2001         swblk_t dvbase;
 2002         u_long mblocks;
 2003 
 2004         /*
 2005          * If we go beyond this, we get overflows in the radix
 2006          * tree bitmap code.
 2007          */
 2008         mblocks = 0x40000000 / BLIST_META_RADIX;
 2009         if (nblks > mblocks) {
 2010                 printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n",
 2011                         mblocks);
 2012                 nblks = mblocks;
 2013         }
 2014         /*
 2015          * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
 2016          * First chop nblks off to page-align it, then convert.
 2017          * 
 2018          * sw->sw_nblks is in page-sized chunks now too.
 2019          */
 2020         nblks &= ~(ctodb(1) - 1);
 2021         nblks = dbtoc(nblks);
 2022 
 2023         sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
 2024         sp->sw_vp = vp;
 2025         sp->sw_id = id;
 2026         sp->sw_dev = dev;
 2027         sp->sw_flags = 0;
 2028         sp->sw_nblks = nblks;
 2029         sp->sw_used = 0;
 2030         sp->sw_strategy = strategy;
 2031         sp->sw_close = close;
 2032 
 2033         sp->sw_blist = blist_create(nblks);
 2034         /*
 2035          * Do not free the first two block in order to avoid overwriting
 2036          * any bsd label at the front of the partition
 2037          */
 2038         blist_free(sp->sw_blist, 2, nblks - 2);
 2039 
 2040         dvbase = 0;
 2041         mtx_lock(&sw_dev_mtx);
 2042         TAILQ_FOREACH(tsp, &swtailq, sw_list) {
 2043                 if (tsp->sw_end >= dvbase) {
 2044                         /*
 2045                          * We put one uncovered page between the devices
 2046                          * in order to definitively prevent any cross-device
 2047                          * I/O requests
 2048                          */
 2049                         dvbase = tsp->sw_end + 1;
 2050                 }
 2051         }
 2052         sp->sw_first = dvbase;
 2053         sp->sw_end = dvbase + nblks;
 2054         TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
 2055         nswapdev++;
 2056         swap_pager_avail += nblks;
 2057         swp_sizecheck();
 2058         mtx_unlock(&sw_dev_mtx);
 2059 }
 2060 
 2061 /*
 2062  * SYSCALL: swapoff(devname)
 2063  *
 2064  * Disable swapping on the given device.
 2065  *
 2066  * XXX: Badly designed system call: it should use a device index
 2067  * rather than filename as specification.  We keep sw_vp around
 2068  * only to make this work.
 2069  */
 2070 #ifndef _SYS_SYSPROTO_H_
 2071 struct swapoff_args {
 2072         char *name;
 2073 };
 2074 #endif
 2075 
 2076 /*
 2077  * MPSAFE
 2078  */
 2079 /* ARGSUSED */
 2080 int
 2081 swapoff(struct thread *td, struct swapoff_args *uap)
 2082 {
 2083         struct vnode *vp;
 2084         struct nameidata nd;
 2085         struct swdevt *sp;
 2086         int error;
 2087 
 2088         error = suser(td);
 2089         if (error)
 2090                 return (error);
 2091 
 2092         mtx_lock(&Giant);
 2093         while (swdev_syscall_active)
 2094             tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2095         swdev_syscall_active = 1;
 2096 
 2097         NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, uap->name, td);
 2098         error = namei(&nd);
 2099         if (error)
 2100                 goto done;
 2101         NDFREE(&nd, NDF_ONLY_PNBUF);
 2102         vp = nd.ni_vp;
 2103 
 2104         mtx_lock(&sw_dev_mtx);
 2105         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2106                 if (sp->sw_vp == vp)
 2107                         break;
 2108         }
 2109         mtx_unlock(&sw_dev_mtx);
 2110         if (sp == NULL) {
 2111                 error = EINVAL;
 2112                 goto done;
 2113         }
 2114         error = swapoff_one(sp, td);
 2115 done:
 2116         swdev_syscall_active = 0;
 2117         wakeup_one(&swdev_syscall_active);
 2118         mtx_unlock(&Giant);
 2119         return (error);
 2120 }
 2121 
 2122 static int
 2123 swapoff_one(struct swdevt *sp, struct thread *td)
 2124 {
 2125         u_long nblks, dvbase;
 2126 #ifdef MAC
 2127         int error;
 2128 #endif
 2129 
 2130         mtx_assert(&Giant, MA_OWNED);
 2131 #ifdef MAC
 2132         (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY, td);
 2133         error = mac_check_system_swapoff(td->td_ucred, sp->sw_vp);
 2134         (void) VOP_UNLOCK(sp->sw_vp, 0, td);
 2135         if (error != 0)
 2136                 return (error);
 2137 #endif
 2138         nblks = sp->sw_nblks;
 2139 
 2140         /*
 2141          * We can turn off this swap device safely only if the
 2142          * available virtual memory in the system will fit the amount
 2143          * of data we will have to page back in, plus an epsilon so
 2144          * the system doesn't become critically low on swap space.
 2145          */
 2146         if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
 2147             nblks + nswap_lowat) {
 2148                 return (ENOMEM);
 2149         }
 2150 
 2151         /*
 2152          * Prevent further allocations on this device.
 2153          */
 2154         mtx_lock(&sw_dev_mtx);
 2155         sp->sw_flags |= SW_CLOSING;
 2156         for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
 2157                 swap_pager_avail -= blist_fill(sp->sw_blist,
 2158                      dvbase, dmmax);
 2159         }
 2160         mtx_unlock(&sw_dev_mtx);
 2161 
 2162         /*
 2163          * Page in the contents of the device and close it.
 2164          */
 2165         swap_pager_swapoff(sp);
 2166 
 2167         sp->sw_close(td, sp);
 2168         sp->sw_id = NULL;
 2169         mtx_lock(&sw_dev_mtx);
 2170         TAILQ_REMOVE(&swtailq, sp, sw_list);
 2171         nswapdev--;
 2172         if (nswapdev == 0) {
 2173                 swap_pager_full = 2;
 2174                 swap_pager_almost_full = 1;
 2175         }
 2176         if (swdevhd == sp)
 2177                 swdevhd = NULL;
 2178         mtx_unlock(&sw_dev_mtx);
 2179         blist_destroy(sp->sw_blist);
 2180         free(sp, M_VMPGDATA);
 2181         return (0);
 2182 }
 2183 
 2184 void
 2185 swapoff_all(void)
 2186 {
 2187         struct swdevt *sp, *spt;
 2188         const char *devname;
 2189         int error;
 2190  
 2191         mtx_lock(&Giant);
 2192         while (swdev_syscall_active)
 2193                 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2194         swdev_syscall_active = 1;
 2195  
 2196         mtx_lock(&sw_dev_mtx);
 2197         TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
 2198                 mtx_unlock(&sw_dev_mtx);
 2199                 if (vn_isdisk(sp->sw_vp, NULL))
 2200                         devname = sp->sw_vp->v_rdev->si_name;
 2201                 else
 2202                         devname = "[file]";
 2203                 error = swapoff_one(sp, &thread0);
 2204                 if (error != 0) {
 2205                         printf("Cannot remove swap device %s (error=%d), "
 2206                             "skipping.\n", devname, error);
 2207                 } else if (bootverbose) {
 2208                         printf("Swap device %s removed.\n", devname);
 2209                 }
 2210                 mtx_lock(&sw_dev_mtx);
 2211         }
 2212         mtx_unlock(&sw_dev_mtx);
 2213  
 2214         swdev_syscall_active = 0;
 2215         wakeup_one(&swdev_syscall_active);
 2216         mtx_unlock(&Giant);
 2217 }
 2218 
 2219 void
 2220 swap_pager_status(int *total, int *used)
 2221 {
 2222         struct swdevt *sp;
 2223 
 2224         *total = 0;
 2225         *used = 0;
 2226         mtx_lock(&sw_dev_mtx);
 2227         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2228                 *total += sp->sw_nblks;
 2229                 *used += sp->sw_used;
 2230         }
 2231         mtx_unlock(&sw_dev_mtx);
 2232 }
 2233 
 2234 static int
 2235 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
 2236 {
 2237         int     *name = (int *)arg1;
 2238         int     error, n;
 2239         struct xswdev xs;
 2240         struct swdevt *sp;
 2241 
 2242         if (arg2 != 1) /* name length */
 2243                 return (EINVAL);
 2244 
 2245         n = 0;
 2246         mtx_lock(&sw_dev_mtx);
 2247         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2248                 if (n == *name) {
 2249                         mtx_unlock(&sw_dev_mtx);
 2250                         xs.xsw_version = XSWDEV_VERSION;
 2251                         xs.xsw_dev = sp->sw_dev;
 2252                         xs.xsw_flags = sp->sw_flags;
 2253                         xs.xsw_nblks = sp->sw_nblks;
 2254                         xs.xsw_used = sp->sw_used;
 2255 
 2256                         error = SYSCTL_OUT(req, &xs, sizeof(xs));
 2257                         return (error);
 2258                 }
 2259                 n++;
 2260         }
 2261         mtx_unlock(&sw_dev_mtx);
 2262         return (ENOENT);
 2263 }
 2264 
 2265 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
 2266     "Number of swap devices");
 2267 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
 2268     "Swap statistics by device");
 2269 
 2270 /*
 2271  * vmspace_swap_count() - count the approximate swap useage in pages for a
 2272  *                        vmspace.
 2273  *
 2274  *      The map must be locked.
 2275  *
 2276  *      Swap useage is determined by taking the proportional swap used by
 2277  *      VM objects backing the VM map.  To make up for fractional losses,
 2278  *      if the VM object has any swap use at all the associated map entries
 2279  *      count for at least 1 swap page.
 2280  */
 2281 int
 2282 vmspace_swap_count(struct vmspace *vmspace)
 2283 {
 2284         vm_map_t map = &vmspace->vm_map;
 2285         vm_map_entry_t cur;
 2286         int count = 0;
 2287 
 2288         for (cur = map->header.next; cur != &map->header; cur = cur->next) {
 2289                 vm_object_t object;
 2290 
 2291                 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2292                     (object = cur->object.vm_object) != NULL) {
 2293                         VM_OBJECT_LOCK(object);
 2294                         if (object->type == OBJT_SWAP &&
 2295                             object->un_pager.swp.swp_bcount != 0) {
 2296                                 int n = (cur->end - cur->start) / PAGE_SIZE;
 2297 
 2298                                 count += object->un_pager.swp.swp_bcount *
 2299                                     SWAP_META_PAGES * n / object->size + 1;
 2300                         }
 2301                         VM_OBJECT_UNLOCK(object);
 2302                 }
 2303         }
 2304         return (count);
 2305 }
 2306 
 2307 /*
 2308  * GEOM backend
 2309  *
 2310  * Swapping onto disk devices.
 2311  *
 2312  */
 2313 
 2314 static g_orphan_t swapgeom_orphan;
 2315 
 2316 static struct g_class g_swap_class = {
 2317         .name = "SWAP",
 2318         .version = G_VERSION,
 2319         .orphan = swapgeom_orphan,
 2320 };
 2321 
 2322 DECLARE_GEOM_CLASS(g_swap_class, g_class);
 2323 
 2324 
 2325 static void
 2326 swapgeom_done(struct bio *bp2)
 2327 {
 2328         struct buf *bp;
 2329 
 2330         bp = bp2->bio_caller2;
 2331         bp->b_ioflags = bp2->bio_flags;
 2332         if (bp2->bio_error)
 2333                 bp->b_ioflags |= BIO_ERROR;
 2334         bp->b_resid = bp->b_bcount - bp2->bio_completed;
 2335         bp->b_error = bp2->bio_error;
 2336         bufdone(bp);
 2337         g_destroy_bio(bp2);
 2338 }
 2339 
 2340 static void
 2341 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
 2342 {
 2343         struct bio *bio;
 2344         struct g_consumer *cp;
 2345 
 2346         cp = sp->sw_id;
 2347         if (cp == NULL) {
 2348                 bp->b_error = ENXIO;
 2349                 bp->b_ioflags |= BIO_ERROR;
 2350                 bufdone(bp);
 2351                 return;
 2352         }
 2353         bio = g_alloc_bio();
 2354 #if 0
 2355         /*
 2356          * XXX: We shouldn't really sleep here when we run out of buffers
 2357          * XXX: but the alternative is worse right now.
 2358          */
 2359         if (bio == NULL) {
 2360                 bp->b_error = ENOMEM;
 2361                 bp->b_ioflags |= BIO_ERROR;
 2362                 bufdone(bp);
 2363                 return;
 2364         }
 2365 #endif
 2366         bio->bio_caller2 = bp;
 2367         bio->bio_cmd = bp->b_iocmd;
 2368         bio->bio_data = bp->b_data;
 2369         bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
 2370         bio->bio_length = bp->b_bcount;
 2371         bio->bio_done = swapgeom_done;
 2372         g_io_request(bio, cp);
 2373         return;
 2374 }
 2375 
 2376 static void
 2377 swapgeom_orphan(struct g_consumer *cp)
 2378 {
 2379         struct swdevt *sp;
 2380 
 2381         mtx_lock(&sw_dev_mtx);
 2382         TAILQ_FOREACH(sp, &swtailq, sw_list)
 2383                 if (sp->sw_id == cp)
 2384                         sp->sw_id = NULL;
 2385         mtx_unlock(&sw_dev_mtx);
 2386 }
 2387 
 2388 static void
 2389 swapgeom_close_ev(void *arg, int flags)
 2390 {
 2391         struct g_consumer *cp;
 2392 
 2393         cp = arg;
 2394         g_access(cp, -1, -1, 0);
 2395         g_detach(cp);
 2396         g_destroy_consumer(cp);
 2397 }
 2398 
 2399 static void
 2400 swapgeom_close(struct thread *td, struct swdevt *sw)
 2401 {
 2402 
 2403         /* XXX: direct call when Giant untangled */
 2404         g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
 2405 }
 2406 
 2407 
 2408 struct swh0h0 {
 2409         struct cdev *dev;
 2410         struct vnode *vp;
 2411         int     error;
 2412 };
 2413 
 2414 static void
 2415 swapongeom_ev(void *arg, int flags)
 2416 {
 2417         struct swh0h0 *swh;
 2418         struct g_provider *pp;
 2419         struct g_consumer *cp;
 2420         static struct g_geom *gp;
 2421         struct swdevt *sp;
 2422         u_long nblks;
 2423         int error;
 2424 
 2425         swh = arg;
 2426         swh->error = 0;
 2427         pp = g_dev_getprovider(swh->dev);
 2428         if (pp == NULL) {
 2429                 swh->error = ENODEV;
 2430                 return;
 2431         }
 2432         mtx_lock(&sw_dev_mtx);
 2433         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2434                 cp = sp->sw_id;
 2435                 if (cp != NULL && cp->provider == pp) {
 2436                         mtx_unlock(&sw_dev_mtx);
 2437                         swh->error = EBUSY;
 2438                         return;
 2439                 }
 2440         }
 2441         mtx_unlock(&sw_dev_mtx);
 2442         if (gp == NULL)
 2443                 gp = g_new_geomf(&g_swap_class, "swap", NULL);
 2444         cp = g_new_consumer(gp);
 2445         g_attach(cp, pp);
 2446         /*
 2447          * XXX: Everytime you think you can improve the margin for
 2448          * footshooting, somebody depends on the ability to do so:
 2449          * savecore(8) wants to write to our swapdev so we cannot
 2450          * set an exclusive count :-(
 2451          */
 2452         error = g_access(cp, 1, 1, 0);
 2453         if (error) {
 2454                 g_detach(cp);
 2455                 g_destroy_consumer(cp);
 2456                 swh->error = error;
 2457                 return;
 2458         }
 2459         nblks = pp->mediasize / DEV_BSIZE;
 2460         swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
 2461             swapgeom_close, dev2udev(swh->dev));
 2462         swh->error = 0;
 2463         return;
 2464 }
 2465 
 2466 static int
 2467 swapongeom(struct thread *td, struct vnode *vp)
 2468 {
 2469         int error;
 2470         struct swh0h0 swh;
 2471 
 2472         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
 2473 
 2474         swh.dev = vp->v_rdev;
 2475         swh.vp = vp;
 2476         swh.error = 0;
 2477         /* XXX: direct call when Giant untangled */
 2478         error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
 2479         if (!error)
 2480                 error = swh.error;
 2481         VOP_UNLOCK(vp, 0, td);
 2482         return (error);
 2483 }
 2484 
 2485 /*
 2486  * VNODE backend
 2487  *
 2488  * This is used mainly for network filesystem (read: probably only tested
 2489  * with NFS) swapfiles.
 2490  *
 2491  */
 2492 
 2493 static void
 2494 swapdev_strategy(struct buf *bp, struct swdevt *sp)
 2495 {
 2496         struct vnode *vp2;
 2497 
 2498         bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
 2499 
 2500         vp2 = sp->sw_id;
 2501         vhold(vp2);
 2502         if (bp->b_iocmd == BIO_WRITE) {
 2503                 if (bp->b_bufobj) /* XXX: should always be true /phk */
 2504                         bufobj_wdrop(bp->b_bufobj);
 2505                 bufobj_wref(&vp2->v_bufobj);
 2506         }
 2507         if (bp->b_bufobj != &vp2->v_bufobj)
 2508                 bp->b_bufobj = &vp2->v_bufobj;
 2509         bp->b_vp = vp2;
 2510         bp->b_iooffset = dbtob(bp->b_blkno);
 2511         bstrategy(bp);
 2512         return;
 2513 }
 2514 
 2515 static void
 2516 swapdev_close(struct thread *td, struct swdevt *sp)
 2517 {
 2518 
 2519         VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
 2520         vrele(sp->sw_vp);
 2521 }
 2522 
 2523 
 2524 static int
 2525 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
 2526 {
 2527         struct swdevt *sp;
 2528         int error;
 2529 
 2530         if (nblks == 0)
 2531                 return (ENXIO);
 2532         mtx_lock(&sw_dev_mtx);
 2533         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2534                 if (sp->sw_id == vp) {
 2535                         mtx_unlock(&sw_dev_mtx);
 2536                         return (EBUSY);
 2537                 }
 2538         }
 2539         mtx_unlock(&sw_dev_mtx);
 2540     
 2541         (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
 2542 #ifdef MAC
 2543         error = mac_check_system_swapon(td->td_ucred, vp);
 2544         if (error == 0)
 2545 #endif
 2546                 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, -1);
 2547         (void) VOP_UNLOCK(vp, 0, td);
 2548         if (error)
 2549                 return (error);
 2550 
 2551         swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
 2552             NODEV);
 2553         return (0);
 2554 }

Cache object: bb51bb27333b06f75253a5eb6d53e826


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.