The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/vm/swap_pager.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1998 Matthew Dillon,
    3  * Copyright (c) 1994 John S. Dyson
    4  * Copyright (c) 1990 University of Utah.
    5  * Copyright (c) 1982, 1986, 1989, 1993
    6  *      The Regents of the University of California.  All rights reserved.
    7  *
    8  * This code is derived from software contributed to Berkeley by
    9  * the Systems Programming Group of the University of Utah Computer
   10  * Science Department.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. All advertising materials mentioning features or use of this software
   21  *    must display the following acknowledgement:
   22  *      This product includes software developed by the University of
   23  *      California, Berkeley and its contributors.
   24  * 4. Neither the name of the University nor the names of its contributors
   25  *    may be used to endorse or promote products derived from this software
   26  *    without specific prior written permission.
   27  *
   28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   38  * SUCH DAMAGE.
   39  *
   40  *                              New Swap System
   41  *                              Matthew Dillon
   42  *
   43  * Radix Bitmap 'blists'.
   44  *
   45  *      - The new swapper uses the new radix bitmap code.  This should scale
   46  *        to arbitrarily small or arbitrarily large swap spaces and an almost
   47  *        arbitrary degree of fragmentation.
   48  *
   49  * Features:
   50  *
   51  *      - on the fly reallocation of swap during putpages.  The new system
   52  *        does not try to keep previously allocated swap blocks for dirty
   53  *        pages.  
   54  *
   55  *      - on the fly deallocation of swap
   56  *
   57  *      - No more garbage collection required.  Unnecessarily allocated swap
   58  *        blocks only exist for dirty vm_page_t's now and these are already
   59  *        cycled (in a high-load system) by the pager.  We also do on-the-fly
   60  *        removal of invalidated swap blocks when a page is destroyed
   61  *        or renamed.
   62  *
   63  * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$
   64  *
   65  *      @(#)swap_pager.c        8.9 (Berkeley) 3/21/94
   66  *      @(#)vm_swap.c   8.5 (Berkeley) 2/17/94
   67  */
   68 
   69 #include <sys/cdefs.h>
   70 __FBSDID("$FreeBSD$");
   71 
   72 #include "opt_mac.h"
   73 #include "opt_swap.h"
   74 #include "opt_vm.h"
   75 
   76 #include <sys/param.h>
   77 #include <sys/systm.h>
   78 #include <sys/conf.h>
   79 #include <sys/kernel.h>
   80 #include <sys/priv.h>
   81 #include <sys/proc.h>
   82 #include <sys/bio.h>
   83 #include <sys/buf.h>
   84 #include <sys/disk.h>
   85 #include <sys/fcntl.h>
   86 #include <sys/mount.h>
   87 #include <sys/namei.h>
   88 #include <sys/vnode.h>
   89 #include <sys/malloc.h>
   90 #include <sys/sysctl.h>
   91 #include <sys/sysproto.h>
   92 #include <sys/blist.h>
   93 #include <sys/lock.h>
   94 #include <sys/sx.h>
   95 #include <sys/vmmeter.h>
   96 
   97 #include <security/mac/mac_framework.h>
   98 
   99 #include <vm/vm.h>
  100 #include <vm/pmap.h>
  101 #include <vm/vm_map.h>
  102 #include <vm/vm_kern.h>
  103 #include <vm/vm_object.h>
  104 #include <vm/vm_page.h>
  105 #include <vm/vm_pager.h>
  106 #include <vm/vm_pageout.h>
  107 #include <vm/vm_param.h>
  108 #include <vm/swap_pager.h>
  109 #include <vm/vm_extern.h>
  110 #include <vm/uma.h>
  111 
  112 #include <geom/geom.h>
  113 
  114 /*
  115  * SWB_NPAGES must be a power of 2.  It may be set to 1, 2, 4, 8, or 16
  116  * pages per allocation.  We recommend you stick with the default of 8.
  117  * The 16-page limit is due to the radix code (kern/subr_blist.c).
  118  */
  119 #ifndef MAX_PAGEOUT_CLUSTER
  120 #define MAX_PAGEOUT_CLUSTER 16
  121 #endif
  122 
  123 #if !defined(SWB_NPAGES)
  124 #define SWB_NPAGES      MAX_PAGEOUT_CLUSTER
  125 #endif
  126 
  127 /*
  128  * Piecemeal swap metadata structure.  Swap is stored in a radix tree.
  129  *
  130  * If SWB_NPAGES is 8 and sizeof(char *) == sizeof(daddr_t), our radix
  131  * is basically 8.  Assuming PAGE_SIZE == 4096, one tree level represents
  132  * 32K worth of data, two levels represent 256K, three levels represent
  133  * 2 MBytes.   This is acceptable.
  134  *
  135  * Overall memory utilization is about the same as the old swap structure.
  136  */
  137 #define SWCORRECT(n) (sizeof(void *) * (n) / sizeof(daddr_t))
  138 #define SWAP_META_PAGES         (SWB_NPAGES * 2)
  139 #define SWAP_META_MASK          (SWAP_META_PAGES - 1)
  140 
  141 struct swblock {
  142         struct swblock  *swb_hnext;
  143         vm_object_t     swb_object;
  144         vm_pindex_t     swb_index;
  145         int             swb_count;
  146         daddr_t         swb_pages[SWAP_META_PAGES];
  147 };
  148 
  149 static struct mtx sw_dev_mtx;
  150 static TAILQ_HEAD(, swdevt) swtailq = TAILQ_HEAD_INITIALIZER(swtailq);
  151 static struct swdevt *swdevhd;  /* Allocate from here next */
  152 static int nswapdev;            /* Number of swap devices */
  153 int swap_pager_avail;
  154 static int swdev_syscall_active = 0; /* serialize swap(on|off) */
  155 
  156 static void swapdev_strategy(struct buf *, struct swdevt *sw);
  157 
  158 #define SWM_FREE        0x02    /* free, period                 */
  159 #define SWM_POP         0x04    /* pop out                      */
  160 
  161 int swap_pager_full = 2;        /* swap space exhaustion (task killing) */
  162 static int swap_pager_almost_full = 1; /* swap space exhaustion (w/hysteresis)*/
  163 static int nsw_rcount;          /* free read buffers                    */
  164 static int nsw_wcount_sync;     /* limit write buffers / synchronous    */
  165 static int nsw_wcount_async;    /* limit write buffers / asynchronous   */
  166 static int nsw_wcount_async_max;/* assigned maximum                     */
  167 static int nsw_cluster_max;     /* maximum VOP I/O allowed              */
  168 
  169 static struct swblock **swhash;
  170 static int swhash_mask;
  171 static struct mtx swhash_mtx;
  172 
  173 static int swap_async_max = 4;  /* maximum in-progress async I/O's      */
  174 static struct sx sw_alloc_sx;
  175 
  176 
  177 SYSCTL_INT(_vm, OID_AUTO, swap_async_max,
  178         CTLFLAG_RW, &swap_async_max, 0, "Maximum running async swap ops");
  179 
  180 /*
  181  * "named" and "unnamed" anon region objects.  Try to reduce the overhead
  182  * of searching a named list by hashing it just a little.
  183  */
  184 
  185 #define NOBJLISTS               8
  186 
  187 #define NOBJLIST(handle)        \
  188         (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
  189 
  190 static struct mtx sw_alloc_mtx; /* protect list manipulation */ 
  191 static struct pagerlst  swap_pager_object_list[NOBJLISTS];
  192 static uma_zone_t       swap_zone;
  193 static struct vm_object swap_zone_obj;
  194 
  195 /*
  196  * pagerops for OBJT_SWAP - "swap pager".  Some ops are also global procedure
  197  * calls hooked from other parts of the VM system and do not appear here.
  198  * (see vm/swap_pager.h).
  199  */
  200 static vm_object_t
  201                 swap_pager_alloc(void *handle, vm_ooffset_t size,
  202                                       vm_prot_t prot, vm_ooffset_t offset);
  203 static void     swap_pager_dealloc(vm_object_t object);
  204 static int      swap_pager_getpages(vm_object_t, vm_page_t *, int, int);
  205 static void     swap_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *);
  206 static boolean_t
  207                 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
  208 static void     swap_pager_init(void);
  209 static void     swap_pager_unswapped(vm_page_t);
  210 static void     swap_pager_swapoff(struct swdevt *sp);
  211 
  212 struct pagerops swappagerops = {
  213         .pgo_init =     swap_pager_init,        /* early system initialization of pager */
  214         .pgo_alloc =    swap_pager_alloc,       /* allocate an OBJT_SWAP object         */
  215         .pgo_dealloc =  swap_pager_dealloc,     /* deallocate an OBJT_SWAP object       */
  216         .pgo_getpages = swap_pager_getpages,    /* pagein                               */
  217         .pgo_putpages = swap_pager_putpages,    /* pageout                              */
  218         .pgo_haspage =  swap_pager_haspage,     /* get backing store status for page    */
  219         .pgo_pageunswapped = swap_pager_unswapped,      /* remove swap related to page          */
  220 };
  221 
  222 /*
  223  * dmmax is in page-sized chunks with the new swap system.  It was
  224  * dev-bsized chunks in the old.  dmmax is always a power of 2.
  225  *
  226  * swap_*() routines are externally accessible.  swp_*() routines are
  227  * internal.
  228  */
  229 static int dmmax;
  230 static int nswap_lowat = 128;   /* in pages, swap_pager_almost_full warn */
  231 static int nswap_hiwat = 512;   /* in pages, swap_pager_almost_full warn */
  232 
  233 SYSCTL_INT(_vm, OID_AUTO, dmmax,
  234         CTLFLAG_RD, &dmmax, 0, "Maximum size of a swap block");
  235 
  236 static void     swp_sizecheck(void);
  237 static void     swp_pager_async_iodone(struct buf *bp);
  238 static int      swapongeom(struct thread *, struct vnode *);
  239 static int      swaponvp(struct thread *, struct vnode *, u_long);
  240 static int      swapoff_one(struct swdevt *sp, struct thread *td);
  241 
  242 /*
  243  * Swap bitmap functions
  244  */
  245 static void     swp_pager_freeswapspace(daddr_t blk, int npages);
  246 static daddr_t  swp_pager_getswapspace(int npages);
  247 
  248 /*
  249  * Metadata functions
  250  */
  251 static struct swblock **swp_pager_hash(vm_object_t object, vm_pindex_t index);
  252 static void swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t);
  253 static void swp_pager_meta_free(vm_object_t, vm_pindex_t, daddr_t);
  254 static void swp_pager_meta_free_all(vm_object_t);
  255 static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int);
  256 
  257 /*
  258  * SWP_SIZECHECK() -    update swap_pager_full indication
  259  *      
  260  *      update the swap_pager_almost_full indication and warn when we are
  261  *      about to run out of swap space, using lowat/hiwat hysteresis.
  262  *
  263  *      Clear swap_pager_full ( task killing ) indication when lowat is met.
  264  *
  265  *      No restrictions on call
  266  *      This routine may not block.
  267  *      This routine must be called at splvm()
  268  */
  269 static void
  270 swp_sizecheck(void)
  271 {
  272 
  273         if (swap_pager_avail < nswap_lowat) {
  274                 if (swap_pager_almost_full == 0) {
  275                         printf("swap_pager: out of swap space\n");
  276                         swap_pager_almost_full = 1;
  277                 }
  278         } else {
  279                 swap_pager_full = 0;
  280                 if (swap_pager_avail > nswap_hiwat)
  281                         swap_pager_almost_full = 0;
  282         }
  283 }
  284 
  285 /*
  286  * SWP_PAGER_HASH() -   hash swap meta data
  287  *
  288  *      This is an helper function which hashes the swapblk given
  289  *      the object and page index.  It returns a pointer to a pointer
  290  *      to the object, or a pointer to a NULL pointer if it could not
  291  *      find a swapblk.
  292  *
  293  *      This routine must be called at splvm().
  294  */
  295 static struct swblock **
  296 swp_pager_hash(vm_object_t object, vm_pindex_t index)
  297 {
  298         struct swblock **pswap;
  299         struct swblock *swap;
  300 
  301         index &= ~(vm_pindex_t)SWAP_META_MASK;
  302         pswap = &swhash[(index ^ (int)(intptr_t)object) & swhash_mask];
  303         while ((swap = *pswap) != NULL) {
  304                 if (swap->swb_object == object &&
  305                     swap->swb_index == index
  306                 ) {
  307                         break;
  308                 }
  309                 pswap = &swap->swb_hnext;
  310         }
  311         return (pswap);
  312 }
  313 
  314 /*
  315  * SWAP_PAGER_INIT() -  initialize the swap pager!
  316  *
  317  *      Expected to be started from system init.  NOTE:  This code is run 
  318  *      before much else so be careful what you depend on.  Most of the VM
  319  *      system has yet to be initialized at this point.
  320  */
  321 static void
  322 swap_pager_init(void)
  323 {
  324         /*
  325          * Initialize object lists
  326          */
  327         int i;
  328 
  329         for (i = 0; i < NOBJLISTS; ++i)
  330                 TAILQ_INIT(&swap_pager_object_list[i]);
  331         mtx_init(&sw_alloc_mtx, "swap_pager list", NULL, MTX_DEF);
  332         mtx_init(&sw_dev_mtx, "swapdev", NULL, MTX_DEF);
  333 
  334         /*
  335          * Device Stripe, in PAGE_SIZE'd blocks
  336          */
  337         dmmax = SWB_NPAGES * 2;
  338 }
  339 
  340 /*
  341  * SWAP_PAGER_SWAP_INIT() - swap pager initialization from pageout process
  342  *
  343  *      Expected to be started from pageout process once, prior to entering
  344  *      its main loop.
  345  */
  346 void
  347 swap_pager_swap_init(void)
  348 {
  349         int n, n2;
  350 
  351         /*
  352          * Number of in-transit swap bp operations.  Don't
  353          * exhaust the pbufs completely.  Make sure we
  354          * initialize workable values (0 will work for hysteresis
  355          * but it isn't very efficient).
  356          *
  357          * The nsw_cluster_max is constrained by the bp->b_pages[]
  358          * array (MAXPHYS/PAGE_SIZE) and our locally defined
  359          * MAX_PAGEOUT_CLUSTER.   Also be aware that swap ops are
  360          * constrained by the swap device interleave stripe size.
  361          *
  362          * Currently we hardwire nsw_wcount_async to 4.  This limit is 
  363          * designed to prevent other I/O from having high latencies due to
  364          * our pageout I/O.  The value 4 works well for one or two active swap
  365          * devices but is probably a little low if you have more.  Even so,
  366          * a higher value would probably generate only a limited improvement
  367          * with three or four active swap devices since the system does not
  368          * typically have to pageout at extreme bandwidths.   We will want
  369          * at least 2 per swap devices, and 4 is a pretty good value if you
  370          * have one NFS swap device due to the command/ack latency over NFS.
  371          * So it all works out pretty well.
  372          */
  373         nsw_cluster_max = min((MAXPHYS/PAGE_SIZE), MAX_PAGEOUT_CLUSTER);
  374 
  375         mtx_lock(&pbuf_mtx);
  376         nsw_rcount = (nswbuf + 1) / 2;
  377         nsw_wcount_sync = (nswbuf + 3) / 4;
  378         nsw_wcount_async = 4;
  379         nsw_wcount_async_max = nsw_wcount_async;
  380         mtx_unlock(&pbuf_mtx);
  381 
  382         /*
  383          * Initialize our zone.  Right now I'm just guessing on the number
  384          * we need based on the number of pages in the system.  Each swblock
  385          * can hold 16 pages, so this is probably overkill.  This reservation
  386          * is typically limited to around 32MB by default.
  387          */
  388         n = cnt.v_page_count / 2;
  389         if (maxswzone && n > maxswzone / sizeof(struct swblock))
  390                 n = maxswzone / sizeof(struct swblock);
  391         n2 = n;
  392         swap_zone = uma_zcreate("SWAPMETA", sizeof(struct swblock), NULL, NULL,
  393             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
  394         if (swap_zone == NULL)
  395                 panic("failed to create swap_zone.");
  396         do {
  397                 if (uma_zone_set_obj(swap_zone, &swap_zone_obj, n))
  398                         break;
  399                 /*
  400                  * if the allocation failed, try a zone two thirds the
  401                  * size of the previous attempt.
  402                  */
  403                 n -= ((n + 2) / 3);
  404         } while (n > 0);
  405         if (n2 != n)
  406                 printf("Swap zone entries reduced from %d to %d.\n", n2, n);
  407         n2 = n;
  408 
  409         /*
  410          * Initialize our meta-data hash table.  The swapper does not need to
  411          * be quite as efficient as the VM system, so we do not use an 
  412          * oversized hash table.
  413          *
  414          *      n:              size of hash table, must be power of 2
  415          *      swhash_mask:    hash table index mask
  416          */
  417         for (n = 1; n < n2 / 8; n *= 2)
  418                 ;
  419         swhash = malloc(sizeof(struct swblock *) * n, M_VMPGDATA, M_WAITOK | M_ZERO);
  420         swhash_mask = n - 1;
  421         mtx_init(&swhash_mtx, "swap_pager swhash", NULL, MTX_DEF);
  422 }
  423 
  424 /*
  425  * SWAP_PAGER_ALLOC() - allocate a new OBJT_SWAP VM object and instantiate
  426  *                      its metadata structures.
  427  *
  428  *      This routine is called from the mmap and fork code to create a new
  429  *      OBJT_SWAP object.  We do this by creating an OBJT_DEFAULT object
  430  *      and then converting it with swp_pager_meta_build().
  431  *
  432  *      This routine may block in vm_object_allocate() and create a named
  433  *      object lookup race, so we must interlock.   We must also run at
  434  *      splvm() for the object lookup to handle races with interrupts, but
  435  *      we do not have to maintain splvm() in between the lookup and the
  436  *      add because (I believe) it is not possible to attempt to create
  437  *      a new swap object w/handle when a default object with that handle
  438  *      already exists.
  439  *
  440  * MPSAFE
  441  */
  442 static vm_object_t
  443 swap_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
  444                  vm_ooffset_t offset)
  445 {
  446         vm_object_t object;
  447         vm_pindex_t pindex;
  448 
  449         pindex = OFF_TO_IDX(offset + PAGE_MASK + size);
  450 
  451         if (handle) {
  452                 mtx_lock(&Giant);
  453                 /*
  454                  * Reference existing named region or allocate new one.  There
  455                  * should not be a race here against swp_pager_meta_build()
  456                  * as called from vm_page_remove() in regards to the lookup
  457                  * of the handle.
  458                  */
  459                 sx_xlock(&sw_alloc_sx);
  460                 object = vm_pager_object_lookup(NOBJLIST(handle), handle);
  461 
  462                 if (object == NULL) {
  463                         object = vm_object_allocate(OBJT_DEFAULT, pindex);
  464                         object->handle = handle;
  465 
  466                         VM_OBJECT_LOCK(object);
  467                         swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  468                         VM_OBJECT_UNLOCK(object);
  469                 }
  470                 sx_xunlock(&sw_alloc_sx);
  471                 mtx_unlock(&Giant);
  472         } else {
  473                 object = vm_object_allocate(OBJT_DEFAULT, pindex);
  474 
  475                 VM_OBJECT_LOCK(object);
  476                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
  477                 VM_OBJECT_UNLOCK(object);
  478         }
  479         return (object);
  480 }
  481 
  482 /*
  483  * SWAP_PAGER_DEALLOC() -       remove swap metadata from object
  484  *
  485  *      The swap backing for the object is destroyed.  The code is 
  486  *      designed such that we can reinstantiate it later, but this
  487  *      routine is typically called only when the entire object is
  488  *      about to be destroyed.
  489  *
  490  *      This routine may block, but no longer does. 
  491  *
  492  *      The object must be locked or unreferenceable.
  493  */
  494 static void
  495 swap_pager_dealloc(vm_object_t object)
  496 {
  497 
  498         /*
  499          * Remove from list right away so lookups will fail if we block for
  500          * pageout completion.
  501          */
  502         if (object->handle != NULL) {
  503                 mtx_lock(&sw_alloc_mtx);
  504                 TAILQ_REMOVE(NOBJLIST(object->handle), object, pager_object_list);
  505                 mtx_unlock(&sw_alloc_mtx);
  506         }
  507 
  508         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  509         vm_object_pip_wait(object, "swpdea");
  510 
  511         /*
  512          * Free all remaining metadata.  We only bother to free it from 
  513          * the swap meta data.  We do not attempt to free swapblk's still
  514          * associated with vm_page_t's for this object.  We do not care
  515          * if paging is still in progress on some objects.
  516          */
  517         swp_pager_meta_free_all(object);
  518 }
  519 
  520 /************************************************************************
  521  *                      SWAP PAGER BITMAP ROUTINES                      *
  522  ************************************************************************/
  523 
  524 /*
  525  * SWP_PAGER_GETSWAPSPACE() -   allocate raw swap space
  526  *
  527  *      Allocate swap for the requested number of pages.  The starting
  528  *      swap block number (a page index) is returned or SWAPBLK_NONE
  529  *      if the allocation failed.
  530  *
  531  *      Also has the side effect of advising that somebody made a mistake
  532  *      when they configured swap and didn't configure enough.
  533  *
  534  *      Must be called at splvm() to avoid races with bitmap frees from
  535  *      vm_page_remove() aka swap_pager_page_removed().
  536  *
  537  *      This routine may not block
  538  *      This routine must be called at splvm().
  539  *
  540  *      We allocate in round-robin fashion from the configured devices.
  541  */
  542 static daddr_t
  543 swp_pager_getswapspace(int npages)
  544 {
  545         daddr_t blk;
  546         struct swdevt *sp;
  547         int i;
  548 
  549         blk = SWAPBLK_NONE;
  550         mtx_lock(&sw_dev_mtx);
  551         sp = swdevhd;
  552         for (i = 0; i < nswapdev; i++) {
  553                 if (sp == NULL)
  554                         sp = TAILQ_FIRST(&swtailq);
  555                 if (!(sp->sw_flags & SW_CLOSING)) {
  556                         blk = blist_alloc(sp->sw_blist, npages);
  557                         if (blk != SWAPBLK_NONE) {
  558                                 blk += sp->sw_first;
  559                                 sp->sw_used += npages;
  560                                 swap_pager_avail -= npages;
  561                                 swp_sizecheck();
  562                                 swdevhd = TAILQ_NEXT(sp, sw_list);
  563                                 goto done;
  564                         }
  565                 }
  566                 sp = TAILQ_NEXT(sp, sw_list);
  567         }
  568         if (swap_pager_full != 2) {
  569                 printf("swap_pager_getswapspace(%d): failed\n", npages);
  570                 swap_pager_full = 2;
  571                 swap_pager_almost_full = 1;
  572         }
  573         swdevhd = NULL;
  574 done:
  575         mtx_unlock(&sw_dev_mtx);
  576         return (blk);
  577 }
  578 
  579 static int
  580 swp_pager_isondev(daddr_t blk, struct swdevt *sp)
  581 {
  582 
  583         return (blk >= sp->sw_first && blk < sp->sw_end);
  584 }
  585         
  586 static void
  587 swp_pager_strategy(struct buf *bp)
  588 {
  589         struct swdevt *sp;
  590 
  591         mtx_lock(&sw_dev_mtx);
  592         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  593                 if (bp->b_blkno >= sp->sw_first && bp->b_blkno < sp->sw_end) {
  594                         mtx_unlock(&sw_dev_mtx);
  595                         sp->sw_strategy(bp, sp);
  596                         return;
  597                 }
  598         }
  599         panic("Swapdev not found");
  600 }
  601         
  602 
  603 /*
  604  * SWP_PAGER_FREESWAPSPACE() -  free raw swap space 
  605  *
  606  *      This routine returns the specified swap blocks back to the bitmap.
  607  *
  608  *      Note:  This routine may not block (it could in the old swap code),
  609  *      and through the use of the new blist routines it does not block.
  610  *
  611  *      We must be called at splvm() to avoid races with bitmap frees from
  612  *      vm_page_remove() aka swap_pager_page_removed().
  613  *
  614  *      This routine may not block
  615  *      This routine must be called at splvm().
  616  */
  617 static void
  618 swp_pager_freeswapspace(daddr_t blk, int npages)
  619 {
  620         struct swdevt *sp;
  621 
  622         mtx_lock(&sw_dev_mtx);
  623         TAILQ_FOREACH(sp, &swtailq, sw_list) {
  624                 if (blk >= sp->sw_first && blk < sp->sw_end) {
  625                         sp->sw_used -= npages;
  626                         /*
  627                          * If we are attempting to stop swapping on
  628                          * this device, we don't want to mark any
  629                          * blocks free lest they be reused.  
  630                          */
  631                         if ((sp->sw_flags & SW_CLOSING) == 0) {
  632                                 blist_free(sp->sw_blist, blk - sp->sw_first,
  633                                     npages);
  634                                 swap_pager_avail += npages;
  635                                 swp_sizecheck();
  636                         }
  637                         mtx_unlock(&sw_dev_mtx);
  638                         return;
  639                 }
  640         }
  641         panic("Swapdev not found");
  642 }
  643 
  644 /*
  645  * SWAP_PAGER_FREESPACE() -     frees swap blocks associated with a page
  646  *                              range within an object.
  647  *
  648  *      This is a globally accessible routine.
  649  *
  650  *      This routine removes swapblk assignments from swap metadata.
  651  *
  652  *      The external callers of this routine typically have already destroyed 
  653  *      or renamed vm_page_t's associated with this range in the object so 
  654  *      we should be ok.
  655  *
  656  *      This routine may be called at any spl.  We up our spl to splvm temporarily
  657  *      in order to perform the metadata removal.
  658  */
  659 void
  660 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
  661 {
  662 
  663         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  664         swp_pager_meta_free(object, start, size);
  665 }
  666 
  667 /*
  668  * SWAP_PAGER_RESERVE() - reserve swap blocks in object
  669  *
  670  *      Assigns swap blocks to the specified range within the object.  The 
  671  *      swap blocks are not zerod.  Any previous swap assignment is destroyed.
  672  *
  673  *      Returns 0 on success, -1 on failure.
  674  */
  675 int
  676 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_size_t size)
  677 {
  678         int n = 0;
  679         daddr_t blk = SWAPBLK_NONE;
  680         vm_pindex_t beg = start;        /* save start index */
  681 
  682         VM_OBJECT_LOCK(object);
  683         while (size) {
  684                 if (n == 0) {
  685                         n = BLIST_MAX_ALLOC;
  686                         while ((blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE) {
  687                                 n >>= 1;
  688                                 if (n == 0) {
  689                                         swp_pager_meta_free(object, beg, start - beg);
  690                                         VM_OBJECT_UNLOCK(object);
  691                                         return (-1);
  692                                 }
  693                         }
  694                 }
  695                 swp_pager_meta_build(object, start, blk);
  696                 --size;
  697                 ++start;
  698                 ++blk;
  699                 --n;
  700         }
  701         swp_pager_meta_free(object, start, n);
  702         VM_OBJECT_UNLOCK(object);
  703         return (0);
  704 }
  705 
  706 /*
  707  * SWAP_PAGER_COPY() -  copy blocks from source pager to destination pager
  708  *                      and destroy the source.
  709  *
  710  *      Copy any valid swapblks from the source to the destination.  In
  711  *      cases where both the source and destination have a valid swapblk,
  712  *      we keep the destination's.
  713  *
  714  *      This routine is allowed to block.  It may block allocating metadata
  715  *      indirectly through swp_pager_meta_build() or if paging is still in
  716  *      progress on the source. 
  717  *
  718  *      This routine can be called at any spl
  719  *
  720  *      XXX vm_page_collapse() kinda expects us not to block because we 
  721  *      supposedly do not need to allocate memory, but for the moment we
  722  *      *may* have to get a little memory from the zone allocator, but
  723  *      it is taken from the interrupt memory.  We should be ok. 
  724  *
  725  *      The source object contains no vm_page_t's (which is just as well)
  726  *
  727  *      The source object is of type OBJT_SWAP.
  728  *
  729  *      The source and destination objects must be locked or 
  730  *      inaccessible (XXX are they ?)
  731  */
  732 void
  733 swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject,
  734     vm_pindex_t offset, int destroysource)
  735 {
  736         vm_pindex_t i;
  737 
  738         VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED);
  739         VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED);
  740 
  741         /*
  742          * If destroysource is set, we remove the source object from the 
  743          * swap_pager internal queue now. 
  744          */
  745         if (destroysource) {
  746                 if (srcobject->handle != NULL) {
  747                         mtx_lock(&sw_alloc_mtx);
  748                         TAILQ_REMOVE(
  749                             NOBJLIST(srcobject->handle),
  750                             srcobject,
  751                             pager_object_list
  752                         );
  753                         mtx_unlock(&sw_alloc_mtx);
  754                 }
  755         }
  756 
  757         /*
  758          * transfer source to destination.
  759          */
  760         for (i = 0; i < dstobject->size; ++i) {
  761                 daddr_t dstaddr;
  762 
  763                 /*
  764                  * Locate (without changing) the swapblk on the destination,
  765                  * unless it is invalid in which case free it silently, or
  766                  * if the destination is a resident page, in which case the
  767                  * source is thrown away.
  768                  */
  769                 dstaddr = swp_pager_meta_ctl(dstobject, i, 0);
  770 
  771                 if (dstaddr == SWAPBLK_NONE) {
  772                         /*
  773                          * Destination has no swapblk and is not resident,
  774                          * copy source.
  775                          */
  776                         daddr_t srcaddr;
  777 
  778                         srcaddr = swp_pager_meta_ctl(
  779                             srcobject, 
  780                             i + offset,
  781                             SWM_POP
  782                         );
  783 
  784                         if (srcaddr != SWAPBLK_NONE) {
  785                                 /*
  786                                  * swp_pager_meta_build() can sleep.
  787                                  */
  788                                 vm_object_pip_add(srcobject, 1);
  789                                 VM_OBJECT_UNLOCK(srcobject);
  790                                 vm_object_pip_add(dstobject, 1);
  791                                 swp_pager_meta_build(dstobject, i, srcaddr);
  792                                 vm_object_pip_wakeup(dstobject);
  793                                 VM_OBJECT_LOCK(srcobject);
  794                                 vm_object_pip_wakeup(srcobject);
  795                         }
  796                 } else {
  797                         /*
  798                          * Destination has valid swapblk or it is represented
  799                          * by a resident page.  We destroy the sourceblock.
  800                          */
  801                         
  802                         swp_pager_meta_ctl(srcobject, i + offset, SWM_FREE);
  803                 }
  804         }
  805 
  806         /*
  807          * Free left over swap blocks in source.
  808          *
  809          * We have to revert the type to OBJT_DEFAULT so we do not accidently
  810          * double-remove the object from the swap queues.
  811          */
  812         if (destroysource) {
  813                 swp_pager_meta_free_all(srcobject);
  814                 /*
  815                  * Reverting the type is not necessary, the caller is going
  816                  * to destroy srcobject directly, but I'm doing it here
  817                  * for consistency since we've removed the object from its
  818                  * queues.
  819                  */
  820                 srcobject->type = OBJT_DEFAULT;
  821         }
  822 }
  823 
  824 /*
  825  * SWAP_PAGER_HASPAGE() -       determine if we have good backing store for
  826  *                              the requested page.
  827  *
  828  *      We determine whether good backing store exists for the requested
  829  *      page and return TRUE if it does, FALSE if it doesn't.
  830  *
  831  *      If TRUE, we also try to determine how much valid, contiguous backing
  832  *      store exists before and after the requested page within a reasonable
  833  *      distance.  We do not try to restrict it to the swap device stripe
  834  *      (that is handled in getpages/putpages).  It probably isn't worth
  835  *      doing here.
  836  */
  837 static boolean_t
  838 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after)
  839 {
  840         daddr_t blk0;
  841 
  842         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
  843         /*
  844          * do we have good backing store at the requested index ?
  845          */
  846         blk0 = swp_pager_meta_ctl(object, pindex, 0);
  847 
  848         if (blk0 == SWAPBLK_NONE) {
  849                 if (before)
  850                         *before = 0;
  851                 if (after)
  852                         *after = 0;
  853                 return (FALSE);
  854         }
  855 
  856         /*
  857          * find backwards-looking contiguous good backing store
  858          */
  859         if (before != NULL) {
  860                 int i;
  861 
  862                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
  863                         daddr_t blk;
  864 
  865                         if (i > pindex)
  866                                 break;
  867                         blk = swp_pager_meta_ctl(object, pindex - i, 0);
  868                         if (blk != blk0 - i)
  869                                 break;
  870                 }
  871                 *before = (i - 1);
  872         }
  873 
  874         /*
  875          * find forward-looking contiguous good backing store
  876          */
  877         if (after != NULL) {
  878                 int i;
  879 
  880                 for (i = 1; i < (SWB_NPAGES/2); ++i) {
  881                         daddr_t blk;
  882 
  883                         blk = swp_pager_meta_ctl(object, pindex + i, 0);
  884                         if (blk != blk0 + i)
  885                                 break;
  886                 }
  887                 *after = (i - 1);
  888         }
  889         return (TRUE);
  890 }
  891 
  892 /*
  893  * SWAP_PAGER_PAGE_UNSWAPPED() - remove swap backing store related to page
  894  *
  895  *      This removes any associated swap backing store, whether valid or
  896  *      not, from the page.  
  897  *
  898  *      This routine is typically called when a page is made dirty, at
  899  *      which point any associated swap can be freed.  MADV_FREE also
  900  *      calls us in a special-case situation
  901  *
  902  *      NOTE!!!  If the page is clean and the swap was valid, the caller
  903  *      should make the page dirty before calling this routine.  This routine
  904  *      does NOT change the m->dirty status of the page.  Also: MADV_FREE
  905  *      depends on it.
  906  *
  907  *      This routine may not block
  908  *      This routine must be called at splvm()
  909  */
  910 static void
  911 swap_pager_unswapped(vm_page_t m)
  912 {
  913 
  914         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
  915         swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE);
  916 }
  917 
  918 /*
  919  * SWAP_PAGER_GETPAGES() - bring pages in from swap
  920  *
  921  *      Attempt to retrieve (m, count) pages from backing store, but make
  922  *      sure we retrieve at least m[reqpage].  We try to load in as large
  923  *      a chunk surrounding m[reqpage] as is contiguous in swap and which
  924  *      belongs to the same object.
  925  *
  926  *      The code is designed for asynchronous operation and 
  927  *      immediate-notification of 'reqpage' but tends not to be
  928  *      used that way.  Please do not optimize-out this algorithmic
  929  *      feature, I intend to improve on it in the future.
  930  *
  931  *      The parent has a single vm_object_pip_add() reference prior to
  932  *      calling us and we should return with the same.
  933  *
  934  *      The parent has BUSY'd the pages.  We should return with 'm'
  935  *      left busy, but the others adjusted.
  936  */
  937 static int
  938 swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
  939 {
  940         struct buf *bp;
  941         vm_page_t mreq;
  942         int i;
  943         int j;
  944         daddr_t blk;
  945 
  946         mreq = m[reqpage];
  947 
  948         KASSERT(mreq->object == object,
  949             ("swap_pager_getpages: object mismatch %p/%p",
  950             object, mreq->object));
  951 
  952         /*
  953          * Calculate range to retrieve.  The pages have already been assigned
  954          * their swapblks.  We require a *contiguous* range but we know it to
  955          * not span devices.   If we do not supply it, bad things
  956          * happen.  Note that blk, iblk & jblk can be SWAPBLK_NONE, but the 
  957          * loops are set up such that the case(s) are handled implicitly.
  958          *
  959          * The swp_*() calls must be made at splvm().  vm_page_free() does
  960          * not need to be, but it will go a little faster if it is.
  961          */
  962         blk = swp_pager_meta_ctl(mreq->object, mreq->pindex, 0);
  963 
  964         for (i = reqpage - 1; i >= 0; --i) {
  965                 daddr_t iblk;
  966 
  967                 iblk = swp_pager_meta_ctl(m[i]->object, m[i]->pindex, 0);
  968                 if (blk != iblk + (reqpage - i))
  969                         break;
  970         }
  971         ++i;
  972 
  973         for (j = reqpage + 1; j < count; ++j) {
  974                 daddr_t jblk;
  975 
  976                 jblk = swp_pager_meta_ctl(m[j]->object, m[j]->pindex, 0);
  977                 if (blk != jblk - (j - reqpage))
  978                         break;
  979         }
  980 
  981         /*
  982          * free pages outside our collection range.   Note: we never free
  983          * mreq, it must remain busy throughout.
  984          */
  985         if (0 < i || j < count) {
  986                 int k;
  987 
  988                 vm_page_lock_queues();
  989                 for (k = 0; k < i; ++k)
  990                         vm_page_free(m[k]);
  991                 for (k = j; k < count; ++k)
  992                         vm_page_free(m[k]);
  993                 vm_page_unlock_queues();
  994         }
  995 
  996         /*
  997          * Return VM_PAGER_FAIL if we have nothing to do.  Return mreq 
  998          * still busy, but the others unbusied.
  999          */
 1000         if (blk == SWAPBLK_NONE)
 1001                 return (VM_PAGER_FAIL);
 1002 
 1003         /*
 1004          * Getpbuf() can sleep.
 1005          */
 1006         VM_OBJECT_UNLOCK(object);
 1007         /*
 1008          * Get a swap buffer header to perform the IO
 1009          */
 1010         bp = getpbuf(&nsw_rcount);
 1011         bp->b_flags |= B_PAGING;
 1012 
 1013         /*
 1014          * map our page(s) into kva for input
 1015          */
 1016         pmap_qenter((vm_offset_t)bp->b_data, m + i, j - i);
 1017 
 1018         bp->b_iocmd = BIO_READ;
 1019         bp->b_iodone = swp_pager_async_iodone;
 1020         bp->b_rcred = crhold(thread0.td_ucred);
 1021         bp->b_wcred = crhold(thread0.td_ucred);
 1022         bp->b_blkno = blk - (reqpage - i);
 1023         bp->b_bcount = PAGE_SIZE * (j - i);
 1024         bp->b_bufsize = PAGE_SIZE * (j - i);
 1025         bp->b_pager.pg_reqpage = reqpage - i;
 1026 
 1027         VM_OBJECT_LOCK(object);
 1028         {
 1029                 int k;
 1030 
 1031                 for (k = i; k < j; ++k) {
 1032                         bp->b_pages[k - i] = m[k];
 1033                         m[k]->oflags |= VPO_SWAPINPROG;
 1034                 }
 1035         }
 1036         bp->b_npages = j - i;
 1037 
 1038         PCPU_INC(cnt.v_swapin);
 1039         PCPU_ADD(cnt.v_swappgsin, bp->b_npages);
 1040 
 1041         /*
 1042          * We still hold the lock on mreq, and our automatic completion routine
 1043          * does not remove it.
 1044          */
 1045         vm_object_pip_add(object, bp->b_npages);
 1046         VM_OBJECT_UNLOCK(object);
 1047 
 1048         /*
 1049          * perform the I/O.  NOTE!!!  bp cannot be considered valid after
 1050          * this point because we automatically release it on completion.
 1051          * Instead, we look at the one page we are interested in which we
 1052          * still hold a lock on even through the I/O completion.
 1053          *
 1054          * The other pages in our m[] array are also released on completion,
 1055          * so we cannot assume they are valid anymore either.
 1056          *
 1057          * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1058          */
 1059         BUF_KERNPROC(bp);
 1060         swp_pager_strategy(bp);
 1061 
 1062         /*
 1063          * wait for the page we want to complete.  VPO_SWAPINPROG is always
 1064          * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
 1065          * is set in the meta-data.
 1066          */
 1067         VM_OBJECT_LOCK(object);
 1068         while ((mreq->oflags & VPO_SWAPINPROG) != 0) {
 1069                 mreq->oflags |= VPO_WANTED;
 1070                 vm_page_lock_queues();
 1071                 vm_page_flag_set(mreq, PG_REFERENCED);
 1072                 vm_page_unlock_queues();
 1073                 PCPU_INC(cnt.v_intrans);
 1074                 if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) {
 1075                         printf(
 1076 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
 1077                             bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
 1078                 }
 1079         }
 1080 
 1081         /*
 1082          * mreq is left busied after completion, but all the other pages
 1083          * are freed.  If we had an unrecoverable read error the page will
 1084          * not be valid.
 1085          */
 1086         if (mreq->valid != VM_PAGE_BITS_ALL) {
 1087                 return (VM_PAGER_ERROR);
 1088         } else {
 1089                 return (VM_PAGER_OK);
 1090         }
 1091 
 1092         /*
 1093          * A final note: in a low swap situation, we cannot deallocate swap
 1094          * and mark a page dirty here because the caller is likely to mark
 1095          * the page clean when we return, causing the page to possibly revert 
 1096          * to all-zero's later.
 1097          */
 1098 }
 1099 
 1100 /*
 1101  *      swap_pager_putpages: 
 1102  *
 1103  *      Assign swap (if necessary) and initiate I/O on the specified pages.
 1104  *
 1105  *      We support both OBJT_DEFAULT and OBJT_SWAP objects.  DEFAULT objects
 1106  *      are automatically converted to SWAP objects.
 1107  *
 1108  *      In a low memory situation we may block in VOP_STRATEGY(), but the new 
 1109  *      vm_page reservation system coupled with properly written VFS devices 
 1110  *      should ensure that no low-memory deadlock occurs.  This is an area
 1111  *      which needs work.
 1112  *
 1113  *      The parent has N vm_object_pip_add() references prior to
 1114  *      calling us and will remove references for rtvals[] that are
 1115  *      not set to VM_PAGER_PEND.  We need to remove the rest on I/O
 1116  *      completion.
 1117  *
 1118  *      The parent has soft-busy'd the pages it passes us and will unbusy
 1119  *      those whos rtvals[] entry is not set to VM_PAGER_PEND on return.
 1120  *      We need to unbusy the rest on I/O completion.
 1121  */
 1122 void
 1123 swap_pager_putpages(vm_object_t object, vm_page_t *m, int count,
 1124     boolean_t sync, int *rtvals)
 1125 {
 1126         int i;
 1127         int n = 0;
 1128 
 1129         if (count && m[0]->object != object) {
 1130                 panic("swap_pager_getpages: object mismatch %p/%p", 
 1131                     object, 
 1132                     m[0]->object
 1133                 );
 1134         }
 1135 
 1136         /*
 1137          * Step 1
 1138          *
 1139          * Turn object into OBJT_SWAP
 1140          * check for bogus sysops
 1141          * force sync if not pageout process
 1142          */
 1143         if (object->type != OBJT_SWAP)
 1144                 swp_pager_meta_build(object, 0, SWAPBLK_NONE);
 1145         VM_OBJECT_UNLOCK(object);
 1146 
 1147         if (curproc != pageproc)
 1148                 sync = TRUE;
 1149 
 1150         /*
 1151          * Step 2
 1152          *
 1153          * Update nsw parameters from swap_async_max sysctl values.  
 1154          * Do not let the sysop crash the machine with bogus numbers.
 1155          */
 1156         mtx_lock(&pbuf_mtx);
 1157         if (swap_async_max != nsw_wcount_async_max) {
 1158                 int n;
 1159 
 1160                 /*
 1161                  * limit range
 1162                  */
 1163                 if ((n = swap_async_max) > nswbuf / 2)
 1164                         n = nswbuf / 2;
 1165                 if (n < 1)
 1166                         n = 1;
 1167                 swap_async_max = n;
 1168 
 1169                 /*
 1170                  * Adjust difference ( if possible ).  If the current async
 1171                  * count is too low, we may not be able to make the adjustment
 1172                  * at this time.
 1173                  */
 1174                 n -= nsw_wcount_async_max;
 1175                 if (nsw_wcount_async + n >= 0) {
 1176                         nsw_wcount_async += n;
 1177                         nsw_wcount_async_max += n;
 1178                         wakeup(&nsw_wcount_async);
 1179                 }
 1180         }
 1181         mtx_unlock(&pbuf_mtx);
 1182 
 1183         /*
 1184          * Step 3
 1185          *
 1186          * Assign swap blocks and issue I/O.  We reallocate swap on the fly.
 1187          * The page is left dirty until the pageout operation completes
 1188          * successfully.
 1189          */
 1190         for (i = 0; i < count; i += n) {
 1191                 int j;
 1192                 struct buf *bp;
 1193                 daddr_t blk;
 1194 
 1195                 /*
 1196                  * Maximum I/O size is limited by a number of factors.
 1197                  */
 1198                 n = min(BLIST_MAX_ALLOC, count - i);
 1199                 n = min(n, nsw_cluster_max);
 1200 
 1201                 /*
 1202                  * Get biggest block of swap we can.  If we fail, fall
 1203                  * back and try to allocate a smaller block.  Don't go
 1204                  * overboard trying to allocate space if it would overly
 1205                  * fragment swap.
 1206                  */
 1207                 while (
 1208                     (blk = swp_pager_getswapspace(n)) == SWAPBLK_NONE &&
 1209                     n > 4
 1210                 ) {
 1211                         n >>= 1;
 1212                 }
 1213                 if (blk == SWAPBLK_NONE) {
 1214                         for (j = 0; j < n; ++j)
 1215                                 rtvals[i+j] = VM_PAGER_FAIL;
 1216                         continue;
 1217                 }
 1218 
 1219                 /*
 1220                  * All I/O parameters have been satisfied, build the I/O
 1221                  * request and assign the swap space.
 1222                  */
 1223                 if (sync == TRUE) {
 1224                         bp = getpbuf(&nsw_wcount_sync);
 1225                 } else {
 1226                         bp = getpbuf(&nsw_wcount_async);
 1227                         bp->b_flags = B_ASYNC;
 1228                 }
 1229                 bp->b_flags |= B_PAGING;
 1230                 bp->b_iocmd = BIO_WRITE;
 1231 
 1232                 pmap_qenter((vm_offset_t)bp->b_data, &m[i], n);
 1233 
 1234                 bp->b_rcred = crhold(thread0.td_ucred);
 1235                 bp->b_wcred = crhold(thread0.td_ucred);
 1236                 bp->b_bcount = PAGE_SIZE * n;
 1237                 bp->b_bufsize = PAGE_SIZE * n;
 1238                 bp->b_blkno = blk;
 1239 
 1240                 VM_OBJECT_LOCK(object);
 1241                 for (j = 0; j < n; ++j) {
 1242                         vm_page_t mreq = m[i+j];
 1243 
 1244                         swp_pager_meta_build(
 1245                             mreq->object, 
 1246                             mreq->pindex,
 1247                             blk + j
 1248                         );
 1249                         vm_page_dirty(mreq);
 1250                         rtvals[i+j] = VM_PAGER_OK;
 1251 
 1252                         mreq->oflags |= VPO_SWAPINPROG;
 1253                         bp->b_pages[j] = mreq;
 1254                 }
 1255                 VM_OBJECT_UNLOCK(object);
 1256                 bp->b_npages = n;
 1257                 /*
 1258                  * Must set dirty range for NFS to work.
 1259                  */
 1260                 bp->b_dirtyoff = 0;
 1261                 bp->b_dirtyend = bp->b_bcount;
 1262 
 1263                 PCPU_INC(cnt.v_swapout);
 1264                 PCPU_ADD(cnt.v_swappgsout, bp->b_npages);
 1265 
 1266                 /*
 1267                  * asynchronous
 1268                  *
 1269                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1270                  */
 1271                 if (sync == FALSE) {
 1272                         bp->b_iodone = swp_pager_async_iodone;
 1273                         BUF_KERNPROC(bp);
 1274                         swp_pager_strategy(bp);
 1275 
 1276                         for (j = 0; j < n; ++j)
 1277                                 rtvals[i+j] = VM_PAGER_PEND;
 1278                         /* restart outter loop */
 1279                         continue;
 1280                 }
 1281 
 1282                 /*
 1283                  * synchronous
 1284                  *
 1285                  * NOTE: b_blkno is destroyed by the call to swapdev_strategy
 1286                  */
 1287                 bp->b_iodone = bdone;
 1288                 swp_pager_strategy(bp);
 1289 
 1290                 /*
 1291                  * Wait for the sync I/O to complete, then update rtvals.
 1292                  * We just set the rtvals[] to VM_PAGER_PEND so we can call
 1293                  * our async completion routine at the end, thus avoiding a
 1294                  * double-free.
 1295                  */
 1296                 bwait(bp, PVM, "swwrt");
 1297                 for (j = 0; j < n; ++j)
 1298                         rtvals[i+j] = VM_PAGER_PEND;
 1299                 /*
 1300                  * Now that we are through with the bp, we can call the
 1301                  * normal async completion, which frees everything up.
 1302                  */
 1303                 swp_pager_async_iodone(bp);
 1304         }
 1305         VM_OBJECT_LOCK(object);
 1306 }
 1307 
 1308 /*
 1309  *      swp_pager_async_iodone:
 1310  *
 1311  *      Completion routine for asynchronous reads and writes from/to swap.
 1312  *      Also called manually by synchronous code to finish up a bp.
 1313  *
 1314  *      For READ operations, the pages are PG_BUSY'd.  For WRITE operations, 
 1315  *      the pages are vm_page_t->busy'd.  For READ operations, we PG_BUSY 
 1316  *      unbusy all pages except the 'main' request page.  For WRITE 
 1317  *      operations, we vm_page_t->busy'd unbusy all pages ( we can do this 
 1318  *      because we marked them all VM_PAGER_PEND on return from putpages ).
 1319  *
 1320  *      This routine may not block.
 1321  *      This routine is called at splbio() or better
 1322  *
 1323  *      We up ourselves to splvm() as required for various vm_page related
 1324  *      calls.
 1325  */
 1326 static void
 1327 swp_pager_async_iodone(struct buf *bp)
 1328 {
 1329         int i;
 1330         vm_object_t object = NULL;
 1331 
 1332         /*
 1333          * report error
 1334          */
 1335         if (bp->b_ioflags & BIO_ERROR) {
 1336                 printf(
 1337                     "swap_pager: I/O error - %s failed; blkno %ld,"
 1338                         "size %ld, error %d\n",
 1339                     ((bp->b_iocmd == BIO_READ) ? "pagein" : "pageout"),
 1340                     (long)bp->b_blkno, 
 1341                     (long)bp->b_bcount,
 1342                     bp->b_error
 1343                 );
 1344         }
 1345 
 1346         /*
 1347          * remove the mapping for kernel virtual
 1348          */
 1349         pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
 1350 
 1351         if (bp->b_npages) {
 1352                 object = bp->b_pages[0]->object;
 1353                 VM_OBJECT_LOCK(object);
 1354         }
 1355         vm_page_lock_queues();
 1356         /*
 1357          * cleanup pages.  If an error occurs writing to swap, we are in
 1358          * very serious trouble.  If it happens to be a disk error, though,
 1359          * we may be able to recover by reassigning the swap later on.  So
 1360          * in this case we remove the m->swapblk assignment for the page 
 1361          * but do not free it in the rlist.  The errornous block(s) are thus
 1362          * never reallocated as swap.  Redirty the page and continue.
 1363          */
 1364         for (i = 0; i < bp->b_npages; ++i) {
 1365                 vm_page_t m = bp->b_pages[i];
 1366 
 1367                 m->oflags &= ~VPO_SWAPINPROG;
 1368 
 1369                 if (bp->b_ioflags & BIO_ERROR) {
 1370                         /*
 1371                          * If an error occurs I'd love to throw the swapblk
 1372                          * away without freeing it back to swapspace, so it
 1373                          * can never be used again.  But I can't from an 
 1374                          * interrupt.
 1375                          */
 1376                         if (bp->b_iocmd == BIO_READ) {
 1377                                 /*
 1378                                  * When reading, reqpage needs to stay
 1379                                  * locked for the parent, but all other
 1380                                  * pages can be freed.  We still want to
 1381                                  * wakeup the parent waiting on the page,
 1382                                  * though.  ( also: pg_reqpage can be -1 and 
 1383                                  * not match anything ).
 1384                                  *
 1385                                  * We have to wake specifically requested pages
 1386                                  * up too because we cleared VPO_SWAPINPROG and
 1387                                  * someone may be waiting for that.
 1388                                  *
 1389                                  * NOTE: for reads, m->dirty will probably
 1390                                  * be overridden by the original caller of
 1391                                  * getpages so don't play cute tricks here.
 1392                                  */
 1393                                 m->valid = 0;
 1394                                 if (i != bp->b_pager.pg_reqpage)
 1395                                         vm_page_free(m);
 1396                                 else
 1397                                         vm_page_flash(m);
 1398                                 /*
 1399                                  * If i == bp->b_pager.pg_reqpage, do not wake 
 1400                                  * the page up.  The caller needs to.
 1401                                  */
 1402                         } else {
 1403                                 /*
 1404                                  * If a write error occurs, reactivate page
 1405                                  * so it doesn't clog the inactive list,
 1406                                  * then finish the I/O.
 1407                                  */
 1408                                 vm_page_dirty(m);
 1409                                 vm_page_activate(m);
 1410                                 vm_page_io_finish(m);
 1411                         }
 1412                 } else if (bp->b_iocmd == BIO_READ) {
 1413                         /*
 1414                          * For read success, clear dirty bits.  Nobody should
 1415                          * have this page mapped but don't take any chances,
 1416                          * make sure the pmap modify bits are also cleared.
 1417                          *
 1418                          * NOTE: for reads, m->dirty will probably be 
 1419                          * overridden by the original caller of getpages so
 1420                          * we cannot set them in order to free the underlying
 1421                          * swap in a low-swap situation.  I don't think we'd
 1422                          * want to do that anyway, but it was an optimization
 1423                          * that existed in the old swapper for a time before
 1424                          * it got ripped out due to precisely this problem.
 1425                          *
 1426                          * If not the requested page then deactivate it.
 1427                          *
 1428                          * Note that the requested page, reqpage, is left
 1429                          * busied, but we still have to wake it up.  The
 1430                          * other pages are released (unbusied) by 
 1431                          * vm_page_wakeup().  We do not set reqpage's
 1432                          * valid bits here, it is up to the caller.
 1433                          */
 1434                         pmap_clear_modify(m);
 1435                         m->valid = VM_PAGE_BITS_ALL;
 1436                         vm_page_undirty(m);
 1437 
 1438                         /*
 1439                          * We have to wake specifically requested pages
 1440                          * up too because we cleared VPO_SWAPINPROG and
 1441                          * could be waiting for it in getpages.  However,
 1442                          * be sure to not unbusy getpages specifically
 1443                          * requested page - getpages expects it to be 
 1444                          * left busy.
 1445                          */
 1446                         if (i != bp->b_pager.pg_reqpage) {
 1447                                 vm_page_deactivate(m);
 1448                                 vm_page_wakeup(m);
 1449                         } else {
 1450                                 vm_page_flash(m);
 1451                         }
 1452                 } else {
 1453                         /*
 1454                          * For write success, clear the modify and dirty 
 1455                          * status, then finish the I/O ( which decrements the 
 1456                          * busy count and possibly wakes waiter's up ).
 1457                          */
 1458                         pmap_clear_modify(m);
 1459                         vm_page_undirty(m);
 1460                         vm_page_io_finish(m);
 1461                         if (vm_page_count_severe())
 1462                                 vm_page_try_to_cache(m);
 1463                 }
 1464         }
 1465         vm_page_unlock_queues();
 1466 
 1467         /*
 1468          * adjust pip.  NOTE: the original parent may still have its own
 1469          * pip refs on the object.
 1470          */
 1471         if (object != NULL) {
 1472                 vm_object_pip_wakeupn(object, bp->b_npages);
 1473                 VM_OBJECT_UNLOCK(object);
 1474         }
 1475 
 1476         /* 
 1477          * swapdev_strategy() manually sets b_vp and b_bufobj before calling 
 1478          * bstrategy(). Set them back to NULL now we're done with it, or we'll
 1479          * trigger a KASSERT in relpbuf().
 1480          */
 1481         if (bp->b_vp) {
 1482                     bp->b_vp = NULL;
 1483                     bp->b_bufobj = NULL;
 1484         }
 1485         /*
 1486          * release the physical I/O buffer
 1487          */
 1488         relpbuf(
 1489             bp, 
 1490             ((bp->b_iocmd == BIO_READ) ? &nsw_rcount : 
 1491                 ((bp->b_flags & B_ASYNC) ? 
 1492                     &nsw_wcount_async : 
 1493                     &nsw_wcount_sync
 1494                 )
 1495             )
 1496         );
 1497 }
 1498 
 1499 /*
 1500  *      swap_pager_isswapped:
 1501  *
 1502  *      Return 1 if at least one page in the given object is paged
 1503  *      out to the given swap device.
 1504  *
 1505  *      This routine may not block.
 1506  */
 1507 int
 1508 swap_pager_isswapped(vm_object_t object, struct swdevt *sp)
 1509 {
 1510         daddr_t index = 0;
 1511         int bcount;
 1512         int i;
 1513 
 1514         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1515         if (object->type != OBJT_SWAP)
 1516                 return (0);
 1517 
 1518         mtx_lock(&swhash_mtx);
 1519         for (bcount = 0; bcount < object->un_pager.swp.swp_bcount; bcount++) {
 1520                 struct swblock *swap;
 1521 
 1522                 if ((swap = *swp_pager_hash(object, index)) != NULL) {
 1523                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1524                                 if (swp_pager_isondev(swap->swb_pages[i], sp)) {
 1525                                         mtx_unlock(&swhash_mtx);
 1526                                         return (1);
 1527                                 }
 1528                         }
 1529                 }
 1530                 index += SWAP_META_PAGES;
 1531                 if (index > 0x20000000)
 1532                         panic("swap_pager_isswapped: failed to locate all swap meta blocks");
 1533         }
 1534         mtx_unlock(&swhash_mtx);
 1535         return (0);
 1536 }
 1537 
 1538 /*
 1539  * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in
 1540  *
 1541  *      This routine dissociates the page at the given index within a
 1542  *      swap block from its backing store, paging it in if necessary.
 1543  *      If the page is paged in, it is placed in the inactive queue,
 1544  *      since it had its backing store ripped out from under it.
 1545  *      We also attempt to swap in all other pages in the swap block,
 1546  *      we only guarantee that the one at the specified index is
 1547  *      paged in.
 1548  *
 1549  *      XXX - The code to page the whole block in doesn't work, so we
 1550  *            revert to the one-by-one behavior for now.  Sigh.
 1551  */
 1552 static inline void
 1553 swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex)
 1554 {
 1555         vm_page_t m;
 1556 
 1557         vm_object_pip_add(object, 1);
 1558         m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY);
 1559         if (m->valid == VM_PAGE_BITS_ALL) {
 1560                 vm_object_pip_subtract(object, 1);
 1561                 vm_page_lock_queues();
 1562                 vm_page_activate(m);
 1563                 vm_page_dirty(m);
 1564                 vm_page_unlock_queues();
 1565                 vm_page_wakeup(m);
 1566                 vm_pager_page_unswapped(m);
 1567                 return;
 1568         }
 1569 
 1570         if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK)
 1571                 panic("swap_pager_force_pagein: read from swap failed");/*XXX*/
 1572         vm_object_pip_subtract(object, 1);
 1573         vm_page_lock_queues();
 1574         vm_page_dirty(m);
 1575         vm_page_dontneed(m);
 1576         vm_page_unlock_queues();
 1577         vm_page_wakeup(m);
 1578         vm_pager_page_unswapped(m);
 1579 }
 1580 
 1581 /*
 1582  *      swap_pager_swapoff:
 1583  *
 1584  *      Page in all of the pages that have been paged out to the
 1585  *      given device.  The corresponding blocks in the bitmap must be
 1586  *      marked as allocated and the device must be flagged SW_CLOSING.
 1587  *      There may be no processes swapped out to the device.
 1588  *
 1589  *      This routine may block.
 1590  */
 1591 static void
 1592 swap_pager_swapoff(struct swdevt *sp)
 1593 {
 1594         struct swblock *swap;
 1595         int i, j, retries;
 1596 
 1597         GIANT_REQUIRED;
 1598 
 1599         retries = 0;
 1600 full_rescan:
 1601         mtx_lock(&swhash_mtx);
 1602         for (i = 0; i <= swhash_mask; i++) { /* '<=' is correct here */
 1603 restart:
 1604                 for (swap = swhash[i]; swap != NULL; swap = swap->swb_hnext) {
 1605                         vm_object_t object = swap->swb_object;
 1606                         vm_pindex_t pindex = swap->swb_index;
 1607                         for (j = 0; j < SWAP_META_PAGES; ++j) {
 1608                                 if (swp_pager_isondev(swap->swb_pages[j], sp)) {
 1609                                         /* avoid deadlock */
 1610                                         if (!VM_OBJECT_TRYLOCK(object)) {
 1611                                                 break;
 1612                                         } else {
 1613                                                 mtx_unlock(&swhash_mtx);
 1614                                                 swp_pager_force_pagein(object,
 1615                                                     pindex + j);
 1616                                                 VM_OBJECT_UNLOCK(object);
 1617                                                 mtx_lock(&swhash_mtx);
 1618                                                 goto restart;
 1619                                         }
 1620                                 }
 1621                         }
 1622                 }
 1623         }
 1624         mtx_unlock(&swhash_mtx);
 1625         if (sp->sw_used) {
 1626                 /*
 1627                  * Objects may be locked or paging to the device being
 1628                  * removed, so we will miss their pages and need to
 1629                  * make another pass.  We have marked this device as
 1630                  * SW_CLOSING, so the activity should finish soon.
 1631                  */
 1632                 retries++;
 1633                 if (retries > 100) {
 1634                         panic("swapoff: failed to locate %d swap blocks",
 1635                             sp->sw_used);
 1636                 }
 1637                 pause("swpoff", hz / 20);
 1638                 goto full_rescan;
 1639         }
 1640 }
 1641 
 1642 /************************************************************************
 1643  *                              SWAP META DATA                          *
 1644  ************************************************************************
 1645  *
 1646  *      These routines manipulate the swap metadata stored in the 
 1647  *      OBJT_SWAP object.  All swp_*() routines must be called at
 1648  *      splvm() because swap can be freed up by the low level vm_page
 1649  *      code which might be called from interrupts beyond what splbio() covers.
 1650  *
 1651  *      Swap metadata is implemented with a global hash and not directly
 1652  *      linked into the object.  Instead the object simply contains
 1653  *      appropriate tracking counters.
 1654  */
 1655 
 1656 /*
 1657  * SWP_PAGER_META_BUILD() -     add swap block to swap meta data for object
 1658  *
 1659  *      We first convert the object to a swap object if it is a default
 1660  *      object.
 1661  *
 1662  *      The specified swapblk is added to the object's swap metadata.  If
 1663  *      the swapblk is not valid, it is freed instead.  Any previously
 1664  *      assigned swapblk is freed.
 1665  *
 1666  *      This routine must be called at splvm(), except when used to convert
 1667  *      an OBJT_DEFAULT object into an OBJT_SWAP object.
 1668  */
 1669 static void
 1670 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
 1671 {
 1672         struct swblock *swap;
 1673         struct swblock **pswap;
 1674         int idx;
 1675 
 1676         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1677         /*
 1678          * Convert default object to swap object if necessary
 1679          */
 1680         if (object->type != OBJT_SWAP) {
 1681                 object->type = OBJT_SWAP;
 1682                 object->un_pager.swp.swp_bcount = 0;
 1683 
 1684                 if (object->handle != NULL) {
 1685                         mtx_lock(&sw_alloc_mtx);
 1686                         TAILQ_INSERT_TAIL(
 1687                             NOBJLIST(object->handle),
 1688                             object, 
 1689                             pager_object_list
 1690                         );
 1691                         mtx_unlock(&sw_alloc_mtx);
 1692                 }
 1693         }
 1694         
 1695         /*
 1696          * Locate hash entry.  If not found create, but if we aren't adding
 1697          * anything just return.  If we run out of space in the map we wait
 1698          * and, since the hash table may have changed, retry.
 1699          */
 1700 retry:
 1701         mtx_lock(&swhash_mtx);
 1702         pswap = swp_pager_hash(object, pindex);
 1703 
 1704         if ((swap = *pswap) == NULL) {
 1705                 int i;
 1706 
 1707                 if (swapblk == SWAPBLK_NONE)
 1708                         goto done;
 1709 
 1710                 swap = *pswap = uma_zalloc(swap_zone, M_NOWAIT);
 1711                 if (swap == NULL) {
 1712                         mtx_unlock(&swhash_mtx);
 1713                         VM_OBJECT_UNLOCK(object);
 1714                         if (uma_zone_exhausted(swap_zone))
 1715                                 printf("swap zone exhausted, increase kern.maxswzone\n");
 1716                         VM_WAIT;
 1717                         VM_OBJECT_LOCK(object);
 1718                         goto retry;
 1719                 }
 1720 
 1721                 swap->swb_hnext = NULL;
 1722                 swap->swb_object = object;
 1723                 swap->swb_index = pindex & ~(vm_pindex_t)SWAP_META_MASK;
 1724                 swap->swb_count = 0;
 1725 
 1726                 ++object->un_pager.swp.swp_bcount;
 1727 
 1728                 for (i = 0; i < SWAP_META_PAGES; ++i)
 1729                         swap->swb_pages[i] = SWAPBLK_NONE;
 1730         }
 1731 
 1732         /*
 1733          * Delete prior contents of metadata
 1734          */
 1735         idx = pindex & SWAP_META_MASK;
 1736 
 1737         if (swap->swb_pages[idx] != SWAPBLK_NONE) {
 1738                 swp_pager_freeswapspace(swap->swb_pages[idx], 1);
 1739                 --swap->swb_count;
 1740         }
 1741 
 1742         /*
 1743          * Enter block into metadata
 1744          */
 1745         swap->swb_pages[idx] = swapblk;
 1746         if (swapblk != SWAPBLK_NONE)
 1747                 ++swap->swb_count;
 1748 done:
 1749         mtx_unlock(&swhash_mtx);
 1750 }
 1751 
 1752 /*
 1753  * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata
 1754  *
 1755  *      The requested range of blocks is freed, with any associated swap 
 1756  *      returned to the swap bitmap.
 1757  *
 1758  *      This routine will free swap metadata structures as they are cleaned 
 1759  *      out.  This routine does *NOT* operate on swap metadata associated
 1760  *      with resident pages.
 1761  *
 1762  *      This routine must be called at splvm()
 1763  */
 1764 static void
 1765 swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count)
 1766 {
 1767 
 1768         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1769         if (object->type != OBJT_SWAP)
 1770                 return;
 1771 
 1772         while (count > 0) {
 1773                 struct swblock **pswap;
 1774                 struct swblock *swap;
 1775 
 1776                 mtx_lock(&swhash_mtx);
 1777                 pswap = swp_pager_hash(object, index);
 1778 
 1779                 if ((swap = *pswap) != NULL) {
 1780                         daddr_t v = swap->swb_pages[index & SWAP_META_MASK];
 1781 
 1782                         if (v != SWAPBLK_NONE) {
 1783                                 swp_pager_freeswapspace(v, 1);
 1784                                 swap->swb_pages[index & SWAP_META_MASK] =
 1785                                         SWAPBLK_NONE;
 1786                                 if (--swap->swb_count == 0) {
 1787                                         *pswap = swap->swb_hnext;
 1788                                         uma_zfree(swap_zone, swap);
 1789                                         --object->un_pager.swp.swp_bcount;
 1790                                 }
 1791                         }
 1792                         --count;
 1793                         ++index;
 1794                 } else {
 1795                         int n = SWAP_META_PAGES - (index & SWAP_META_MASK);
 1796                         count -= n;
 1797                         index += n;
 1798                 }
 1799                 mtx_unlock(&swhash_mtx);
 1800         }
 1801 }
 1802 
 1803 /*
 1804  * SWP_PAGER_META_FREE_ALL() - destroy all swap metadata associated with object
 1805  *
 1806  *      This routine locates and destroys all swap metadata associated with
 1807  *      an object.
 1808  *
 1809  *      This routine must be called at splvm()
 1810  */
 1811 static void
 1812 swp_pager_meta_free_all(vm_object_t object)
 1813 {
 1814         daddr_t index = 0;
 1815 
 1816         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1817         if (object->type != OBJT_SWAP)
 1818                 return;
 1819 
 1820         while (object->un_pager.swp.swp_bcount) {
 1821                 struct swblock **pswap;
 1822                 struct swblock *swap;
 1823 
 1824                 mtx_lock(&swhash_mtx);
 1825                 pswap = swp_pager_hash(object, index);
 1826                 if ((swap = *pswap) != NULL) {
 1827                         int i;
 1828 
 1829                         for (i = 0; i < SWAP_META_PAGES; ++i) {
 1830                                 daddr_t v = swap->swb_pages[i];
 1831                                 if (v != SWAPBLK_NONE) {
 1832                                         --swap->swb_count;
 1833                                         swp_pager_freeswapspace(v, 1);
 1834                                 }
 1835                         }
 1836                         if (swap->swb_count != 0)
 1837                                 panic("swap_pager_meta_free_all: swb_count != 0");
 1838                         *pswap = swap->swb_hnext;
 1839                         uma_zfree(swap_zone, swap);
 1840                         --object->un_pager.swp.swp_bcount;
 1841                 }
 1842                 mtx_unlock(&swhash_mtx);
 1843                 index += SWAP_META_PAGES;
 1844                 if (index > 0x20000000)
 1845                         panic("swp_pager_meta_free_all: failed to locate all swap meta blocks");
 1846         }
 1847 }
 1848 
 1849 /*
 1850  * SWP_PAGER_METACTL() -  misc control of swap and vm_page_t meta data.
 1851  *
 1852  *      This routine is capable of looking up, popping, or freeing
 1853  *      swapblk assignments in the swap meta data or in the vm_page_t.
 1854  *      The routine typically returns the swapblk being looked-up, or popped,
 1855  *      or SWAPBLK_NONE if the block was freed, or SWAPBLK_NONE if the block
 1856  *      was invalid.  This routine will automatically free any invalid 
 1857  *      meta-data swapblks.
 1858  *
 1859  *      It is not possible to store invalid swapblks in the swap meta data
 1860  *      (other then a literal 'SWAPBLK_NONE'), so we don't bother checking.
 1861  *
 1862  *      When acting on a busy resident page and paging is in progress, we 
 1863  *      have to wait until paging is complete but otherwise can act on the 
 1864  *      busy page.
 1865  *
 1866  *      This routine must be called at splvm().
 1867  *
 1868  *      SWM_FREE        remove and free swap block from metadata
 1869  *      SWM_POP         remove from meta data but do not free.. pop it out
 1870  */
 1871 static daddr_t
 1872 swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags)
 1873 {
 1874         struct swblock **pswap;
 1875         struct swblock *swap;
 1876         daddr_t r1;
 1877         int idx;
 1878 
 1879         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 1880         /*
 1881          * The meta data only exists of the object is OBJT_SWAP 
 1882          * and even then might not be allocated yet.
 1883          */
 1884         if (object->type != OBJT_SWAP)
 1885                 return (SWAPBLK_NONE);
 1886 
 1887         r1 = SWAPBLK_NONE;
 1888         mtx_lock(&swhash_mtx);
 1889         pswap = swp_pager_hash(object, pindex);
 1890 
 1891         if ((swap = *pswap) != NULL) {
 1892                 idx = pindex & SWAP_META_MASK;
 1893                 r1 = swap->swb_pages[idx];
 1894 
 1895                 if (r1 != SWAPBLK_NONE) {
 1896                         if (flags & SWM_FREE) {
 1897                                 swp_pager_freeswapspace(r1, 1);
 1898                                 r1 = SWAPBLK_NONE;
 1899                         }
 1900                         if (flags & (SWM_FREE|SWM_POP)) {
 1901                                 swap->swb_pages[idx] = SWAPBLK_NONE;
 1902                                 if (--swap->swb_count == 0) {
 1903                                         *pswap = swap->swb_hnext;
 1904                                         uma_zfree(swap_zone, swap);
 1905                                         --object->un_pager.swp.swp_bcount;
 1906                                 }
 1907                         } 
 1908                 }
 1909         }
 1910         mtx_unlock(&swhash_mtx);
 1911         return (r1);
 1912 }
 1913 
 1914 /*
 1915  * System call swapon(name) enables swapping on device name,
 1916  * which must be in the swdevsw.  Return EBUSY
 1917  * if already swapping on this device.
 1918  */
 1919 #ifndef _SYS_SYSPROTO_H_
 1920 struct swapon_args {
 1921         char *name;
 1922 };
 1923 #endif
 1924 
 1925 /* 
 1926  * MPSAFE
 1927  */
 1928 /* ARGSUSED */
 1929 int
 1930 swapon(struct thread *td, struct swapon_args *uap)
 1931 {
 1932         struct vattr attr;
 1933         struct vnode *vp;
 1934         struct nameidata nd;
 1935         int error;
 1936 
 1937         error = priv_check(td, PRIV_SWAPON);
 1938         if (error)
 1939                 return (error);
 1940 
 1941         mtx_lock(&Giant);
 1942         while (swdev_syscall_active)
 1943             tsleep(&swdev_syscall_active, PUSER - 1, "swpon", 0);
 1944         swdev_syscall_active = 1;
 1945 
 1946         /*
 1947          * Swap metadata may not fit in the KVM if we have physical
 1948          * memory of >1GB.
 1949          */
 1950         if (swap_zone == NULL) {
 1951                 error = ENOMEM;
 1952                 goto done;
 1953         }
 1954 
 1955         NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | AUDITVNODE1, UIO_USERSPACE,
 1956             uap->name, td);
 1957         error = namei(&nd);
 1958         if (error)
 1959                 goto done;
 1960 
 1961         NDFREE(&nd, NDF_ONLY_PNBUF);
 1962         vp = nd.ni_vp;
 1963 
 1964         if (vn_isdisk(vp, &error)) {
 1965                 error = swapongeom(td, vp);
 1966         } else if (vp->v_type == VREG &&
 1967             (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
 1968             (error = VOP_GETATTR(vp, &attr, td->td_ucred, td)) == 0) {
 1969                 /*
 1970                  * Allow direct swapping to NFS regular files in the same
 1971                  * way that nfs_mountroot() sets up diskless swapping.
 1972                  */
 1973                 error = swaponvp(td, vp, attr.va_size / DEV_BSIZE);
 1974         }
 1975 
 1976         if (error)
 1977                 vrele(vp);
 1978 done:
 1979         swdev_syscall_active = 0;
 1980         wakeup_one(&swdev_syscall_active);
 1981         mtx_unlock(&Giant);
 1982         return (error);
 1983 }
 1984 
 1985 static void
 1986 swaponsomething(struct vnode *vp, void *id, u_long nblks, sw_strategy_t *strategy, sw_close_t *close, dev_t dev)
 1987 {
 1988         struct swdevt *sp, *tsp;
 1989         swblk_t dvbase;
 1990         u_long mblocks;
 1991 
 1992         /*
 1993          * If we go beyond this, we get overflows in the radix
 1994          * tree bitmap code.
 1995          */
 1996         mblocks = 0x40000000 / BLIST_META_RADIX;
 1997         if (nblks > mblocks) {
 1998                 printf("WARNING: reducing size to maximum of %lu blocks per swap unit\n",
 1999                         mblocks);
 2000                 nblks = mblocks;
 2001         }
 2002         /*
 2003          * nblks is in DEV_BSIZE'd chunks, convert to PAGE_SIZE'd chunks.
 2004          * First chop nblks off to page-align it, then convert.
 2005          * 
 2006          * sw->sw_nblks is in page-sized chunks now too.
 2007          */
 2008         nblks &= ~(ctodb(1) - 1);
 2009         nblks = dbtoc(nblks);
 2010 
 2011         sp = malloc(sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
 2012         sp->sw_vp = vp;
 2013         sp->sw_id = id;
 2014         sp->sw_dev = dev;
 2015         sp->sw_flags = 0;
 2016         sp->sw_nblks = nblks;
 2017         sp->sw_used = 0;
 2018         sp->sw_strategy = strategy;
 2019         sp->sw_close = close;
 2020 
 2021         sp->sw_blist = blist_create(nblks);
 2022         /*
 2023          * Do not free the first two block in order to avoid overwriting
 2024          * any bsd label at the front of the partition
 2025          */
 2026         blist_free(sp->sw_blist, 2, nblks - 2);
 2027 
 2028         dvbase = 0;
 2029         mtx_lock(&sw_dev_mtx);
 2030         TAILQ_FOREACH(tsp, &swtailq, sw_list) {
 2031                 if (tsp->sw_end >= dvbase) {
 2032                         /*
 2033                          * We put one uncovered page between the devices
 2034                          * in order to definitively prevent any cross-device
 2035                          * I/O requests
 2036                          */
 2037                         dvbase = tsp->sw_end + 1;
 2038                 }
 2039         }
 2040         sp->sw_first = dvbase;
 2041         sp->sw_end = dvbase + nblks;
 2042         TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
 2043         nswapdev++;
 2044         swap_pager_avail += nblks;
 2045         swp_sizecheck();
 2046         mtx_unlock(&sw_dev_mtx);
 2047 }
 2048 
 2049 /*
 2050  * SYSCALL: swapoff(devname)
 2051  *
 2052  * Disable swapping on the given device.
 2053  *
 2054  * XXX: Badly designed system call: it should use a device index
 2055  * rather than filename as specification.  We keep sw_vp around
 2056  * only to make this work.
 2057  */
 2058 #ifndef _SYS_SYSPROTO_H_
 2059 struct swapoff_args {
 2060         char *name;
 2061 };
 2062 #endif
 2063 
 2064 /*
 2065  * MPSAFE
 2066  */
 2067 /* ARGSUSED */
 2068 int
 2069 swapoff(struct thread *td, struct swapoff_args *uap)
 2070 {
 2071         struct vnode *vp;
 2072         struct nameidata nd;
 2073         struct swdevt *sp;
 2074         int error;
 2075 
 2076         error = priv_check(td, PRIV_SWAPOFF);
 2077         if (error)
 2078                 return (error);
 2079 
 2080         mtx_lock(&Giant);
 2081         while (swdev_syscall_active)
 2082             tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2083         swdev_syscall_active = 1;
 2084 
 2085         NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->name,
 2086             td);
 2087         error = namei(&nd);
 2088         if (error)
 2089                 goto done;
 2090         NDFREE(&nd, NDF_ONLY_PNBUF);
 2091         vp = nd.ni_vp;
 2092 
 2093         mtx_lock(&sw_dev_mtx);
 2094         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2095                 if (sp->sw_vp == vp)
 2096                         break;
 2097         }
 2098         mtx_unlock(&sw_dev_mtx);
 2099         if (sp == NULL) {
 2100                 error = EINVAL;
 2101                 goto done;
 2102         }
 2103         error = swapoff_one(sp, td);
 2104 done:
 2105         swdev_syscall_active = 0;
 2106         wakeup_one(&swdev_syscall_active);
 2107         mtx_unlock(&Giant);
 2108         return (error);
 2109 }
 2110 
 2111 static int
 2112 swapoff_one(struct swdevt *sp, struct thread *td)
 2113 {
 2114         u_long nblks, dvbase;
 2115 #ifdef MAC
 2116         int error;
 2117 #endif
 2118 
 2119         mtx_assert(&Giant, MA_OWNED);
 2120 #ifdef MAC
 2121         (void) vn_lock(sp->sw_vp, LK_EXCLUSIVE | LK_RETRY, td);
 2122         error = mac_check_system_swapoff(td->td_ucred, sp->sw_vp);
 2123         (void) VOP_UNLOCK(sp->sw_vp, 0, td);
 2124         if (error != 0)
 2125                 return (error);
 2126 #endif
 2127         nblks = sp->sw_nblks;
 2128 
 2129         /*
 2130          * We can turn off this swap device safely only if the
 2131          * available virtual memory in the system will fit the amount
 2132          * of data we will have to page back in, plus an epsilon so
 2133          * the system doesn't become critically low on swap space.
 2134          */
 2135         if (cnt.v_free_count + cnt.v_cache_count + swap_pager_avail <
 2136             nblks + nswap_lowat) {
 2137                 return (ENOMEM);
 2138         }
 2139 
 2140         /*
 2141          * Prevent further allocations on this device.
 2142          */
 2143         mtx_lock(&sw_dev_mtx);
 2144         sp->sw_flags |= SW_CLOSING;
 2145         for (dvbase = 0; dvbase < sp->sw_end; dvbase += dmmax) {
 2146                 swap_pager_avail -= blist_fill(sp->sw_blist,
 2147                      dvbase, dmmax);
 2148         }
 2149         mtx_unlock(&sw_dev_mtx);
 2150 
 2151         /*
 2152          * Page in the contents of the device and close it.
 2153          */
 2154         swap_pager_swapoff(sp);
 2155 
 2156         sp->sw_close(td, sp);
 2157         sp->sw_id = NULL;
 2158         mtx_lock(&sw_dev_mtx);
 2159         TAILQ_REMOVE(&swtailq, sp, sw_list);
 2160         nswapdev--;
 2161         if (nswapdev == 0) {
 2162                 swap_pager_full = 2;
 2163                 swap_pager_almost_full = 1;
 2164         }
 2165         if (swdevhd == sp)
 2166                 swdevhd = NULL;
 2167         mtx_unlock(&sw_dev_mtx);
 2168         blist_destroy(sp->sw_blist);
 2169         free(sp, M_VMPGDATA);
 2170         return (0);
 2171 }
 2172 
 2173 void
 2174 swapoff_all(void)
 2175 {
 2176         struct swdevt *sp, *spt;
 2177         const char *devname;
 2178         int error;
 2179  
 2180         mtx_lock(&Giant);
 2181         while (swdev_syscall_active)
 2182                 tsleep(&swdev_syscall_active, PUSER - 1, "swpoff", 0);
 2183         swdev_syscall_active = 1;
 2184  
 2185         mtx_lock(&sw_dev_mtx);
 2186         TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
 2187                 mtx_unlock(&sw_dev_mtx);
 2188                 if (vn_isdisk(sp->sw_vp, NULL))
 2189                         devname = sp->sw_vp->v_rdev->si_name;
 2190                 else
 2191                         devname = "[file]";
 2192                 error = swapoff_one(sp, &thread0);
 2193                 if (error != 0) {
 2194                         printf("Cannot remove swap device %s (error=%d), "
 2195                             "skipping.\n", devname, error);
 2196                 } else if (bootverbose) {
 2197                         printf("Swap device %s removed.\n", devname);
 2198                 }
 2199                 mtx_lock(&sw_dev_mtx);
 2200         }
 2201         mtx_unlock(&sw_dev_mtx);
 2202  
 2203         swdev_syscall_active = 0;
 2204         wakeup_one(&swdev_syscall_active);
 2205         mtx_unlock(&Giant);
 2206 }
 2207 
 2208 void
 2209 swap_pager_status(int *total, int *used)
 2210 {
 2211         struct swdevt *sp;
 2212 
 2213         *total = 0;
 2214         *used = 0;
 2215         mtx_lock(&sw_dev_mtx);
 2216         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2217                 *total += sp->sw_nblks;
 2218                 *used += sp->sw_used;
 2219         }
 2220         mtx_unlock(&sw_dev_mtx);
 2221 }
 2222 
 2223 static int
 2224 sysctl_vm_swap_info(SYSCTL_HANDLER_ARGS)
 2225 {
 2226         int     *name = (int *)arg1;
 2227         int     error, n;
 2228         struct xswdev xs;
 2229         struct swdevt *sp;
 2230 
 2231         if (arg2 != 1) /* name length */
 2232                 return (EINVAL);
 2233 
 2234         n = 0;
 2235         mtx_lock(&sw_dev_mtx);
 2236         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2237                 if (n == *name) {
 2238                         mtx_unlock(&sw_dev_mtx);
 2239                         xs.xsw_version = XSWDEV_VERSION;
 2240                         xs.xsw_dev = sp->sw_dev;
 2241                         xs.xsw_flags = sp->sw_flags;
 2242                         xs.xsw_nblks = sp->sw_nblks;
 2243                         xs.xsw_used = sp->sw_used;
 2244 
 2245                         error = SYSCTL_OUT(req, &xs, sizeof(xs));
 2246                         return (error);
 2247                 }
 2248                 n++;
 2249         }
 2250         mtx_unlock(&sw_dev_mtx);
 2251         return (ENOENT);
 2252 }
 2253 
 2254 SYSCTL_INT(_vm, OID_AUTO, nswapdev, CTLFLAG_RD, &nswapdev, 0,
 2255     "Number of swap devices");
 2256 SYSCTL_NODE(_vm, OID_AUTO, swap_info, CTLFLAG_RD, sysctl_vm_swap_info,
 2257     "Swap statistics by device");
 2258 
 2259 /*
 2260  * vmspace_swap_count() - count the approximate swap useage in pages for a
 2261  *                        vmspace.
 2262  *
 2263  *      The map must be locked.
 2264  *
 2265  *      Swap useage is determined by taking the proportional swap used by
 2266  *      VM objects backing the VM map.  To make up for fractional losses,
 2267  *      if the VM object has any swap use at all the associated map entries
 2268  *      count for at least 1 swap page.
 2269  */
 2270 int
 2271 vmspace_swap_count(struct vmspace *vmspace)
 2272 {
 2273         vm_map_t map = &vmspace->vm_map;
 2274         vm_map_entry_t cur;
 2275         int count = 0;
 2276 
 2277         for (cur = map->header.next; cur != &map->header; cur = cur->next) {
 2278                 vm_object_t object;
 2279 
 2280                 if ((cur->eflags & MAP_ENTRY_IS_SUB_MAP) == 0 &&
 2281                     (object = cur->object.vm_object) != NULL) {
 2282                         VM_OBJECT_LOCK(object);
 2283                         if (object->type == OBJT_SWAP &&
 2284                             object->un_pager.swp.swp_bcount != 0) {
 2285                                 int n = (cur->end - cur->start) / PAGE_SIZE;
 2286 
 2287                                 count += object->un_pager.swp.swp_bcount *
 2288                                     SWAP_META_PAGES * n / object->size + 1;
 2289                         }
 2290                         VM_OBJECT_UNLOCK(object);
 2291                 }
 2292         }
 2293         return (count);
 2294 }
 2295 
 2296 /*
 2297  * GEOM backend
 2298  *
 2299  * Swapping onto disk devices.
 2300  *
 2301  */
 2302 
 2303 static g_orphan_t swapgeom_orphan;
 2304 
 2305 static struct g_class g_swap_class = {
 2306         .name = "SWAP",
 2307         .version = G_VERSION,
 2308         .orphan = swapgeom_orphan,
 2309 };
 2310 
 2311 DECLARE_GEOM_CLASS(g_swap_class, g_class);
 2312 
 2313 
 2314 static void
 2315 swapgeom_done(struct bio *bp2)
 2316 {
 2317         struct buf *bp;
 2318 
 2319         bp = bp2->bio_caller2;
 2320         bp->b_ioflags = bp2->bio_flags;
 2321         if (bp2->bio_error)
 2322                 bp->b_ioflags |= BIO_ERROR;
 2323         bp->b_resid = bp->b_bcount - bp2->bio_completed;
 2324         bp->b_error = bp2->bio_error;
 2325         bufdone(bp);
 2326         g_destroy_bio(bp2);
 2327 }
 2328 
 2329 static void
 2330 swapgeom_strategy(struct buf *bp, struct swdevt *sp)
 2331 {
 2332         struct bio *bio;
 2333         struct g_consumer *cp;
 2334 
 2335         cp = sp->sw_id;
 2336         if (cp == NULL) {
 2337                 bp->b_error = ENXIO;
 2338                 bp->b_ioflags |= BIO_ERROR;
 2339                 bufdone(bp);
 2340                 return;
 2341         }
 2342         bio = g_alloc_bio();
 2343 #if 0
 2344         /*
 2345          * XXX: We shouldn't really sleep here when we run out of buffers
 2346          * XXX: but the alternative is worse right now.
 2347          */
 2348         if (bio == NULL) {
 2349                 bp->b_error = ENOMEM;
 2350                 bp->b_ioflags |= BIO_ERROR;
 2351                 bufdone(bp);
 2352                 return;
 2353         }
 2354 #endif
 2355         bio->bio_caller2 = bp;
 2356         bio->bio_cmd = bp->b_iocmd;
 2357         bio->bio_data = bp->b_data;
 2358         bio->bio_offset = (bp->b_blkno - sp->sw_first) * PAGE_SIZE;
 2359         bio->bio_length = bp->b_bcount;
 2360         bio->bio_done = swapgeom_done;
 2361         g_io_request(bio, cp);
 2362         return;
 2363 }
 2364 
 2365 static void
 2366 swapgeom_orphan(struct g_consumer *cp)
 2367 {
 2368         struct swdevt *sp;
 2369 
 2370         mtx_lock(&sw_dev_mtx);
 2371         TAILQ_FOREACH(sp, &swtailq, sw_list)
 2372                 if (sp->sw_id == cp)
 2373                         sp->sw_id = NULL;
 2374         mtx_unlock(&sw_dev_mtx);
 2375 }
 2376 
 2377 static void
 2378 swapgeom_close_ev(void *arg, int flags)
 2379 {
 2380         struct g_consumer *cp;
 2381 
 2382         cp = arg;
 2383         g_access(cp, -1, -1, 0);
 2384         g_detach(cp);
 2385         g_destroy_consumer(cp);
 2386 }
 2387 
 2388 static void
 2389 swapgeom_close(struct thread *td, struct swdevt *sw)
 2390 {
 2391 
 2392         /* XXX: direct call when Giant untangled */
 2393         g_waitfor_event(swapgeom_close_ev, sw->sw_id, M_WAITOK, NULL);
 2394 }
 2395 
 2396 
 2397 struct swh0h0 {
 2398         struct cdev *dev;
 2399         struct vnode *vp;
 2400         int     error;
 2401 };
 2402 
 2403 static void
 2404 swapongeom_ev(void *arg, int flags)
 2405 {
 2406         struct swh0h0 *swh;
 2407         struct g_provider *pp;
 2408         struct g_consumer *cp;
 2409         static struct g_geom *gp;
 2410         struct swdevt *sp;
 2411         u_long nblks;
 2412         int error;
 2413 
 2414         swh = arg;
 2415         swh->error = 0;
 2416         pp = g_dev_getprovider(swh->dev);
 2417         if (pp == NULL) {
 2418                 swh->error = ENODEV;
 2419                 return;
 2420         }
 2421         mtx_lock(&sw_dev_mtx);
 2422         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2423                 cp = sp->sw_id;
 2424                 if (cp != NULL && cp->provider == pp) {
 2425                         mtx_unlock(&sw_dev_mtx);
 2426                         swh->error = EBUSY;
 2427                         return;
 2428                 }
 2429         }
 2430         mtx_unlock(&sw_dev_mtx);
 2431         if (gp == NULL)
 2432                 gp = g_new_geomf(&g_swap_class, "swap", NULL);
 2433         cp = g_new_consumer(gp);
 2434         g_attach(cp, pp);
 2435         /*
 2436          * XXX: Everytime you think you can improve the margin for
 2437          * footshooting, somebody depends on the ability to do so:
 2438          * savecore(8) wants to write to our swapdev so we cannot
 2439          * set an exclusive count :-(
 2440          */
 2441         error = g_access(cp, 1, 1, 0);
 2442         if (error) {
 2443                 g_detach(cp);
 2444                 g_destroy_consumer(cp);
 2445                 swh->error = error;
 2446                 return;
 2447         }
 2448         nblks = pp->mediasize / DEV_BSIZE;
 2449         swaponsomething(swh->vp, cp, nblks, swapgeom_strategy,
 2450             swapgeom_close, dev2udev(swh->dev));
 2451         swh->error = 0;
 2452         return;
 2453 }
 2454 
 2455 static int
 2456 swapongeom(struct thread *td, struct vnode *vp)
 2457 {
 2458         int error;
 2459         struct swh0h0 swh;
 2460 
 2461         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
 2462 
 2463         swh.dev = vp->v_rdev;
 2464         swh.vp = vp;
 2465         swh.error = 0;
 2466         /* XXX: direct call when Giant untangled */
 2467         error = g_waitfor_event(swapongeom_ev, &swh, M_WAITOK, NULL);
 2468         if (!error)
 2469                 error = swh.error;
 2470         VOP_UNLOCK(vp, 0, td);
 2471         return (error);
 2472 }
 2473 
 2474 /*
 2475  * VNODE backend
 2476  *
 2477  * This is used mainly for network filesystem (read: probably only tested
 2478  * with NFS) swapfiles.
 2479  *
 2480  */
 2481 
 2482 static void
 2483 swapdev_strategy(struct buf *bp, struct swdevt *sp)
 2484 {
 2485         struct vnode *vp2;
 2486 
 2487         bp->b_blkno = ctodb(bp->b_blkno - sp->sw_first);
 2488 
 2489         vp2 = sp->sw_id;
 2490         vhold(vp2);
 2491         if (bp->b_iocmd == BIO_WRITE) {
 2492                 if (bp->b_bufobj)
 2493                         bufobj_wdrop(bp->b_bufobj);
 2494                 bufobj_wref(&vp2->v_bufobj);
 2495         }
 2496         if (bp->b_bufobj != &vp2->v_bufobj)
 2497                 bp->b_bufobj = &vp2->v_bufobj;
 2498         bp->b_vp = vp2;
 2499         bp->b_iooffset = dbtob(bp->b_blkno);
 2500         bstrategy(bp);
 2501         return;
 2502 }
 2503 
 2504 static void
 2505 swapdev_close(struct thread *td, struct swdevt *sp)
 2506 {
 2507 
 2508         VOP_CLOSE(sp->sw_vp, FREAD | FWRITE, td->td_ucred, td);
 2509         vrele(sp->sw_vp);
 2510 }
 2511 
 2512 
 2513 static int
 2514 swaponvp(struct thread *td, struct vnode *vp, u_long nblks)
 2515 {
 2516         struct swdevt *sp;
 2517         int error;
 2518 
 2519         if (nblks == 0)
 2520                 return (ENXIO);
 2521         mtx_lock(&sw_dev_mtx);
 2522         TAILQ_FOREACH(sp, &swtailq, sw_list) {
 2523                 if (sp->sw_id == vp) {
 2524                         mtx_unlock(&sw_dev_mtx);
 2525                         return (EBUSY);
 2526                 }
 2527         }
 2528         mtx_unlock(&sw_dev_mtx);
 2529     
 2530         (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
 2531 #ifdef MAC
 2532         error = mac_check_system_swapon(td->td_ucred, vp);
 2533         if (error == 0)
 2534 #endif
 2535                 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
 2536         (void) VOP_UNLOCK(vp, 0, td);
 2537         if (error)
 2538                 return (error);
 2539 
 2540         swaponsomething(vp, vp, nblks, swapdev_strategy, swapdev_close,
 2541             NODEV);
 2542         return (0);
 2543 }

Cache object: 440942734944fe5ec5b6ee9401c8d254


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.