The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_swap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: uvm_swap.c,v 1.85.2.1 2004/05/15 13:48:49 tron Exp $   */
    2 
    3 /*
    4  * Copyright (c) 1995, 1996, 1997 Matthew R. Green
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  * 3. The name of the author may not be used to endorse or promote products
   16  *    derived from this software without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
   23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
   25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp
   31  * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.85.2.1 2004/05/15 13:48:49 tron Exp $");
   36 
   37 #include "fs_nfs.h"
   38 #include "opt_uvmhist.h"
   39 #include "opt_compat_netbsd.h"
   40 #include "opt_ddb.h"
   41 
   42 #include <sys/param.h>
   43 #include <sys/systm.h>
   44 #include <sys/buf.h>
   45 #include <sys/conf.h>
   46 #include <sys/proc.h>
   47 #include <sys/namei.h>
   48 #include <sys/disklabel.h>
   49 #include <sys/errno.h>
   50 #include <sys/kernel.h>
   51 #include <sys/malloc.h>
   52 #include <sys/vnode.h>
   53 #include <sys/file.h>
   54 #include <sys/extent.h>
   55 #include <sys/mount.h>
   56 #include <sys/pool.h>
   57 #include <sys/sa.h>
   58 #include <sys/syscallargs.h>
   59 #include <sys/swap.h>
   60 
   61 #include <uvm/uvm.h>
   62 
   63 #include <miscfs/specfs/specdev.h>
   64 
   65 /*
   66  * uvm_swap.c: manage configuration and i/o to swap space.
   67  */
   68 
   69 /*
   70  * swap space is managed in the following way:
   71  *
   72  * each swap partition or file is described by a "swapdev" structure.
   73  * each "swapdev" structure contains a "swapent" structure which contains
   74  * information that is passed up to the user (via system calls).
   75  *
   76  * each swap partition is assigned a "priority" (int) which controls
   77  * swap parition usage.
   78  *
   79  * the system maintains a global data structure describing all swap
   80  * partitions/files.   there is a sorted LIST of "swappri" structures
   81  * which describe "swapdev"'s at that priority.   this LIST is headed
   82  * by the "swap_priority" global var.    each "swappri" contains a
   83  * CIRCLEQ of "swapdev" structures at that priority.
   84  *
   85  * locking:
   86  *  - swap_syscall_lock (sleep lock): this lock serializes the swapctl
   87  *    system call and prevents the swap priority list from changing
   88  *    while we are in the middle of a system call (e.g. SWAP_STATS).
   89  *  - uvm.swap_data_lock (simple_lock): this lock protects all swap data
   90  *    structures including the priority list, the swapdev structures,
   91  *    and the swapmap extent.
   92  *
   93  * each swap device has the following info:
   94  *  - swap device in use (could be disabled, preventing future use)
   95  *  - swap enabled (allows new allocations on swap)
   96  *  - map info in /dev/drum
   97  *  - vnode pointer
   98  * for swap files only:
   99  *  - block size
  100  *  - max byte count in buffer
  101  *  - buffer
  102  *
  103  * userland controls and configures swap with the swapctl(2) system call.
  104  * the sys_swapctl performs the following operations:
  105  *  [1] SWAP_NSWAP: returns the number of swap devices currently configured
  106  *  [2] SWAP_STATS: given a pointer to an array of swapent structures
  107  *      (passed in via "arg") of a size passed in via "misc" ... we load
  108  *      the current swap config into the array. The actual work is done
  109  *      in the uvm_swap_stats(9) function.
  110  *  [3] SWAP_ON: given a pathname in arg (could be device or file) and a
  111  *      priority in "misc", start swapping on it.
  112  *  [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device
  113  *  [5] SWAP_CTL: changes the priority of a swap device (new priority in
  114  *      "misc")
  115  */
  116 
  117 /*
  118  * swapdev: describes a single swap partition/file
  119  *
  120  * note the following should be true:
  121  * swd_inuse <= swd_nblks  [number of blocks in use is <= total blocks]
  122  * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel]
  123  */
  124 struct swapdev {
  125         struct oswapent swd_ose;
  126 #define swd_dev         swd_ose.ose_dev         /* device id */
  127 #define swd_flags       swd_ose.ose_flags       /* flags:inuse/enable/fake */
  128 #define swd_priority    swd_ose.ose_priority    /* our priority */
  129         /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */
  130         char                    *swd_path;      /* saved pathname of device */
  131         int                     swd_pathlen;    /* length of pathname */
  132         int                     swd_npages;     /* #pages we can use */
  133         int                     swd_npginuse;   /* #pages in use */
  134         int                     swd_npgbad;     /* #pages bad */
  135         int                     swd_drumoffset; /* page0 offset in drum */
  136         int                     swd_drumsize;   /* #pages in drum */
  137         struct extent           *swd_ex;        /* extent for this swapdev */
  138         char                    swd_exname[12]; /* name of extent above */
  139         struct vnode            *swd_vp;        /* backing vnode */
  140         CIRCLEQ_ENTRY(swapdev)  swd_next;       /* priority circleq */
  141 
  142         int                     swd_bsize;      /* blocksize (bytes) */
  143         int                     swd_maxactive;  /* max active i/o reqs */
  144         struct bufq_state       swd_tab;        /* buffer list */
  145         int                     swd_active;     /* number of active buffers */
  146 };
  147 
  148 /*
  149  * swap device priority entry; the list is kept sorted on `spi_priority'.
  150  */
  151 struct swappri {
  152         int                     spi_priority;     /* priority */
  153         CIRCLEQ_HEAD(spi_swapdev, swapdev)      spi_swapdev;
  154         /* circleq of swapdevs at this priority */
  155         LIST_ENTRY(swappri)     spi_swappri;      /* global list of pri's */
  156 };
  157 
  158 /*
  159  * The following two structures are used to keep track of data transfers
  160  * on swap devices associated with regular files.
  161  * NOTE: this code is more or less a copy of vnd.c; we use the same
  162  * structure names here to ease porting..
  163  */
  164 struct vndxfer {
  165         struct buf      *vx_bp;         /* Pointer to parent buffer */
  166         struct swapdev  *vx_sdp;
  167         int             vx_error;
  168         int             vx_pending;     /* # of pending aux buffers */
  169         int             vx_flags;
  170 #define VX_BUSY         1
  171 #define VX_DEAD         2
  172 };
  173 
  174 struct vndbuf {
  175         struct buf      vb_buf;
  176         struct vndxfer  *vb_xfer;
  177 };
  178 
  179 
  180 /*
  181  * We keep a of pool vndbuf's and vndxfer structures.
  182  */
  183 static struct pool vndxfer_pool;
  184 static struct pool vndbuf_pool;
  185 
  186 #define getvndxfer(vnx) do {                                            \
  187         int s = splbio();                                               \
  188         vnx = pool_get(&vndxfer_pool, PR_WAITOK);                       \
  189         splx(s);                                                        \
  190 } while (/*CONSTCOND*/ 0)
  191 
  192 #define putvndxfer(vnx) {                                               \
  193         pool_put(&vndxfer_pool, (void *)(vnx));                         \
  194 }
  195 
  196 #define getvndbuf(vbp)  do {                                            \
  197         int s = splbio();                                               \
  198         vbp = pool_get(&vndbuf_pool, PR_WAITOK);                        \
  199         splx(s);                                                        \
  200 } while (/*CONSTCOND*/ 0)
  201 
  202 #define putvndbuf(vbp) {                                                \
  203         pool_put(&vndbuf_pool, (void *)(vbp));                          \
  204 }
  205 
  206 /*
  207  * local variables
  208  */
  209 static struct extent *swapmap;          /* controls the mapping of /dev/drum */
  210 
  211 MALLOC_DEFINE(M_VMSWAP, "VM swap", "VM swap structures");
  212 
  213 /* list of all active swap devices [by priority] */
  214 LIST_HEAD(swap_priority, swappri);
  215 static struct swap_priority swap_priority;
  216 
  217 /* locks */
  218 struct lock swap_syscall_lock;
  219 
  220 /*
  221  * prototypes
  222  */
  223 static struct swapdev   *swapdrum_getsdp(int);
  224 
  225 static struct swapdev   *swaplist_find(struct vnode *, int);
  226 static void              swaplist_insert(struct swapdev *,
  227                                          struct swappri *, int);
  228 static void              swaplist_trim(void);
  229 
  230 static int swap_on(struct proc *, struct swapdev *);
  231 static int swap_off(struct proc *, struct swapdev *);
  232 
  233 static void sw_reg_strategy(struct swapdev *, struct buf *, int);
  234 static void sw_reg_iodone(struct buf *);
  235 static void sw_reg_start(struct swapdev *);
  236 
  237 static int uvm_swap_io(struct vm_page **, int, int, int);
  238 
  239 dev_type_read(swread);
  240 dev_type_write(swwrite);
  241 dev_type_strategy(swstrategy);
  242 
  243 const struct bdevsw swap_bdevsw = {
  244         noopen, noclose, swstrategy, noioctl, nodump, nosize,
  245 };
  246 
  247 const struct cdevsw swap_cdevsw = {
  248         nullopen, nullclose, swread, swwrite, noioctl,
  249         nostop, notty, nopoll, nommap, nokqfilter
  250 };
  251 
  252 /*
  253  * uvm_swap_init: init the swap system data structures and locks
  254  *
  255  * => called at boot time from init_main.c after the filesystems
  256  *      are brought up (which happens after uvm_init())
  257  */
  258 void
  259 uvm_swap_init()
  260 {
  261         UVMHIST_FUNC("uvm_swap_init");
  262 
  263         UVMHIST_CALLED(pdhist);
  264         /*
  265          * first, init the swap list, its counter, and its lock.
  266          * then get a handle on the vnode for /dev/drum by using
  267          * the its dev_t number ("swapdev", from MD conf.c).
  268          */
  269 
  270         LIST_INIT(&swap_priority);
  271         uvmexp.nswapdev = 0;
  272         lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0);
  273         simple_lock_init(&uvm.swap_data_lock);
  274 
  275         if (bdevvp(swapdev, &swapdev_vp))
  276                 panic("uvm_swap_init: can't get vnode for swap device");
  277 
  278         /*
  279          * create swap block resource map to map /dev/drum.   the range
  280          * from 1 to INT_MAX allows 2 gigablocks of swap space.  note
  281          * that block 0 is reserved (used to indicate an allocation
  282          * failure, or no allocation).
  283          */
  284         swapmap = extent_create("swapmap", 1, INT_MAX,
  285                                 M_VMSWAP, 0, 0, EX_NOWAIT);
  286         if (swapmap == 0)
  287                 panic("uvm_swap_init: extent_create failed");
  288 
  289         /*
  290          * allocate pools for structures used for swapping to files.
  291          */
  292 
  293         pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0,
  294             "swp vnx", NULL);
  295 
  296         pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0,
  297             "swp vnd", NULL);
  298 
  299         /*
  300          * done!
  301          */
  302         UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0);
  303 }
  304 
  305 /*
  306  * swaplist functions: functions that operate on the list of swap
  307  * devices on the system.
  308  */
  309 
  310 /*
  311  * swaplist_insert: insert swap device "sdp" into the global list
  312  *
  313  * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
  314  * => caller must provide a newly malloc'd swappri structure (we will
  315  *      FREE it if we don't need it... this it to prevent malloc blocking
  316  *      here while adding swap)
  317  */
  318 static void
  319 swaplist_insert(sdp, newspp, priority)
  320         struct swapdev *sdp;
  321         struct swappri *newspp;
  322         int priority;
  323 {
  324         struct swappri *spp, *pspp;
  325         UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist);
  326 
  327         /*
  328          * find entry at or after which to insert the new device.
  329          */
  330         pspp = NULL;
  331         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  332                 if (priority <= spp->spi_priority)
  333                         break;
  334                 pspp = spp;
  335         }
  336 
  337         /*
  338          * new priority?
  339          */
  340         if (spp == NULL || spp->spi_priority != priority) {
  341                 spp = newspp;  /* use newspp! */
  342                 UVMHIST_LOG(pdhist, "created new swappri = %d",
  343                             priority, 0, 0, 0);
  344 
  345                 spp->spi_priority = priority;
  346                 CIRCLEQ_INIT(&spp->spi_swapdev);
  347 
  348                 if (pspp)
  349                         LIST_INSERT_AFTER(pspp, spp, spi_swappri);
  350                 else
  351                         LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri);
  352         } else {
  353                 /* we don't need a new priority structure, free it */
  354                 FREE(newspp, M_VMSWAP);
  355         }
  356 
  357         /*
  358          * priority found (or created).   now insert on the priority's
  359          * circleq list and bump the total number of swapdevs.
  360          */
  361         sdp->swd_priority = priority;
  362         CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
  363         uvmexp.nswapdev++;
  364 }
  365 
  366 /*
  367  * swaplist_find: find and optionally remove a swap device from the
  368  *      global list.
  369  *
  370  * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
  371  * => we return the swapdev we found (and removed)
  372  */
  373 static struct swapdev *
  374 swaplist_find(vp, remove)
  375         struct vnode *vp;
  376         boolean_t remove;
  377 {
  378         struct swapdev *sdp;
  379         struct swappri *spp;
  380 
  381         /*
  382          * search the lists for the requested vp
  383          */
  384 
  385         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  386                 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
  387                         if (sdp->swd_vp == vp) {
  388                                 if (remove) {
  389                                         CIRCLEQ_REMOVE(&spp->spi_swapdev,
  390                                             sdp, swd_next);
  391                                         uvmexp.nswapdev--;
  392                                 }
  393                                 return(sdp);
  394                         }
  395                 }
  396         }
  397         return (NULL);
  398 }
  399 
  400 
  401 /*
  402  * swaplist_trim: scan priority list for empty priority entries and kill
  403  *      them.
  404  *
  405  * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
  406  */
  407 static void
  408 swaplist_trim()
  409 {
  410         struct swappri *spp, *nextspp;
  411 
  412         for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) {
  413                 nextspp = LIST_NEXT(spp, spi_swappri);
  414                 if (CIRCLEQ_FIRST(&spp->spi_swapdev) !=
  415                     (void *)&spp->spi_swapdev)
  416                         continue;
  417                 LIST_REMOVE(spp, spi_swappri);
  418                 free(spp, M_VMSWAP);
  419         }
  420 }
  421 
  422 /*
  423  * swapdrum_getsdp: given a page offset in /dev/drum, convert it back
  424  *      to the "swapdev" that maps that section of the drum.
  425  *
  426  * => each swapdev takes one big contig chunk of the drum
  427  * => caller must hold uvm.swap_data_lock
  428  */
  429 static struct swapdev *
  430 swapdrum_getsdp(pgno)
  431         int pgno;
  432 {
  433         struct swapdev *sdp;
  434         struct swappri *spp;
  435 
  436         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  437                 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
  438                         if (sdp->swd_flags & SWF_FAKE)
  439                                 continue;
  440                         if (pgno >= sdp->swd_drumoffset &&
  441                             pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) {
  442                                 return sdp;
  443                         }
  444                 }
  445         }
  446         return NULL;
  447 }
  448 
  449 
  450 /*
  451  * sys_swapctl: main entry point for swapctl(2) system call
  452  *      [with two helper functions: swap_on and swap_off]
  453  */
  454 int
  455 sys_swapctl(l, v, retval)
  456         struct lwp *l;
  457         void *v;
  458         register_t *retval;
  459 {
  460         struct sys_swapctl_args /* {
  461                 syscallarg(int) cmd;
  462                 syscallarg(void *) arg;
  463                 syscallarg(int) misc;
  464         } */ *uap = (struct sys_swapctl_args *)v;
  465         struct proc *p = l->l_proc;
  466         struct vnode *vp;
  467         struct nameidata nd;
  468         struct swappri *spp;
  469         struct swapdev *sdp;
  470         struct swapent *sep;
  471         char    userpath[PATH_MAX + 1];
  472         size_t  len;
  473         int     error, misc;
  474         int     priority;
  475         UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist);
  476 
  477         misc = SCARG(uap, misc);
  478 
  479         /*
  480          * ensure serialized syscall access by grabbing the swap_syscall_lock
  481          */
  482         lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL);
  483 
  484         /*
  485          * we handle the non-priv NSWAP and STATS request first.
  486          *
  487          * SWAP_NSWAP: return number of config'd swap devices
  488          * [can also be obtained with uvmexp sysctl]
  489          */
  490         if (SCARG(uap, cmd) == SWAP_NSWAP) {
  491                 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev,
  492                     0, 0, 0);
  493                 *retval = uvmexp.nswapdev;
  494                 error = 0;
  495                 goto out;
  496         }
  497 
  498         /*
  499          * SWAP_STATS: get stats on current # of configured swap devs
  500          *
  501          * note that the swap_priority list can't change as long
  502          * as we are holding the swap_syscall_lock.  we don't want
  503          * to grab the uvm.swap_data_lock because we may fault&sleep during
  504          * copyout() and we don't want to be holding that lock then!
  505          */
  506         if (SCARG(uap, cmd) == SWAP_STATS
  507 #if defined(COMPAT_13)
  508             || SCARG(uap, cmd) == SWAP_OSTATS
  509 #endif
  510             ) {
  511                 if ((size_t)misc > (size_t)uvmexp.nswapdev)
  512                         misc = uvmexp.nswapdev;
  513 #if defined(COMPAT_13)
  514                 if (SCARG(uap, cmd) == SWAP_OSTATS)
  515                         len = sizeof(struct oswapent) * misc;
  516                 else
  517 #endif
  518                         len = sizeof(struct swapent) * misc;
  519                 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK);
  520 
  521                 uvm_swap_stats(SCARG(uap, cmd), sep, misc, retval);
  522                 error = copyout(sep, (void *)SCARG(uap, arg), len);
  523 
  524                 free(sep, M_TEMP);
  525                 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0);
  526                 goto out;
  527         }
  528         if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) {
  529                 dev_t   *devp = (dev_t *)SCARG(uap, arg);
  530 
  531                 error = copyout(&dumpdev, devp, sizeof(dumpdev));
  532                 goto out;
  533         }
  534 
  535         /*
  536          * all other requests require superuser privs.   verify.
  537          */
  538         if ((error = suser(p->p_ucred, &p->p_acflag)))
  539                 goto out;
  540 
  541         /*
  542          * at this point we expect a path name in arg.   we will
  543          * use namei() to gain a vnode reference (vref), and lock
  544          * the vnode (VOP_LOCK).
  545          *
  546          * XXX: a NULL arg means use the root vnode pointer (e.g. for
  547          * miniroot)
  548          */
  549         if (SCARG(uap, arg) == NULL) {
  550                 vp = rootvp;            /* miniroot */
  551                 if (vget(vp, LK_EXCLUSIVE)) {
  552                         error = EBUSY;
  553                         goto out;
  554                 }
  555                 if (SCARG(uap, cmd) == SWAP_ON &&
  556                     copystr("miniroot", userpath, sizeof userpath, &len))
  557                         panic("swapctl: miniroot copy failed");
  558         } else {
  559                 int     space;
  560                 char    *where;
  561 
  562                 if (SCARG(uap, cmd) == SWAP_ON) {
  563                         if ((error = copyinstr(SCARG(uap, arg), userpath,
  564                             sizeof userpath, &len)))
  565                                 goto out;
  566                         space = UIO_SYSSPACE;
  567                         where = userpath;
  568                 } else {
  569                         space = UIO_USERSPACE;
  570                         where = (char *)SCARG(uap, arg);
  571                 }
  572                 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p);
  573                 if ((error = namei(&nd)))
  574                         goto out;
  575                 vp = nd.ni_vp;
  576         }
  577         /* note: "vp" is referenced and locked */
  578 
  579         error = 0;              /* assume no error */
  580         switch(SCARG(uap, cmd)) {
  581 
  582         case SWAP_DUMPDEV:
  583                 if (vp->v_type != VBLK) {
  584                         error = ENOTBLK;
  585                         break;
  586                 }
  587                 dumpdev = vp->v_rdev;
  588                 cpu_dumpconf();
  589                 break;
  590 
  591         case SWAP_CTL:
  592                 /*
  593                  * get new priority, remove old entry (if any) and then
  594                  * reinsert it in the correct place.  finally, prune out
  595                  * any empty priority structures.
  596                  */
  597                 priority = SCARG(uap, misc);
  598                 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
  599                 simple_lock(&uvm.swap_data_lock);
  600                 if ((sdp = swaplist_find(vp, 1)) == NULL) {
  601                         error = ENOENT;
  602                 } else {
  603                         swaplist_insert(sdp, spp, priority);
  604                         swaplist_trim();
  605                 }
  606                 simple_unlock(&uvm.swap_data_lock);
  607                 if (error)
  608                         free(spp, M_VMSWAP);
  609                 break;
  610 
  611         case SWAP_ON:
  612 
  613                 /*
  614                  * check for duplicates.   if none found, then insert a
  615                  * dummy entry on the list to prevent someone else from
  616                  * trying to enable this device while we are working on
  617                  * it.
  618                  */
  619 
  620                 priority = SCARG(uap, misc);
  621                 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK);
  622                 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
  623                 memset(sdp, 0, sizeof(*sdp));
  624                 sdp->swd_flags = SWF_FAKE;
  625                 sdp->swd_vp = vp;
  626                 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV;
  627                 bufq_alloc(&sdp->swd_tab, BUFQ_DISKSORT|BUFQ_SORT_RAWBLOCK);
  628                 simple_lock(&uvm.swap_data_lock);
  629                 if (swaplist_find(vp, 0) != NULL) {
  630                         error = EBUSY;
  631                         simple_unlock(&uvm.swap_data_lock);
  632                         bufq_free(&sdp->swd_tab);
  633                         free(sdp, M_VMSWAP);
  634                         free(spp, M_VMSWAP);
  635                         break;
  636                 }
  637                 swaplist_insert(sdp, spp, priority);
  638                 simple_unlock(&uvm.swap_data_lock);
  639 
  640                 sdp->swd_pathlen = len;
  641                 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK);
  642                 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0)
  643                         panic("swapctl: copystr");
  644 
  645                 /*
  646                  * we've now got a FAKE placeholder in the swap list.
  647                  * now attempt to enable swap on it.  if we fail, undo
  648                  * what we've done and kill the fake entry we just inserted.
  649                  * if swap_on is a success, it will clear the SWF_FAKE flag
  650                  */
  651 
  652                 if ((error = swap_on(p, sdp)) != 0) {
  653                         simple_lock(&uvm.swap_data_lock);
  654                         (void) swaplist_find(vp, 1);  /* kill fake entry */
  655                         swaplist_trim();
  656                         simple_unlock(&uvm.swap_data_lock);
  657                         bufq_free(&sdp->swd_tab);
  658                         free(sdp->swd_path, M_VMSWAP);
  659                         free(sdp, M_VMSWAP);
  660                         break;
  661                 }
  662                 break;
  663 
  664         case SWAP_OFF:
  665                 simple_lock(&uvm.swap_data_lock);
  666                 if ((sdp = swaplist_find(vp, 0)) == NULL) {
  667                         simple_unlock(&uvm.swap_data_lock);
  668                         error = ENXIO;
  669                         break;
  670                 }
  671 
  672                 /*
  673                  * If a device isn't in use or enabled, we
  674                  * can't stop swapping from it (again).
  675                  */
  676                 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) {
  677                         simple_unlock(&uvm.swap_data_lock);
  678                         error = EBUSY;
  679                         break;
  680                 }
  681 
  682                 /*
  683                  * do the real work.
  684                  */
  685                 error = swap_off(p, sdp);
  686                 break;
  687 
  688         default:
  689                 error = EINVAL;
  690         }
  691 
  692         /*
  693          * done!  release the ref gained by namei() and unlock.
  694          */
  695         vput(vp);
  696 
  697 out:
  698         lockmgr(&swap_syscall_lock, LK_RELEASE, NULL);
  699 
  700         UVMHIST_LOG(pdhist, "<- done!  error=%d", error, 0, 0, 0);
  701         return (error);
  702 }
  703 
  704 /*
  705  * swap_stats: implements swapctl(SWAP_STATS). The function is kept
  706  * away from sys_swapctl() in order to allow COMPAT_* swapctl()
  707  * emulation to use it directly without going through sys_swapctl().
  708  * The problem with using sys_swapctl() there is that it involves
  709  * copying the swapent array to the stackgap, and this array's size
  710  * is not known at build time. Hence it would not be possible to
  711  * ensure it would fit in the stackgap in any case.
  712  */
  713 void
  714 uvm_swap_stats(cmd, sep, sec, retval)
  715         int cmd;
  716         struct swapent *sep;
  717         int sec;
  718         register_t *retval;
  719 {
  720         struct swappri *spp;
  721         struct swapdev *sdp;
  722         int count = 0;
  723 
  724         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  725                 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev);
  726                      sdp != (void *)&spp->spi_swapdev && sec-- > 0;
  727                      sdp = CIRCLEQ_NEXT(sdp, swd_next)) {
  728                         /*
  729                          * backwards compatibility for system call.
  730                          * note that we use 'struct oswapent' as an
  731                          * overlay into both 'struct swapdev' and
  732                          * the userland 'struct swapent', as we
  733                          * want to retain backwards compatibility
  734                          * with NetBSD 1.3.
  735                          */
  736                         sdp->swd_ose.ose_inuse =
  737                             btodb((u_int64_t)sdp->swd_npginuse <<
  738                             PAGE_SHIFT);
  739                         (void)memcpy(sep, &sdp->swd_ose,
  740                             sizeof(struct oswapent));
  741 
  742                         /* now copy out the path if necessary */
  743 #if defined(COMPAT_13)
  744                         if (cmd == SWAP_STATS)
  745 #endif
  746                                 (void)memcpy(&sep->se_path, sdp->swd_path,
  747                                     sdp->swd_pathlen);
  748 
  749                         count++;
  750 #if defined(COMPAT_13)
  751                         if (cmd == SWAP_OSTATS)
  752                                 sep = (struct swapent *)
  753                                     ((struct oswapent *)sep + 1);
  754                         else
  755 #endif
  756                                 sep++;
  757                 }
  758         }
  759 
  760         *retval = count;
  761         return;
  762 }
  763 
  764 /*
  765  * swap_on: attempt to enable a swapdev for swapping.   note that the
  766  *      swapdev is already on the global list, but disabled (marked
  767  *      SWF_FAKE).
  768  *
  769  * => we avoid the start of the disk (to protect disk labels)
  770  * => we also avoid the miniroot, if we are swapping to root.
  771  * => caller should leave uvm.swap_data_lock unlocked, we may lock it
  772  *      if needed.
  773  */
  774 static int
  775 swap_on(p, sdp)
  776         struct proc *p;
  777         struct swapdev *sdp;
  778 {
  779         static int count = 0;   /* static */
  780         struct vnode *vp;
  781         int error, npages, nblocks, size;
  782         long addr;
  783         u_long result;
  784         struct vattr va;
  785 #ifdef NFS
  786         extern int (**nfsv2_vnodeop_p)(void *);
  787 #endif /* NFS */
  788         const struct bdevsw *bdev;
  789         dev_t dev;
  790         UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist);
  791 
  792         /*
  793          * we want to enable swapping on sdp.   the swd_vp contains
  794          * the vnode we want (locked and ref'd), and the swd_dev
  795          * contains the dev_t of the file, if it a block device.
  796          */
  797 
  798         vp = sdp->swd_vp;
  799         dev = sdp->swd_dev;
  800 
  801         /*
  802          * open the swap file (mostly useful for block device files to
  803          * let device driver know what is up).
  804          *
  805          * we skip the open/close for root on swap because the root
  806          * has already been opened when root was mounted (mountroot).
  807          */
  808         if (vp != rootvp) {
  809                 if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p)))
  810                         return (error);
  811         }
  812 
  813         /* XXX this only works for block devices */
  814         UVMHIST_LOG(pdhist, "  dev=%d, major(dev)=%d", dev, major(dev), 0,0);
  815 
  816         /*
  817          * we now need to determine the size of the swap area.   for
  818          * block specials we can call the d_psize function.
  819          * for normal files, we must stat [get attrs].
  820          *
  821          * we put the result in nblks.
  822          * for normal files, we also want the filesystem block size
  823          * (which we get with statfs).
  824          */
  825         switch (vp->v_type) {
  826         case VBLK:
  827                 bdev = bdevsw_lookup(dev);
  828                 if (bdev == NULL || bdev->d_psize == NULL ||
  829                     (nblocks = (*bdev->d_psize)(dev)) == -1) {
  830                         error = ENXIO;
  831                         goto bad;
  832                 }
  833                 break;
  834 
  835         case VREG:
  836                 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p)))
  837                         goto bad;
  838                 nblocks = (int)btodb(va.va_size);
  839                 if ((error =
  840                      VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0)
  841                         goto bad;
  842 
  843                 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize;
  844                 /*
  845                  * limit the max # of outstanding I/O requests we issue
  846                  * at any one time.   take it easy on NFS servers.
  847                  */
  848 #ifdef NFS
  849                 if (vp->v_op == nfsv2_vnodeop_p)
  850                         sdp->swd_maxactive = 2; /* XXX */
  851                 else
  852 #endif /* NFS */
  853                         sdp->swd_maxactive = 8; /* XXX */
  854                 break;
  855 
  856         default:
  857                 error = ENXIO;
  858                 goto bad;
  859         }
  860 
  861         /*
  862          * save nblocks in a safe place and convert to pages.
  863          */
  864 
  865         sdp->swd_ose.ose_nblks = nblocks;
  866         npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT;
  867 
  868         /*
  869          * for block special files, we want to make sure that leave
  870          * the disklabel and bootblocks alone, so we arrange to skip
  871          * over them (arbitrarily choosing to skip PAGE_SIZE bytes).
  872          * note that because of this the "size" can be less than the
  873          * actual number of blocks on the device.
  874          */
  875         if (vp->v_type == VBLK) {
  876                 /* we use pages 1 to (size - 1) [inclusive] */
  877                 size = npages - 1;
  878                 addr = 1;
  879         } else {
  880                 /* we use pages 0 to (size - 1) [inclusive] */
  881                 size = npages;
  882                 addr = 0;
  883         }
  884 
  885         /*
  886          * make sure we have enough blocks for a reasonable sized swap
  887          * area.   we want at least one page.
  888          */
  889 
  890         if (size < 1) {
  891                 UVMHIST_LOG(pdhist, "  size <= 1!!", 0, 0, 0, 0);
  892                 error = EINVAL;
  893                 goto bad;
  894         }
  895 
  896         UVMHIST_LOG(pdhist, "  dev=%x: size=%d addr=%ld\n", dev, size, addr, 0);
  897 
  898         /*
  899          * now we need to allocate an extent to manage this swap device
  900          */
  901         snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x",
  902             count++);
  903 
  904         /* note that extent_create's 3rd arg is inclusive, thus "- 1" */
  905         sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP,
  906                                     0, 0, EX_WAITOK);
  907         /* allocate the `saved' region from the extent so it won't be used */
  908         if (addr) {
  909                 if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK))
  910                         panic("disklabel region");
  911         }
  912 
  913         /*
  914          * if the vnode we are swapping to is the root vnode
  915          * (i.e. we are swapping to the miniroot) then we want
  916          * to make sure we don't overwrite it.   do a statfs to
  917          * find its size and skip over it.
  918          */
  919         if (vp == rootvp) {
  920                 struct mount *mp;
  921                 struct statfs *sp;
  922                 int rootblocks, rootpages;
  923 
  924                 mp = rootvnode->v_mount;
  925                 sp = &mp->mnt_stat;
  926                 rootblocks = sp->f_blocks * btodb(sp->f_bsize);
  927                 /*
  928                  * XXX: sp->f_blocks isn't the total number of
  929                  * blocks in the filesystem, it's the number of
  930                  * data blocks.  so, our rootblocks almost
  931                  * definitely underestimates the total size
  932                  * of the filesystem - how badly depends on the
  933                  * details of the filesystem type.  there isn't
  934                  * an obvious way to deal with this cleanly
  935                  * and perfectly, so for now we just pad our
  936                  * rootblocks estimate with an extra 5 percent.
  937                  */
  938                 rootblocks += (rootblocks >> 5) +
  939                         (rootblocks >> 6) +
  940                         (rootblocks >> 7);
  941                 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT;
  942                 if (rootpages > size)
  943                         panic("swap_on: miniroot larger than swap?");
  944 
  945                 if (extent_alloc_region(sdp->swd_ex, addr,
  946                                         rootpages, EX_WAITOK))
  947                         panic("swap_on: unable to preserve miniroot");
  948 
  949                 size -= rootpages;
  950                 printf("Preserved %d pages of miniroot ", rootpages);
  951                 printf("leaving %d pages of swap\n", size);
  952         }
  953 
  954         /*
  955          * try to add anons to reflect the new swap space.
  956          */
  957 
  958         error = uvm_anon_add(size);
  959         if (error) {
  960                 goto bad;
  961         }
  962 
  963         /*
  964          * add a ref to vp to reflect usage as a swap device.
  965          */
  966         vref(vp);
  967 
  968         /*
  969          * now add the new swapdev to the drum and enable.
  970          */
  971         if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY,
  972             EX_WAITOK, &result))
  973                 panic("swapdrum_add");
  974 
  975         sdp->swd_drumoffset = (int)result;
  976         sdp->swd_drumsize = npages;
  977         sdp->swd_npages = size;
  978         simple_lock(&uvm.swap_data_lock);
  979         sdp->swd_flags &= ~SWF_FAKE;    /* going live */
  980         sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE);
  981         uvmexp.swpages += size;
  982         uvmexp.swpgavail += size;
  983         simple_unlock(&uvm.swap_data_lock);
  984         return (0);
  985 
  986         /*
  987          * failure: clean up and return error.
  988          */
  989 
  990 bad:
  991         if (sdp->swd_ex) {
  992                 extent_destroy(sdp->swd_ex);
  993         }
  994         if (vp != rootvp) {
  995                 (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p);
  996         }
  997         return (error);
  998 }
  999 
 1000 /*
 1001  * swap_off: stop swapping on swapdev
 1002  *
 1003  * => swap data should be locked, we will unlock.
 1004  */
 1005 static int
 1006 swap_off(p, sdp)
 1007         struct proc *p;
 1008         struct swapdev *sdp;
 1009 {
 1010         int npages =  sdp->swd_npages;
 1011 
 1012         UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist);
 1013         UVMHIST_LOG(pdhist, "  dev=%x, npages=%d", sdp->swd_dev,npages,0,0);
 1014 
 1015         /* disable the swap area being removed */
 1016         sdp->swd_flags &= ~SWF_ENABLE;
 1017         uvmexp.swpgavail -= npages;
 1018         simple_unlock(&uvm.swap_data_lock);
 1019 
 1020         /*
 1021          * the idea is to find all the pages that are paged out to this
 1022          * device, and page them all in.  in uvm, swap-backed pageable
 1023          * memory can take two forms: aobjs and anons.  call the
 1024          * swapoff hook for each subsystem to bring in pages.
 1025          */
 1026 
 1027         if (uao_swap_off(sdp->swd_drumoffset,
 1028                          sdp->swd_drumoffset + sdp->swd_drumsize) ||
 1029             anon_swap_off(sdp->swd_drumoffset,
 1030                           sdp->swd_drumoffset + sdp->swd_drumsize)) {
 1031 
 1032                 simple_lock(&uvm.swap_data_lock);
 1033                 sdp->swd_flags |= SWF_ENABLE;
 1034                 uvmexp.swpgavail += npages;
 1035                 simple_unlock(&uvm.swap_data_lock);
 1036                 return ENOMEM;
 1037         }
 1038         KASSERT(sdp->swd_npginuse == sdp->swd_npgbad);
 1039 
 1040         /*
 1041          * done with the vnode.
 1042          * drop our ref on the vnode before calling VOP_CLOSE()
 1043          * so that spec_close() can tell if this is the last close.
 1044          */
 1045         vrele(sdp->swd_vp);
 1046         if (sdp->swd_vp != rootvp) {
 1047                 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p);
 1048         }
 1049 
 1050         /* remove anons from the system */
 1051         uvm_anon_remove(npages);
 1052 
 1053         simple_lock(&uvm.swap_data_lock);
 1054         uvmexp.swpages -= npages;
 1055         uvmexp.swpginuse -= sdp->swd_npgbad;
 1056 
 1057         if (swaplist_find(sdp->swd_vp, 1) == NULL)
 1058                 panic("swap_off: swapdev not in list");
 1059         swaplist_trim();
 1060         simple_unlock(&uvm.swap_data_lock);
 1061 
 1062         /*
 1063          * free all resources!
 1064          */
 1065         extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize,
 1066                     EX_WAITOK);
 1067         extent_destroy(sdp->swd_ex);
 1068         bufq_free(&sdp->swd_tab);
 1069         free(sdp, M_VMSWAP);
 1070         return (0);
 1071 }
 1072 
 1073 /*
 1074  * /dev/drum interface and i/o functions
 1075  */
 1076 
 1077 /*
 1078  * swread: the read function for the drum (just a call to physio)
 1079  */
 1080 /*ARGSUSED*/
 1081 int
 1082 swread(dev, uio, ioflag)
 1083         dev_t dev;
 1084         struct uio *uio;
 1085         int ioflag;
 1086 {
 1087         UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist);
 1088 
 1089         UVMHIST_LOG(pdhist, "  dev=%x offset=%qx", dev, uio->uio_offset, 0, 0);
 1090         return (physio(swstrategy, NULL, dev, B_READ, minphys, uio));
 1091 }
 1092 
 1093 /*
 1094  * swwrite: the write function for the drum (just a call to physio)
 1095  */
 1096 /*ARGSUSED*/
 1097 int
 1098 swwrite(dev, uio, ioflag)
 1099         dev_t dev;
 1100         struct uio *uio;
 1101         int ioflag;
 1102 {
 1103         UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist);
 1104 
 1105         UVMHIST_LOG(pdhist, "  dev=%x offset=%qx", dev, uio->uio_offset, 0, 0);
 1106         return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio));
 1107 }
 1108 
 1109 /*
 1110  * swstrategy: perform I/O on the drum
 1111  *
 1112  * => we must map the i/o request from the drum to the correct swapdev.
 1113  */
 1114 void
 1115 swstrategy(bp)
 1116         struct buf *bp;
 1117 {
 1118         struct swapdev *sdp;
 1119         struct vnode *vp;
 1120         int s, pageno, bn;
 1121         UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist);
 1122 
 1123         /*
 1124          * convert block number to swapdev.   note that swapdev can't
 1125          * be yanked out from under us because we are holding resources
 1126          * in it (i.e. the blocks we are doing I/O on).
 1127          */
 1128         pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT;
 1129         simple_lock(&uvm.swap_data_lock);
 1130         sdp = swapdrum_getsdp(pageno);
 1131         simple_unlock(&uvm.swap_data_lock);
 1132         if (sdp == NULL) {
 1133                 bp->b_error = EINVAL;
 1134                 bp->b_flags |= B_ERROR;
 1135                 biodone(bp);
 1136                 UVMHIST_LOG(pdhist, "  failed to get swap device", 0, 0, 0, 0);
 1137                 return;
 1138         }
 1139 
 1140         /*
 1141          * convert drum page number to block number on this swapdev.
 1142          */
 1143 
 1144         pageno -= sdp->swd_drumoffset;  /* page # on swapdev */
 1145         bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */
 1146 
 1147         UVMHIST_LOG(pdhist, "  %s: mapoff=%x bn=%x bcount=%ld",
 1148                 ((bp->b_flags & B_READ) == 0) ? "write" : "read",
 1149                 sdp->swd_drumoffset, bn, bp->b_bcount);
 1150 
 1151         /*
 1152          * for block devices we finish up here.
 1153          * for regular files we have to do more work which we delegate
 1154          * to sw_reg_strategy().
 1155          */
 1156 
 1157         switch (sdp->swd_vp->v_type) {
 1158         default:
 1159                 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type);
 1160 
 1161         case VBLK:
 1162 
 1163                 /*
 1164                  * must convert "bp" from an I/O on /dev/drum to an I/O
 1165                  * on the swapdev (sdp).
 1166                  */
 1167                 s = splbio();
 1168                 bp->b_blkno = bn;               /* swapdev block number */
 1169                 vp = sdp->swd_vp;               /* swapdev vnode pointer */
 1170                 bp->b_dev = sdp->swd_dev;       /* swapdev dev_t */
 1171 
 1172                 /*
 1173                  * if we are doing a write, we have to redirect the i/o on
 1174                  * drum's v_numoutput counter to the swapdevs.
 1175                  */
 1176                 if ((bp->b_flags & B_READ) == 0) {
 1177                         vwakeup(bp);    /* kills one 'v_numoutput' on drum */
 1178                         V_INCR_NUMOUTPUT(vp);   /* put it on swapdev */
 1179                 }
 1180 
 1181                 /*
 1182                  * finally plug in swapdev vnode and start I/O
 1183                  */
 1184                 bp->b_vp = vp;
 1185                 splx(s);
 1186                 VOP_STRATEGY(vp, bp);
 1187                 return;
 1188 
 1189         case VREG:
 1190                 /*
 1191                  * delegate to sw_reg_strategy function.
 1192                  */
 1193                 sw_reg_strategy(sdp, bp, bn);
 1194                 return;
 1195         }
 1196         /* NOTREACHED */
 1197 }
 1198 
 1199 /*
 1200  * sw_reg_strategy: handle swap i/o to regular files
 1201  */
 1202 static void
 1203 sw_reg_strategy(sdp, bp, bn)
 1204         struct swapdev  *sdp;
 1205         struct buf      *bp;
 1206         int             bn;
 1207 {
 1208         struct vnode    *vp;
 1209         struct vndxfer  *vnx;
 1210         daddr_t         nbn;
 1211         caddr_t         addr;
 1212         off_t           byteoff;
 1213         int             s, off, nra, error, sz, resid;
 1214         UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist);
 1215 
 1216         /*
 1217          * allocate a vndxfer head for this transfer and point it to
 1218          * our buffer.
 1219          */
 1220         getvndxfer(vnx);
 1221         vnx->vx_flags = VX_BUSY;
 1222         vnx->vx_error = 0;
 1223         vnx->vx_pending = 0;
 1224         vnx->vx_bp = bp;
 1225         vnx->vx_sdp = sdp;
 1226 
 1227         /*
 1228          * setup for main loop where we read filesystem blocks into
 1229          * our buffer.
 1230          */
 1231         error = 0;
 1232         bp->b_resid = bp->b_bcount;     /* nothing transfered yet! */
 1233         addr = bp->b_data;              /* current position in buffer */
 1234         byteoff = dbtob((u_int64_t)bn);
 1235 
 1236         for (resid = bp->b_resid; resid; resid -= sz) {
 1237                 struct vndbuf   *nbp;
 1238 
 1239                 /*
 1240                  * translate byteoffset into block number.  return values:
 1241                  *   vp = vnode of underlying device
 1242                  *  nbn = new block number (on underlying vnode dev)
 1243                  *  nra = num blocks we can read-ahead (excludes requested
 1244                  *      block)
 1245                  */
 1246                 nra = 0;
 1247                 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize,
 1248                                         &vp, &nbn, &nra);
 1249 
 1250                 if (error == 0 && nbn == (daddr_t)-1) {
 1251                         /*
 1252                          * this used to just set error, but that doesn't
 1253                          * do the right thing.  Instead, it causes random
 1254                          * memory errors.  The panic() should remain until
 1255                          * this condition doesn't destabilize the system.
 1256                          */
 1257 #if 1
 1258                         panic("sw_reg_strategy: swap to sparse file");
 1259 #else
 1260                         error = EIO;    /* failure */
 1261 #endif
 1262                 }
 1263 
 1264                 /*
 1265                  * punt if there was an error or a hole in the file.
 1266                  * we must wait for any i/o ops we have already started
 1267                  * to finish before returning.
 1268                  *
 1269                  * XXX we could deal with holes here but it would be
 1270                  * a hassle (in the write case).
 1271                  */
 1272                 if (error) {
 1273                         s = splbio();
 1274                         vnx->vx_error = error;  /* pass error up */
 1275                         goto out;
 1276                 }
 1277 
 1278                 /*
 1279                  * compute the size ("sz") of this transfer (in bytes).
 1280                  */
 1281                 off = byteoff % sdp->swd_bsize;
 1282                 sz = (1 + nra) * sdp->swd_bsize - off;
 1283                 if (sz > resid)
 1284                         sz = resid;
 1285 
 1286                 UVMHIST_LOG(pdhist, "sw_reg_strategy: "
 1287                             "vp %p/%p offset 0x%x/0x%x",
 1288                             sdp->swd_vp, vp, byteoff, nbn);
 1289 
 1290                 /*
 1291                  * now get a buf structure.   note that the vb_buf is
 1292                  * at the front of the nbp structure so that you can
 1293                  * cast pointers between the two structure easily.
 1294                  */
 1295                 getvndbuf(nbp);
 1296                 BUF_INIT(&nbp->vb_buf);
 1297                 nbp->vb_buf.b_flags    = bp->b_flags | B_CALL;
 1298                 nbp->vb_buf.b_bcount   = sz;
 1299                 nbp->vb_buf.b_bufsize  = sz;
 1300                 nbp->vb_buf.b_error    = 0;
 1301                 nbp->vb_buf.b_data     = addr;
 1302                 nbp->vb_buf.b_lblkno   = 0;
 1303                 nbp->vb_buf.b_blkno    = nbn + btodb(off);
 1304                 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno;
 1305                 nbp->vb_buf.b_iodone   = sw_reg_iodone;
 1306                 nbp->vb_buf.b_vp       = vp;
 1307                 if (vp->v_type == VBLK) {
 1308                         nbp->vb_buf.b_dev = vp->v_rdev;
 1309                 }
 1310 
 1311                 nbp->vb_xfer = vnx;     /* patch it back in to vnx */
 1312 
 1313                 /*
 1314                  * Just sort by block number
 1315                  */
 1316                 s = splbio();
 1317                 if (vnx->vx_error != 0) {
 1318                         putvndbuf(nbp);
 1319                         goto out;
 1320                 }
 1321                 vnx->vx_pending++;
 1322 
 1323                 /* sort it in and start I/O if we are not over our limit */
 1324                 BUFQ_PUT(&sdp->swd_tab, &nbp->vb_buf);
 1325                 sw_reg_start(sdp);
 1326                 splx(s);
 1327 
 1328                 /*
 1329                  * advance to the next I/O
 1330                  */
 1331                 byteoff += sz;
 1332                 addr += sz;
 1333         }
 1334 
 1335         s = splbio();
 1336 
 1337 out: /* Arrive here at splbio */
 1338         vnx->vx_flags &= ~VX_BUSY;
 1339         if (vnx->vx_pending == 0) {
 1340                 if (vnx->vx_error != 0) {
 1341                         bp->b_error = vnx->vx_error;
 1342                         bp->b_flags |= B_ERROR;
 1343                 }
 1344                 putvndxfer(vnx);
 1345                 biodone(bp);
 1346         }
 1347         splx(s);
 1348 }
 1349 
 1350 /*
 1351  * sw_reg_start: start an I/O request on the requested swapdev
 1352  *
 1353  * => reqs are sorted by b_rawblkno (above)
 1354  */
 1355 static void
 1356 sw_reg_start(sdp)
 1357         struct swapdev  *sdp;
 1358 {
 1359         struct buf      *bp;
 1360         UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist);
 1361 
 1362         /* recursion control */
 1363         if ((sdp->swd_flags & SWF_BUSY) != 0)
 1364                 return;
 1365 
 1366         sdp->swd_flags |= SWF_BUSY;
 1367 
 1368         while (sdp->swd_active < sdp->swd_maxactive) {
 1369                 bp = BUFQ_GET(&sdp->swd_tab);
 1370                 if (bp == NULL)
 1371                         break;
 1372                 sdp->swd_active++;
 1373 
 1374                 UVMHIST_LOG(pdhist,
 1375                     "sw_reg_start:  bp %p vp %p blkno %p cnt %lx",
 1376                     bp, bp->b_vp, bp->b_blkno, bp->b_bcount);
 1377                 if ((bp->b_flags & B_READ) == 0)
 1378                         V_INCR_NUMOUTPUT(bp->b_vp);
 1379 
 1380                 VOP_STRATEGY(bp->b_vp, bp);
 1381         }
 1382         sdp->swd_flags &= ~SWF_BUSY;
 1383 }
 1384 
 1385 /*
 1386  * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup
 1387  *
 1388  * => note that we can recover the vndbuf struct by casting the buf ptr
 1389  */
 1390 static void
 1391 sw_reg_iodone(bp)
 1392         struct buf *bp;
 1393 {
 1394         struct vndbuf *vbp = (struct vndbuf *) bp;
 1395         struct vndxfer *vnx = vbp->vb_xfer;
 1396         struct buf *pbp = vnx->vx_bp;           /* parent buffer */
 1397         struct swapdev  *sdp = vnx->vx_sdp;
 1398         int s, resid, error;
 1399         UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist);
 1400 
 1401         UVMHIST_LOG(pdhist, "  vbp=%p vp=%p blkno=%x addr=%p",
 1402             vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data);
 1403         UVMHIST_LOG(pdhist, "  cnt=%lx resid=%lx",
 1404             vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0);
 1405 
 1406         /*
 1407          * protect vbp at splbio and update.
 1408          */
 1409 
 1410         s = splbio();
 1411         resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid;
 1412         pbp->b_resid -= resid;
 1413         vnx->vx_pending--;
 1414 
 1415         if (vbp->vb_buf.b_flags & B_ERROR) {
 1416                 /* pass error upward */
 1417                 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO;
 1418                 UVMHIST_LOG(pdhist, "  got error=%d !", error, 0, 0, 0);
 1419                 vnx->vx_error = error;
 1420         }
 1421 
 1422         /*
 1423          * kill vbp structure
 1424          */
 1425         putvndbuf(vbp);
 1426 
 1427         /*
 1428          * wrap up this transaction if it has run to completion or, in
 1429          * case of an error, when all auxiliary buffers have returned.
 1430          */
 1431         if (vnx->vx_error != 0) {
 1432                 /* pass error upward */
 1433                 pbp->b_flags |= B_ERROR;
 1434                 pbp->b_error = vnx->vx_error;
 1435                 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) {
 1436                         putvndxfer(vnx);
 1437                         biodone(pbp);
 1438                 }
 1439         } else if (pbp->b_resid == 0) {
 1440                 KASSERT(vnx->vx_pending == 0);
 1441                 if ((vnx->vx_flags & VX_BUSY) == 0) {
 1442                         UVMHIST_LOG(pdhist, "  iodone error=%d !",
 1443                             pbp, vnx->vx_error, 0, 0);
 1444                         putvndxfer(vnx);
 1445                         biodone(pbp);
 1446                 }
 1447         }
 1448 
 1449         /*
 1450          * done!   start next swapdev I/O if one is pending
 1451          */
 1452         sdp->swd_active--;
 1453         sw_reg_start(sdp);
 1454         splx(s);
 1455 }
 1456 
 1457 
 1458 /*
 1459  * uvm_swap_alloc: allocate space on swap
 1460  *
 1461  * => allocation is done "round robin" down the priority list, as we
 1462  *      allocate in a priority we "rotate" the circle queue.
 1463  * => space can be freed with uvm_swap_free
 1464  * => we return the page slot number in /dev/drum (0 == invalid slot)
 1465  * => we lock uvm.swap_data_lock
 1466  * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM
 1467  */
 1468 int
 1469 uvm_swap_alloc(nslots, lessok)
 1470         int *nslots;    /* IN/OUT */
 1471         boolean_t lessok;
 1472 {
 1473         struct swapdev *sdp;
 1474         struct swappri *spp;
 1475         u_long  result;
 1476         UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist);
 1477 
 1478         /*
 1479          * no swap devices configured yet?   definite failure.
 1480          */
 1481         if (uvmexp.nswapdev < 1)
 1482                 return 0;
 1483 
 1484         /*
 1485          * lock data lock, convert slots into blocks, and enter loop
 1486          */
 1487         simple_lock(&uvm.swap_data_lock);
 1488 
 1489 ReTry:  /* XXXMRG */
 1490         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
 1491                 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
 1492                         /* if it's not enabled, then we can't swap from it */
 1493                         if ((sdp->swd_flags & SWF_ENABLE) == 0)
 1494                                 continue;
 1495                         if (sdp->swd_npginuse + *nslots > sdp->swd_npages)
 1496                                 continue;
 1497                         if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN,
 1498                                          EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT,
 1499                                          &result) != 0) {
 1500                                 continue;
 1501                         }
 1502 
 1503                         /*
 1504                          * successful allocation!  now rotate the circleq.
 1505                          */
 1506                         CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next);
 1507                         CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
 1508                         sdp->swd_npginuse += *nslots;
 1509                         uvmexp.swpginuse += *nslots;
 1510                         simple_unlock(&uvm.swap_data_lock);
 1511                         /* done!  return drum slot number */
 1512                         UVMHIST_LOG(pdhist,
 1513                             "success!  returning %d slots starting at %d",
 1514                             *nslots, result + sdp->swd_drumoffset, 0, 0);
 1515                         return (result + sdp->swd_drumoffset);
 1516                 }
 1517         }
 1518 
 1519         /* XXXMRG: BEGIN HACK */
 1520         if (*nslots > 1 && lessok) {
 1521                 *nslots = 1;
 1522                 goto ReTry;     /* XXXMRG: ugh!  extent should support this for us */
 1523         }
 1524         /* XXXMRG: END HACK */
 1525 
 1526         simple_unlock(&uvm.swap_data_lock);
 1527         return 0;
 1528 }
 1529 
 1530 boolean_t
 1531 uvm_swapisfull(void)
 1532 {
 1533         boolean_t rv;
 1534 
 1535         simple_lock(&uvm.swap_data_lock);
 1536         KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
 1537         rv = (uvmexp.swpgonly >= uvmexp.swpgavail);
 1538         simple_unlock(&uvm.swap_data_lock);
 1539 
 1540         return (rv);
 1541 }
 1542 
 1543 /*
 1544  * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors
 1545  *
 1546  * => we lock uvm.swap_data_lock
 1547  */
 1548 void
 1549 uvm_swap_markbad(startslot, nslots)
 1550         int startslot;
 1551         int nslots;
 1552 {
 1553         struct swapdev *sdp;
 1554         UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist);
 1555 
 1556         simple_lock(&uvm.swap_data_lock);
 1557         sdp = swapdrum_getsdp(startslot);
 1558         KASSERT(sdp != NULL);
 1559 
 1560         /*
 1561          * we just keep track of how many pages have been marked bad
 1562          * in this device, to make everything add up in swap_off().
 1563          * we assume here that the range of slots will all be within
 1564          * one swap device.
 1565          */
 1566 
 1567         KASSERT(uvmexp.swpgonly >= nslots);
 1568         uvmexp.swpgonly -= nslots;
 1569         sdp->swd_npgbad += nslots;
 1570         UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0);
 1571         simple_unlock(&uvm.swap_data_lock);
 1572 }
 1573 
 1574 /*
 1575  * uvm_swap_free: free swap slots
 1576  *
 1577  * => this can be all or part of an allocation made by uvm_swap_alloc
 1578  * => we lock uvm.swap_data_lock
 1579  */
 1580 void
 1581 uvm_swap_free(startslot, nslots)
 1582         int startslot;
 1583         int nslots;
 1584 {
 1585         struct swapdev *sdp;
 1586         UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist);
 1587 
 1588         UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots,
 1589             startslot, 0, 0);
 1590 
 1591         /*
 1592          * ignore attempts to free the "bad" slot.
 1593          */
 1594 
 1595         if (startslot == SWSLOT_BAD) {
 1596                 return;
 1597         }
 1598 
 1599         /*
 1600          * convert drum slot offset back to sdp, free the blocks
 1601          * in the extent, and return.   must hold pri lock to do
 1602          * lookup and access the extent.
 1603          */
 1604 
 1605         simple_lock(&uvm.swap_data_lock);
 1606         sdp = swapdrum_getsdp(startslot);
 1607         KASSERT(uvmexp.nswapdev >= 1);
 1608         KASSERT(sdp != NULL);
 1609         KASSERT(sdp->swd_npginuse >= nslots);
 1610         if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots,
 1611                         EX_MALLOCOK|EX_NOWAIT) != 0) {
 1612                 printf("warning: resource shortage: %d pages of swap lost\n",
 1613                         nslots);
 1614         }
 1615         sdp->swd_npginuse -= nslots;
 1616         uvmexp.swpginuse -= nslots;
 1617         simple_unlock(&uvm.swap_data_lock);
 1618 }
 1619 
 1620 /*
 1621  * uvm_swap_put: put any number of pages into a contig place on swap
 1622  *
 1623  * => can be sync or async
 1624  */
 1625 
 1626 int
 1627 uvm_swap_put(swslot, ppsp, npages, flags)
 1628         int swslot;
 1629         struct vm_page **ppsp;
 1630         int npages;
 1631         int flags;
 1632 {
 1633         int error;
 1634 
 1635         error = uvm_swap_io(ppsp, swslot, npages, B_WRITE |
 1636             ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
 1637         return error;
 1638 }
 1639 
 1640 /*
 1641  * uvm_swap_get: get a single page from swap
 1642  *
 1643  * => usually a sync op (from fault)
 1644  */
 1645 
 1646 int
 1647 uvm_swap_get(page, swslot, flags)
 1648         struct vm_page *page;
 1649         int swslot, flags;
 1650 {
 1651         int error;
 1652 
 1653         uvmexp.nswget++;
 1654         KASSERT(flags & PGO_SYNCIO);
 1655         if (swslot == SWSLOT_BAD) {
 1656                 return EIO;
 1657         }
 1658 
 1659         error = uvm_swap_io(&page, swslot, 1, B_READ |
 1660             ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
 1661         if (error == 0) {
 1662 
 1663                 /*
 1664                  * this page is no longer only in swap.
 1665                  */
 1666 
 1667                 simple_lock(&uvm.swap_data_lock);
 1668                 KASSERT(uvmexp.swpgonly > 0);
 1669                 uvmexp.swpgonly--;
 1670                 simple_unlock(&uvm.swap_data_lock);
 1671         }
 1672         return error;
 1673 }
 1674 
 1675 /*
 1676  * uvm_swap_io: do an i/o operation to swap
 1677  */
 1678 
 1679 static int
 1680 uvm_swap_io(pps, startslot, npages, flags)
 1681         struct vm_page **pps;
 1682         int startslot, npages, flags;
 1683 {
 1684         daddr_t startblk;
 1685         struct  buf *bp;
 1686         vaddr_t kva;
 1687         int     error, s, mapinflags;
 1688         boolean_t write, async;
 1689         UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist);
 1690 
 1691         UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d",
 1692             startslot, npages, flags, 0);
 1693 
 1694         write = (flags & B_READ) == 0;
 1695         async = (flags & B_ASYNC) != 0;
 1696 
 1697         /*
 1698          * convert starting drum slot to block number
 1699          */
 1700 
 1701         startblk = btodb((u_int64_t)startslot << PAGE_SHIFT);
 1702 
 1703         /*
 1704          * first, map the pages into the kernel.
 1705          */
 1706 
 1707         mapinflags = !write ?
 1708                 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ :
 1709                 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE;
 1710         kva = uvm_pagermapin(pps, npages, mapinflags);
 1711 
 1712         /*
 1713          * now allocate a buf for the i/o.
 1714          */
 1715 
 1716         s = splbio();
 1717         bp = pool_get(&bufpool, PR_WAITOK);
 1718         splx(s);
 1719 
 1720         /*
 1721          * fill in the bp/sbp.   we currently route our i/o through
 1722          * /dev/drum's vnode [swapdev_vp].
 1723          */
 1724 
 1725         BUF_INIT(bp);
 1726         bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC));
 1727         bp->b_proc = &proc0;    /* XXX */
 1728         bp->b_vnbufs.le_next = NOLIST;
 1729         bp->b_data = (caddr_t)kva;
 1730         bp->b_blkno = startblk;
 1731         bp->b_vp = swapdev_vp;
 1732         bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT;
 1733 
 1734         /*
 1735          * bump v_numoutput (counter of number of active outputs).
 1736          */
 1737 
 1738         if (write) {
 1739                 s = splbio();
 1740                 V_INCR_NUMOUTPUT(swapdev_vp);
 1741                 splx(s);
 1742         }
 1743 
 1744         /*
 1745          * for async ops we must set up the iodone handler.
 1746          */
 1747 
 1748         if (async) {
 1749                 bp->b_flags |= B_CALL;
 1750                 bp->b_iodone = uvm_aio_biodone;
 1751                 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0);
 1752                 if (curproc == uvm.pagedaemon_proc)
 1753                         BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
 1754                 else
 1755                         BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
 1756         } else {
 1757                 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
 1758         }
 1759         UVMHIST_LOG(pdhist,
 1760             "about to start io: data = %p blkno = 0x%x, bcount = %ld",
 1761             bp->b_data, bp->b_blkno, bp->b_bcount, 0);
 1762 
 1763         /*
 1764          * now we start the I/O, and if async, return.
 1765          */
 1766 
 1767         VOP_STRATEGY(swapdev_vp, bp);
 1768         if (async)
 1769                 return 0;
 1770 
 1771         /*
 1772          * must be sync i/o.   wait for it to finish
 1773          */
 1774 
 1775         error = biowait(bp);
 1776 
 1777         /*
 1778          * kill the pager mapping
 1779          */
 1780 
 1781         uvm_pagermapout(kva, npages);
 1782 
 1783         /*
 1784          * now dispose of the buf and we're done.
 1785          */
 1786 
 1787         s = splbio();
 1788         if (write)
 1789                 vwakeup(bp);
 1790         pool_put(&bufpool, bp);
 1791         splx(s);
 1792         UVMHIST_LOG(pdhist, "<- done (sync)  error=%d", error, 0, 0, 0);
 1793         return (error);
 1794 }

Cache object: 5069355804842c1bfee4d693d312c104


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.