The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_swap.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: uvm_swap.c,v 1.89.10.2 2006/04/06 19:30:53 tron Exp $  */
    2 
    3 /*
    4  * Copyright (c) 1995, 1996, 1997 Matthew R. Green
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  * 3. The name of the author may not be used to endorse or promote products
   16  *    derived from this software without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
   22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
   23  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   24  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
   25  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
   26  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp
   31  * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.89.10.2 2006/04/06 19:30:53 tron Exp $");
   36 
   37 #include "fs_nfs.h"
   38 #include "opt_uvmhist.h"
   39 #include "opt_compat_netbsd.h"
   40 #include "opt_ddb.h"
   41 
   42 #include <sys/param.h>
   43 #include <sys/systm.h>
   44 #include <sys/buf.h>
   45 #include <sys/bufq.h>
   46 #include <sys/conf.h>
   47 #include <sys/proc.h>
   48 #include <sys/namei.h>
   49 #include <sys/disklabel.h>
   50 #include <sys/errno.h>
   51 #include <sys/kernel.h>
   52 #include <sys/malloc.h>
   53 #include <sys/vnode.h>
   54 #include <sys/file.h>
   55 #include <sys/extent.h>
   56 #include <sys/mount.h>
   57 #include <sys/pool.h>
   58 #include <sys/sa.h>
   59 #include <sys/syscallargs.h>
   60 #include <sys/swap.h>
   61 
   62 #include <uvm/uvm.h>
   63 
   64 #include <miscfs/specfs/specdev.h>
   65 
   66 /*
   67  * uvm_swap.c: manage configuration and i/o to swap space.
   68  */
   69 
   70 /*
   71  * swap space is managed in the following way:
   72  *
   73  * each swap partition or file is described by a "swapdev" structure.
   74  * each "swapdev" structure contains a "swapent" structure which contains
   75  * information that is passed up to the user (via system calls).
   76  *
   77  * each swap partition is assigned a "priority" (int) which controls
   78  * swap parition usage.
   79  *
   80  * the system maintains a global data structure describing all swap
   81  * partitions/files.   there is a sorted LIST of "swappri" structures
   82  * which describe "swapdev"'s at that priority.   this LIST is headed
   83  * by the "swap_priority" global var.    each "swappri" contains a
   84  * CIRCLEQ of "swapdev" structures at that priority.
   85  *
   86  * locking:
   87  *  - swap_syscall_lock (sleep lock): this lock serializes the swapctl
   88  *    system call and prevents the swap priority list from changing
   89  *    while we are in the middle of a system call (e.g. SWAP_STATS).
   90  *  - uvm.swap_data_lock (simple_lock): this lock protects all swap data
   91  *    structures including the priority list, the swapdev structures,
   92  *    and the swapmap extent.
   93  *
   94  * each swap device has the following info:
   95  *  - swap device in use (could be disabled, preventing future use)
   96  *  - swap enabled (allows new allocations on swap)
   97  *  - map info in /dev/drum
   98  *  - vnode pointer
   99  * for swap files only:
  100  *  - block size
  101  *  - max byte count in buffer
  102  *  - buffer
  103  *
  104  * userland controls and configures swap with the swapctl(2) system call.
  105  * the sys_swapctl performs the following operations:
  106  *  [1] SWAP_NSWAP: returns the number of swap devices currently configured
  107  *  [2] SWAP_STATS: given a pointer to an array of swapent structures
  108  *      (passed in via "arg") of a size passed in via "misc" ... we load
  109  *      the current swap config into the array. The actual work is done
  110  *      in the uvm_swap_stats(9) function.
  111  *  [3] SWAP_ON: given a pathname in arg (could be device or file) and a
  112  *      priority in "misc", start swapping on it.
  113  *  [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device
  114  *  [5] SWAP_CTL: changes the priority of a swap device (new priority in
  115  *      "misc")
  116  */
  117 
  118 /*
  119  * swapdev: describes a single swap partition/file
  120  *
  121  * note the following should be true:
  122  * swd_inuse <= swd_nblks  [number of blocks in use is <= total blocks]
  123  * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel]
  124  */
  125 struct swapdev {
  126         struct oswapent swd_ose;
  127 #define swd_dev         swd_ose.ose_dev         /* device id */
  128 #define swd_flags       swd_ose.ose_flags       /* flags:inuse/enable/fake */
  129 #define swd_priority    swd_ose.ose_priority    /* our priority */
  130         /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */
  131         char                    *swd_path;      /* saved pathname of device */
  132         int                     swd_pathlen;    /* length of pathname */
  133         int                     swd_npages;     /* #pages we can use */
  134         int                     swd_npginuse;   /* #pages in use */
  135         int                     swd_npgbad;     /* #pages bad */
  136         int                     swd_drumoffset; /* page0 offset in drum */
  137         int                     swd_drumsize;   /* #pages in drum */
  138         struct extent           *swd_ex;        /* extent for this swapdev */
  139         char                    swd_exname[12]; /* name of extent above */
  140         struct vnode            *swd_vp;        /* backing vnode */
  141         CIRCLEQ_ENTRY(swapdev)  swd_next;       /* priority circleq */
  142 
  143         int                     swd_bsize;      /* blocksize (bytes) */
  144         int                     swd_maxactive;  /* max active i/o reqs */
  145         struct bufq_state       swd_tab;        /* buffer list */
  146         int                     swd_active;     /* number of active buffers */
  147 };
  148 
  149 /*
  150  * swap device priority entry; the list is kept sorted on `spi_priority'.
  151  */
  152 struct swappri {
  153         int                     spi_priority;     /* priority */
  154         CIRCLEQ_HEAD(spi_swapdev, swapdev)      spi_swapdev;
  155         /* circleq of swapdevs at this priority */
  156         LIST_ENTRY(swappri)     spi_swappri;      /* global list of pri's */
  157 };
  158 
  159 /*
  160  * The following two structures are used to keep track of data transfers
  161  * on swap devices associated with regular files.
  162  * NOTE: this code is more or less a copy of vnd.c; we use the same
  163  * structure names here to ease porting..
  164  */
  165 struct vndxfer {
  166         struct buf      *vx_bp;         /* Pointer to parent buffer */
  167         struct swapdev  *vx_sdp;
  168         int             vx_error;
  169         int             vx_pending;     /* # of pending aux buffers */
  170         int             vx_flags;
  171 #define VX_BUSY         1
  172 #define VX_DEAD         2
  173 };
  174 
  175 struct vndbuf {
  176         struct buf      vb_buf;
  177         struct vndxfer  *vb_xfer;
  178 };
  179 
  180 
  181 /*
  182  * We keep a of pool vndbuf's and vndxfer structures.
  183  */
  184 POOL_INIT(vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", NULL);
  185 POOL_INIT(vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", NULL);
  186 
  187 #define getvndxfer(vnx) do {                                            \
  188         int s = splbio();                                               \
  189         vnx = pool_get(&vndxfer_pool, PR_WAITOK);                       \
  190         splx(s);                                                        \
  191 } while (/*CONSTCOND*/ 0)
  192 
  193 #define putvndxfer(vnx) {                                               \
  194         pool_put(&vndxfer_pool, (void *)(vnx));                         \
  195 }
  196 
  197 #define getvndbuf(vbp)  do {                                            \
  198         int s = splbio();                                               \
  199         vbp = pool_get(&vndbuf_pool, PR_WAITOK);                        \
  200         splx(s);                                                        \
  201 } while (/*CONSTCOND*/ 0)
  202 
  203 #define putvndbuf(vbp) {                                                \
  204         pool_put(&vndbuf_pool, (void *)(vbp));                          \
  205 }
  206 
  207 /*
  208  * local variables
  209  */
  210 static struct extent *swapmap;          /* controls the mapping of /dev/drum */
  211 
  212 MALLOC_DEFINE(M_VMSWAP, "VM swap", "VM swap structures");
  213 
  214 /* list of all active swap devices [by priority] */
  215 LIST_HEAD(swap_priority, swappri);
  216 static struct swap_priority swap_priority;
  217 
  218 /* locks */
  219 struct lock swap_syscall_lock;
  220 
  221 /*
  222  * prototypes
  223  */
  224 static struct swapdev   *swapdrum_getsdp(int);
  225 
  226 static struct swapdev   *swaplist_find(struct vnode *, int);
  227 static void              swaplist_insert(struct swapdev *,
  228                                          struct swappri *, int);
  229 static void              swaplist_trim(void);
  230 
  231 static int swap_on(struct proc *, struct swapdev *);
  232 static int swap_off(struct proc *, struct swapdev *);
  233 
  234 static void sw_reg_strategy(struct swapdev *, struct buf *, int);
  235 static void sw_reg_iodone(struct buf *);
  236 static void sw_reg_start(struct swapdev *);
  237 
  238 static int uvm_swap_io(struct vm_page **, int, int, int);
  239 
  240 dev_type_read(swread);
  241 dev_type_write(swwrite);
  242 dev_type_strategy(swstrategy);
  243 
  244 const struct bdevsw swap_bdevsw = {
  245         noopen, noclose, swstrategy, noioctl, nodump, nosize,
  246 };
  247 
  248 const struct cdevsw swap_cdevsw = {
  249         nullopen, nullclose, swread, swwrite, noioctl,
  250         nostop, notty, nopoll, nommap, nokqfilter
  251 };
  252 
  253 /*
  254  * uvm_swap_init: init the swap system data structures and locks
  255  *
  256  * => called at boot time from init_main.c after the filesystems
  257  *      are brought up (which happens after uvm_init())
  258  */
  259 void
  260 uvm_swap_init()
  261 {
  262         UVMHIST_FUNC("uvm_swap_init");
  263 
  264         UVMHIST_CALLED(pdhist);
  265         /*
  266          * first, init the swap list, its counter, and its lock.
  267          * then get a handle on the vnode for /dev/drum by using
  268          * the its dev_t number ("swapdev", from MD conf.c).
  269          */
  270 
  271         LIST_INIT(&swap_priority);
  272         uvmexp.nswapdev = 0;
  273         lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0);
  274         simple_lock_init(&uvm.swap_data_lock);
  275 
  276         if (bdevvp(swapdev, &swapdev_vp))
  277                 panic("uvm_swap_init: can't get vnode for swap device");
  278 
  279         /*
  280          * create swap block resource map to map /dev/drum.   the range
  281          * from 1 to INT_MAX allows 2 gigablocks of swap space.  note
  282          * that block 0 is reserved (used to indicate an allocation
  283          * failure, or no allocation).
  284          */
  285         swapmap = extent_create("swapmap", 1, INT_MAX,
  286                                 M_VMSWAP, 0, 0, EX_NOWAIT);
  287         if (swapmap == 0)
  288                 panic("uvm_swap_init: extent_create failed");
  289 
  290         /*
  291          * done!
  292          */
  293         UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0);
  294 }
  295 
  296 /*
  297  * swaplist functions: functions that operate on the list of swap
  298  * devices on the system.
  299  */
  300 
  301 /*
  302  * swaplist_insert: insert swap device "sdp" into the global list
  303  *
  304  * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
  305  * => caller must provide a newly malloc'd swappri structure (we will
  306  *      FREE it if we don't need it... this it to prevent malloc blocking
  307  *      here while adding swap)
  308  */
  309 static void
  310 swaplist_insert(sdp, newspp, priority)
  311         struct swapdev *sdp;
  312         struct swappri *newspp;
  313         int priority;
  314 {
  315         struct swappri *spp, *pspp;
  316         UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist);
  317 
  318         /*
  319          * find entry at or after which to insert the new device.
  320          */
  321         pspp = NULL;
  322         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  323                 if (priority <= spp->spi_priority)
  324                         break;
  325                 pspp = spp;
  326         }
  327 
  328         /*
  329          * new priority?
  330          */
  331         if (spp == NULL || spp->spi_priority != priority) {
  332                 spp = newspp;  /* use newspp! */
  333                 UVMHIST_LOG(pdhist, "created new swappri = %d",
  334                             priority, 0, 0, 0);
  335 
  336                 spp->spi_priority = priority;
  337                 CIRCLEQ_INIT(&spp->spi_swapdev);
  338 
  339                 if (pspp)
  340                         LIST_INSERT_AFTER(pspp, spp, spi_swappri);
  341                 else
  342                         LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri);
  343         } else {
  344                 /* we don't need a new priority structure, free it */
  345                 FREE(newspp, M_VMSWAP);
  346         }
  347 
  348         /*
  349          * priority found (or created).   now insert on the priority's
  350          * circleq list and bump the total number of swapdevs.
  351          */
  352         sdp->swd_priority = priority;
  353         CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
  354         uvmexp.nswapdev++;
  355 }
  356 
  357 /*
  358  * swaplist_find: find and optionally remove a swap device from the
  359  *      global list.
  360  *
  361  * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
  362  * => we return the swapdev we found (and removed)
  363  */
  364 static struct swapdev *
  365 swaplist_find(vp, remove)
  366         struct vnode *vp;
  367         boolean_t remove;
  368 {
  369         struct swapdev *sdp;
  370         struct swappri *spp;
  371 
  372         /*
  373          * search the lists for the requested vp
  374          */
  375 
  376         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  377                 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
  378                         if (sdp->swd_vp == vp) {
  379                                 if (remove) {
  380                                         CIRCLEQ_REMOVE(&spp->spi_swapdev,
  381                                             sdp, swd_next);
  382                                         uvmexp.nswapdev--;
  383                                 }
  384                                 return(sdp);
  385                         }
  386                 }
  387         }
  388         return (NULL);
  389 }
  390 
  391 
  392 /*
  393  * swaplist_trim: scan priority list for empty priority entries and kill
  394  *      them.
  395  *
  396  * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
  397  */
  398 static void
  399 swaplist_trim()
  400 {
  401         struct swappri *spp, *nextspp;
  402 
  403         for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) {
  404                 nextspp = LIST_NEXT(spp, spi_swappri);
  405                 if (CIRCLEQ_FIRST(&spp->spi_swapdev) !=
  406                     (void *)&spp->spi_swapdev)
  407                         continue;
  408                 LIST_REMOVE(spp, spi_swappri);
  409                 free(spp, M_VMSWAP);
  410         }
  411 }
  412 
  413 /*
  414  * swapdrum_getsdp: given a page offset in /dev/drum, convert it back
  415  *      to the "swapdev" that maps that section of the drum.
  416  *
  417  * => each swapdev takes one big contig chunk of the drum
  418  * => caller must hold uvm.swap_data_lock
  419  */
  420 static struct swapdev *
  421 swapdrum_getsdp(pgno)
  422         int pgno;
  423 {
  424         struct swapdev *sdp;
  425         struct swappri *spp;
  426 
  427         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  428                 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
  429                         if (sdp->swd_flags & SWF_FAKE)
  430                                 continue;
  431                         if (pgno >= sdp->swd_drumoffset &&
  432                             pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) {
  433                                 return sdp;
  434                         }
  435                 }
  436         }
  437         return NULL;
  438 }
  439 
  440 
  441 /*
  442  * sys_swapctl: main entry point for swapctl(2) system call
  443  *      [with two helper functions: swap_on and swap_off]
  444  */
  445 int
  446 sys_swapctl(l, v, retval)
  447         struct lwp *l;
  448         void *v;
  449         register_t *retval;
  450 {
  451         struct sys_swapctl_args /* {
  452                 syscallarg(int) cmd;
  453                 syscallarg(void *) arg;
  454                 syscallarg(int) misc;
  455         } */ *uap = (struct sys_swapctl_args *)v;
  456         struct proc *p = l->l_proc;
  457         struct vnode *vp;
  458         struct nameidata nd;
  459         struct swappri *spp;
  460         struct swapdev *sdp;
  461         struct swapent *sep;
  462         char    userpath[PATH_MAX + 1];
  463         size_t  len;
  464         int     error, misc;
  465         int     priority;
  466         UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist);
  467 
  468         misc = SCARG(uap, misc);
  469 
  470         /*
  471          * ensure serialized syscall access by grabbing the swap_syscall_lock
  472          */
  473         lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL);
  474 
  475         /*
  476          * we handle the non-priv NSWAP and STATS request first.
  477          *
  478          * SWAP_NSWAP: return number of config'd swap devices
  479          * [can also be obtained with uvmexp sysctl]
  480          */
  481         if (SCARG(uap, cmd) == SWAP_NSWAP) {
  482                 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev,
  483                     0, 0, 0);
  484                 *retval = uvmexp.nswapdev;
  485                 error = 0;
  486                 goto out;
  487         }
  488 
  489         /*
  490          * SWAP_STATS: get stats on current # of configured swap devs
  491          *
  492          * note that the swap_priority list can't change as long
  493          * as we are holding the swap_syscall_lock.  we don't want
  494          * to grab the uvm.swap_data_lock because we may fault&sleep during
  495          * copyout() and we don't want to be holding that lock then!
  496          */
  497         if (SCARG(uap, cmd) == SWAP_STATS
  498 #if defined(COMPAT_13)
  499             || SCARG(uap, cmd) == SWAP_OSTATS
  500 #endif
  501             ) {
  502                 if ((size_t)misc > (size_t)uvmexp.nswapdev)
  503                         misc = uvmexp.nswapdev;
  504 #if defined(COMPAT_13)
  505                 if (SCARG(uap, cmd) == SWAP_OSTATS)
  506                         len = sizeof(struct oswapent) * misc;
  507                 else
  508 #endif
  509                         len = sizeof(struct swapent) * misc;
  510                 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK);
  511 
  512                 uvm_swap_stats(SCARG(uap, cmd), sep, misc, retval);
  513                 error = copyout(sep, (void *)SCARG(uap, arg), len);
  514 
  515                 free(sep, M_TEMP);
  516                 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0);
  517                 goto out;
  518         }
  519         if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) {
  520                 dev_t   *devp = (dev_t *)SCARG(uap, arg);
  521 
  522                 error = copyout(&dumpdev, devp, sizeof(dumpdev));
  523                 goto out;
  524         }
  525 
  526         /*
  527          * all other requests require superuser privs.   verify.
  528          */
  529         if ((error = suser(p->p_ucred, &p->p_acflag)))
  530                 goto out;
  531 
  532         /*
  533          * at this point we expect a path name in arg.   we will
  534          * use namei() to gain a vnode reference (vref), and lock
  535          * the vnode (VOP_LOCK).
  536          *
  537          * XXX: a NULL arg means use the root vnode pointer (e.g. for
  538          * miniroot)
  539          */
  540         if (SCARG(uap, arg) == NULL) {
  541                 vp = rootvp;            /* miniroot */
  542                 if (vget(vp, LK_EXCLUSIVE)) {
  543                         error = EBUSY;
  544                         goto out;
  545                 }
  546                 if (SCARG(uap, cmd) == SWAP_ON &&
  547                     copystr("miniroot", userpath, sizeof userpath, &len))
  548                         panic("swapctl: miniroot copy failed");
  549         } else {
  550                 int     space;
  551                 char    *where;
  552 
  553                 if (SCARG(uap, cmd) == SWAP_ON) {
  554                         if ((error = copyinstr(SCARG(uap, arg), userpath,
  555                             sizeof userpath, &len)))
  556                                 goto out;
  557                         space = UIO_SYSSPACE;
  558                         where = userpath;
  559                 } else {
  560                         space = UIO_USERSPACE;
  561                         where = (char *)SCARG(uap, arg);
  562                 }
  563                 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p);
  564                 if ((error = namei(&nd)))
  565                         goto out;
  566                 vp = nd.ni_vp;
  567         }
  568         /* note: "vp" is referenced and locked */
  569 
  570         error = 0;              /* assume no error */
  571         switch(SCARG(uap, cmd)) {
  572 
  573         case SWAP_DUMPDEV:
  574                 if (vp->v_type != VBLK) {
  575                         error = ENOTBLK;
  576                         break;
  577                 }
  578                 dumpdev = vp->v_rdev;
  579                 cpu_dumpconf();
  580                 break;
  581 
  582         case SWAP_CTL:
  583                 /*
  584                  * get new priority, remove old entry (if any) and then
  585                  * reinsert it in the correct place.  finally, prune out
  586                  * any empty priority structures.
  587                  */
  588                 priority = SCARG(uap, misc);
  589                 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
  590                 simple_lock(&uvm.swap_data_lock);
  591                 if ((sdp = swaplist_find(vp, 1)) == NULL) {
  592                         error = ENOENT;
  593                 } else {
  594                         swaplist_insert(sdp, spp, priority);
  595                         swaplist_trim();
  596                 }
  597                 simple_unlock(&uvm.swap_data_lock);
  598                 if (error)
  599                         free(spp, M_VMSWAP);
  600                 break;
  601 
  602         case SWAP_ON:
  603 
  604                 /*
  605                  * check for duplicates.   if none found, then insert a
  606                  * dummy entry on the list to prevent someone else from
  607                  * trying to enable this device while we are working on
  608                  * it.
  609                  */
  610 
  611                 priority = SCARG(uap, misc);
  612                 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK);
  613                 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
  614                 memset(sdp, 0, sizeof(*sdp));
  615                 sdp->swd_flags = SWF_FAKE;
  616                 sdp->swd_vp = vp;
  617                 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV;
  618                 bufq_alloc(&sdp->swd_tab, BUFQ_DISKSORT|BUFQ_SORT_RAWBLOCK);
  619                 simple_lock(&uvm.swap_data_lock);
  620                 if (swaplist_find(vp, 0) != NULL) {
  621                         error = EBUSY;
  622                         simple_unlock(&uvm.swap_data_lock);
  623                         bufq_free(&sdp->swd_tab);
  624                         free(sdp, M_VMSWAP);
  625                         free(spp, M_VMSWAP);
  626                         break;
  627                 }
  628                 swaplist_insert(sdp, spp, priority);
  629                 simple_unlock(&uvm.swap_data_lock);
  630 
  631                 sdp->swd_pathlen = len;
  632                 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK);
  633                 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0)
  634                         panic("swapctl: copystr");
  635 
  636                 /*
  637                  * we've now got a FAKE placeholder in the swap list.
  638                  * now attempt to enable swap on it.  if we fail, undo
  639                  * what we've done and kill the fake entry we just inserted.
  640                  * if swap_on is a success, it will clear the SWF_FAKE flag
  641                  */
  642 
  643                 if ((error = swap_on(p, sdp)) != 0) {
  644                         simple_lock(&uvm.swap_data_lock);
  645                         (void) swaplist_find(vp, 1);  /* kill fake entry */
  646                         swaplist_trim();
  647                         simple_unlock(&uvm.swap_data_lock);
  648                         bufq_free(&sdp->swd_tab);
  649                         free(sdp->swd_path, M_VMSWAP);
  650                         free(sdp, M_VMSWAP);
  651                         break;
  652                 }
  653                 break;
  654 
  655         case SWAP_OFF:
  656                 simple_lock(&uvm.swap_data_lock);
  657                 if ((sdp = swaplist_find(vp, 0)) == NULL) {
  658                         simple_unlock(&uvm.swap_data_lock);
  659                         error = ENXIO;
  660                         break;
  661                 }
  662 
  663                 /*
  664                  * If a device isn't in use or enabled, we
  665                  * can't stop swapping from it (again).
  666                  */
  667                 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) {
  668                         simple_unlock(&uvm.swap_data_lock);
  669                         error = EBUSY;
  670                         break;
  671                 }
  672 
  673                 /*
  674                  * do the real work.
  675                  */
  676                 error = swap_off(p, sdp);
  677                 break;
  678 
  679         default:
  680                 error = EINVAL;
  681         }
  682 
  683         /*
  684          * done!  release the ref gained by namei() and unlock.
  685          */
  686         vput(vp);
  687 
  688 out:
  689         lockmgr(&swap_syscall_lock, LK_RELEASE, NULL);
  690 
  691         UVMHIST_LOG(pdhist, "<- done!  error=%d", error, 0, 0, 0);
  692         return (error);
  693 }
  694 
  695 /*
  696  * swap_stats: implements swapctl(SWAP_STATS). The function is kept
  697  * away from sys_swapctl() in order to allow COMPAT_* swapctl()
  698  * emulation to use it directly without going through sys_swapctl().
  699  * The problem with using sys_swapctl() there is that it involves
  700  * copying the swapent array to the stackgap, and this array's size
  701  * is not known at build time. Hence it would not be possible to
  702  * ensure it would fit in the stackgap in any case.
  703  */
  704 void
  705 uvm_swap_stats(cmd, sep, sec, retval)
  706         int cmd;
  707         struct swapent *sep;
  708         int sec;
  709         register_t *retval;
  710 {
  711         struct swappri *spp;
  712         struct swapdev *sdp;
  713         int count = 0;
  714 
  715         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
  716                 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev);
  717                      sdp != (void *)&spp->spi_swapdev && sec-- > 0;
  718                      sdp = CIRCLEQ_NEXT(sdp, swd_next)) {
  719                         /*
  720                          * backwards compatibility for system call.
  721                          * note that we use 'struct oswapent' as an
  722                          * overlay into both 'struct swapdev' and
  723                          * the userland 'struct swapent', as we
  724                          * want to retain backwards compatibility
  725                          * with NetBSD 1.3.
  726                          */
  727                         sdp->swd_ose.ose_inuse =
  728                             btodb((u_int64_t)sdp->swd_npginuse <<
  729                             PAGE_SHIFT);
  730                         (void)memcpy(sep, &sdp->swd_ose,
  731                             sizeof(struct oswapent));
  732 
  733                         /* now copy out the path if necessary */
  734 #if defined(COMPAT_13)
  735                         if (cmd == SWAP_STATS)
  736 #endif
  737                                 (void)memcpy(&sep->se_path, sdp->swd_path,
  738                                     sdp->swd_pathlen);
  739 
  740                         count++;
  741 #if defined(COMPAT_13)
  742                         if (cmd == SWAP_OSTATS)
  743                                 sep = (struct swapent *)
  744                                     ((struct oswapent *)sep + 1);
  745                         else
  746 #endif
  747                                 sep++;
  748                 }
  749         }
  750 
  751         *retval = count;
  752         return;
  753 }
  754 
  755 /*
  756  * swap_on: attempt to enable a swapdev for swapping.   note that the
  757  *      swapdev is already on the global list, but disabled (marked
  758  *      SWF_FAKE).
  759  *
  760  * => we avoid the start of the disk (to protect disk labels)
  761  * => we also avoid the miniroot, if we are swapping to root.
  762  * => caller should leave uvm.swap_data_lock unlocked, we may lock it
  763  *      if needed.
  764  */
  765 static int
  766 swap_on(p, sdp)
  767         struct proc *p;
  768         struct swapdev *sdp;
  769 {
  770         static int count = 0;   /* static */
  771         struct vnode *vp;
  772         int error, npages, nblocks, size;
  773         long addr;
  774         u_long result;
  775         struct vattr va;
  776 #ifdef NFS
  777         extern int (**nfsv2_vnodeop_p)(void *);
  778 #endif /* NFS */
  779         const struct bdevsw *bdev;
  780         dev_t dev;
  781         UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist);
  782 
  783         /*
  784          * we want to enable swapping on sdp.   the swd_vp contains
  785          * the vnode we want (locked and ref'd), and the swd_dev
  786          * contains the dev_t of the file, if it a block device.
  787          */
  788 
  789         vp = sdp->swd_vp;
  790         dev = sdp->swd_dev;
  791 
  792         /*
  793          * open the swap file (mostly useful for block device files to
  794          * let device driver know what is up).
  795          *
  796          * we skip the open/close for root on swap because the root
  797          * has already been opened when root was mounted (mountroot).
  798          */
  799         if (vp != rootvp) {
  800                 if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p)))
  801                         return (error);
  802         }
  803 
  804         /* XXX this only works for block devices */
  805         UVMHIST_LOG(pdhist, "  dev=%d, major(dev)=%d", dev, major(dev), 0,0);
  806 
  807         /*
  808          * we now need to determine the size of the swap area.   for
  809          * block specials we can call the d_psize function.
  810          * for normal files, we must stat [get attrs].
  811          *
  812          * we put the result in nblks.
  813          * for normal files, we also want the filesystem block size
  814          * (which we get with statfs).
  815          */
  816         switch (vp->v_type) {
  817         case VBLK:
  818                 bdev = bdevsw_lookup(dev);
  819                 if (bdev == NULL || bdev->d_psize == NULL ||
  820                     (nblocks = (*bdev->d_psize)(dev)) == -1) {
  821                         error = ENXIO;
  822                         goto bad;
  823                 }
  824                 break;
  825 
  826         case VREG:
  827                 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p)))
  828                         goto bad;
  829                 nblocks = (int)btodb(va.va_size);
  830                 if ((error =
  831                      VFS_STATVFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0)
  832                         goto bad;
  833 
  834                 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize;
  835                 /*
  836                  * limit the max # of outstanding I/O requests we issue
  837                  * at any one time.   take it easy on NFS servers.
  838                  */
  839 #ifdef NFS
  840                 if (vp->v_op == nfsv2_vnodeop_p)
  841                         sdp->swd_maxactive = 2; /* XXX */
  842                 else
  843 #endif /* NFS */
  844                         sdp->swd_maxactive = 8; /* XXX */
  845                 break;
  846 
  847         default:
  848                 error = ENXIO;
  849                 goto bad;
  850         }
  851 
  852         /*
  853          * save nblocks in a safe place and convert to pages.
  854          */
  855 
  856         sdp->swd_ose.ose_nblks = nblocks;
  857         npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT;
  858 
  859         /*
  860          * for block special files, we want to make sure that leave
  861          * the disklabel and bootblocks alone, so we arrange to skip
  862          * over them (arbitrarily choosing to skip PAGE_SIZE bytes).
  863          * note that because of this the "size" can be less than the
  864          * actual number of blocks on the device.
  865          */
  866         if (vp->v_type == VBLK) {
  867                 /* we use pages 1 to (size - 1) [inclusive] */
  868                 size = npages - 1;
  869                 addr = 1;
  870         } else {
  871                 /* we use pages 0 to (size - 1) [inclusive] */
  872                 size = npages;
  873                 addr = 0;
  874         }
  875 
  876         /*
  877          * make sure we have enough blocks for a reasonable sized swap
  878          * area.   we want at least one page.
  879          */
  880 
  881         if (size < 1) {
  882                 UVMHIST_LOG(pdhist, "  size <= 1!!", 0, 0, 0, 0);
  883                 error = EINVAL;
  884                 goto bad;
  885         }
  886 
  887         UVMHIST_LOG(pdhist, "  dev=%x: size=%d addr=%ld\n", dev, size, addr, 0);
  888 
  889         /*
  890          * now we need to allocate an extent to manage this swap device
  891          */
  892         snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x",
  893             count++);
  894 
  895         /* note that extent_create's 3rd arg is inclusive, thus "- 1" */
  896         sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP,
  897                                     0, 0, EX_WAITOK);
  898         /* allocate the `saved' region from the extent so it won't be used */
  899         if (addr) {
  900                 if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK))
  901                         panic("disklabel region");
  902         }
  903 
  904         /*
  905          * if the vnode we are swapping to is the root vnode
  906          * (i.e. we are swapping to the miniroot) then we want
  907          * to make sure we don't overwrite it.   do a statfs to
  908          * find its size and skip over it.
  909          */
  910         if (vp == rootvp) {
  911                 struct mount *mp;
  912                 struct statvfs *sp;
  913                 int rootblocks, rootpages;
  914 
  915                 mp = rootvnode->v_mount;
  916                 sp = &mp->mnt_stat;
  917                 rootblocks = sp->f_blocks * btodb(sp->f_frsize);
  918                 /*
  919                  * XXX: sp->f_blocks isn't the total number of
  920                  * blocks in the filesystem, it's the number of
  921                  * data blocks.  so, our rootblocks almost
  922                  * definitely underestimates the total size
  923                  * of the filesystem - how badly depends on the
  924                  * details of the filesystem type.  there isn't
  925                  * an obvious way to deal with this cleanly
  926                  * and perfectly, so for now we just pad our
  927                  * rootblocks estimate with an extra 5 percent.
  928                  */
  929                 rootblocks += (rootblocks >> 5) +
  930                         (rootblocks >> 6) +
  931                         (rootblocks >> 7);
  932                 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT;
  933                 if (rootpages > size)
  934                         panic("swap_on: miniroot larger than swap?");
  935 
  936                 if (extent_alloc_region(sdp->swd_ex, addr,
  937                                         rootpages, EX_WAITOK))
  938                         panic("swap_on: unable to preserve miniroot");
  939 
  940                 size -= rootpages;
  941                 printf("Preserved %d pages of miniroot ", rootpages);
  942                 printf("leaving %d pages of swap\n", size);
  943         }
  944 
  945         /*
  946          * try to add anons to reflect the new swap space.
  947          */
  948 
  949         error = uvm_anon_add(size);
  950         if (error) {
  951                 goto bad;
  952         }
  953 
  954         /*
  955          * add a ref to vp to reflect usage as a swap device.
  956          */
  957         vref(vp);
  958 
  959         /*
  960          * now add the new swapdev to the drum and enable.
  961          */
  962         if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY,
  963             EX_WAITOK, &result))
  964                 panic("swapdrum_add");
  965 
  966         sdp->swd_drumoffset = (int)result;
  967         sdp->swd_drumsize = npages;
  968         sdp->swd_npages = size;
  969         simple_lock(&uvm.swap_data_lock);
  970         sdp->swd_flags &= ~SWF_FAKE;    /* going live */
  971         sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE);
  972         uvmexp.swpages += size;
  973         uvmexp.swpgavail += size;
  974         simple_unlock(&uvm.swap_data_lock);
  975         return (0);
  976 
  977         /*
  978          * failure: clean up and return error.
  979          */
  980 
  981 bad:
  982         if (sdp->swd_ex) {
  983                 extent_destroy(sdp->swd_ex);
  984         }
  985         if (vp != rootvp) {
  986                 (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p);
  987         }
  988         return (error);
  989 }
  990 
  991 /*
  992  * swap_off: stop swapping on swapdev
  993  *
  994  * => swap data should be locked, we will unlock.
  995  */
  996 static int
  997 swap_off(p, sdp)
  998         struct proc *p;
  999         struct swapdev *sdp;
 1000 {
 1001         int npages =  sdp->swd_npages;
 1002 
 1003         UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist);
 1004         UVMHIST_LOG(pdhist, "  dev=%x, npages=%d", sdp->swd_dev,npages,0,0);
 1005 
 1006         /* disable the swap area being removed */
 1007         sdp->swd_flags &= ~SWF_ENABLE;
 1008         uvmexp.swpgavail -= npages;
 1009         simple_unlock(&uvm.swap_data_lock);
 1010 
 1011         /*
 1012          * the idea is to find all the pages that are paged out to this
 1013          * device, and page them all in.  in uvm, swap-backed pageable
 1014          * memory can take two forms: aobjs and anons.  call the
 1015          * swapoff hook for each subsystem to bring in pages.
 1016          */
 1017 
 1018         if (uao_swap_off(sdp->swd_drumoffset,
 1019                          sdp->swd_drumoffset + sdp->swd_drumsize) ||
 1020             anon_swap_off(sdp->swd_drumoffset,
 1021                           sdp->swd_drumoffset + sdp->swd_drumsize)) {
 1022 
 1023                 simple_lock(&uvm.swap_data_lock);
 1024                 sdp->swd_flags |= SWF_ENABLE;
 1025                 uvmexp.swpgavail += npages;
 1026                 simple_unlock(&uvm.swap_data_lock);
 1027                 return ENOMEM;
 1028         }
 1029         KASSERT(sdp->swd_npginuse == sdp->swd_npgbad);
 1030 
 1031         /*
 1032          * done with the vnode.
 1033          * drop our ref on the vnode before calling VOP_CLOSE()
 1034          * so that spec_close() can tell if this is the last close.
 1035          */
 1036         vrele(sdp->swd_vp);
 1037         if (sdp->swd_vp != rootvp) {
 1038                 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p);
 1039         }
 1040 
 1041         /* remove anons from the system */
 1042         uvm_anon_remove(npages);
 1043 
 1044         simple_lock(&uvm.swap_data_lock);
 1045         uvmexp.swpages -= npages;
 1046         uvmexp.swpginuse -= sdp->swd_npgbad;
 1047 
 1048         if (swaplist_find(sdp->swd_vp, 1) == NULL)
 1049                 panic("swap_off: swapdev not in list");
 1050         swaplist_trim();
 1051         simple_unlock(&uvm.swap_data_lock);
 1052 
 1053         /*
 1054          * free all resources!
 1055          */
 1056         extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize,
 1057                     EX_WAITOK);
 1058         extent_destroy(sdp->swd_ex);
 1059         bufq_free(&sdp->swd_tab);
 1060         free(sdp, M_VMSWAP);
 1061         return (0);
 1062 }
 1063 
 1064 /*
 1065  * /dev/drum interface and i/o functions
 1066  */
 1067 
 1068 /*
 1069  * swread: the read function for the drum (just a call to physio)
 1070  */
 1071 /*ARGSUSED*/
 1072 int
 1073 swread(dev, uio, ioflag)
 1074         dev_t dev;
 1075         struct uio *uio;
 1076         int ioflag;
 1077 {
 1078         UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist);
 1079 
 1080         UVMHIST_LOG(pdhist, "  dev=%x offset=%qx", dev, uio->uio_offset, 0, 0);
 1081         return (physio(swstrategy, NULL, dev, B_READ, minphys, uio));
 1082 }
 1083 
 1084 /*
 1085  * swwrite: the write function for the drum (just a call to physio)
 1086  */
 1087 /*ARGSUSED*/
 1088 int
 1089 swwrite(dev, uio, ioflag)
 1090         dev_t dev;
 1091         struct uio *uio;
 1092         int ioflag;
 1093 {
 1094         UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist);
 1095 
 1096         UVMHIST_LOG(pdhist, "  dev=%x offset=%qx", dev, uio->uio_offset, 0, 0);
 1097         return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio));
 1098 }
 1099 
 1100 /*
 1101  * swstrategy: perform I/O on the drum
 1102  *
 1103  * => we must map the i/o request from the drum to the correct swapdev.
 1104  */
 1105 void
 1106 swstrategy(bp)
 1107         struct buf *bp;
 1108 {
 1109         struct swapdev *sdp;
 1110         struct vnode *vp;
 1111         int s, pageno, bn;
 1112         UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist);
 1113 
 1114         /*
 1115          * convert block number to swapdev.   note that swapdev can't
 1116          * be yanked out from under us because we are holding resources
 1117          * in it (i.e. the blocks we are doing I/O on).
 1118          */
 1119         pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT;
 1120         simple_lock(&uvm.swap_data_lock);
 1121         sdp = swapdrum_getsdp(pageno);
 1122         simple_unlock(&uvm.swap_data_lock);
 1123         if (sdp == NULL) {
 1124                 bp->b_error = EINVAL;
 1125                 bp->b_flags |= B_ERROR;
 1126                 biodone(bp);
 1127                 UVMHIST_LOG(pdhist, "  failed to get swap device", 0, 0, 0, 0);
 1128                 return;
 1129         }
 1130 
 1131         /*
 1132          * convert drum page number to block number on this swapdev.
 1133          */
 1134 
 1135         pageno -= sdp->swd_drumoffset;  /* page # on swapdev */
 1136         bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */
 1137 
 1138         UVMHIST_LOG(pdhist, "  %s: mapoff=%x bn=%x bcount=%ld",
 1139                 ((bp->b_flags & B_READ) == 0) ? "write" : "read",
 1140                 sdp->swd_drumoffset, bn, bp->b_bcount);
 1141 
 1142         /*
 1143          * for block devices we finish up here.
 1144          * for regular files we have to do more work which we delegate
 1145          * to sw_reg_strategy().
 1146          */
 1147 
 1148         switch (sdp->swd_vp->v_type) {
 1149         default:
 1150                 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type);
 1151 
 1152         case VBLK:
 1153 
 1154                 /*
 1155                  * must convert "bp" from an I/O on /dev/drum to an I/O
 1156                  * on the swapdev (sdp).
 1157                  */
 1158                 s = splbio();
 1159                 bp->b_blkno = bn;               /* swapdev block number */
 1160                 vp = sdp->swd_vp;               /* swapdev vnode pointer */
 1161                 bp->b_dev = sdp->swd_dev;       /* swapdev dev_t */
 1162 
 1163                 /*
 1164                  * if we are doing a write, we have to redirect the i/o on
 1165                  * drum's v_numoutput counter to the swapdevs.
 1166                  */
 1167                 if ((bp->b_flags & B_READ) == 0) {
 1168                         vwakeup(bp);    /* kills one 'v_numoutput' on drum */
 1169                         V_INCR_NUMOUTPUT(vp);   /* put it on swapdev */
 1170                 }
 1171 
 1172                 /*
 1173                  * finally plug in swapdev vnode and start I/O
 1174                  */
 1175                 bp->b_vp = vp;
 1176                 splx(s);
 1177                 VOP_STRATEGY(vp, bp);
 1178                 return;
 1179 
 1180         case VREG:
 1181                 /*
 1182                  * delegate to sw_reg_strategy function.
 1183                  */
 1184                 sw_reg_strategy(sdp, bp, bn);
 1185                 return;
 1186         }
 1187         /* NOTREACHED */
 1188 }
 1189 
 1190 /*
 1191  * sw_reg_strategy: handle swap i/o to regular files
 1192  */
 1193 static void
 1194 sw_reg_strategy(sdp, bp, bn)
 1195         struct swapdev  *sdp;
 1196         struct buf      *bp;
 1197         int             bn;
 1198 {
 1199         struct vnode    *vp;
 1200         struct vndxfer  *vnx;
 1201         daddr_t         nbn;
 1202         caddr_t         addr;
 1203         off_t           byteoff;
 1204         int             s, off, nra, error, sz, resid;
 1205         UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist);
 1206 
 1207         /*
 1208          * allocate a vndxfer head for this transfer and point it to
 1209          * our buffer.
 1210          */
 1211         getvndxfer(vnx);
 1212         vnx->vx_flags = VX_BUSY;
 1213         vnx->vx_error = 0;
 1214         vnx->vx_pending = 0;
 1215         vnx->vx_bp = bp;
 1216         vnx->vx_sdp = sdp;
 1217 
 1218         /*
 1219          * setup for main loop where we read filesystem blocks into
 1220          * our buffer.
 1221          */
 1222         error = 0;
 1223         bp->b_resid = bp->b_bcount;     /* nothing transfered yet! */
 1224         addr = bp->b_data;              /* current position in buffer */
 1225         byteoff = dbtob((u_int64_t)bn);
 1226 
 1227         for (resid = bp->b_resid; resid; resid -= sz) {
 1228                 struct vndbuf   *nbp;
 1229 
 1230                 /*
 1231                  * translate byteoffset into block number.  return values:
 1232                  *   vp = vnode of underlying device
 1233                  *  nbn = new block number (on underlying vnode dev)
 1234                  *  nra = num blocks we can read-ahead (excludes requested
 1235                  *      block)
 1236                  */
 1237                 nra = 0;
 1238                 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize,
 1239                                         &vp, &nbn, &nra);
 1240 
 1241                 if (error == 0 && nbn == (daddr_t)-1) {
 1242                         /*
 1243                          * this used to just set error, but that doesn't
 1244                          * do the right thing.  Instead, it causes random
 1245                          * memory errors.  The panic() should remain until
 1246                          * this condition doesn't destabilize the system.
 1247                          */
 1248 #if 1
 1249                         panic("sw_reg_strategy: swap to sparse file");
 1250 #else
 1251                         error = EIO;    /* failure */
 1252 #endif
 1253                 }
 1254 
 1255                 /*
 1256                  * punt if there was an error or a hole in the file.
 1257                  * we must wait for any i/o ops we have already started
 1258                  * to finish before returning.
 1259                  *
 1260                  * XXX we could deal with holes here but it would be
 1261                  * a hassle (in the write case).
 1262                  */
 1263                 if (error) {
 1264                         s = splbio();
 1265                         vnx->vx_error = error;  /* pass error up */
 1266                         goto out;
 1267                 }
 1268 
 1269                 /*
 1270                  * compute the size ("sz") of this transfer (in bytes).
 1271                  */
 1272                 off = byteoff % sdp->swd_bsize;
 1273                 sz = (1 + nra) * sdp->swd_bsize - off;
 1274                 if (sz > resid)
 1275                         sz = resid;
 1276 
 1277                 UVMHIST_LOG(pdhist, "sw_reg_strategy: "
 1278                             "vp %p/%p offset 0x%x/0x%x",
 1279                             sdp->swd_vp, vp, byteoff, nbn);
 1280 
 1281                 /*
 1282                  * now get a buf structure.   note that the vb_buf is
 1283                  * at the front of the nbp structure so that you can
 1284                  * cast pointers between the two structure easily.
 1285                  */
 1286                 getvndbuf(nbp);
 1287                 BUF_INIT(&nbp->vb_buf);
 1288                 nbp->vb_buf.b_flags    = bp->b_flags | B_CALL;
 1289                 nbp->vb_buf.b_bcount   = sz;
 1290                 nbp->vb_buf.b_bufsize  = sz;
 1291                 nbp->vb_buf.b_error    = 0;
 1292                 nbp->vb_buf.b_data     = addr;
 1293                 nbp->vb_buf.b_lblkno   = 0;
 1294                 nbp->vb_buf.b_blkno    = nbn + btodb(off);
 1295                 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno;
 1296                 nbp->vb_buf.b_iodone   = sw_reg_iodone;
 1297                 nbp->vb_buf.b_vp       = vp;
 1298                 if (vp->v_type == VBLK) {
 1299                         nbp->vb_buf.b_dev = vp->v_rdev;
 1300                 }
 1301 
 1302                 nbp->vb_xfer = vnx;     /* patch it back in to vnx */
 1303 
 1304                 /*
 1305                  * Just sort by block number
 1306                  */
 1307                 s = splbio();
 1308                 if (vnx->vx_error != 0) {
 1309                         putvndbuf(nbp);
 1310                         goto out;
 1311                 }
 1312                 vnx->vx_pending++;
 1313 
 1314                 /* sort it in and start I/O if we are not over our limit */
 1315                 BUFQ_PUT(&sdp->swd_tab, &nbp->vb_buf);
 1316                 sw_reg_start(sdp);
 1317                 splx(s);
 1318 
 1319                 /*
 1320                  * advance to the next I/O
 1321                  */
 1322                 byteoff += sz;
 1323                 addr += sz;
 1324         }
 1325 
 1326         s = splbio();
 1327 
 1328 out: /* Arrive here at splbio */
 1329         vnx->vx_flags &= ~VX_BUSY;
 1330         if (vnx->vx_pending == 0) {
 1331                 if (vnx->vx_error != 0) {
 1332                         bp->b_error = vnx->vx_error;
 1333                         bp->b_flags |= B_ERROR;
 1334                 }
 1335                 putvndxfer(vnx);
 1336                 biodone(bp);
 1337         }
 1338         splx(s);
 1339 }
 1340 
 1341 /*
 1342  * sw_reg_start: start an I/O request on the requested swapdev
 1343  *
 1344  * => reqs are sorted by b_rawblkno (above)
 1345  */
 1346 static void
 1347 sw_reg_start(sdp)
 1348         struct swapdev  *sdp;
 1349 {
 1350         struct buf      *bp;
 1351         UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist);
 1352 
 1353         /* recursion control */
 1354         if ((sdp->swd_flags & SWF_BUSY) != 0)
 1355                 return;
 1356 
 1357         sdp->swd_flags |= SWF_BUSY;
 1358 
 1359         while (sdp->swd_active < sdp->swd_maxactive) {
 1360                 bp = BUFQ_GET(&sdp->swd_tab);
 1361                 if (bp == NULL)
 1362                         break;
 1363                 sdp->swd_active++;
 1364 
 1365                 UVMHIST_LOG(pdhist,
 1366                     "sw_reg_start:  bp %p vp %p blkno %p cnt %lx",
 1367                     bp, bp->b_vp, bp->b_blkno, bp->b_bcount);
 1368                 if ((bp->b_flags & B_READ) == 0)
 1369                         V_INCR_NUMOUTPUT(bp->b_vp);
 1370 
 1371                 VOP_STRATEGY(bp->b_vp, bp);
 1372         }
 1373         sdp->swd_flags &= ~SWF_BUSY;
 1374 }
 1375 
 1376 /*
 1377  * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup
 1378  *
 1379  * => note that we can recover the vndbuf struct by casting the buf ptr
 1380  */
 1381 static void
 1382 sw_reg_iodone(bp)
 1383         struct buf *bp;
 1384 {
 1385         struct vndbuf *vbp = (struct vndbuf *) bp;
 1386         struct vndxfer *vnx = vbp->vb_xfer;
 1387         struct buf *pbp = vnx->vx_bp;           /* parent buffer */
 1388         struct swapdev  *sdp = vnx->vx_sdp;
 1389         int s, resid, error;
 1390         UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist);
 1391 
 1392         UVMHIST_LOG(pdhist, "  vbp=%p vp=%p blkno=%x addr=%p",
 1393             vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data);
 1394         UVMHIST_LOG(pdhist, "  cnt=%lx resid=%lx",
 1395             vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0);
 1396 
 1397         /*
 1398          * protect vbp at splbio and update.
 1399          */
 1400 
 1401         s = splbio();
 1402         resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid;
 1403         pbp->b_resid -= resid;
 1404         vnx->vx_pending--;
 1405 
 1406         if (vbp->vb_buf.b_flags & B_ERROR) {
 1407                 /* pass error upward */
 1408                 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO;
 1409                 UVMHIST_LOG(pdhist, "  got error=%d !", error, 0, 0, 0);
 1410                 vnx->vx_error = error;
 1411         }
 1412 
 1413         /*
 1414          * kill vbp structure
 1415          */
 1416         putvndbuf(vbp);
 1417 
 1418         /*
 1419          * wrap up this transaction if it has run to completion or, in
 1420          * case of an error, when all auxiliary buffers have returned.
 1421          */
 1422         if (vnx->vx_error != 0) {
 1423                 /* pass error upward */
 1424                 pbp->b_flags |= B_ERROR;
 1425                 pbp->b_error = vnx->vx_error;
 1426                 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) {
 1427                         putvndxfer(vnx);
 1428                         biodone(pbp);
 1429                 }
 1430         } else if (pbp->b_resid == 0) {
 1431                 KASSERT(vnx->vx_pending == 0);
 1432                 if ((vnx->vx_flags & VX_BUSY) == 0) {
 1433                         UVMHIST_LOG(pdhist, "  iodone error=%d !",
 1434                             pbp, vnx->vx_error, 0, 0);
 1435                         putvndxfer(vnx);
 1436                         biodone(pbp);
 1437                 }
 1438         }
 1439 
 1440         /*
 1441          * done!   start next swapdev I/O if one is pending
 1442          */
 1443         sdp->swd_active--;
 1444         sw_reg_start(sdp);
 1445         splx(s);
 1446 }
 1447 
 1448 
 1449 /*
 1450  * uvm_swap_alloc: allocate space on swap
 1451  *
 1452  * => allocation is done "round robin" down the priority list, as we
 1453  *      allocate in a priority we "rotate" the circle queue.
 1454  * => space can be freed with uvm_swap_free
 1455  * => we return the page slot number in /dev/drum (0 == invalid slot)
 1456  * => we lock uvm.swap_data_lock
 1457  * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM
 1458  */
 1459 int
 1460 uvm_swap_alloc(nslots, lessok)
 1461         int *nslots;    /* IN/OUT */
 1462         boolean_t lessok;
 1463 {
 1464         struct swapdev *sdp;
 1465         struct swappri *spp;
 1466         u_long  result;
 1467         UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist);
 1468 
 1469         /*
 1470          * no swap devices configured yet?   definite failure.
 1471          */
 1472         if (uvmexp.nswapdev < 1)
 1473                 return 0;
 1474 
 1475         /*
 1476          * lock data lock, convert slots into blocks, and enter loop
 1477          */
 1478         simple_lock(&uvm.swap_data_lock);
 1479 
 1480 ReTry:  /* XXXMRG */
 1481         LIST_FOREACH(spp, &swap_priority, spi_swappri) {
 1482                 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
 1483                         /* if it's not enabled, then we can't swap from it */
 1484                         if ((sdp->swd_flags & SWF_ENABLE) == 0)
 1485                                 continue;
 1486                         if (sdp->swd_npginuse + *nslots > sdp->swd_npages)
 1487                                 continue;
 1488                         if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN,
 1489                                          EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT,
 1490                                          &result) != 0) {
 1491                                 continue;
 1492                         }
 1493 
 1494                         /*
 1495                          * successful allocation!  now rotate the circleq.
 1496                          */
 1497                         CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next);
 1498                         CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
 1499                         sdp->swd_npginuse += *nslots;
 1500                         uvmexp.swpginuse += *nslots;
 1501                         simple_unlock(&uvm.swap_data_lock);
 1502                         /* done!  return drum slot number */
 1503                         UVMHIST_LOG(pdhist,
 1504                             "success!  returning %d slots starting at %d",
 1505                             *nslots, result + sdp->swd_drumoffset, 0, 0);
 1506                         return (result + sdp->swd_drumoffset);
 1507                 }
 1508         }
 1509 
 1510         /* XXXMRG: BEGIN HACK */
 1511         if (*nslots > 1 && lessok) {
 1512                 *nslots = 1;
 1513                 goto ReTry;     /* XXXMRG: ugh!  extent should support this for us */
 1514         }
 1515         /* XXXMRG: END HACK */
 1516 
 1517         simple_unlock(&uvm.swap_data_lock);
 1518         return 0;
 1519 }
 1520 
 1521 boolean_t
 1522 uvm_swapisfull(void)
 1523 {
 1524         boolean_t rv;
 1525 
 1526         simple_lock(&uvm.swap_data_lock);
 1527         KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
 1528         rv = (uvmexp.swpgonly >= uvmexp.swpgavail);
 1529         simple_unlock(&uvm.swap_data_lock);
 1530 
 1531         return (rv);
 1532 }
 1533 
 1534 /*
 1535  * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors
 1536  *
 1537  * => we lock uvm.swap_data_lock
 1538  */
 1539 void
 1540 uvm_swap_markbad(startslot, nslots)
 1541         int startslot;
 1542         int nslots;
 1543 {
 1544         struct swapdev *sdp;
 1545         UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist);
 1546 
 1547         simple_lock(&uvm.swap_data_lock);
 1548         sdp = swapdrum_getsdp(startslot);
 1549         KASSERT(sdp != NULL);
 1550 
 1551         /*
 1552          * we just keep track of how many pages have been marked bad
 1553          * in this device, to make everything add up in swap_off().
 1554          * we assume here that the range of slots will all be within
 1555          * one swap device.
 1556          */
 1557 
 1558         KASSERT(uvmexp.swpgonly >= nslots);
 1559         uvmexp.swpgonly -= nslots;
 1560         sdp->swd_npgbad += nslots;
 1561         UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0);
 1562         simple_unlock(&uvm.swap_data_lock);
 1563 }
 1564 
 1565 /*
 1566  * uvm_swap_free: free swap slots
 1567  *
 1568  * => this can be all or part of an allocation made by uvm_swap_alloc
 1569  * => we lock uvm.swap_data_lock
 1570  */
 1571 void
 1572 uvm_swap_free(startslot, nslots)
 1573         int startslot;
 1574         int nslots;
 1575 {
 1576         struct swapdev *sdp;
 1577         UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist);
 1578 
 1579         UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots,
 1580             startslot, 0, 0);
 1581 
 1582         /*
 1583          * ignore attempts to free the "bad" slot.
 1584          */
 1585 
 1586         if (startslot == SWSLOT_BAD) {
 1587                 return;
 1588         }
 1589 
 1590         /*
 1591          * convert drum slot offset back to sdp, free the blocks
 1592          * in the extent, and return.   must hold pri lock to do
 1593          * lookup and access the extent.
 1594          */
 1595 
 1596         simple_lock(&uvm.swap_data_lock);
 1597         sdp = swapdrum_getsdp(startslot);
 1598         KASSERT(uvmexp.nswapdev >= 1);
 1599         KASSERT(sdp != NULL);
 1600         KASSERT(sdp->swd_npginuse >= nslots);
 1601         if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots,
 1602                         EX_MALLOCOK|EX_NOWAIT) != 0) {
 1603                 printf("warning: resource shortage: %d pages of swap lost\n",
 1604                         nslots);
 1605         }
 1606         sdp->swd_npginuse -= nslots;
 1607         uvmexp.swpginuse -= nslots;
 1608         simple_unlock(&uvm.swap_data_lock);
 1609 }
 1610 
 1611 /*
 1612  * uvm_swap_put: put any number of pages into a contig place on swap
 1613  *
 1614  * => can be sync or async
 1615  */
 1616 
 1617 int
 1618 uvm_swap_put(swslot, ppsp, npages, flags)
 1619         int swslot;
 1620         struct vm_page **ppsp;
 1621         int npages;
 1622         int flags;
 1623 {
 1624         int error;
 1625 
 1626         error = uvm_swap_io(ppsp, swslot, npages, B_WRITE |
 1627             ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
 1628         return error;
 1629 }
 1630 
 1631 /*
 1632  * uvm_swap_get: get a single page from swap
 1633  *
 1634  * => usually a sync op (from fault)
 1635  */
 1636 
 1637 int
 1638 uvm_swap_get(page, swslot, flags)
 1639         struct vm_page *page;
 1640         int swslot, flags;
 1641 {
 1642         int error;
 1643 
 1644         uvmexp.nswget++;
 1645         KASSERT(flags & PGO_SYNCIO);
 1646         if (swslot == SWSLOT_BAD) {
 1647                 return EIO;
 1648         }
 1649 
 1650         error = uvm_swap_io(&page, swslot, 1, B_READ |
 1651             ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
 1652         if (error == 0) {
 1653 
 1654                 /*
 1655                  * this page is no longer only in swap.
 1656                  */
 1657 
 1658                 simple_lock(&uvm.swap_data_lock);
 1659                 KASSERT(uvmexp.swpgonly > 0);
 1660                 uvmexp.swpgonly--;
 1661                 simple_unlock(&uvm.swap_data_lock);
 1662         }
 1663         return error;
 1664 }
 1665 
 1666 /*
 1667  * uvm_swap_io: do an i/o operation to swap
 1668  */
 1669 
 1670 static int
 1671 uvm_swap_io(pps, startslot, npages, flags)
 1672         struct vm_page **pps;
 1673         int startslot, npages, flags;
 1674 {
 1675         daddr_t startblk;
 1676         struct  buf *bp;
 1677         vaddr_t kva;
 1678         int     error, s, mapinflags;
 1679         boolean_t write, async;
 1680         UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist);
 1681 
 1682         UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d",
 1683             startslot, npages, flags, 0);
 1684 
 1685         write = (flags & B_READ) == 0;
 1686         async = (flags & B_ASYNC) != 0;
 1687 
 1688         /*
 1689          * convert starting drum slot to block number
 1690          */
 1691 
 1692         startblk = btodb((u_int64_t)startslot << PAGE_SHIFT);
 1693 
 1694         /*
 1695          * first, map the pages into the kernel.
 1696          */
 1697 
 1698         mapinflags = !write ?
 1699                 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ :
 1700                 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE;
 1701         kva = uvm_pagermapin(pps, npages, mapinflags);
 1702 
 1703         /*
 1704          * now allocate a buf for the i/o.
 1705          */
 1706 
 1707         s = splbio();
 1708         bp = pool_get(&bufpool, PR_WAITOK);
 1709         splx(s);
 1710 
 1711         /*
 1712          * fill in the bp/sbp.   we currently route our i/o through
 1713          * /dev/drum's vnode [swapdev_vp].
 1714          */
 1715 
 1716         BUF_INIT(bp);
 1717         bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC));
 1718         bp->b_proc = &proc0;    /* XXX */
 1719         bp->b_vnbufs.le_next = NOLIST;
 1720         bp->b_data = (caddr_t)kva;
 1721         bp->b_blkno = startblk;
 1722         bp->b_vp = swapdev_vp;
 1723         bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT;
 1724 
 1725         /*
 1726          * bump v_numoutput (counter of number of active outputs).
 1727          */
 1728 
 1729         if (write) {
 1730                 s = splbio();
 1731                 V_INCR_NUMOUTPUT(swapdev_vp);
 1732                 splx(s);
 1733         }
 1734 
 1735         /*
 1736          * for async ops we must set up the iodone handler.
 1737          */
 1738 
 1739         if (async) {
 1740                 bp->b_flags |= B_CALL;
 1741                 bp->b_iodone = uvm_aio_biodone;
 1742                 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0);
 1743                 if (curproc == uvm.pagedaemon_proc)
 1744                         BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
 1745                 else
 1746                         BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
 1747         } else {
 1748                 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
 1749         }
 1750         UVMHIST_LOG(pdhist,
 1751             "about to start io: data = %p blkno = 0x%x, bcount = %ld",
 1752             bp->b_data, bp->b_blkno, bp->b_bcount, 0);
 1753 
 1754         /*
 1755          * now we start the I/O, and if async, return.
 1756          */
 1757 
 1758         VOP_STRATEGY(swapdev_vp, bp);
 1759         if (async)
 1760                 return 0;
 1761 
 1762         /*
 1763          * must be sync i/o.   wait for it to finish
 1764          */
 1765 
 1766         error = biowait(bp);
 1767 
 1768         /*
 1769          * kill the pager mapping
 1770          */
 1771 
 1772         uvm_pagermapout(kva, npages);
 1773 
 1774         /*
 1775          * now dispose of the buf and we're done.
 1776          */
 1777 
 1778         s = splbio();
 1779         if (write)
 1780                 vwakeup(bp);
 1781         pool_put(&bufpool, bp);
 1782         splx(s);
 1783         UVMHIST_LOG(pdhist, "<- done (sync)  error=%d", error, 0, 0, 0);
 1784         return (error);
 1785 }

Cache object: 603c0381e92cde33f0c458f6bbc65acb


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.