The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_bio.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2004 Poul-Henning Kamp
    5  * Copyright (c) 1994,1997 John S. Dyson
    6  * Copyright (c) 2013 The FreeBSD Foundation
    7  * All rights reserved.
    8  *
    9  * Portions of this software were developed by Konstantin Belousov
   10  * under sponsorship from the FreeBSD Foundation.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  *
   21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   31  * SUCH DAMAGE.
   32  */
   33 
   34 /*
   35  * this file contains a new buffer I/O scheme implementing a coherent
   36  * VM object and buffer cache scheme.  Pains have been taken to make
   37  * sure that the performance degradation associated with schemes such
   38  * as this is not realized.
   39  *
   40  * Author:  John S. Dyson
   41  * Significant help during the development and debugging phases
   42  * had been provided by David Greenman, also of the FreeBSD core team.
   43  *
   44  * see man buf(9) for more info.
   45  */
   46 
   47 #include <sys/cdefs.h>
   48 __FBSDID("$FreeBSD: head/sys/kern/vfs_bio.c 334506 2018-06-01 23:49:32Z markj $");
   49 
   50 #include <sys/param.h>
   51 #include <sys/systm.h>
   52 #include <sys/bio.h>
   53 #include <sys/bitset.h>
   54 #include <sys/conf.h>
   55 #include <sys/counter.h>
   56 #include <sys/buf.h>
   57 #include <sys/devicestat.h>
   58 #include <sys/eventhandler.h>
   59 #include <sys/fail.h>
   60 #include <sys/limits.h>
   61 #include <sys/lock.h>
   62 #include <sys/malloc.h>
   63 #include <sys/mount.h>
   64 #include <sys/mutex.h>
   65 #include <sys/kernel.h>
   66 #include <sys/kthread.h>
   67 #include <sys/proc.h>
   68 #include <sys/racct.h>
   69 #include <sys/resourcevar.h>
   70 #include <sys/rwlock.h>
   71 #include <sys/smp.h>
   72 #include <sys/sysctl.h>
   73 #include <sys/sysproto.h>
   74 #include <sys/vmem.h>
   75 #include <sys/vmmeter.h>
   76 #include <sys/vnode.h>
   77 #include <sys/watchdog.h>
   78 #include <geom/geom.h>
   79 #include <vm/vm.h>
   80 #include <vm/vm_param.h>
   81 #include <vm/vm_kern.h>
   82 #include <vm/vm_object.h>
   83 #include <vm/vm_page.h>
   84 #include <vm/vm_pageout.h>
   85 #include <vm/vm_pager.h>
   86 #include <vm/vm_extern.h>
   87 #include <vm/vm_map.h>
   88 #include <vm/swap_pager.h>
   89 #include "opt_swap.h"
   90 
   91 static MALLOC_DEFINE(M_BIOBUF, "biobuf", "BIO buffer");
   92 
   93 struct  bio_ops bioops;         /* I/O operation notification */
   94 
   95 struct  buf_ops buf_ops_bio = {
   96         .bop_name       =       "buf_ops_bio",
   97         .bop_write      =       bufwrite,
   98         .bop_strategy   =       bufstrategy,
   99         .bop_sync       =       bufsync,
  100         .bop_bdflush    =       bufbdflush,
  101 };
  102 
  103 struct bufqueue {
  104         struct mtx_padalign     bq_lock;
  105         TAILQ_HEAD(, buf)       bq_queue;
  106         uint8_t                 bq_index;
  107         uint16_t                bq_subqueue;
  108         int                     bq_len;
  109 } __aligned(CACHE_LINE_SIZE);
  110 
  111 #define BQ_LOCKPTR(bq)          (&(bq)->bq_lock)
  112 #define BQ_LOCK(bq)             mtx_lock(BQ_LOCKPTR((bq)))
  113 #define BQ_UNLOCK(bq)           mtx_unlock(BQ_LOCKPTR((bq)))
  114 #define BQ_ASSERT_LOCKED(bq)    mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED)
  115 
  116 struct bufdomain {
  117         struct bufqueue bd_subq[MAXCPU + 1]; /* Per-cpu sub queues + global */
  118         struct bufqueue bd_dirtyq;
  119         struct bufqueue *bd_cleanq;
  120         struct mtx_padalign bd_run_lock;
  121         /* Constants */
  122         long            bd_maxbufspace;
  123         long            bd_hibufspace;
  124         long            bd_lobufspace;
  125         long            bd_bufspacethresh;
  126         int             bd_hifreebuffers;
  127         int             bd_lofreebuffers;
  128         int             bd_hidirtybuffers;
  129         int             bd_lodirtybuffers;
  130         int             bd_dirtybufthresh;
  131         int             bd_lim;
  132         /* atomics */
  133         int             bd_wanted;
  134         int __aligned(CACHE_LINE_SIZE)  bd_numdirtybuffers;
  135         int __aligned(CACHE_LINE_SIZE)  bd_running;
  136         long __aligned(CACHE_LINE_SIZE) bd_bufspace;
  137         int __aligned(CACHE_LINE_SIZE)  bd_freebuffers;
  138 } __aligned(CACHE_LINE_SIZE);
  139 
  140 #define BD_LOCKPTR(bd)          (&(bd)->bd_cleanq->bq_lock)
  141 #define BD_LOCK(bd)             mtx_lock(BD_LOCKPTR((bd)))
  142 #define BD_UNLOCK(bd)           mtx_unlock(BD_LOCKPTR((bd)))
  143 #define BD_ASSERT_LOCKED(bd)    mtx_assert(BD_LOCKPTR((bd)), MA_OWNED)
  144 #define BD_RUN_LOCKPTR(bd)      (&(bd)->bd_run_lock)
  145 #define BD_RUN_LOCK(bd)         mtx_lock(BD_RUN_LOCKPTR((bd)))
  146 #define BD_RUN_UNLOCK(bd)       mtx_unlock(BD_RUN_LOCKPTR((bd)))
  147 #define BD_DOMAIN(bd)           (bd - bdomain)
  148 
  149 static struct buf *buf;         /* buffer header pool */
  150 extern struct buf *swbuf;       /* Swap buffer header pool. */
  151 caddr_t unmapped_buf;
  152 
  153 /* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
  154 struct proc *bufdaemonproc;
  155 
  156 static int inmem(struct vnode *vp, daddr_t blkno);
  157 static void vm_hold_free_pages(struct buf *bp, int newbsize);
  158 static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
  159                 vm_offset_t to);
  160 static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
  161 static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
  162                 vm_page_t m);
  163 static void vfs_clean_pages_dirty_buf(struct buf *bp);
  164 static void vfs_setdirty_locked_object(struct buf *bp);
  165 static void vfs_vmio_invalidate(struct buf *bp);
  166 static void vfs_vmio_truncate(struct buf *bp, int npages);
  167 static void vfs_vmio_extend(struct buf *bp, int npages, int size);
  168 static int vfs_bio_clcheck(struct vnode *vp, int size,
  169                 daddr_t lblkno, daddr_t blkno);
  170 static void breada(struct vnode *, daddr_t *, int *, int, struct ucred *, int,
  171                 void (*)(struct buf *));
  172 static int buf_flush(struct vnode *vp, struct bufdomain *, int);
  173 static int flushbufqueues(struct vnode *, struct bufdomain *, int, int);
  174 static void buf_daemon(void);
  175 static __inline void bd_wakeup(void);
  176 static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
  177 static void bufkva_reclaim(vmem_t *, int);
  178 static void bufkva_free(struct buf *);
  179 static int buf_import(void *, void **, int, int, int);
  180 static void buf_release(void *, void **, int);
  181 static void maxbcachebuf_adjust(void);
  182 static inline struct bufdomain *bufdomain(struct buf *);
  183 static void bq_remove(struct bufqueue *bq, struct buf *bp);
  184 static void bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock);
  185 static int buf_recycle(struct bufdomain *, bool kva);
  186 static void bq_init(struct bufqueue *bq, int qindex, int cpu,
  187             const char *lockname);
  188 static void bd_init(struct bufdomain *bd);
  189 static int bd_flushall(struct bufdomain *bd);
  190 static int sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS);
  191 static int sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS);
  192 
  193 static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
  194 int vmiodirenable = TRUE;
  195 SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW, &vmiodirenable, 0,
  196     "Use the VM system for directory writes");
  197 long runningbufspace;
  198 SYSCTL_LONG(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, &runningbufspace, 0,
  199     "Amount of presently outstanding async buffer io");
  200 SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RD,
  201     NULL, 0, sysctl_bufspace, "L", "Physical memory used for buffers");
  202 static counter_u64_t bufkvaspace;
  203 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace,
  204     "Kernel virtual memory used for buffers");
  205 static long maxbufspace;
  206 SYSCTL_PROC(_vfs, OID_AUTO, maxbufspace,
  207     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &maxbufspace,
  208     __offsetof(struct bufdomain, bd_maxbufspace), sysctl_bufdomain_long, "L",
  209     "Maximum allowed value of bufspace (including metadata)");
  210 static long bufmallocspace;
  211 SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
  212     "Amount of malloced memory for buffers");
  213 static long maxbufmallocspace;
  214 SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
  215     0, "Maximum amount of malloced memory for buffers");
  216 static long lobufspace;
  217 SYSCTL_PROC(_vfs, OID_AUTO, lobufspace,
  218     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &lobufspace,
  219     __offsetof(struct bufdomain, bd_lobufspace), sysctl_bufdomain_long, "L",
  220     "Minimum amount of buffers we want to have");
  221 long hibufspace;
  222 SYSCTL_PROC(_vfs, OID_AUTO, hibufspace,
  223     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &hibufspace,
  224     __offsetof(struct bufdomain, bd_hibufspace), sysctl_bufdomain_long, "L",
  225     "Maximum allowed value of bufspace (excluding metadata)");
  226 long bufspacethresh;
  227 SYSCTL_PROC(_vfs, OID_AUTO, bufspacethresh,
  228     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &bufspacethresh,
  229     __offsetof(struct bufdomain, bd_bufspacethresh), sysctl_bufdomain_long, "L",
  230     "Bufspace consumed before waking the daemon to free some");
  231 static counter_u64_t buffreekvacnt;
  232 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt,
  233     "Number of times we have freed the KVA space from some buffer");
  234 static counter_u64_t bufdefragcnt;
  235 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW, &bufdefragcnt,
  236     "Number of times we have had to repeat buffer allocation to defragment");
  237 static long lorunningspace;
  238 SYSCTL_PROC(_vfs, OID_AUTO, lorunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
  239     CTLFLAG_RW, &lorunningspace, 0, sysctl_runningspace, "L",
  240     "Minimum preferred space used for in-progress I/O");
  241 static long hirunningspace;
  242 SYSCTL_PROC(_vfs, OID_AUTO, hirunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
  243     CTLFLAG_RW, &hirunningspace, 0, sysctl_runningspace, "L",
  244     "Maximum amount of space to use for in-progress I/O");
  245 int dirtybufferflushes;
  246 SYSCTL_INT(_vfs, OID_AUTO, dirtybufferflushes, CTLFLAG_RW, &dirtybufferflushes,
  247     0, "Number of bdwrite to bawrite conversions to limit dirty buffers");
  248 int bdwriteskip;
  249 SYSCTL_INT(_vfs, OID_AUTO, bdwriteskip, CTLFLAG_RW, &bdwriteskip,
  250     0, "Number of buffers supplied to bdwrite with snapshot deadlock risk");
  251 int altbufferflushes;
  252 SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFLAG_RW, &altbufferflushes,
  253     0, "Number of fsync flushes to limit dirty buffers");
  254 static int recursiveflushes;
  255 SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW, &recursiveflushes,
  256     0, "Number of flushes skipped due to being recursive");
  257 static int sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS);
  258 SYSCTL_PROC(_vfs, OID_AUTO, numdirtybuffers,
  259     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RD, NULL, 0, sysctl_numdirtybuffers, "I",
  260     "Number of buffers that are dirty (has unwritten changes) at the moment");
  261 static int lodirtybuffers;
  262 SYSCTL_PROC(_vfs, OID_AUTO, lodirtybuffers,
  263     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lodirtybuffers,
  264     __offsetof(struct bufdomain, bd_lodirtybuffers), sysctl_bufdomain_int, "I",
  265     "How many buffers we want to have free before bufdaemon can sleep");
  266 static int hidirtybuffers;
  267 SYSCTL_PROC(_vfs, OID_AUTO, hidirtybuffers,
  268     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hidirtybuffers,
  269     __offsetof(struct bufdomain, bd_hidirtybuffers), sysctl_bufdomain_int, "I",
  270     "When the number of dirty buffers is considered severe");
  271 int dirtybufthresh;
  272 SYSCTL_PROC(_vfs, OID_AUTO, dirtybufthresh,
  273     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &dirtybufthresh,
  274     __offsetof(struct bufdomain, bd_dirtybufthresh), sysctl_bufdomain_int, "I",
  275     "Number of bdwrite to bawrite conversions to clear dirty buffers");
  276 static int numfreebuffers;
  277 SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
  278     "Number of free buffers");
  279 static int lofreebuffers;
  280 SYSCTL_PROC(_vfs, OID_AUTO, lofreebuffers,
  281     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lofreebuffers,
  282     __offsetof(struct bufdomain, bd_lofreebuffers), sysctl_bufdomain_int, "I",
  283    "Target number of free buffers");
  284 static int hifreebuffers;
  285 SYSCTL_PROC(_vfs, OID_AUTO, hifreebuffers,
  286     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hifreebuffers,
  287     __offsetof(struct bufdomain, bd_hifreebuffers), sysctl_bufdomain_int, "I",
  288    "Threshold for clean buffer recycling");
  289 static counter_u64_t getnewbufcalls;
  290 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RD,
  291    &getnewbufcalls, "Number of calls to getnewbuf");
  292 static counter_u64_t getnewbufrestarts;
  293 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RD,
  294     &getnewbufrestarts,
  295     "Number of times getnewbuf has had to restart a buffer acquisition");
  296 static counter_u64_t mappingrestarts;
  297 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RD,
  298     &mappingrestarts,
  299     "Number of times getblk has had to restart a buffer mapping for "
  300     "unmapped buffer");
  301 static counter_u64_t numbufallocfails;
  302 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW,
  303     &numbufallocfails, "Number of times buffer allocations failed");
  304 static int flushbufqtarget = 100;
  305 SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
  306     "Amount of work to do in flushbufqueues when helping bufdaemon");
  307 static counter_u64_t notbufdflushes;
  308 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, notbufdflushes, CTLFLAG_RD, &notbufdflushes,
  309     "Number of dirty buffer flushes done by the bufdaemon helpers");
  310 static long barrierwrites;
  311 SYSCTL_LONG(_vfs, OID_AUTO, barrierwrites, CTLFLAG_RW, &barrierwrites, 0,
  312     "Number of barrier writes");
  313 SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed, CTLFLAG_RD,
  314     &unmapped_buf_allowed, 0,
  315     "Permit the use of the unmapped i/o");
  316 int maxbcachebuf = MAXBCACHEBUF;
  317 SYSCTL_INT(_vfs, OID_AUTO, maxbcachebuf, CTLFLAG_RDTUN, &maxbcachebuf, 0,
  318     "Maximum size of a buffer cache block");
  319 
  320 /*
  321  * This lock synchronizes access to bd_request.
  322  */
  323 static struct mtx_padalign __exclusive_cache_line bdlock;
  324 
  325 /*
  326  * This lock protects the runningbufreq and synchronizes runningbufwakeup and
  327  * waitrunningbufspace().
  328  */
  329 static struct mtx_padalign __exclusive_cache_line rbreqlock;
  330 
  331 /*
  332  * Lock that protects bdirtywait.
  333  */
  334 static struct mtx_padalign __exclusive_cache_line bdirtylock;
  335 
  336 /*
  337  * Wakeup point for bufdaemon, as well as indicator of whether it is already
  338  * active.  Set to 1 when the bufdaemon is already "on" the queue, 0 when it
  339  * is idling.
  340  */
  341 static int bd_request;
  342 
  343 /*
  344  * Request for the buf daemon to write more buffers than is indicated by
  345  * lodirtybuf.  This may be necessary to push out excess dependencies or
  346  * defragment the address space where a simple count of the number of dirty
  347  * buffers is insufficient to characterize the demand for flushing them.
  348  */
  349 static int bd_speedupreq;
  350 
  351 /*
  352  * Synchronization (sleep/wakeup) variable for active buffer space requests.
  353  * Set when wait starts, cleared prior to wakeup().
  354  * Used in runningbufwakeup() and waitrunningbufspace().
  355  */
  356 static int runningbufreq;
  357 
  358 /*
  359  * Synchronization for bwillwrite() waiters.
  360  */
  361 static int bdirtywait;
  362 
  363 /*
  364  * Definitions for the buffer free lists.
  365  */
  366 #define QUEUE_NONE      0       /* on no queue */
  367 #define QUEUE_EMPTY     1       /* empty buffer headers */
  368 #define QUEUE_DIRTY     2       /* B_DELWRI buffers */
  369 #define QUEUE_CLEAN     3       /* non-B_DELWRI buffers */
  370 #define QUEUE_SENTINEL  4       /* not an queue index, but mark for sentinel */
  371 
  372 /* Maximum number of buffer domains. */
  373 #define BUF_DOMAINS     8
  374 
  375 struct bufdomainset bdlodirty;          /* Domains > lodirty */
  376 struct bufdomainset bdhidirty;          /* Domains > hidirty */
  377 
  378 /* Configured number of clean queues. */
  379 static int __read_mostly buf_domains;
  380 
  381 BITSET_DEFINE(bufdomainset, BUF_DOMAINS);
  382 struct bufdomain __exclusive_cache_line bdomain[BUF_DOMAINS];
  383 struct bufqueue __exclusive_cache_line bqempty;
  384 
  385 /*
  386  * per-cpu empty buffer cache.
  387  */
  388 uma_zone_t buf_zone;
  389 
  390 /*
  391  * Single global constant for BUF_WMESG, to avoid getting multiple references.
  392  * buf_wmesg is referred from macros.
  393  */
  394 const char *buf_wmesg = BUF_WMESG;
  395 
  396 static int
  397 sysctl_runningspace(SYSCTL_HANDLER_ARGS)
  398 {
  399         long value;
  400         int error;
  401 
  402         value = *(long *)arg1;
  403         error = sysctl_handle_long(oidp, &value, 0, req);
  404         if (error != 0 || req->newptr == NULL)
  405                 return (error);
  406         mtx_lock(&rbreqlock);
  407         if (arg1 == &hirunningspace) {
  408                 if (value < lorunningspace)
  409                         error = EINVAL;
  410                 else
  411                         hirunningspace = value;
  412         } else {
  413                 KASSERT(arg1 == &lorunningspace,
  414                     ("%s: unknown arg1", __func__));
  415                 if (value > hirunningspace)
  416                         error = EINVAL;
  417                 else
  418                         lorunningspace = value;
  419         }
  420         mtx_unlock(&rbreqlock);
  421         return (error);
  422 }
  423 
  424 static int
  425 sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS)
  426 {
  427         int error;
  428         int value;
  429         int i;
  430 
  431         value = *(int *)arg1;
  432         error = sysctl_handle_int(oidp, &value, 0, req);
  433         if (error != 0 || req->newptr == NULL)
  434                 return (error);
  435         *(int *)arg1 = value;
  436         for (i = 0; i < buf_domains; i++)
  437                 *(int *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
  438                     value / buf_domains;
  439 
  440         return (error);
  441 }
  442 
  443 static int
  444 sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS)
  445 {
  446         long value;
  447         int error;
  448         int i;
  449 
  450         value = *(long *)arg1;
  451         error = sysctl_handle_long(oidp, &value, 0, req);
  452         if (error != 0 || req->newptr == NULL)
  453                 return (error);
  454         *(long *)arg1 = value;
  455         for (i = 0; i < buf_domains; i++)
  456                 *(long *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
  457                     value / buf_domains;
  458 
  459         return (error);
  460 }
  461 
  462 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
  463     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
  464 static int
  465 sysctl_bufspace(SYSCTL_HANDLER_ARGS)
  466 {
  467         long lvalue;
  468         int ivalue;
  469         int i;
  470 
  471         lvalue = 0;
  472         for (i = 0; i < buf_domains; i++)
  473                 lvalue += bdomain[i].bd_bufspace;
  474         if (sizeof(int) == sizeof(long) || req->oldlen >= sizeof(long))
  475                 return (sysctl_handle_long(oidp, &lvalue, 0, req));
  476         if (lvalue > INT_MAX)
  477                 /* On overflow, still write out a long to trigger ENOMEM. */
  478                 return (sysctl_handle_long(oidp, &lvalue, 0, req));
  479         ivalue = lvalue;
  480         return (sysctl_handle_int(oidp, &ivalue, 0, req));
  481 }
  482 #else
  483 static int
  484 sysctl_bufspace(SYSCTL_HANDLER_ARGS)
  485 {
  486         long lvalue;
  487         int i;
  488 
  489         lvalue = 0;
  490         for (i = 0; i < buf_domains; i++)
  491                 lvalue += bdomain[i].bd_bufspace;
  492         return (sysctl_handle_long(oidp, &lvalue, 0, req));
  493 }
  494 #endif
  495 
  496 static int
  497 sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS)
  498 {
  499         int value;
  500         int i;
  501 
  502         value = 0;
  503         for (i = 0; i < buf_domains; i++)
  504                 value += bdomain[i].bd_numdirtybuffers;
  505         return (sysctl_handle_int(oidp, &value, 0, req));
  506 }
  507 
  508 /*
  509  *      bdirtywakeup:
  510  *
  511  *      Wakeup any bwillwrite() waiters.
  512  */
  513 static void
  514 bdirtywakeup(void)
  515 {
  516         mtx_lock(&bdirtylock);
  517         if (bdirtywait) {
  518                 bdirtywait = 0;
  519                 wakeup(&bdirtywait);
  520         }
  521         mtx_unlock(&bdirtylock);
  522 }
  523 
  524 /*
  525  *      bd_clear:
  526  *
  527  *      Clear a domain from the appropriate bitsets when dirtybuffers
  528  *      is decremented.
  529  */
  530 static void
  531 bd_clear(struct bufdomain *bd)
  532 {
  533 
  534         mtx_lock(&bdirtylock);
  535         if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers)
  536                 BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
  537         if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers)
  538                 BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
  539         mtx_unlock(&bdirtylock);
  540 }
  541 
  542 /*
  543  *      bd_set:
  544  *
  545  *      Set a domain in the appropriate bitsets when dirtybuffers
  546  *      is incremented.
  547  */
  548 static void
  549 bd_set(struct bufdomain *bd)
  550 {
  551 
  552         mtx_lock(&bdirtylock);
  553         if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers)
  554                 BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
  555         if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers)
  556                 BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
  557         mtx_unlock(&bdirtylock);
  558 }
  559 
  560 /*
  561  *      bdirtysub:
  562  *
  563  *      Decrement the numdirtybuffers count by one and wakeup any
  564  *      threads blocked in bwillwrite().
  565  */
  566 static void
  567 bdirtysub(struct buf *bp)
  568 {
  569         struct bufdomain *bd;
  570         int num;
  571 
  572         bd = bufdomain(bp);
  573         num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1);
  574         if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
  575                 bdirtywakeup();
  576         if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
  577                 bd_clear(bd);
  578 }
  579 
  580 /*
  581  *      bdirtyadd:
  582  *
  583  *      Increment the numdirtybuffers count by one and wakeup the buf 
  584  *      daemon if needed.
  585  */
  586 static void
  587 bdirtyadd(struct buf *bp)
  588 {
  589         struct bufdomain *bd;
  590         int num;
  591 
  592         /*
  593          * Only do the wakeup once as we cross the boundary.  The
  594          * buf daemon will keep running until the condition clears.
  595          */
  596         bd = bufdomain(bp);
  597         num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1);
  598         if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
  599                 bd_wakeup();
  600         if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
  601                 bd_set(bd);
  602 }
  603 
  604 /*
  605  *      bufspace_daemon_wakeup:
  606  *
  607  *      Wakeup the daemons responsible for freeing clean bufs.
  608  */
  609 static void
  610 bufspace_daemon_wakeup(struct bufdomain *bd)
  611 {
  612 
  613         /*
  614          * avoid the lock if the daemon is running.
  615          */
  616         if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) {
  617                 BD_RUN_LOCK(bd);
  618                 atomic_store_int(&bd->bd_running, 1);
  619                 wakeup(&bd->bd_running);
  620                 BD_RUN_UNLOCK(bd);
  621         }
  622 }
  623 
  624 /*
  625  *      bufspace_daemon_wait:
  626  *
  627  *      Sleep until the domain falls below a limit or one second passes.
  628  */
  629 static void
  630 bufspace_daemon_wait(struct bufdomain *bd)
  631 {
  632         /*
  633          * Re-check our limits and sleep.  bd_running must be
  634          * cleared prior to checking the limits to avoid missed
  635          * wakeups.  The waker will adjust one of bufspace or
  636          * freebuffers prior to checking bd_running.
  637          */
  638         BD_RUN_LOCK(bd);
  639         atomic_store_int(&bd->bd_running, 0);
  640         if (bd->bd_bufspace < bd->bd_bufspacethresh &&
  641             bd->bd_freebuffers > bd->bd_lofreebuffers) {
  642                 msleep(&bd->bd_running, BD_RUN_LOCKPTR(bd), PRIBIO|PDROP,
  643                     "-", hz);
  644         } else {
  645                 /* Avoid spurious wakeups while running. */
  646                 atomic_store_int(&bd->bd_running, 1);
  647                 BD_RUN_UNLOCK(bd);
  648         }
  649 }
  650 
  651 /*
  652  *      bufspace_adjust:
  653  *
  654  *      Adjust the reported bufspace for a KVA managed buffer, possibly
  655  *      waking any waiters.
  656  */
  657 static void
  658 bufspace_adjust(struct buf *bp, int bufsize)
  659 {
  660         struct bufdomain *bd;
  661         long space;
  662         int diff;
  663 
  664         KASSERT((bp->b_flags & B_MALLOC) == 0,
  665             ("bufspace_adjust: malloc buf %p", bp));
  666         bd = bufdomain(bp);
  667         diff = bufsize - bp->b_bufsize;
  668         if (diff < 0) {
  669                 atomic_subtract_long(&bd->bd_bufspace, -diff);
  670         } else if (diff > 0) {
  671                 space = atomic_fetchadd_long(&bd->bd_bufspace, diff);
  672                 /* Wake up the daemon on the transition. */
  673                 if (space < bd->bd_bufspacethresh &&
  674                     space + diff >= bd->bd_bufspacethresh)
  675                         bufspace_daemon_wakeup(bd);
  676         }
  677         bp->b_bufsize = bufsize;
  678 }
  679 
  680 /*
  681  *      bufspace_reserve:
  682  *
  683  *      Reserve bufspace before calling allocbuf().  metadata has a
  684  *      different space limit than data.
  685  */
  686 static int
  687 bufspace_reserve(struct bufdomain *bd, int size, bool metadata)
  688 {
  689         long limit, new;
  690         long space;
  691 
  692         if (metadata)
  693                 limit = bd->bd_maxbufspace;
  694         else
  695                 limit = bd->bd_hibufspace;
  696         space = atomic_fetchadd_long(&bd->bd_bufspace, size);
  697         new = space + size;
  698         if (new > limit) {
  699                 atomic_subtract_long(&bd->bd_bufspace, size);
  700                 return (ENOSPC);
  701         }
  702 
  703         /* Wake up the daemon on the transition. */
  704         if (space < bd->bd_bufspacethresh && new >= bd->bd_bufspacethresh)
  705                 bufspace_daemon_wakeup(bd);
  706 
  707         return (0);
  708 }
  709 
  710 /*
  711  *      bufspace_release:
  712  *
  713  *      Release reserved bufspace after bufspace_adjust() has consumed it.
  714  */
  715 static void
  716 bufspace_release(struct bufdomain *bd, int size)
  717 {
  718 
  719         atomic_subtract_long(&bd->bd_bufspace, size);
  720 }
  721 
  722 /*
  723  *      bufspace_wait:
  724  *
  725  *      Wait for bufspace, acting as the buf daemon if a locked vnode is
  726  *      supplied.  bd_wanted must be set prior to polling for space.  The
  727  *      operation must be re-tried on return.
  728  */
  729 static void
  730 bufspace_wait(struct bufdomain *bd, struct vnode *vp, int gbflags,
  731     int slpflag, int slptimeo)
  732 {
  733         struct thread *td;
  734         int error, fl, norunbuf;
  735 
  736         if ((gbflags & GB_NOWAIT_BD) != 0)
  737                 return;
  738 
  739         td = curthread;
  740         BD_LOCK(bd);
  741         while (bd->bd_wanted) {
  742                 if (vp != NULL && vp->v_type != VCHR &&
  743                     (td->td_pflags & TDP_BUFNEED) == 0) {
  744                         BD_UNLOCK(bd);
  745                         /*
  746                          * getblk() is called with a vnode locked, and
  747                          * some majority of the dirty buffers may as
  748                          * well belong to the vnode.  Flushing the
  749                          * buffers there would make a progress that
  750                          * cannot be achieved by the buf_daemon, that
  751                          * cannot lock the vnode.
  752                          */
  753                         norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
  754                             (td->td_pflags & TDP_NORUNNINGBUF);
  755 
  756                         /*
  757                          * Play bufdaemon.  The getnewbuf() function
  758                          * may be called while the thread owns lock
  759                          * for another dirty buffer for the same
  760                          * vnode, which makes it impossible to use
  761                          * VOP_FSYNC() there, due to the buffer lock
  762                          * recursion.
  763                          */
  764                         td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
  765                         fl = buf_flush(vp, bd, flushbufqtarget);
  766                         td->td_pflags &= norunbuf;
  767                         BD_LOCK(bd);
  768                         if (fl != 0)
  769                                 continue;
  770                         if (bd->bd_wanted == 0)
  771                                 break;
  772                 }
  773                 error = msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
  774                     (PRIBIO + 4) | slpflag, "newbuf", slptimeo);
  775                 if (error != 0)
  776                         break;
  777         }
  778         BD_UNLOCK(bd);
  779 }
  780 
  781 
  782 /*
  783  *      bufspace_daemon:
  784  *
  785  *      buffer space management daemon.  Tries to maintain some marginal
  786  *      amount of free buffer space so that requesting processes neither
  787  *      block nor work to reclaim buffers.
  788  */
  789 static void
  790 bufspace_daemon(void *arg)
  791 {
  792         struct bufdomain *bd;
  793 
  794         EVENTHANDLER_REGISTER(shutdown_pre_sync, kthread_shutdown, curthread,
  795             SHUTDOWN_PRI_LAST + 100);
  796 
  797         bd = arg;
  798         for (;;) {
  799                 kthread_suspend_check();
  800 
  801                 /*
  802                  * Free buffers from the clean queue until we meet our
  803                  * targets.
  804                  *
  805                  * Theory of operation:  The buffer cache is most efficient
  806                  * when some free buffer headers and space are always
  807                  * available to getnewbuf().  This daemon attempts to prevent
  808                  * the excessive blocking and synchronization associated
  809                  * with shortfall.  It goes through three phases according
  810                  * demand:
  811                  *
  812                  * 1)   The daemon wakes up voluntarily once per-second
  813                  *      during idle periods when the counters are below
  814                  *      the wakeup thresholds (bufspacethresh, lofreebuffers).
  815                  *
  816                  * 2)   The daemon wakes up as we cross the thresholds
  817                  *      ahead of any potential blocking.  This may bounce
  818                  *      slightly according to the rate of consumption and
  819                  *      release.
  820                  *
  821                  * 3)   The daemon and consumers are starved for working
  822                  *      clean buffers.  This is the 'bufspace' sleep below
  823                  *      which will inefficiently trade bufs with bqrelse
  824                  *      until we return to condition 2.
  825                  */
  826                 while (bd->bd_bufspace > bd->bd_lobufspace ||
  827                     bd->bd_freebuffers < bd->bd_hifreebuffers) {
  828                         if (buf_recycle(bd, false) != 0) {
  829                                 if (bd_flushall(bd))
  830                                         continue;
  831                                 /*
  832                                  * Speedup dirty if we've run out of clean
  833                                  * buffers.  This is possible in particular
  834                                  * because softdep may held many bufs locked
  835                                  * pending writes to other bufs which are
  836                                  * marked for delayed write, exhausting
  837                                  * clean space until they are written.
  838                                  */
  839                                 bd_speedup();
  840                                 BD_LOCK(bd);
  841                                 if (bd->bd_wanted) {
  842                                         msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
  843                                             PRIBIO|PDROP, "bufspace", hz/10);
  844                                 } else
  845                                         BD_UNLOCK(bd);
  846                         }
  847                         maybe_yield();
  848                 }
  849                 bufspace_daemon_wait(bd);
  850         }
  851 }
  852 
  853 /*
  854  *      bufmallocadjust:
  855  *
  856  *      Adjust the reported bufspace for a malloc managed buffer, possibly
  857  *      waking any waiters.
  858  */
  859 static void
  860 bufmallocadjust(struct buf *bp, int bufsize)
  861 {
  862         int diff;
  863 
  864         KASSERT((bp->b_flags & B_MALLOC) != 0,
  865             ("bufmallocadjust: non-malloc buf %p", bp));
  866         diff = bufsize - bp->b_bufsize;
  867         if (diff < 0)
  868                 atomic_subtract_long(&bufmallocspace, -diff);
  869         else
  870                 atomic_add_long(&bufmallocspace, diff);
  871         bp->b_bufsize = bufsize;
  872 }
  873 
  874 /*
  875  *      runningwakeup:
  876  *
  877  *      Wake up processes that are waiting on asynchronous writes to fall
  878  *      below lorunningspace.
  879  */
  880 static void
  881 runningwakeup(void)
  882 {
  883 
  884         mtx_lock(&rbreqlock);
  885         if (runningbufreq) {
  886                 runningbufreq = 0;
  887                 wakeup(&runningbufreq);
  888         }
  889         mtx_unlock(&rbreqlock);
  890 }
  891 
  892 /*
  893  *      runningbufwakeup:
  894  *
  895  *      Decrement the outstanding write count according.
  896  */
  897 void
  898 runningbufwakeup(struct buf *bp)
  899 {
  900         long space, bspace;
  901 
  902         bspace = bp->b_runningbufspace;
  903         if (bspace == 0)
  904                 return;
  905         space = atomic_fetchadd_long(&runningbufspace, -bspace);
  906         KASSERT(space >= bspace, ("runningbufspace underflow %ld %ld",
  907             space, bspace));
  908         bp->b_runningbufspace = 0;
  909         /*
  910          * Only acquire the lock and wakeup on the transition from exceeding
  911          * the threshold to falling below it.
  912          */
  913         if (space < lorunningspace)
  914                 return;
  915         if (space - bspace > lorunningspace)
  916                 return;
  917         runningwakeup();
  918 }
  919 
  920 /*
  921  *      waitrunningbufspace()
  922  *
  923  *      runningbufspace is a measure of the amount of I/O currently
  924  *      running.  This routine is used in async-write situations to
  925  *      prevent creating huge backups of pending writes to a device.
  926  *      Only asynchronous writes are governed by this function.
  927  *
  928  *      This does NOT turn an async write into a sync write.  It waits  
  929  *      for earlier writes to complete and generally returns before the
  930  *      caller's write has reached the device.
  931  */
  932 void
  933 waitrunningbufspace(void)
  934 {
  935 
  936         mtx_lock(&rbreqlock);
  937         while (runningbufspace > hirunningspace) {
  938                 runningbufreq = 1;
  939                 msleep(&runningbufreq, &rbreqlock, PVM, "wdrain", 0);
  940         }
  941         mtx_unlock(&rbreqlock);
  942 }
  943 
  944 
  945 /*
  946  *      vfs_buf_test_cache:
  947  *
  948  *      Called when a buffer is extended.  This function clears the B_CACHE
  949  *      bit if the newly extended portion of the buffer does not contain
  950  *      valid data.
  951  */
  952 static __inline void
  953 vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
  954     vm_offset_t size, vm_page_t m)
  955 {
  956 
  957         VM_OBJECT_ASSERT_LOCKED(m->object);
  958         if (bp->b_flags & B_CACHE) {
  959                 int base = (foff + off) & PAGE_MASK;
  960                 if (vm_page_is_valid(m, base, size) == 0)
  961                         bp->b_flags &= ~B_CACHE;
  962         }
  963 }
  964 
  965 /* Wake up the buffer daemon if necessary */
  966 static void
  967 bd_wakeup(void)
  968 {
  969 
  970         mtx_lock(&bdlock);
  971         if (bd_request == 0) {
  972                 bd_request = 1;
  973                 wakeup(&bd_request);
  974         }
  975         mtx_unlock(&bdlock);
  976 }
  977 
  978 /*
  979  * Adjust the maxbcachbuf tunable.
  980  */
  981 static void
  982 maxbcachebuf_adjust(void)
  983 {
  984         int i;
  985 
  986         /*
  987          * maxbcachebuf must be a power of 2 >= MAXBSIZE.
  988          */
  989         i = 2;
  990         while (i * 2 <= maxbcachebuf)
  991                 i *= 2;
  992         maxbcachebuf = i;
  993         if (maxbcachebuf < MAXBSIZE)
  994                 maxbcachebuf = MAXBSIZE;
  995         if (maxbcachebuf > MAXPHYS)
  996                 maxbcachebuf = MAXPHYS;
  997         if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF)
  998                 printf("maxbcachebuf=%d\n", maxbcachebuf);
  999 }
 1000 
 1001 /*
 1002  * bd_speedup - speedup the buffer cache flushing code
 1003  */
 1004 void
 1005 bd_speedup(void)
 1006 {
 1007         int needwake;
 1008 
 1009         mtx_lock(&bdlock);
 1010         needwake = 0;
 1011         if (bd_speedupreq == 0 || bd_request == 0)
 1012                 needwake = 1;
 1013         bd_speedupreq = 1;
 1014         bd_request = 1;
 1015         if (needwake)
 1016                 wakeup(&bd_request);
 1017         mtx_unlock(&bdlock);
 1018 }
 1019 
 1020 #ifndef NSWBUF_MIN
 1021 #define NSWBUF_MIN      16
 1022 #endif
 1023 
 1024 #ifdef __i386__
 1025 #define TRANSIENT_DENOM 5
 1026 #else
 1027 #define TRANSIENT_DENOM 10
 1028 #endif
 1029 
 1030 /*
 1031  * Calculating buffer cache scaling values and reserve space for buffer
 1032  * headers.  This is called during low level kernel initialization and
 1033  * may be called more then once.  We CANNOT write to the memory area
 1034  * being reserved at this time.
 1035  */
 1036 caddr_t
 1037 kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est)
 1038 {
 1039         int tuned_nbuf;
 1040         long maxbuf, maxbuf_sz, buf_sz, biotmap_sz;
 1041 
 1042         /*
 1043          * physmem_est is in pages.  Convert it to kilobytes (assumes
 1044          * PAGE_SIZE is >= 1K)
 1045          */
 1046         physmem_est = physmem_est * (PAGE_SIZE / 1024);
 1047 
 1048         maxbcachebuf_adjust();
 1049         /*
 1050          * The nominal buffer size (and minimum KVA allocation) is BKVASIZE.
 1051          * For the first 64MB of ram nominally allocate sufficient buffers to
 1052          * cover 1/4 of our ram.  Beyond the first 64MB allocate additional
 1053          * buffers to cover 1/10 of our ram over 64MB.  When auto-sizing
 1054          * the buffer cache we limit the eventual kva reservation to
 1055          * maxbcache bytes.
 1056          *
 1057          * factor represents the 1/4 x ram conversion.
 1058          */
 1059         if (nbuf == 0) {
 1060                 int factor = 4 * BKVASIZE / 1024;
 1061 
 1062                 nbuf = 50;
 1063                 if (physmem_est > 4096)
 1064                         nbuf += min((physmem_est - 4096) / factor,
 1065                             65536 / factor);
 1066                 if (physmem_est > 65536)
 1067                         nbuf += min((physmem_est - 65536) * 2 / (factor * 5),
 1068                             32 * 1024 * 1024 / (factor * 5));
 1069 
 1070                 if (maxbcache && nbuf > maxbcache / BKVASIZE)
 1071                         nbuf = maxbcache / BKVASIZE;
 1072                 tuned_nbuf = 1;
 1073         } else
 1074                 tuned_nbuf = 0;
 1075 
 1076         /* XXX Avoid unsigned long overflows later on with maxbufspace. */
 1077         maxbuf = (LONG_MAX / 3) / BKVASIZE;
 1078         if (nbuf > maxbuf) {
 1079                 if (!tuned_nbuf)
 1080                         printf("Warning: nbufs lowered from %d to %ld\n", nbuf,
 1081                             maxbuf);
 1082                 nbuf = maxbuf;
 1083         }
 1084 
 1085         /*
 1086          * Ideal allocation size for the transient bio submap is 10%
 1087          * of the maximal space buffer map.  This roughly corresponds
 1088          * to the amount of the buffer mapped for typical UFS load.
 1089          *
 1090          * Clip the buffer map to reserve space for the transient
 1091          * BIOs, if its extent is bigger than 90% (80% on i386) of the
 1092          * maximum buffer map extent on the platform.
 1093          *
 1094          * The fall-back to the maxbuf in case of maxbcache unset,
 1095          * allows to not trim the buffer KVA for the architectures
 1096          * with ample KVA space.
 1097          */
 1098         if (bio_transient_maxcnt == 0 && unmapped_buf_allowed) {
 1099                 maxbuf_sz = maxbcache != 0 ? maxbcache : maxbuf * BKVASIZE;
 1100                 buf_sz = (long)nbuf * BKVASIZE;
 1101                 if (buf_sz < maxbuf_sz / TRANSIENT_DENOM *
 1102                     (TRANSIENT_DENOM - 1)) {
 1103                         /*
 1104                          * There is more KVA than memory.  Do not
 1105                          * adjust buffer map size, and assign the rest
 1106                          * of maxbuf to transient map.
 1107                          */
 1108                         biotmap_sz = maxbuf_sz - buf_sz;
 1109                 } else {
 1110                         /*
 1111                          * Buffer map spans all KVA we could afford on
 1112                          * this platform.  Give 10% (20% on i386) of
 1113                          * the buffer map to the transient bio map.
 1114                          */
 1115                         biotmap_sz = buf_sz / TRANSIENT_DENOM;
 1116                         buf_sz -= biotmap_sz;
 1117                 }
 1118                 if (biotmap_sz / INT_MAX > MAXPHYS)
 1119                         bio_transient_maxcnt = INT_MAX;
 1120                 else
 1121                         bio_transient_maxcnt = biotmap_sz / MAXPHYS;
 1122                 /*
 1123                  * Artificially limit to 1024 simultaneous in-flight I/Os
 1124                  * using the transient mapping.
 1125                  */
 1126                 if (bio_transient_maxcnt > 1024)
 1127                         bio_transient_maxcnt = 1024;
 1128                 if (tuned_nbuf)
 1129                         nbuf = buf_sz / BKVASIZE;
 1130         }
 1131 
 1132         /*
 1133          * swbufs are used as temporary holders for I/O, such as paging I/O.
 1134          * We have no less then 16 and no more then 256.
 1135          */
 1136         nswbuf = min(nbuf / 4, 256);
 1137         TUNABLE_INT_FETCH("kern.nswbuf", &nswbuf);
 1138         if (nswbuf < NSWBUF_MIN)
 1139                 nswbuf = NSWBUF_MIN;
 1140 
 1141         /*
 1142          * Reserve space for the buffer cache buffers
 1143          */
 1144         swbuf = (void *)v;
 1145         v = (caddr_t)(swbuf + nswbuf);
 1146         buf = (void *)v;
 1147         v = (caddr_t)(buf + nbuf);
 1148 
 1149         return(v);
 1150 }
 1151 
 1152 /* Initialize the buffer subsystem.  Called before use of any buffers. */
 1153 void
 1154 bufinit(void)
 1155 {
 1156         struct buf *bp;
 1157         int i;
 1158 
 1159         KASSERT(maxbcachebuf >= MAXBSIZE,
 1160             ("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf,
 1161             MAXBSIZE));
 1162         bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock");
 1163         mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
 1164         mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
 1165         mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF);
 1166 
 1167         unmapped_buf = (caddr_t)kva_alloc(MAXPHYS);
 1168 
 1169         /* finally, initialize each buffer header and stick on empty q */
 1170         for (i = 0; i < nbuf; i++) {
 1171                 bp = &buf[i];
 1172                 bzero(bp, sizeof *bp);
 1173                 bp->b_flags = B_INVAL;
 1174                 bp->b_rcred = NOCRED;
 1175                 bp->b_wcred = NOCRED;
 1176                 bp->b_qindex = QUEUE_NONE;
 1177                 bp->b_domain = -1;
 1178                 bp->b_subqueue = mp_maxid + 1;
 1179                 bp->b_xflags = 0;
 1180                 bp->b_data = bp->b_kvabase = unmapped_buf;
 1181                 LIST_INIT(&bp->b_dep);
 1182                 BUF_LOCKINIT(bp);
 1183                 bq_insert(&bqempty, bp, false);
 1184         }
 1185 
 1186         /*
 1187          * maxbufspace is the absolute maximum amount of buffer space we are 
 1188          * allowed to reserve in KVM and in real terms.  The absolute maximum
 1189          * is nominally used by metadata.  hibufspace is the nominal maximum
 1190          * used by most other requests.  The differential is required to 
 1191          * ensure that metadata deadlocks don't occur.
 1192          *
 1193          * maxbufspace is based on BKVASIZE.  Allocating buffers larger then
 1194          * this may result in KVM fragmentation which is not handled optimally
 1195          * by the system. XXX This is less true with vmem.  We could use
 1196          * PAGE_SIZE.
 1197          */
 1198         maxbufspace = (long)nbuf * BKVASIZE;
 1199         hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - maxbcachebuf * 10);
 1200         lobufspace = (hibufspace / 20) * 19; /* 95% */
 1201         bufspacethresh = lobufspace + (hibufspace - lobufspace) / 2;
 1202 
 1203         /*
 1204          * Note: The 16 MiB upper limit for hirunningspace was chosen
 1205          * arbitrarily and may need further tuning. It corresponds to
 1206          * 128 outstanding write IO requests (if IO size is 128 KiB),
 1207          * which fits with many RAID controllers' tagged queuing limits.
 1208          * The lower 1 MiB limit is the historical upper limit for
 1209          * hirunningspace.
 1210          */
 1211         hirunningspace = lmax(lmin(roundup(hibufspace / 64, maxbcachebuf),
 1212             16 * 1024 * 1024), 1024 * 1024);
 1213         lorunningspace = roundup((hirunningspace * 2) / 3, maxbcachebuf);
 1214 
 1215         /*
 1216          * Limit the amount of malloc memory since it is wired permanently into
 1217          * the kernel space.  Even though this is accounted for in the buffer
 1218          * allocation, we don't want the malloced region to grow uncontrolled.
 1219          * The malloc scheme improves memory utilization significantly on
 1220          * average (small) directories.
 1221          */
 1222         maxbufmallocspace = hibufspace / 20;
 1223 
 1224         /*
 1225          * Reduce the chance of a deadlock occurring by limiting the number
 1226          * of delayed-write dirty buffers we allow to stack up.
 1227          */
 1228         hidirtybuffers = nbuf / 4 + 20;
 1229         dirtybufthresh = hidirtybuffers * 9 / 10;
 1230         /*
 1231          * To support extreme low-memory systems, make sure hidirtybuffers
 1232          * cannot eat up all available buffer space.  This occurs when our
 1233          * minimum cannot be met.  We try to size hidirtybuffers to 3/4 our
 1234          * buffer space assuming BKVASIZE'd buffers.
 1235          */
 1236         while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
 1237                 hidirtybuffers >>= 1;
 1238         }
 1239         lodirtybuffers = hidirtybuffers / 2;
 1240 
 1241         /*
 1242          * lofreebuffers should be sufficient to avoid stalling waiting on
 1243          * buf headers under heavy utilization.  The bufs in per-cpu caches
 1244          * are counted as free but will be unavailable to threads executing
 1245          * on other cpus.
 1246          *
 1247          * hifreebuffers is the free target for the bufspace daemon.  This
 1248          * should be set appropriately to limit work per-iteration.
 1249          */
 1250         lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus);
 1251         hifreebuffers = (3 * lofreebuffers) / 2;
 1252         numfreebuffers = nbuf;
 1253 
 1254         /* Setup the kva and free list allocators. */
 1255         vmem_set_reclaim(buffer_arena, bufkva_reclaim);
 1256         buf_zone = uma_zcache_create("buf free cache", sizeof(struct buf),
 1257             NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0);
 1258 
 1259         /*
 1260          * Size the clean queue according to the amount of buffer space.
 1261          * One queue per-256mb up to the max.  More queues gives better
 1262          * concurrency but less accurate LRU.
 1263          */
 1264         buf_domains = MIN(howmany(maxbufspace, 256*1024*1024), BUF_DOMAINS);
 1265         for (i = 0 ; i < buf_domains; i++) {
 1266                 struct bufdomain *bd;
 1267 
 1268                 bd = &bdomain[i];
 1269                 bd_init(bd);
 1270                 bd->bd_freebuffers = nbuf / buf_domains;
 1271                 bd->bd_hifreebuffers = hifreebuffers / buf_domains;
 1272                 bd->bd_lofreebuffers = lofreebuffers / buf_domains;
 1273                 bd->bd_bufspace = 0;
 1274                 bd->bd_maxbufspace = maxbufspace / buf_domains;
 1275                 bd->bd_hibufspace = hibufspace / buf_domains;
 1276                 bd->bd_lobufspace = lobufspace / buf_domains;
 1277                 bd->bd_bufspacethresh = bufspacethresh / buf_domains;
 1278                 bd->bd_numdirtybuffers = 0;
 1279                 bd->bd_hidirtybuffers = hidirtybuffers / buf_domains;
 1280                 bd->bd_lodirtybuffers = lodirtybuffers / buf_domains;
 1281                 bd->bd_dirtybufthresh = dirtybufthresh / buf_domains;
 1282                 /* Don't allow more than 2% of bufs in the per-cpu caches. */
 1283                 bd->bd_lim = nbuf / buf_domains / 50 / mp_ncpus;
 1284         }
 1285         getnewbufcalls = counter_u64_alloc(M_WAITOK);
 1286         getnewbufrestarts = counter_u64_alloc(M_WAITOK);
 1287         mappingrestarts = counter_u64_alloc(M_WAITOK);
 1288         numbufallocfails = counter_u64_alloc(M_WAITOK);
 1289         notbufdflushes = counter_u64_alloc(M_WAITOK);
 1290         buffreekvacnt = counter_u64_alloc(M_WAITOK);
 1291         bufdefragcnt = counter_u64_alloc(M_WAITOK);
 1292         bufkvaspace = counter_u64_alloc(M_WAITOK);
 1293 }
 1294 
 1295 #ifdef INVARIANTS
 1296 static inline void
 1297 vfs_buf_check_mapped(struct buf *bp)
 1298 {
 1299 
 1300         KASSERT(bp->b_kvabase != unmapped_buf,
 1301             ("mapped buf: b_kvabase was not updated %p", bp));
 1302         KASSERT(bp->b_data != unmapped_buf,
 1303             ("mapped buf: b_data was not updated %p", bp));
 1304         KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf +
 1305             MAXPHYS, ("b_data + b_offset unmapped %p", bp));
 1306 }
 1307 
 1308 static inline void
 1309 vfs_buf_check_unmapped(struct buf *bp)
 1310 {
 1311 
 1312         KASSERT(bp->b_data == unmapped_buf,
 1313             ("unmapped buf: corrupted b_data %p", bp));
 1314 }
 1315 
 1316 #define BUF_CHECK_MAPPED(bp) vfs_buf_check_mapped(bp)
 1317 #define BUF_CHECK_UNMAPPED(bp) vfs_buf_check_unmapped(bp)
 1318 #else
 1319 #define BUF_CHECK_MAPPED(bp) do {} while (0)
 1320 #define BUF_CHECK_UNMAPPED(bp) do {} while (0)
 1321 #endif
 1322 
 1323 static int
 1324 isbufbusy(struct buf *bp)
 1325 {
 1326         if (((bp->b_flags & B_INVAL) == 0 && BUF_ISLOCKED(bp)) ||
 1327             ((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI))
 1328                 return (1);
 1329         return (0);
 1330 }
 1331 
 1332 /*
 1333  * Shutdown the system cleanly to prepare for reboot, halt, or power off.
 1334  */
 1335 void
 1336 bufshutdown(int show_busybufs)
 1337 {
 1338         static int first_buf_printf = 1;
 1339         struct buf *bp;
 1340         int iter, nbusy, pbusy;
 1341 #ifndef PREEMPTION
 1342         int subiter;
 1343 #endif
 1344 
 1345         /* 
 1346          * Sync filesystems for shutdown
 1347          */
 1348         wdog_kern_pat(WD_LASTVAL);
 1349         sys_sync(curthread, NULL);
 1350 
 1351         /*
 1352          * With soft updates, some buffers that are
 1353          * written will be remarked as dirty until other
 1354          * buffers are written.
 1355          */
 1356         for (iter = pbusy = 0; iter < 20; iter++) {
 1357                 nbusy = 0;
 1358                 for (bp = &buf[nbuf]; --bp >= buf; )
 1359                         if (isbufbusy(bp))
 1360                                 nbusy++;
 1361                 if (nbusy == 0) {
 1362                         if (first_buf_printf)
 1363                                 printf("All buffers synced.");
 1364                         break;
 1365                 }
 1366                 if (first_buf_printf) {
 1367                         printf("Syncing disks, buffers remaining... ");
 1368                         first_buf_printf = 0;
 1369                 }
 1370                 printf("%d ", nbusy);
 1371                 if (nbusy < pbusy)
 1372                         iter = 0;
 1373                 pbusy = nbusy;
 1374 
 1375                 wdog_kern_pat(WD_LASTVAL);
 1376                 sys_sync(curthread, NULL);
 1377 
 1378 #ifdef PREEMPTION
 1379                 /*
 1380                  * Spin for a while to allow interrupt threads to run.
 1381                  */
 1382                 DELAY(50000 * iter);
 1383 #else
 1384                 /*
 1385                  * Context switch several times to allow interrupt
 1386                  * threads to run.
 1387                  */
 1388                 for (subiter = 0; subiter < 50 * iter; subiter++) {
 1389                         thread_lock(curthread);
 1390                         mi_switch(SW_VOL, NULL);
 1391                         thread_unlock(curthread);
 1392                         DELAY(1000);
 1393                 }
 1394 #endif
 1395         }
 1396         printf("\n");
 1397         /*
 1398          * Count only busy local buffers to prevent forcing 
 1399          * a fsck if we're just a client of a wedged NFS server
 1400          */
 1401         nbusy = 0;
 1402         for (bp = &buf[nbuf]; --bp >= buf; ) {
 1403                 if (isbufbusy(bp)) {
 1404 #if 0
 1405 /* XXX: This is bogus.  We should probably have a BO_REMOTE flag instead */
 1406                         if (bp->b_dev == NULL) {
 1407                                 TAILQ_REMOVE(&mountlist,
 1408                                     bp->b_vp->v_mount, mnt_list);
 1409                                 continue;
 1410                         }
 1411 #endif
 1412                         nbusy++;
 1413                         if (show_busybufs > 0) {
 1414                                 printf(
 1415             "%d: buf:%p, vnode:%p, flags:%0x, blkno:%jd, lblkno:%jd, buflock:",
 1416                                     nbusy, bp, bp->b_vp, bp->b_flags,
 1417                                     (intmax_t)bp->b_blkno,
 1418                                     (intmax_t)bp->b_lblkno);
 1419                                 BUF_LOCKPRINTINFO(bp);
 1420                                 if (show_busybufs > 1)
 1421                                         vn_printf(bp->b_vp,
 1422                                             "vnode content: ");
 1423                         }
 1424                 }
 1425         }
 1426         if (nbusy) {
 1427                 /*
 1428                  * Failed to sync all blocks. Indicate this and don't
 1429                  * unmount filesystems (thus forcing an fsck on reboot).
 1430                  */
 1431                 printf("Giving up on %d buffers\n", nbusy);
 1432                 DELAY(5000000); /* 5 seconds */
 1433         } else {
 1434                 if (!first_buf_printf)
 1435                         printf("Final sync complete\n");
 1436                 /*
 1437                  * Unmount filesystems
 1438                  */
 1439                 if (panicstr == NULL)
 1440                         vfs_unmountall();
 1441         }
 1442         swapoff_all();
 1443         DELAY(100000);          /* wait for console output to finish */
 1444 }
 1445 
 1446 static void
 1447 bpmap_qenter(struct buf *bp)
 1448 {
 1449 
 1450         BUF_CHECK_MAPPED(bp);
 1451 
 1452         /*
 1453          * bp->b_data is relative to bp->b_offset, but
 1454          * bp->b_offset may be offset into the first page.
 1455          */
 1456         bp->b_data = (caddr_t)trunc_page((vm_offset_t)bp->b_data);
 1457         pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
 1458         bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
 1459             (vm_offset_t)(bp->b_offset & PAGE_MASK));
 1460 }
 1461 
 1462 static inline struct bufdomain *
 1463 bufdomain(struct buf *bp)
 1464 {
 1465 
 1466         return (&bdomain[bp->b_domain]);
 1467 }
 1468 
 1469 static struct bufqueue *
 1470 bufqueue(struct buf *bp)
 1471 {
 1472 
 1473         switch (bp->b_qindex) {
 1474         case QUEUE_NONE:
 1475                 /* FALLTHROUGH */
 1476         case QUEUE_SENTINEL:
 1477                 return (NULL);
 1478         case QUEUE_EMPTY:
 1479                 return (&bqempty);
 1480         case QUEUE_DIRTY:
 1481                 return (&bufdomain(bp)->bd_dirtyq);
 1482         case QUEUE_CLEAN:
 1483                 return (&bufdomain(bp)->bd_subq[bp->b_subqueue]);
 1484         default:
 1485                 break;
 1486         }
 1487         panic("bufqueue(%p): Unhandled type %d\n", bp, bp->b_qindex);
 1488 }
 1489 
 1490 /*
 1491  * Return the locked bufqueue that bp is a member of.
 1492  */
 1493 static struct bufqueue *
 1494 bufqueue_acquire(struct buf *bp)
 1495 {
 1496         struct bufqueue *bq, *nbq;
 1497 
 1498         /*
 1499          * bp can be pushed from a per-cpu queue to the
 1500          * cleanq while we're waiting on the lock.  Retry
 1501          * if the queues don't match.
 1502          */
 1503         bq = bufqueue(bp);
 1504         BQ_LOCK(bq);
 1505         for (;;) {
 1506                 nbq = bufqueue(bp);
 1507                 if (bq == nbq)
 1508                         break;
 1509                 BQ_UNLOCK(bq);
 1510                 BQ_LOCK(nbq);
 1511                 bq = nbq;
 1512         }
 1513         return (bq);
 1514 }
 1515 
 1516 /*
 1517  *      binsfree:
 1518  *
 1519  *      Insert the buffer into the appropriate free list.  Requires a
 1520  *      locked buffer on entry and buffer is unlocked before return.
 1521  */
 1522 static void
 1523 binsfree(struct buf *bp, int qindex)
 1524 {
 1525         struct bufdomain *bd;
 1526         struct bufqueue *bq;
 1527 
 1528         KASSERT(qindex == QUEUE_CLEAN || qindex == QUEUE_DIRTY,
 1529             ("binsfree: Invalid qindex %d", qindex));
 1530         BUF_ASSERT_XLOCKED(bp);
 1531 
 1532         /*
 1533          * Handle delayed bremfree() processing.
 1534          */
 1535         if (bp->b_flags & B_REMFREE) {
 1536                 if (bp->b_qindex == qindex) {
 1537                         bp->b_flags |= B_REUSE;
 1538                         bp->b_flags &= ~B_REMFREE;
 1539                         BUF_UNLOCK(bp);
 1540                         return;
 1541                 }
 1542                 bq = bufqueue_acquire(bp);
 1543                 bq_remove(bq, bp);
 1544                 BQ_UNLOCK(bq);
 1545         }
 1546         bd = bufdomain(bp);
 1547         if (qindex == QUEUE_CLEAN) {
 1548                 if (bd->bd_lim != 0)
 1549                         bq = &bd->bd_subq[PCPU_GET(cpuid)];
 1550                 else
 1551                         bq = bd->bd_cleanq;
 1552         } else
 1553                 bq = &bd->bd_dirtyq;
 1554         bq_insert(bq, bp, true);
 1555 }
 1556 
 1557 /*
 1558  * buf_free:
 1559  *
 1560  *      Free a buffer to the buf zone once it no longer has valid contents.
 1561  */
 1562 static void
 1563 buf_free(struct buf *bp)
 1564 {
 1565 
 1566         if (bp->b_flags & B_REMFREE)
 1567                 bremfreef(bp);
 1568         if (bp->b_vflags & BV_BKGRDINPROG)
 1569                 panic("losing buffer 1");
 1570         if (bp->b_rcred != NOCRED) {
 1571                 crfree(bp->b_rcred);
 1572                 bp->b_rcred = NOCRED;
 1573         }
 1574         if (bp->b_wcred != NOCRED) {
 1575                 crfree(bp->b_wcred);
 1576                 bp->b_wcred = NOCRED;
 1577         }
 1578         if (!LIST_EMPTY(&bp->b_dep))
 1579                 buf_deallocate(bp);
 1580         bufkva_free(bp);
 1581         atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1);
 1582         BUF_UNLOCK(bp);
 1583         uma_zfree(buf_zone, bp);
 1584 }
 1585 
 1586 /*
 1587  * buf_import:
 1588  *
 1589  *      Import bufs into the uma cache from the buf list.  The system still
 1590  *      expects a static array of bufs and much of the synchronization
 1591  *      around bufs assumes type stable storage.  As a result, UMA is used
 1592  *      only as a per-cpu cache of bufs still maintained on a global list.
 1593  */
 1594 static int
 1595 buf_import(void *arg, void **store, int cnt, int domain, int flags)
 1596 {
 1597         struct buf *bp;
 1598         int i;
 1599 
 1600         BQ_LOCK(&bqempty);
 1601         for (i = 0; i < cnt; i++) {
 1602                 bp = TAILQ_FIRST(&bqempty.bq_queue);
 1603                 if (bp == NULL)
 1604                         break;
 1605                 bq_remove(&bqempty, bp);
 1606                 store[i] = bp;
 1607         }
 1608         BQ_UNLOCK(&bqempty);
 1609 
 1610         return (i);
 1611 }
 1612 
 1613 /*
 1614  * buf_release:
 1615  *
 1616  *      Release bufs from the uma cache back to the buffer queues.
 1617  */
 1618 static void
 1619 buf_release(void *arg, void **store, int cnt)
 1620 {
 1621         struct bufqueue *bq;
 1622         struct buf *bp;
 1623         int i;
 1624 
 1625         bq = &bqempty;
 1626         BQ_LOCK(bq);
 1627         for (i = 0; i < cnt; i++) {
 1628                 bp = store[i];
 1629                 /* Inline bq_insert() to batch locking. */
 1630                 TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
 1631                 bp->b_flags &= ~(B_AGE | B_REUSE);
 1632                 bq->bq_len++;
 1633                 bp->b_qindex = bq->bq_index;
 1634         }
 1635         BQ_UNLOCK(bq);
 1636 }
 1637 
 1638 /*
 1639  * buf_alloc:
 1640  *
 1641  *      Allocate an empty buffer header.
 1642  */
 1643 static struct buf *
 1644 buf_alloc(struct bufdomain *bd)
 1645 {
 1646         struct buf *bp;
 1647         int freebufs;
 1648 
 1649         /*
 1650          * We can only run out of bufs in the buf zone if the average buf
 1651          * is less than BKVASIZE.  In this case the actual wait/block will
 1652          * come from buf_reycle() failing to flush one of these small bufs.
 1653          */
 1654         bp = NULL;
 1655         freebufs = atomic_fetchadd_int(&bd->bd_freebuffers, -1);
 1656         if (freebufs > 0)
 1657                 bp = uma_zalloc(buf_zone, M_NOWAIT);
 1658         if (bp == NULL) {
 1659                 atomic_fetchadd_int(&bd->bd_freebuffers, 1);
 1660                 bufspace_daemon_wakeup(bd);
 1661                 counter_u64_add(numbufallocfails, 1);
 1662                 return (NULL);
 1663         }
 1664         /*
 1665          * Wake-up the bufspace daemon on transition below threshold.
 1666          */
 1667         if (freebufs == bd->bd_lofreebuffers)
 1668                 bufspace_daemon_wakeup(bd);
 1669 
 1670         if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
 1671                 panic("getnewbuf_empty: Locked buf %p on free queue.", bp);
 1672         
 1673         KASSERT(bp->b_vp == NULL,
 1674             ("bp: %p still has vnode %p.", bp, bp->b_vp));
 1675         KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
 1676             ("invalid buffer %p flags %#x", bp, bp->b_flags));
 1677         KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
 1678             ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
 1679         KASSERT(bp->b_npages == 0,
 1680             ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
 1681         KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
 1682         KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));
 1683 
 1684         bp->b_domain = BD_DOMAIN(bd);
 1685         bp->b_flags = 0;
 1686         bp->b_ioflags = 0;
 1687         bp->b_xflags = 0;
 1688         bp->b_vflags = 0;
 1689         bp->b_vp = NULL;
 1690         bp->b_blkno = bp->b_lblkno = 0;
 1691         bp->b_offset = NOOFFSET;
 1692         bp->b_iodone = 0;
 1693         bp->b_error = 0;
 1694         bp->b_resid = 0;
 1695         bp->b_bcount = 0;
 1696         bp->b_npages = 0;
 1697         bp->b_dirtyoff = bp->b_dirtyend = 0;
 1698         bp->b_bufobj = NULL;
 1699         bp->b_data = bp->b_kvabase = unmapped_buf;
 1700         bp->b_fsprivate1 = NULL;
 1701         bp->b_fsprivate2 = NULL;
 1702         bp->b_fsprivate3 = NULL;
 1703         LIST_INIT(&bp->b_dep);
 1704 
 1705         return (bp);
 1706 }
 1707 
 1708 /*
 1709  *      buf_recycle:
 1710  *
 1711  *      Free a buffer from the given bufqueue.  kva controls whether the
 1712  *      freed buf must own some kva resources.  This is used for
 1713  *      defragmenting.
 1714  */
 1715 static int
 1716 buf_recycle(struct bufdomain *bd, bool kva)
 1717 {
 1718         struct bufqueue *bq;
 1719         struct buf *bp, *nbp;
 1720 
 1721         if (kva)
 1722                 counter_u64_add(bufdefragcnt, 1);
 1723         nbp = NULL;
 1724         bq = bd->bd_cleanq;
 1725         BQ_LOCK(bq);
 1726         KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd),
 1727             ("buf_recycle: Locks don't match"));
 1728         nbp = TAILQ_FIRST(&bq->bq_queue);
 1729 
 1730         /*
 1731          * Run scan, possibly freeing data and/or kva mappings on the fly
 1732          * depending.
 1733          */
 1734         while ((bp = nbp) != NULL) {
 1735                 /*
 1736                  * Calculate next bp (we can only use it if we do not
 1737                  * release the bqlock).
 1738                  */
 1739                 nbp = TAILQ_NEXT(bp, b_freelist);
 1740 
 1741                 /*
 1742                  * If we are defragging then we need a buffer with 
 1743                  * some kva to reclaim.
 1744                  */
 1745                 if (kva && bp->b_kvasize == 0)
 1746                         continue;
 1747 
 1748                 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
 1749                         continue;
 1750 
 1751                 /*
 1752                  * Implement a second chance algorithm for frequently
 1753                  * accessed buffers.
 1754                  */
 1755                 if ((bp->b_flags & B_REUSE) != 0) {
 1756                         TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
 1757                         TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
 1758                         bp->b_flags &= ~B_REUSE;
 1759                         BUF_UNLOCK(bp);
 1760                         continue;
 1761                 }
 1762 
 1763                 /*
 1764                  * Skip buffers with background writes in progress.
 1765                  */
 1766                 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
 1767                         BUF_UNLOCK(bp);
 1768                         continue;
 1769                 }
 1770 
 1771                 KASSERT(bp->b_qindex == QUEUE_CLEAN,
 1772                     ("buf_recycle: inconsistent queue %d bp %p",
 1773                     bp->b_qindex, bp));
 1774                 KASSERT(bp->b_domain == BD_DOMAIN(bd),
 1775                     ("getnewbuf: queue domain %d doesn't match request %d",
 1776                     bp->b_domain, (int)BD_DOMAIN(bd)));
 1777                 /*
 1778                  * NOTE:  nbp is now entirely invalid.  We can only restart
 1779                  * the scan from this point on.
 1780                  */
 1781                 bq_remove(bq, bp);
 1782                 BQ_UNLOCK(bq);
 1783 
 1784                 /*
 1785                  * Requeue the background write buffer with error and
 1786                  * restart the scan.
 1787                  */
 1788                 if ((bp->b_vflags & BV_BKGRDERR) != 0) {
 1789                         bqrelse(bp);
 1790                         BQ_LOCK(bq);
 1791                         nbp = TAILQ_FIRST(&bq->bq_queue);
 1792                         continue;
 1793                 }
 1794                 bp->b_flags |= B_INVAL;
 1795                 brelse(bp);
 1796                 return (0);
 1797         }
 1798         bd->bd_wanted = 1;
 1799         BQ_UNLOCK(bq);
 1800 
 1801         return (ENOBUFS);
 1802 }
 1803 
 1804 /*
 1805  *      bremfree:
 1806  *
 1807  *      Mark the buffer for removal from the appropriate free list.
 1808  *      
 1809  */
 1810 void
 1811 bremfree(struct buf *bp)
 1812 {
 1813 
 1814         CTR3(KTR_BUF, "bremfree(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
 1815         KASSERT((bp->b_flags & B_REMFREE) == 0,
 1816             ("bremfree: buffer %p already marked for delayed removal.", bp));
 1817         KASSERT(bp->b_qindex != QUEUE_NONE,
 1818             ("bremfree: buffer %p not on a queue.", bp));
 1819         BUF_ASSERT_XLOCKED(bp);
 1820 
 1821         bp->b_flags |= B_REMFREE;
 1822 }
 1823 
 1824 /*
 1825  *      bremfreef:
 1826  *
 1827  *      Force an immediate removal from a free list.  Used only in nfs when
 1828  *      it abuses the b_freelist pointer.
 1829  */
 1830 void
 1831 bremfreef(struct buf *bp)
 1832 {
 1833         struct bufqueue *bq;
 1834 
 1835         bq = bufqueue_acquire(bp);
 1836         bq_remove(bq, bp);
 1837         BQ_UNLOCK(bq);
 1838 }
 1839 
 1840 static void
 1841 bq_init(struct bufqueue *bq, int qindex, int subqueue, const char *lockname)
 1842 {
 1843 
 1844         mtx_init(&bq->bq_lock, lockname, NULL, MTX_DEF);
 1845         TAILQ_INIT(&bq->bq_queue);
 1846         bq->bq_len = 0;
 1847         bq->bq_index = qindex;
 1848         bq->bq_subqueue = subqueue;
 1849 }
 1850 
 1851 static void
 1852 bd_init(struct bufdomain *bd)
 1853 {
 1854         int i;
 1855 
 1856         bd->bd_cleanq = &bd->bd_subq[mp_maxid + 1];
 1857         bq_init(bd->bd_cleanq, QUEUE_CLEAN, mp_maxid + 1, "bufq clean lock");
 1858         bq_init(&bd->bd_dirtyq, QUEUE_DIRTY, -1, "bufq dirty lock");
 1859         for (i = 0; i <= mp_maxid; i++)
 1860                 bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i,
 1861                     "bufq clean subqueue lock");
 1862         mtx_init(&bd->bd_run_lock, "bufspace daemon run lock", NULL, MTX_DEF);
 1863 }
 1864 
 1865 /*
 1866  *      bq_remove:
 1867  *
 1868  *      Removes a buffer from the free list, must be called with the
 1869  *      correct qlock held.
 1870  */
 1871 static void
 1872 bq_remove(struct bufqueue *bq, struct buf *bp)
 1873 {
 1874 
 1875         CTR3(KTR_BUF, "bq_remove(%p) vp %p flags %X",
 1876             bp, bp->b_vp, bp->b_flags);
 1877         KASSERT(bp->b_qindex != QUEUE_NONE,
 1878             ("bq_remove: buffer %p not on a queue.", bp));
 1879         KASSERT(bufqueue(bp) == bq,
 1880             ("bq_remove: Remove buffer %p from wrong queue.", bp));
 1881 
 1882         BQ_ASSERT_LOCKED(bq);
 1883         if (bp->b_qindex != QUEUE_EMPTY) {
 1884                 BUF_ASSERT_XLOCKED(bp);
 1885         }
 1886         KASSERT(bq->bq_len >= 1,
 1887             ("queue %d underflow", bp->b_qindex));
 1888         TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
 1889         bq->bq_len--;
 1890         bp->b_qindex = QUEUE_NONE;
 1891         bp->b_flags &= ~(B_REMFREE | B_REUSE);
 1892 }
 1893 
 1894 static void
 1895 bd_flush(struct bufdomain *bd, struct bufqueue *bq)
 1896 {
 1897         struct buf *bp;
 1898 
 1899         BQ_ASSERT_LOCKED(bq);
 1900         if (bq != bd->bd_cleanq) {
 1901                 BD_LOCK(bd);
 1902                 while ((bp = TAILQ_FIRST(&bq->bq_queue)) != NULL) {
 1903                         TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
 1904                         TAILQ_INSERT_TAIL(&bd->bd_cleanq->bq_queue, bp,
 1905                             b_freelist);
 1906                         bp->b_subqueue = bd->bd_cleanq->bq_subqueue;
 1907                 }
 1908                 bd->bd_cleanq->bq_len += bq->bq_len;
 1909                 bq->bq_len = 0;
 1910         }
 1911         if (bd->bd_wanted) {
 1912                 bd->bd_wanted = 0;
 1913                 wakeup(&bd->bd_wanted);
 1914         }
 1915         if (bq != bd->bd_cleanq)
 1916                 BD_UNLOCK(bd);
 1917 }
 1918 
 1919 static int
 1920 bd_flushall(struct bufdomain *bd)
 1921 {
 1922         struct bufqueue *bq;
 1923         int flushed;
 1924         int i;
 1925 
 1926         if (bd->bd_lim == 0)
 1927                 return (0);
 1928         flushed = 0;
 1929         for (i = 0; i <= mp_maxid; i++) {
 1930                 bq = &bd->bd_subq[i];
 1931                 if (bq->bq_len == 0)
 1932                         continue;
 1933                 BQ_LOCK(bq);
 1934                 bd_flush(bd, bq);
 1935                 BQ_UNLOCK(bq);
 1936                 flushed++;
 1937         }
 1938 
 1939         return (flushed);
 1940 }
 1941 
 1942 static void
 1943 bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock)
 1944 {
 1945         struct bufdomain *bd;
 1946 
 1947         if (bp->b_qindex != QUEUE_NONE)
 1948                 panic("bq_insert: free buffer %p onto another queue?", bp);
 1949 
 1950         bd = bufdomain(bp);
 1951         if (bp->b_flags & B_AGE) {
 1952                 /* Place this buf directly on the real queue. */
 1953                 if (bq->bq_index == QUEUE_CLEAN)
 1954                         bq = bd->bd_cleanq;
 1955                 BQ_LOCK(bq);
 1956                 TAILQ_INSERT_HEAD(&bq->bq_queue, bp, b_freelist);
 1957         } else {
 1958                 BQ_LOCK(bq);
 1959                 TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
 1960         }
 1961         bp->b_flags &= ~(B_AGE | B_REUSE);
 1962         bq->bq_len++;
 1963         bp->b_qindex = bq->bq_index;
 1964         bp->b_subqueue = bq->bq_subqueue;
 1965 
 1966         /*
 1967          * Unlock before we notify so that we don't wakeup a waiter that
 1968          * fails a trylock on the buf and sleeps again.
 1969          */
 1970         if (unlock)
 1971                 BUF_UNLOCK(bp);
 1972 
 1973         if (bp->b_qindex == QUEUE_CLEAN) {
 1974                 /*
 1975                  * Flush the per-cpu queue and notify any waiters.
 1976                  */
 1977                 if (bd->bd_wanted || (bq != bd->bd_cleanq &&
 1978                     bq->bq_len >= bd->bd_lim))
 1979                         bd_flush(bd, bq);
 1980         }
 1981         BQ_UNLOCK(bq);
 1982 }
 1983 
 1984 /*
 1985  *      bufkva_free:
 1986  *
 1987  *      Free the kva allocation for a buffer.
 1988  *
 1989  */
 1990 static void
 1991 bufkva_free(struct buf *bp)
 1992 {
 1993 
 1994 #ifdef INVARIANTS
 1995         if (bp->b_kvasize == 0) {
 1996                 KASSERT(bp->b_kvabase == unmapped_buf &&
 1997                     bp->b_data == unmapped_buf,
 1998                     ("Leaked KVA space on %p", bp));
 1999         } else if (buf_mapped(bp))
 2000                 BUF_CHECK_MAPPED(bp);
 2001         else
 2002                 BUF_CHECK_UNMAPPED(bp);
 2003 #endif
 2004         if (bp->b_kvasize == 0)
 2005                 return;
 2006 
 2007         vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
 2008         counter_u64_add(bufkvaspace, -bp->b_kvasize);
 2009         counter_u64_add(buffreekvacnt, 1);
 2010         bp->b_data = bp->b_kvabase = unmapped_buf;
 2011         bp->b_kvasize = 0;
 2012 }
 2013 
 2014 /*
 2015  *      bufkva_alloc:
 2016  *
 2017  *      Allocate the buffer KVA and set b_kvasize and b_kvabase.
 2018  */
 2019 static int
 2020 bufkva_alloc(struct buf *bp, int maxsize, int gbflags)
 2021 {
 2022         vm_offset_t addr;
 2023         int error;
 2024 
 2025         KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0,
 2026             ("Invalid gbflags 0x%x in %s", gbflags, __func__));
 2027 
 2028         bufkva_free(bp);
 2029 
 2030         addr = 0;
 2031         error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr);
 2032         if (error != 0) {
 2033                 /*
 2034                  * Buffer map is too fragmented.  Request the caller
 2035                  * to defragment the map.
 2036                  */
 2037                 return (error);
 2038         }
 2039         bp->b_kvabase = (caddr_t)addr;
 2040         bp->b_kvasize = maxsize;
 2041         counter_u64_add(bufkvaspace, bp->b_kvasize);
 2042         if ((gbflags & GB_UNMAPPED) != 0) {
 2043                 bp->b_data = unmapped_buf;
 2044                 BUF_CHECK_UNMAPPED(bp);
 2045         } else {
 2046                 bp->b_data = bp->b_kvabase;
 2047                 BUF_CHECK_MAPPED(bp);
 2048         }
 2049         return (0);
 2050 }
 2051 
 2052 /*
 2053  *      bufkva_reclaim:
 2054  *
 2055  *      Reclaim buffer kva by freeing buffers holding kva.  This is a vmem
 2056  *      callback that fires to avoid returning failure.
 2057  */
 2058 static void
 2059 bufkva_reclaim(vmem_t *vmem, int flags)
 2060 {
 2061         bool done;
 2062         int q;
 2063         int i;
 2064 
 2065         done = false;
 2066         for (i = 0; i < 5; i++) {
 2067                 for (q = 0; q < buf_domains; q++)
 2068                         if (buf_recycle(&bdomain[q], true) != 0)
 2069                                 done = true;
 2070                 if (done)
 2071                         break;
 2072         }
 2073         return;
 2074 }
 2075 
 2076 /*
 2077  * Attempt to initiate asynchronous I/O on read-ahead blocks.  We must
 2078  * clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
 2079  * the buffer is valid and we do not have to do anything.
 2080  */
 2081 static void
 2082 breada(struct vnode * vp, daddr_t * rablkno, int * rabsize, int cnt,
 2083     struct ucred * cred, int flags, void (*ckhashfunc)(struct buf *))
 2084 {
 2085         struct buf *rabp;
 2086         int i;
 2087 
 2088         for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
 2089                 if (inmem(vp, *rablkno))
 2090                         continue;
 2091                 rabp = getblk(vp, *rablkno, *rabsize, 0, 0, 0);
 2092                 if ((rabp->b_flags & B_CACHE) != 0) {
 2093                         brelse(rabp);
 2094                         continue;
 2095                 }
 2096                 if (!TD_IS_IDLETHREAD(curthread)) {
 2097 #ifdef RACCT
 2098                         if (racct_enable) {
 2099                                 PROC_LOCK(curproc);
 2100                                 racct_add_buf(curproc, rabp, 0);
 2101                                 PROC_UNLOCK(curproc);
 2102                         }
 2103 #endif /* RACCT */
 2104                         curthread->td_ru.ru_inblock++;
 2105                 }
 2106                 rabp->b_flags |= B_ASYNC;
 2107                 rabp->b_flags &= ~B_INVAL;
 2108                 if ((flags & GB_CKHASH) != 0) {
 2109                         rabp->b_flags |= B_CKHASH;
 2110                         rabp->b_ckhashcalc = ckhashfunc;
 2111                 }
 2112                 rabp->b_ioflags &= ~BIO_ERROR;
 2113                 rabp->b_iocmd = BIO_READ;
 2114                 if (rabp->b_rcred == NOCRED && cred != NOCRED)
 2115                         rabp->b_rcred = crhold(cred);
 2116                 vfs_busy_pages(rabp, 0);
 2117                 BUF_KERNPROC(rabp);
 2118                 rabp->b_iooffset = dbtob(rabp->b_blkno);
 2119                 bstrategy(rabp);
 2120         }
 2121 }
 2122 
 2123 /*
 2124  * Entry point for bread() and breadn() via #defines in sys/buf.h.
 2125  *
 2126  * Get a buffer with the specified data.  Look in the cache first.  We
 2127  * must clear BIO_ERROR and B_INVAL prior to initiating I/O.  If B_CACHE
 2128  * is set, the buffer is valid and we do not have to do anything, see
 2129  * getblk(). Also starts asynchronous I/O on read-ahead blocks.
 2130  *
 2131  * Always return a NULL buffer pointer (in bpp) when returning an error.
 2132  */
 2133 int
 2134 breadn_flags(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablkno,
 2135     int *rabsize, int cnt, struct ucred *cred, int flags,
 2136     void (*ckhashfunc)(struct buf *), struct buf **bpp)
 2137 {
 2138         struct buf *bp;
 2139         struct thread *td;
 2140         int error, readwait, rv;
 2141 
 2142         CTR3(KTR_BUF, "breadn(%p, %jd, %d)", vp, blkno, size);
 2143         td = curthread;
 2144         /*
 2145          * Can only return NULL if GB_LOCK_NOWAIT or GB_SPARSE flags
 2146          * are specified.
 2147          */
 2148         error = getblkx(vp, blkno, size, 0, 0, flags, &bp);
 2149         if (error != 0) {
 2150                 *bpp = NULL;
 2151                 return (error);
 2152         }
 2153         flags &= ~GB_NOSPARSE;
 2154         *bpp = bp;
 2155 
 2156         /*
 2157          * If not found in cache, do some I/O
 2158          */
 2159         readwait = 0;
 2160         if ((bp->b_flags & B_CACHE) == 0) {
 2161                 if (!TD_IS_IDLETHREAD(td)) {
 2162 #ifdef RACCT
 2163                         if (racct_enable) {
 2164                                 PROC_LOCK(td->td_proc);
 2165                                 racct_add_buf(td->td_proc, bp, 0);
 2166                                 PROC_UNLOCK(td->td_proc);
 2167                         }
 2168 #endif /* RACCT */
 2169                         td->td_ru.ru_inblock++;
 2170                 }
 2171                 bp->b_iocmd = BIO_READ;
 2172                 bp->b_flags &= ~B_INVAL;
 2173                 if ((flags & GB_CKHASH) != 0) {
 2174                         bp->b_flags |= B_CKHASH;
 2175                         bp->b_ckhashcalc = ckhashfunc;
 2176                 }
 2177                 bp->b_ioflags &= ~BIO_ERROR;
 2178                 if (bp->b_rcred == NOCRED && cred != NOCRED)
 2179                         bp->b_rcred = crhold(cred);
 2180                 vfs_busy_pages(bp, 0);
 2181                 bp->b_iooffset = dbtob(bp->b_blkno);
 2182                 bstrategy(bp);
 2183                 ++readwait;
 2184         }
 2185 
 2186         /*
 2187          * Attempt to initiate asynchronous I/O on read-ahead blocks.
 2188          */
 2189         breada(vp, rablkno, rabsize, cnt, cred, flags, ckhashfunc);
 2190 
 2191         rv = 0;
 2192         if (readwait) {
 2193                 rv = bufwait(bp);
 2194                 if (rv != 0) {
 2195                         brelse(bp);
 2196                         *bpp = NULL;
 2197                 }
 2198         }
 2199         return (rv);
 2200 }
 2201 
 2202 /*
 2203  * Write, release buffer on completion.  (Done by iodone
 2204  * if async).  Do not bother writing anything if the buffer
 2205  * is invalid.
 2206  *
 2207  * Note that we set B_CACHE here, indicating that buffer is
 2208  * fully valid and thus cacheable.  This is true even of NFS
 2209  * now so we set it generally.  This could be set either here 
 2210  * or in biodone() since the I/O is synchronous.  We put it
 2211  * here.
 2212  */
 2213 int
 2214 bufwrite(struct buf *bp)
 2215 {
 2216         int oldflags;
 2217         struct vnode *vp;
 2218         long space;
 2219         int vp_md;
 2220 
 2221         CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
 2222         if ((bp->b_bufobj->bo_flag & BO_DEAD) != 0) {
 2223                 bp->b_flags |= B_INVAL | B_RELBUF;
 2224                 bp->b_flags &= ~B_CACHE;
 2225                 brelse(bp);
 2226                 return (ENXIO);
 2227         }
 2228         if (bp->b_flags & B_INVAL) {
 2229                 brelse(bp);
 2230                 return (0);
 2231         }
 2232 
 2233         if (bp->b_flags & B_BARRIER)
 2234                 barrierwrites++;
 2235 
 2236         oldflags = bp->b_flags;
 2237 
 2238         BUF_ASSERT_HELD(bp);
 2239 
 2240         KASSERT(!(bp->b_vflags & BV_BKGRDINPROG),
 2241             ("FFS background buffer should not get here %p", bp));
 2242 
 2243         vp = bp->b_vp;
 2244         if (vp)
 2245                 vp_md = vp->v_vflag & VV_MD;
 2246         else
 2247                 vp_md = 0;
 2248 
 2249         /*
 2250          * Mark the buffer clean.  Increment the bufobj write count
 2251          * before bundirty() call, to prevent other thread from seeing
 2252          * empty dirty list and zero counter for writes in progress,
 2253          * falsely indicating that the bufobj is clean.
 2254          */
 2255         bufobj_wref(bp->b_bufobj);
 2256         bundirty(bp);
 2257 
 2258         bp->b_flags &= ~B_DONE;
 2259         bp->b_ioflags &= ~BIO_ERROR;
 2260         bp->b_flags |= B_CACHE;
 2261         bp->b_iocmd = BIO_WRITE;
 2262 
 2263         vfs_busy_pages(bp, 1);
 2264 
 2265         /*
 2266          * Normal bwrites pipeline writes
 2267          */
 2268         bp->b_runningbufspace = bp->b_bufsize;
 2269         space = atomic_fetchadd_long(&runningbufspace, bp->b_runningbufspace);
 2270 
 2271         if (!TD_IS_IDLETHREAD(curthread)) {
 2272 #ifdef RACCT
 2273                 if (racct_enable) {
 2274                         PROC_LOCK(curproc);
 2275                         racct_add_buf(curproc, bp, 1);
 2276                         PROC_UNLOCK(curproc);
 2277                 }
 2278 #endif /* RACCT */
 2279                 curthread->td_ru.ru_oublock++;
 2280         }
 2281         if (oldflags & B_ASYNC)
 2282                 BUF_KERNPROC(bp);
 2283         bp->b_iooffset = dbtob(bp->b_blkno);
 2284         buf_track(bp, __func__);
 2285         bstrategy(bp);
 2286 
 2287         if ((oldflags & B_ASYNC) == 0) {
 2288                 int rtval = bufwait(bp);
 2289                 brelse(bp);
 2290                 return (rtval);
 2291         } else if (space > hirunningspace) {
 2292                 /*
 2293                  * don't allow the async write to saturate the I/O
 2294                  * system.  We will not deadlock here because
 2295                  * we are blocking waiting for I/O that is already in-progress
 2296                  * to complete. We do not block here if it is the update
 2297                  * or syncer daemon trying to clean up as that can lead
 2298                  * to deadlock.
 2299                  */
 2300                 if ((curthread->td_pflags & TDP_NORUNNINGBUF) == 0 && !vp_md)
 2301                         waitrunningbufspace();
 2302         }
 2303 
 2304         return (0);
 2305 }
 2306 
 2307 void
 2308 bufbdflush(struct bufobj *bo, struct buf *bp)
 2309 {
 2310         struct buf *nbp;
 2311 
 2312         if (bo->bo_dirty.bv_cnt > dirtybufthresh + 10) {
 2313                 (void) VOP_FSYNC(bp->b_vp, MNT_NOWAIT, curthread);
 2314                 altbufferflushes++;
 2315         } else if (bo->bo_dirty.bv_cnt > dirtybufthresh) {
 2316                 BO_LOCK(bo);
 2317                 /*
 2318                  * Try to find a buffer to flush.
 2319                  */
 2320                 TAILQ_FOREACH(nbp, &bo->bo_dirty.bv_hd, b_bobufs) {
 2321                         if ((nbp->b_vflags & BV_BKGRDINPROG) ||
 2322                             BUF_LOCK(nbp,
 2323                                      LK_EXCLUSIVE | LK_NOWAIT, NULL))
 2324                                 continue;
 2325                         if (bp == nbp)
 2326                                 panic("bdwrite: found ourselves");
 2327                         BO_UNLOCK(bo);
 2328                         /* Don't countdeps with the bo lock held. */
 2329                         if (buf_countdeps(nbp, 0)) {
 2330                                 BO_LOCK(bo);
 2331                                 BUF_UNLOCK(nbp);
 2332                                 continue;
 2333                         }
 2334                         if (nbp->b_flags & B_CLUSTEROK) {
 2335                                 vfs_bio_awrite(nbp);
 2336                         } else {
 2337                                 bremfree(nbp);
 2338                                 bawrite(nbp);
 2339                         }
 2340                         dirtybufferflushes++;
 2341                         break;
 2342                 }
 2343                 if (nbp == NULL)
 2344                         BO_UNLOCK(bo);
 2345         }
 2346 }
 2347 
 2348 /*
 2349  * Delayed write. (Buffer is marked dirty).  Do not bother writing
 2350  * anything if the buffer is marked invalid.
 2351  *
 2352  * Note that since the buffer must be completely valid, we can safely
 2353  * set B_CACHE.  In fact, we have to set B_CACHE here rather then in
 2354  * biodone() in order to prevent getblk from writing the buffer
 2355  * out synchronously.
 2356  */
 2357 void
 2358 bdwrite(struct buf *bp)
 2359 {
 2360         struct thread *td = curthread;
 2361         struct vnode *vp;
 2362         struct bufobj *bo;
 2363 
 2364         CTR3(KTR_BUF, "bdwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
 2365         KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
 2366         KASSERT((bp->b_flags & B_BARRIER) == 0,
 2367             ("Barrier request in delayed write %p", bp));
 2368         BUF_ASSERT_HELD(bp);
 2369 
 2370         if (bp->b_flags & B_INVAL) {
 2371                 brelse(bp);
 2372                 return;
 2373         }
 2374 
 2375         /*
 2376          * If we have too many dirty buffers, don't create any more.
 2377          * If we are wildly over our limit, then force a complete
 2378          * cleanup. Otherwise, just keep the situation from getting
 2379          * out of control. Note that we have to avoid a recursive
 2380          * disaster and not try to clean up after our own cleanup!
 2381          */
 2382         vp = bp->b_vp;
 2383         bo = bp->b_bufobj;
 2384         if ((td->td_pflags & (TDP_COWINPROGRESS|TDP_INBDFLUSH)) == 0) {
 2385                 td->td_pflags |= TDP_INBDFLUSH;
 2386                 BO_BDFLUSH(bo, bp);
 2387                 td->td_pflags &= ~TDP_INBDFLUSH;
 2388         } else
 2389                 recursiveflushes++;
 2390 
 2391         bdirty(bp);
 2392         /*
 2393          * Set B_CACHE, indicating that the buffer is fully valid.  This is
 2394          * true even of NFS now.
 2395          */
 2396         bp->b_flags |= B_CACHE;
 2397 
 2398         /*
 2399          * This bmap keeps the system from needing to do the bmap later,
 2400          * perhaps when the system is attempting to do a sync.  Since it
 2401          * is likely that the indirect block -- or whatever other datastructure
 2402          * that the filesystem needs is still in memory now, it is a good
 2403          * thing to do this.  Note also, that if the pageout daemon is
 2404          * requesting a sync -- there might not be enough memory to do
 2405          * the bmap then...  So, this is important to do.
 2406          */
 2407         if (vp->v_type != VCHR && bp->b_lblkno == bp->b_blkno) {
 2408                 VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
 2409         }
 2410 
 2411         buf_track(bp, __func__);
 2412 
 2413         /*
 2414          * Set the *dirty* buffer range based upon the VM system dirty
 2415          * pages.
 2416          *
 2417          * Mark the buffer pages as clean.  We need to do this here to
 2418          * satisfy the vnode_pager and the pageout daemon, so that it
 2419          * thinks that the pages have been "cleaned".  Note that since
 2420          * the pages are in a delayed write buffer -- the VFS layer
 2421          * "will" see that the pages get written out on the next sync,
 2422          * or perhaps the cluster will be completed.
 2423          */
 2424         vfs_clean_pages_dirty_buf(bp);
 2425         bqrelse(bp);
 2426 
 2427         /*
 2428          * note: we cannot initiate I/O from a bdwrite even if we wanted to,
 2429          * due to the softdep code.
 2430          */
 2431 }
 2432 
 2433 /*
 2434  *      bdirty:
 2435  *
 2436  *      Turn buffer into delayed write request.  We must clear BIO_READ and
 2437  *      B_RELBUF, and we must set B_DELWRI.  We reassign the buffer to 
 2438  *      itself to properly update it in the dirty/clean lists.  We mark it
 2439  *      B_DONE to ensure that any asynchronization of the buffer properly
 2440  *      clears B_DONE ( else a panic will occur later ).  
 2441  *
 2442  *      bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
 2443  *      might have been set pre-getblk().  Unlike bwrite/bdwrite, bdirty()
 2444  *      should only be called if the buffer is known-good.
 2445  *
 2446  *      Since the buffer is not on a queue, we do not update the numfreebuffers
 2447  *      count.
 2448  *
 2449  *      The buffer must be on QUEUE_NONE.
 2450  */
 2451 void
 2452 bdirty(struct buf *bp)
 2453 {
 2454 
 2455         CTR3(KTR_BUF, "bdirty(%p) vp %p flags %X",
 2456             bp, bp->b_vp, bp->b_flags);
 2457         KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
 2458         KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
 2459             ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
 2460         BUF_ASSERT_HELD(bp);
 2461         bp->b_flags &= ~(B_RELBUF);
 2462         bp->b_iocmd = BIO_WRITE;
 2463 
 2464         if ((bp->b_flags & B_DELWRI) == 0) {
 2465                 bp->b_flags |= /* XXX B_DONE | */ B_DELWRI;
 2466                 reassignbuf(bp);
 2467                 bdirtyadd(bp);
 2468         }
 2469 }
 2470 
 2471 /*
 2472  *      bundirty:
 2473  *
 2474  *      Clear B_DELWRI for buffer.
 2475  *
 2476  *      Since the buffer is not on a queue, we do not update the numfreebuffers
 2477  *      count.
 2478  *      
 2479  *      The buffer must be on QUEUE_NONE.
 2480  */
 2481 
 2482 void
 2483 bundirty(struct buf *bp)
 2484 {
 2485 
 2486         CTR3(KTR_BUF, "bundirty(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
 2487         KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
 2488         KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
 2489             ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
 2490         BUF_ASSERT_HELD(bp);
 2491 
 2492         if (bp->b_flags & B_DELWRI) {
 2493                 bp->b_flags &= ~B_DELWRI;
 2494                 reassignbuf(bp);
 2495                 bdirtysub(bp);
 2496         }
 2497         /*
 2498          * Since it is now being written, we can clear its deferred write flag.
 2499          */
 2500         bp->b_flags &= ~B_DEFERRED;
 2501 }
 2502 
 2503 /*
 2504  *      bawrite:
 2505  *
 2506  *      Asynchronous write.  Start output on a buffer, but do not wait for
 2507  *      it to complete.  The buffer is released when the output completes.
 2508  *
 2509  *      bwrite() ( or the VOP routine anyway ) is responsible for handling 
 2510  *      B_INVAL buffers.  Not us.
 2511  */
 2512 void
 2513 bawrite(struct buf *bp)
 2514 {
 2515 
 2516         bp->b_flags |= B_ASYNC;
 2517         (void) bwrite(bp);
 2518 }
 2519 
 2520 /*
 2521  *      babarrierwrite:
 2522  *
 2523  *      Asynchronous barrier write.  Start output on a buffer, but do not
 2524  *      wait for it to complete.  Place a write barrier after this write so
 2525  *      that this buffer and all buffers written before it are committed to
 2526  *      the disk before any buffers written after this write are committed
 2527  *      to the disk.  The buffer is released when the output completes.
 2528  */
 2529 void
 2530 babarrierwrite(struct buf *bp)
 2531 {
 2532 
 2533         bp->b_flags |= B_ASYNC | B_BARRIER;
 2534         (void) bwrite(bp);
 2535 }
 2536 
 2537 /*
 2538  *      bbarrierwrite:
 2539  *
 2540  *      Synchronous barrier write.  Start output on a buffer and wait for
 2541  *      it to complete.  Place a write barrier after this write so that
 2542  *      this buffer and all buffers written before it are committed to 
 2543  *      the disk before any buffers written after this write are committed
 2544  *      to the disk.  The buffer is released when the output completes.
 2545  */
 2546 int
 2547 bbarrierwrite(struct buf *bp)
 2548 {
 2549 
 2550         bp->b_flags |= B_BARRIER;
 2551         return (bwrite(bp));
 2552 }
 2553 
 2554 /*
 2555  *      bwillwrite:
 2556  *
 2557  *      Called prior to the locking of any vnodes when we are expecting to
 2558  *      write.  We do not want to starve the buffer cache with too many
 2559  *      dirty buffers so we block here.  By blocking prior to the locking
 2560  *      of any vnodes we attempt to avoid the situation where a locked vnode
 2561  *      prevents the various system daemons from flushing related buffers.
 2562  */
 2563 void
 2564 bwillwrite(void)
 2565 {
 2566 
 2567         if (buf_dirty_count_severe()) {
 2568                 mtx_lock(&bdirtylock);
 2569                 while (buf_dirty_count_severe()) {
 2570                         bdirtywait = 1;
 2571                         msleep(&bdirtywait, &bdirtylock, (PRIBIO + 4),
 2572                             "flswai", 0);
 2573                 }
 2574                 mtx_unlock(&bdirtylock);
 2575         }
 2576 }
 2577 
 2578 /*
 2579  * Return true if we have too many dirty buffers.
 2580  */
 2581 int
 2582 buf_dirty_count_severe(void)
 2583 {
 2584 
 2585         return (!BIT_EMPTY(BUF_DOMAINS, &bdhidirty));
 2586 }
 2587 
 2588 /*
 2589  *      brelse:
 2590  *
 2591  *      Release a busy buffer and, if requested, free its resources.  The
 2592  *      buffer will be stashed in the appropriate bufqueue[] allowing it
 2593  *      to be accessed later as a cache entity or reused for other purposes.
 2594  */
 2595 void
 2596 brelse(struct buf *bp)
 2597 {
 2598         struct mount *v_mnt;
 2599         int qindex;
 2600 
 2601         /*
 2602          * Many functions erroneously call brelse with a NULL bp under rare
 2603          * error conditions. Simply return when called with a NULL bp.
 2604          */
 2605         if (bp == NULL)
 2606                 return;
 2607         CTR3(KTR_BUF, "brelse(%p) vp %p flags %X",
 2608             bp, bp->b_vp, bp->b_flags);
 2609         KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
 2610             ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
 2611         KASSERT((bp->b_flags & B_VMIO) != 0 || (bp->b_flags & B_NOREUSE) == 0,
 2612             ("brelse: non-VMIO buffer marked NOREUSE"));
 2613 
 2614         if (BUF_LOCKRECURSED(bp)) {
 2615                 /*
 2616                  * Do not process, in particular, do not handle the
 2617                  * B_INVAL/B_RELBUF and do not release to free list.
 2618                  */
 2619                 BUF_UNLOCK(bp);
 2620                 return;
 2621         }
 2622 
 2623         if (bp->b_flags & B_MANAGED) {
 2624                 bqrelse(bp);
 2625                 return;
 2626         }
 2627 
 2628         if ((bp->b_vflags & (BV_BKGRDINPROG | BV_BKGRDERR)) == BV_BKGRDERR) {
 2629                 BO_LOCK(bp->b_bufobj);
 2630                 bp->b_vflags &= ~BV_BKGRDERR;
 2631                 BO_UNLOCK(bp->b_bufobj);
 2632                 bdirty(bp);
 2633         }
 2634         if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
 2635             (bp->b_error != ENXIO || !LIST_EMPTY(&bp->b_dep)) &&
 2636             !(bp->b_flags & B_INVAL)) {
 2637                 /*
 2638                  * Failed write, redirty.  All errors except ENXIO (which
 2639                  * means the device is gone) are treated as being
 2640                  * transient.
 2641                  *
 2642                  * XXX Treating EIO as transient is not correct; the
 2643                  * contract with the local storage device drivers is that
 2644                  * they will only return EIO once the I/O is no longer
 2645                  * retriable.  Network I/O also respects this through the
 2646                  * guarantees of TCP and/or the internal retries of NFS.
 2647                  * ENOMEM might be transient, but we also have no way of
 2648                  * knowing when its ok to retry/reschedule.  In general,
 2649                  * this entire case should be made obsolete through better
 2650                  * error handling/recovery and resource scheduling.
 2651                  *
 2652                  * Do this also for buffers that failed with ENXIO, but have
 2653                  * non-empty dependencies - the soft updates code might need
 2654                  * to access the buffer to untangle them.
 2655                  *
 2656                  * Must clear BIO_ERROR to prevent pages from being scrapped.
 2657                  */
 2658                 bp->b_ioflags &= ~BIO_ERROR;
 2659                 bdirty(bp);
 2660         } else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
 2661             (bp->b_ioflags & BIO_ERROR) || (bp->b_bufsize <= 0)) {
 2662                 /*
 2663                  * Either a failed read I/O, or we were asked to free or not
 2664                  * cache the buffer, or we failed to write to a device that's
 2665                  * no longer present.
 2666                  */
 2667                 bp->b_flags |= B_INVAL;
 2668                 if (!LIST_EMPTY(&bp->b_dep))
 2669                         buf_deallocate(bp);
 2670                 if (bp->b_flags & B_DELWRI)
 2671                         bdirtysub(bp);
 2672                 bp->b_flags &= ~(B_DELWRI | B_CACHE);
 2673                 if ((bp->b_flags & B_VMIO) == 0) {
 2674                         allocbuf(bp, 0);
 2675                         if (bp->b_vp)
 2676                                 brelvp(bp);
 2677                 }
 2678         }
 2679 
 2680         /*
 2681          * We must clear B_RELBUF if B_DELWRI is set.  If vfs_vmio_truncate() 
 2682          * is called with B_DELWRI set, the underlying pages may wind up
 2683          * getting freed causing a previous write (bdwrite()) to get 'lost'
 2684          * because pages associated with a B_DELWRI bp are marked clean.
 2685          * 
 2686          * We still allow the B_INVAL case to call vfs_vmio_truncate(), even
 2687          * if B_DELWRI is set.
 2688          */
 2689         if (bp->b_flags & B_DELWRI)
 2690                 bp->b_flags &= ~B_RELBUF;
 2691 
 2692         /*
 2693          * VMIO buffer rundown.  It is not very necessary to keep a VMIO buffer
 2694          * constituted, not even NFS buffers now.  Two flags effect this.  If
 2695          * B_INVAL, the struct buf is invalidated but the VM object is kept
 2696          * around ( i.e. so it is trivial to reconstitute the buffer later ).
 2697          *
 2698          * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
 2699          * invalidated.  BIO_ERROR cannot be set for a failed write unless the
 2700          * buffer is also B_INVAL because it hits the re-dirtying code above.
 2701          *
 2702          * Normally we can do this whether a buffer is B_DELWRI or not.  If
 2703          * the buffer is an NFS buffer, it is tracking piecemeal writes or
 2704          * the commit state and we cannot afford to lose the buffer. If the
 2705          * buffer has a background write in progress, we need to keep it
 2706          * around to prevent it from being reconstituted and starting a second
 2707          * background write.
 2708          */
 2709 
 2710         v_mnt = bp->b_vp != NULL ? bp->b_vp->v_mount : NULL;
 2711 
 2712         if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE ||
 2713             (bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
 2714             (v_mnt == NULL || (v_mnt->mnt_vfc->vfc_flags & VFCF_NETWORK) == 0 ||
 2715             vn_isdisk(bp->b_vp, NULL) || (bp->b_flags & B_DELWRI) == 0)) {
 2716                 vfs_vmio_invalidate(bp);
 2717                 allocbuf(bp, 0);
 2718         }
 2719 
 2720         if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0 ||
 2721             (bp->b_flags & (B_DELWRI | B_NOREUSE)) == B_NOREUSE) {
 2722                 allocbuf(bp, 0);
 2723                 bp->b_flags &= ~B_NOREUSE;
 2724                 if (bp->b_vp != NULL)
 2725                         brelvp(bp);
 2726         }
 2727                         
 2728         /*
 2729          * If the buffer has junk contents signal it and eventually
 2730          * clean up B_DELWRI and diassociate the vnode so that gbincore()
 2731          * doesn't find it.
 2732          */
 2733         if (bp->b_bufsize == 0 || (bp->b_ioflags & BIO_ERROR) != 0 ||
 2734             (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF)) != 0)
 2735                 bp->b_flags |= B_INVAL;
 2736         if (bp->b_flags & B_INVAL) {
 2737                 if (bp->b_flags & B_DELWRI)
 2738                         bundirty(bp);
 2739                 if (bp->b_vp)
 2740                         brelvp(bp);
 2741         }
 2742 
 2743         buf_track(bp, __func__);
 2744 
 2745         /* buffers with no memory */
 2746         if (bp->b_bufsize == 0) {
 2747                 buf_free(bp);
 2748                 return;
 2749         }
 2750         /* buffers with junk contents */
 2751         if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
 2752             (bp->b_ioflags & BIO_ERROR)) {
 2753                 bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
 2754                 if (bp->b_vflags & BV_BKGRDINPROG)
 2755                         panic("losing buffer 2");
 2756                 qindex = QUEUE_CLEAN;
 2757                 bp->b_flags |= B_AGE;
 2758         /* remaining buffers */
 2759         } else if (bp->b_flags & B_DELWRI)
 2760                 qindex = QUEUE_DIRTY;
 2761         else
 2762                 qindex = QUEUE_CLEAN;
 2763 
 2764         if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
 2765                 panic("brelse: not dirty");
 2766 
 2767         bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_RELBUF | B_DIRECT);
 2768         /* binsfree unlocks bp. */
 2769         binsfree(bp, qindex);
 2770 }
 2771 
 2772 /*
 2773  * Release a buffer back to the appropriate queue but do not try to free
 2774  * it.  The buffer is expected to be used again soon.
 2775  *
 2776  * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
 2777  * biodone() to requeue an async I/O on completion.  It is also used when
 2778  * known good buffers need to be requeued but we think we may need the data
 2779  * again soon.
 2780  *
 2781  * XXX we should be able to leave the B_RELBUF hint set on completion.
 2782  */
 2783 void
 2784 bqrelse(struct buf *bp)
 2785 {
 2786         int qindex;
 2787 
 2788         CTR3(KTR_BUF, "bqrelse(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
 2789         KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
 2790             ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
 2791 
 2792         qindex = QUEUE_NONE;
 2793         if (BUF_LOCKRECURSED(bp)) {
 2794                 /* do not release to free list */
 2795                 BUF_UNLOCK(bp);
 2796                 return;
 2797         }
 2798         bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
 2799 
 2800         if (bp->b_flags & B_MANAGED) {
 2801                 if (bp->b_flags & B_REMFREE)
 2802                         bremfreef(bp);
 2803                 goto out;
 2804         }
 2805 
 2806         /* buffers with stale but valid contents */
 2807         if ((bp->b_flags & B_DELWRI) != 0 || (bp->b_vflags & (BV_BKGRDINPROG |
 2808             BV_BKGRDERR)) == BV_BKGRDERR) {
 2809                 BO_LOCK(bp->b_bufobj);
 2810                 bp->b_vflags &= ~BV_BKGRDERR;
 2811                 BO_UNLOCK(bp->b_bufobj);
 2812                 qindex = QUEUE_DIRTY;
 2813         } else {
 2814                 if ((bp->b_flags & B_DELWRI) == 0 &&
 2815                     (bp->b_xflags & BX_VNDIRTY))
 2816                         panic("bqrelse: not dirty");
 2817                 if ((bp->b_flags & B_NOREUSE) != 0) {
 2818                         brelse(bp);
 2819                         return;
 2820                 }
 2821                 qindex = QUEUE_CLEAN;
 2822         }
 2823         buf_track(bp, __func__);
 2824         /* binsfree unlocks bp. */
 2825         binsfree(bp, qindex);
 2826         return;
 2827 
 2828 out:
 2829         buf_track(bp, __func__);
 2830         /* unlock */
 2831         BUF_UNLOCK(bp);
 2832 }
 2833 
 2834 /*
 2835  * Complete I/O to a VMIO backed page.  Validate the pages as appropriate,
 2836  * restore bogus pages.
 2837  */
 2838 static void
 2839 vfs_vmio_iodone(struct buf *bp)
 2840 {
 2841         vm_ooffset_t foff;
 2842         vm_page_t m;
 2843         vm_object_t obj;
 2844         struct vnode *vp __unused;
 2845         int i, iosize, resid;
 2846         bool bogus;
 2847 
 2848         obj = bp->b_bufobj->bo_object;
 2849         KASSERT(obj->paging_in_progress >= bp->b_npages,
 2850             ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
 2851             obj->paging_in_progress, bp->b_npages));
 2852 
 2853         vp = bp->b_vp;
 2854         KASSERT(vp->v_holdcnt > 0,
 2855             ("vfs_vmio_iodone: vnode %p has zero hold count", vp));
 2856         KASSERT(vp->v_object != NULL,
 2857             ("vfs_vmio_iodone: vnode %p has no vm_object", vp));
 2858 
 2859         foff = bp->b_offset;
 2860         KASSERT(bp->b_offset != NOOFFSET,
 2861             ("vfs_vmio_iodone: bp %p has no buffer offset", bp));
 2862 
 2863         bogus = false;
 2864         iosize = bp->b_bcount - bp->b_resid;
 2865         VM_OBJECT_WLOCK(obj);
 2866         for (i = 0; i < bp->b_npages; i++) {
 2867                 resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
 2868                 if (resid > iosize)
 2869                         resid = iosize;
 2870 
 2871                 /*
 2872                  * cleanup bogus pages, restoring the originals
 2873                  */
 2874                 m = bp->b_pages[i];
 2875                 if (m == bogus_page) {
 2876                         bogus = true;
 2877                         m = vm_page_lookup(obj, OFF_TO_IDX(foff));
 2878                         if (m == NULL)
 2879                                 panic("biodone: page disappeared!");
 2880                         bp->b_pages[i] = m;
 2881                 } else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
 2882                         /*
 2883                          * In the write case, the valid and clean bits are
 2884                          * already changed correctly ( see bdwrite() ), so we 
 2885                          * only need to do this here in the read case.
 2886                          */
 2887                         KASSERT((m->dirty & vm_page_bits(foff & PAGE_MASK,
 2888                             resid)) == 0, ("vfs_vmio_iodone: page %p "
 2889                             "has unexpected dirty bits", m));
 2890                         vfs_page_set_valid(bp, foff, m);
 2891                 }
 2892                 KASSERT(OFF_TO_IDX(foff) == m->pindex,
 2893                     ("vfs_vmio_iodone: foff(%jd)/pindex(%ju) mismatch",
 2894                     (intmax_t)foff, (uintmax_t)m->pindex));
 2895 
 2896                 vm_page_sunbusy(m);
 2897                 foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
 2898                 iosize -= resid;
 2899         }
 2900         vm_object_pip_wakeupn(obj, bp->b_npages);
 2901         VM_OBJECT_WUNLOCK(obj);
 2902         if (bogus && buf_mapped(bp)) {
 2903                 BUF_CHECK_MAPPED(bp);
 2904                 pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
 2905                     bp->b_pages, bp->b_npages);
 2906         }
 2907 }
 2908 
 2909 /*
 2910  * Unwire a page held by a buf and either free it or update the page queues to
 2911  * reflect its recent use.
 2912  */
 2913 static void
 2914 vfs_vmio_unwire(struct buf *bp, vm_page_t m)
 2915 {
 2916         bool freed;
 2917 
 2918         vm_page_lock(m);
 2919         if (vm_page_unwire_noq(m)) {
 2920                 if ((bp->b_flags & B_DIRECT) != 0)
 2921                         freed = vm_page_try_to_free(m);
 2922                 else
 2923                         freed = false;
 2924                 if (!freed) {
 2925                         /*
 2926                          * Use a racy check of the valid bits to determine
 2927                          * whether we can accelerate reclamation of the page.
 2928                          * The valid bits will be stable unless the page is
 2929                          * being mapped or is referenced by multiple buffers,
 2930                          * and in those cases we expect races to be rare.  At
 2931                          * worst we will either accelerate reclamation of a
 2932                          * valid page and violate LRU, or unnecessarily defer
 2933                          * reclamation of an invalid page.
 2934                          *
 2935                          * The B_NOREUSE flag marks data that is not expected to
 2936                          * be reused, so accelerate reclamation in that case
 2937                          * too.  Otherwise, maintain LRU.
 2938                          */
 2939                         if (m->valid == 0 || (bp->b_flags & B_NOREUSE) != 0)
 2940                                 vm_page_deactivate_noreuse(m);
 2941                         else if (vm_page_active(m))
 2942                                 vm_page_reference(m);
 2943                         else
 2944                                 vm_page_deactivate(m);
 2945                 }
 2946         }
 2947         vm_page_unlock(m);
 2948 }
 2949 
 2950 /*
 2951  * Perform page invalidation when a buffer is released.  The fully invalid
 2952  * pages will be reclaimed later in vfs_vmio_truncate().
 2953  */
 2954 static void
 2955 vfs_vmio_invalidate(struct buf *bp)
 2956 {
 2957         vm_object_t obj;
 2958         vm_page_t m;
 2959         int i, resid, poffset, presid;
 2960 
 2961         if (buf_mapped(bp)) {
 2962                 BUF_CHECK_MAPPED(bp);
 2963                 pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
 2964         } else
 2965                 BUF_CHECK_UNMAPPED(bp);
 2966         /*
 2967          * Get the base offset and length of the buffer.  Note that 
 2968          * in the VMIO case if the buffer block size is not
 2969          * page-aligned then b_data pointer may not be page-aligned.
 2970          * But our b_pages[] array *IS* page aligned.
 2971          *
 2972          * block sizes less then DEV_BSIZE (usually 512) are not 
 2973          * supported due to the page granularity bits (m->valid,
 2974          * m->dirty, etc...). 
 2975          *
 2976          * See man buf(9) for more information
 2977          */
 2978         obj = bp->b_bufobj->bo_object;
 2979         resid = bp->b_bufsize;
 2980         poffset = bp->b_offset & PAGE_MASK;
 2981         VM_OBJECT_WLOCK(obj);
 2982         for (i = 0; i < bp->b_npages; i++) {
 2983                 m = bp->b_pages[i];
 2984                 if (m == bogus_page)
 2985                         panic("vfs_vmio_invalidate: Unexpected bogus page.");
 2986                 bp->b_pages[i] = NULL;
 2987 
 2988                 presid = resid > (PAGE_SIZE - poffset) ?
 2989                     (PAGE_SIZE - poffset) : resid;
 2990                 KASSERT(presid >= 0, ("brelse: extra page"));
 2991                 while (vm_page_xbusied(m)) {
 2992                         vm_page_lock(m);
 2993                         VM_OBJECT_WUNLOCK(obj);
 2994                         vm_page_busy_sleep(m, "mbncsh", true);
 2995                         VM_OBJECT_WLOCK(obj);
 2996                 }
 2997                 if (pmap_page_wired_mappings(m) == 0)
 2998                         vm_page_set_invalid(m, poffset, presid);
 2999                 vfs_vmio_unwire(bp, m);
 3000                 resid -= presid;
 3001                 poffset = 0;
 3002         }
 3003         VM_OBJECT_WUNLOCK(obj);
 3004         bp->b_npages = 0;
 3005 }
 3006 
 3007 /*
 3008  * Page-granular truncation of an existing VMIO buffer.
 3009  */
 3010 static void
 3011 vfs_vmio_truncate(struct buf *bp, int desiredpages)
 3012 {
 3013         vm_object_t obj;
 3014         vm_page_t m;
 3015         int i;
 3016 
 3017         if (bp->b_npages == desiredpages)
 3018                 return;
 3019 
 3020         if (buf_mapped(bp)) {
 3021                 BUF_CHECK_MAPPED(bp);
 3022                 pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
 3023                     (desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
 3024         } else
 3025                 BUF_CHECK_UNMAPPED(bp);
 3026 
 3027         /*
 3028          * The object lock is needed only if we will attempt to free pages.
 3029          */
 3030         obj = (bp->b_flags & B_DIRECT) != 0 ? bp->b_bufobj->bo_object : NULL;
 3031         if (obj != NULL)
 3032                 VM_OBJECT_WLOCK(obj);
 3033         for (i = desiredpages; i < bp->b_npages; i++) {
 3034                 m = bp->b_pages[i];
 3035                 KASSERT(m != bogus_page, ("allocbuf: bogus page found"));
 3036                 bp->b_pages[i] = NULL;
 3037                 vfs_vmio_unwire(bp, m);
 3038         }
 3039         if (obj != NULL)
 3040                 VM_OBJECT_WUNLOCK(obj);
 3041         bp->b_npages = desiredpages;
 3042 }
 3043 
 3044 /*
 3045  * Byte granular extension of VMIO buffers.
 3046  */
 3047 static void
 3048 vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
 3049 {
 3050         /*
 3051          * We are growing the buffer, possibly in a 
 3052          * byte-granular fashion.
 3053          */
 3054         vm_object_t obj;
 3055         vm_offset_t toff;
 3056         vm_offset_t tinc;
 3057         vm_page_t m;
 3058 
 3059         /*
 3060          * Step 1, bring in the VM pages from the object, allocating
 3061          * them if necessary.  We must clear B_CACHE if these pages
 3062          * are not valid for the range covered by the buffer.
 3063          */
 3064         obj = bp->b_bufobj->bo_object;
 3065         VM_OBJECT_WLOCK(obj);
 3066         if (bp->b_npages < desiredpages) {
 3067                 /*
 3068                  * We must allocate system pages since blocking
 3069                  * here could interfere with paging I/O, no
 3070                  * matter which process we are.
 3071                  *
 3072                  * Only exclusive busy can be tested here.
 3073                  * Blocking on shared busy might lead to
 3074                  * deadlocks once allocbuf() is called after
 3075                  * pages are vfs_busy_pages().
 3076                  */
 3077                 (void)vm_page_grab_pages(obj,
 3078                     OFF_TO_IDX(bp->b_offset) + bp->b_npages,
 3079                     VM_ALLOC_SYSTEM | VM_ALLOC_IGN_SBUSY |
 3080                     VM_ALLOC_NOBUSY | VM_ALLOC_WIRED,
 3081                     &bp->b_pages[bp->b_npages], desiredpages - bp->b_npages);
 3082                 bp->b_npages = desiredpages;
 3083         }
 3084 
 3085         /*
 3086          * Step 2.  We've loaded the pages into the buffer,
 3087          * we have to figure out if we can still have B_CACHE
 3088          * set.  Note that B_CACHE is set according to the
 3089          * byte-granular range ( bcount and size ), not the
 3090          * aligned range ( newbsize ).
 3091          *
 3092          * The VM test is against m->valid, which is DEV_BSIZE
 3093          * aligned.  Needless to say, the validity of the data
 3094          * needs to also be DEV_BSIZE aligned.  Note that this
 3095          * fails with NFS if the server or some other client
 3096          * extends the file's EOF.  If our buffer is resized, 
 3097          * B_CACHE may remain set! XXX
 3098          */
 3099         toff = bp->b_bcount;
 3100         tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
 3101         while ((bp->b_flags & B_CACHE) && toff < size) {
 3102                 vm_pindex_t pi;
 3103 
 3104                 if (tinc > (size - toff))
 3105                         tinc = size - toff;
 3106                 pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
 3107                 m = bp->b_pages[pi];
 3108                 vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
 3109                 toff += tinc;
 3110                 tinc = PAGE_SIZE;
 3111         }
 3112         VM_OBJECT_WUNLOCK(obj);
 3113 
 3114         /*
 3115          * Step 3, fixup the KVA pmap.
 3116          */
 3117         if (buf_mapped(bp))
 3118                 bpmap_qenter(bp);
 3119         else
 3120                 BUF_CHECK_UNMAPPED(bp);
 3121 }
 3122 
 3123 /*
 3124  * Check to see if a block at a particular lbn is available for a clustered
 3125  * write.
 3126  */
 3127 static int
 3128 vfs_bio_clcheck(struct vnode *vp, int size, daddr_t lblkno, daddr_t blkno)
 3129 {
 3130         struct buf *bpa;
 3131         int match;
 3132 
 3133         match = 0;
 3134 
 3135         /* If the buf isn't in core skip it */
 3136         if ((bpa = gbincore(&vp->v_bufobj, lblkno)) == NULL)
 3137                 return (0);
 3138 
 3139         /* If the buf is busy we don't want to wait for it */
 3140         if (BUF_LOCK(bpa, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
 3141                 return (0);
 3142 
 3143         /* Only cluster with valid clusterable delayed write buffers */
 3144         if ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) !=
 3145             (B_DELWRI | B_CLUSTEROK))
 3146                 goto done;
 3147 
 3148         if (bpa->b_bufsize != size)
 3149                 goto done;
 3150 
 3151         /*
 3152          * Check to see if it is in the expected place on disk and that the
 3153          * block has been mapped.
 3154          */
 3155         if ((bpa->b_blkno != bpa->b_lblkno) && (bpa->b_blkno == blkno))
 3156                 match = 1;
 3157 done:
 3158         BUF_UNLOCK(bpa);
 3159         return (match);
 3160 }
 3161 
 3162 /*
 3163  *      vfs_bio_awrite:
 3164  *
 3165  *      Implement clustered async writes for clearing out B_DELWRI buffers.
 3166  *      This is much better then the old way of writing only one buffer at
 3167  *      a time.  Note that we may not be presented with the buffers in the 
 3168  *      correct order, so we search for the cluster in both directions.
 3169  */
 3170 int
 3171 vfs_bio_awrite(struct buf *bp)
 3172 {
 3173         struct bufobj *bo;
 3174         int i;
 3175         int j;
 3176         daddr_t lblkno = bp->b_lblkno;
 3177         struct vnode *vp = bp->b_vp;
 3178         int ncl;
 3179         int nwritten;
 3180         int size;
 3181         int maxcl;
 3182         int gbflags;
 3183 
 3184         bo = &vp->v_bufobj;
 3185         gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
 3186         /*
 3187          * right now we support clustered writing only to regular files.  If
 3188          * we find a clusterable block we could be in the middle of a cluster
 3189          * rather then at the beginning.
 3190          */
 3191         if ((vp->v_type == VREG) && 
 3192             (vp->v_mount != 0) && /* Only on nodes that have the size info */
 3193             (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
 3194 
 3195                 size = vp->v_mount->mnt_stat.f_iosize;
 3196                 maxcl = MAXPHYS / size;
 3197 
 3198                 BO_RLOCK(bo);
 3199                 for (i = 1; i < maxcl; i++)
 3200                         if (vfs_bio_clcheck(vp, size, lblkno + i,
 3201                             bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0)
 3202                                 break;
 3203 
 3204                 for (j = 1; i + j <= maxcl && j <= lblkno; j++) 
 3205                         if (vfs_bio_clcheck(vp, size, lblkno - j,
 3206                             bp->b_blkno - ((j * size) >> DEV_BSHIFT)) == 0)
 3207                                 break;
 3208                 BO_RUNLOCK(bo);
 3209                 --j;
 3210                 ncl = i + j;
 3211                 /*
 3212                  * this is a possible cluster write
 3213                  */
 3214                 if (ncl != 1) {
 3215                         BUF_UNLOCK(bp);
 3216                         nwritten = cluster_wbuild(vp, size, lblkno - j, ncl,
 3217                             gbflags);
 3218                         return (nwritten);
 3219                 }
 3220         }
 3221         bremfree(bp);
 3222         bp->b_flags |= B_ASYNC;
 3223         /*
 3224          * default (old) behavior, writing out only one block
 3225          *
 3226          * XXX returns b_bufsize instead of b_bcount for nwritten?
 3227          */
 3228         nwritten = bp->b_bufsize;
 3229         (void) bwrite(bp);
 3230 
 3231         return (nwritten);
 3232 }
 3233 
 3234 /*
 3235  *      getnewbuf_kva:
 3236  *
 3237  *      Allocate KVA for an empty buf header according to gbflags.
 3238  */
 3239 static int
 3240 getnewbuf_kva(struct buf *bp, int gbflags, int maxsize)
 3241 {
 3242 
 3243         if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_UNMAPPED) {
 3244                 /*
 3245                  * In order to keep fragmentation sane we only allocate kva
 3246                  * in BKVASIZE chunks.  XXX with vmem we can do page size.
 3247                  */
 3248                 maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
 3249 
 3250                 if (maxsize != bp->b_kvasize &&
 3251                     bufkva_alloc(bp, maxsize, gbflags))
 3252                         return (ENOSPC);
 3253         }
 3254         return (0);
 3255 }
 3256 
 3257 /*
 3258  *      getnewbuf:
 3259  *
 3260  *      Find and initialize a new buffer header, freeing up existing buffers
 3261  *      in the bufqueues as necessary.  The new buffer is returned locked.
 3262  *
 3263  *      We block if:
 3264  *              We have insufficient buffer headers
 3265  *              We have insufficient buffer space
 3266  *              buffer_arena is too fragmented ( space reservation fails )
 3267  *              If we have to flush dirty buffers ( but we try to avoid this )
 3268  *
 3269  *      The caller is responsible for releasing the reserved bufspace after
 3270  *      allocbuf() is called.
 3271  */
 3272 static struct buf *
 3273 getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int maxsize, int gbflags)
 3274 {
 3275         struct bufdomain *bd;
 3276         struct buf *bp;
 3277         bool metadata, reserved;
 3278 
 3279         bp = NULL;
 3280         KASSERT((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
 3281             ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
 3282         if (!unmapped_buf_allowed)
 3283                 gbflags &= ~(GB_UNMAPPED | GB_KVAALLOC);
 3284 
 3285         if (vp == NULL || (vp->v_vflag & (VV_MD | VV_SYSTEM)) != 0 ||
 3286             vp->v_type == VCHR)
 3287                 metadata = true;
 3288         else
 3289                 metadata = false;
 3290         if (vp == NULL)
 3291                 bd = &bdomain[0];
 3292         else
 3293                 bd = &bdomain[vp->v_bufobj.bo_domain];
 3294 
 3295         counter_u64_add(getnewbufcalls, 1);
 3296         reserved = false;
 3297         do {
 3298                 if (reserved == false &&
 3299                     bufspace_reserve(bd, maxsize, metadata) != 0) {
 3300                         counter_u64_add(getnewbufrestarts, 1);
 3301                         continue;
 3302                 }
 3303                 reserved = true;
 3304                 if ((bp = buf_alloc(bd)) == NULL) {
 3305                         counter_u64_add(getnewbufrestarts, 1);
 3306                         continue;
 3307                 }
 3308                 if (getnewbuf_kva(bp, gbflags, maxsize) == 0)
 3309                         return (bp);
 3310                 break;
 3311         } while (buf_recycle(bd, false) == 0);
 3312 
 3313         if (reserved)
 3314                 bufspace_release(bd, maxsize);
 3315         if (bp != NULL) {
 3316                 bp->b_flags |= B_INVAL;
 3317                 brelse(bp);
 3318         }
 3319         bufspace_wait(bd, vp, gbflags, slpflag, slptimeo);
 3320 
 3321         return (NULL);
 3322 }
 3323 
 3324 /*
 3325  *      buf_daemon:
 3326  *
 3327  *      buffer flushing daemon.  Buffers are normally flushed by the
 3328  *      update daemon but if it cannot keep up this process starts to
 3329  *      take the load in an attempt to prevent getnewbuf() from blocking.
 3330  */
 3331 static struct kproc_desc buf_kp = {
 3332         "bufdaemon",
 3333         buf_daemon,
 3334         &bufdaemonproc
 3335 };
 3336 SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp);
 3337 
 3338 static int
 3339 buf_flush(struct vnode *vp, struct bufdomain *bd, int target)
 3340 {
 3341         int flushed;
 3342 
 3343         flushed = flushbufqueues(vp, bd, target, 0);
 3344         if (flushed == 0) {
 3345                 /*
 3346                  * Could not find any buffers without rollback
 3347                  * dependencies, so just write the first one
 3348                  * in the hopes of eventually making progress.
 3349                  */
 3350                 if (vp != NULL && target > 2)
 3351                         target /= 2;
 3352                 flushbufqueues(vp, bd, target, 1);
 3353         }
 3354         return (flushed);
 3355 }
 3356 
 3357 static void
 3358 buf_daemon()
 3359 {
 3360         struct bufdomain *bd;
 3361         int speedupreq;
 3362         int lodirty;
 3363         int i;
 3364 
 3365         /*
 3366          * This process needs to be suspended prior to shutdown sync.
 3367          */
 3368         EVENTHANDLER_REGISTER(shutdown_pre_sync, kthread_shutdown, curthread,
 3369             SHUTDOWN_PRI_LAST + 100);
 3370 
 3371         /*
 3372          * Start the buf clean daemons as children threads.
 3373          */
 3374         for (i = 0 ; i < buf_domains; i++) {
 3375                 int error;
 3376 
 3377                 error = kthread_add((void (*)(void *))bufspace_daemon,
 3378                     &bdomain[i], curproc, NULL, 0, 0, "bufspacedaemon-%d", i);
 3379                 if (error)
 3380                         panic("error %d spawning bufspace daemon", error);
 3381         }
 3382 
 3383         /*
 3384          * This process is allowed to take the buffer cache to the limit
 3385          */
 3386         curthread->td_pflags |= TDP_NORUNNINGBUF | TDP_BUFNEED;
 3387         mtx_lock(&bdlock);
 3388         for (;;) {
 3389                 bd_request = 0;
 3390                 mtx_unlock(&bdlock);
 3391 
 3392                 kthread_suspend_check();
 3393 
 3394                 /*
 3395                  * Save speedupreq for this pass and reset to capture new
 3396                  * requests.
 3397                  */
 3398                 speedupreq = bd_speedupreq;
 3399                 bd_speedupreq = 0;
 3400 
 3401                 /*
 3402                  * Flush each domain sequentially according to its level and
 3403                  * the speedup request.
 3404                  */
 3405                 for (i = 0; i < buf_domains; i++) {
 3406                         bd = &bdomain[i];
 3407                         if (speedupreq)
 3408                                 lodirty = bd->bd_numdirtybuffers / 2;
 3409                         else
 3410                                 lodirty = bd->bd_lodirtybuffers;
 3411                         while (bd->bd_numdirtybuffers > lodirty) {
 3412                                 if (buf_flush(NULL, bd,
 3413                                     bd->bd_numdirtybuffers - lodirty) == 0)
 3414                                         break;
 3415                                 kern_yield(PRI_USER);
 3416                         }
 3417                 }
 3418 
 3419                 /*
 3420                  * Only clear bd_request if we have reached our low water
 3421                  * mark.  The buf_daemon normally waits 1 second and
 3422                  * then incrementally flushes any dirty buffers that have
 3423                  * built up, within reason.
 3424                  *
 3425                  * If we were unable to hit our low water mark and couldn't
 3426                  * find any flushable buffers, we sleep for a short period
 3427                  * to avoid endless loops on unlockable buffers.
 3428                  */
 3429                 mtx_lock(&bdlock);
 3430                 if (!BIT_EMPTY(BUF_DOMAINS, &bdlodirty)) {
 3431                         /*
 3432                          * We reached our low water mark, reset the
 3433                          * request and sleep until we are needed again.
 3434                          * The sleep is just so the suspend code works.
 3435                          */
 3436                         bd_request = 0;
 3437                         /*
 3438                          * Do an extra wakeup in case dirty threshold
 3439                          * changed via sysctl and the explicit transition
 3440                          * out of shortfall was missed.
 3441                          */
 3442                         bdirtywakeup();
 3443                         if (runningbufspace <= lorunningspace)
 3444                                 runningwakeup();
 3445                         msleep(&bd_request, &bdlock, PVM, "psleep", hz);
 3446                 } else {
 3447                         /*
 3448                          * We couldn't find any flushable dirty buffers but
 3449                          * still have too many dirty buffers, we
 3450                          * have to sleep and try again.  (rare)
 3451                          */
 3452                         msleep(&bd_request, &bdlock, PVM, "qsleep", hz / 10);
 3453                 }
 3454         }
 3455 }
 3456 
 3457 /*
 3458  *      flushbufqueues:
 3459  *
 3460  *      Try to flush a buffer in the dirty queue.  We must be careful to
 3461  *      free up B_INVAL buffers instead of write them, which NFS is 
 3462  *      particularly sensitive to.
 3463  */
 3464 static int flushwithdeps = 0;
 3465 SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_RW, &flushwithdeps,
 3466     0, "Number of buffers flushed with dependecies that require rollbacks");
 3467 
 3468 static int
 3469 flushbufqueues(struct vnode *lvp, struct bufdomain *bd, int target,
 3470     int flushdeps)
 3471 {
 3472         struct bufqueue *bq;
 3473         struct buf *sentinel;
 3474         struct vnode *vp;
 3475         struct mount *mp;
 3476         struct buf *bp;
 3477         int hasdeps;
 3478         int flushed;
 3479         int error;
 3480         bool unlock;
 3481 
 3482         flushed = 0;
 3483         bq = &bd->bd_dirtyq;
 3484         bp = NULL;
 3485         sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK | M_ZERO);
 3486         sentinel->b_qindex = QUEUE_SENTINEL;
 3487         BQ_LOCK(bq);
 3488         TAILQ_INSERT_HEAD(&bq->bq_queue, sentinel, b_freelist);
 3489         BQ_UNLOCK(bq);
 3490         while (flushed != target) {
 3491                 maybe_yield();
 3492                 BQ_LOCK(bq);
 3493                 bp = TAILQ_NEXT(sentinel, b_freelist);
 3494                 if (bp != NULL) {
 3495                         TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
 3496                         TAILQ_INSERT_AFTER(&bq->bq_queue, bp, sentinel,
 3497                             b_freelist);
 3498                 } else {
 3499                         BQ_UNLOCK(bq);
 3500                         break;
 3501                 }
 3502                 /*
 3503                  * Skip sentinels inserted by other invocations of the
 3504                  * flushbufqueues(), taking care to not reorder them.
 3505                  *
 3506                  * Only flush the buffers that belong to the
 3507                  * vnode locked by the curthread.
 3508                  */
 3509                 if (bp->b_qindex == QUEUE_SENTINEL || (lvp != NULL &&
 3510                     bp->b_vp != lvp)) {
 3511                         BQ_UNLOCK(bq);
 3512                         continue;
 3513                 }
 3514                 error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL);
 3515                 BQ_UNLOCK(bq);
 3516                 if (error != 0)
 3517                         continue;
 3518 
 3519                 /*
 3520                  * BKGRDINPROG can only be set with the buf and bufobj
 3521                  * locks both held.  We tolerate a race to clear it here.
 3522                  */
 3523                 if ((bp->b_vflags & BV_BKGRDINPROG) != 0 ||
 3524                     (bp->b_flags & B_DELWRI) == 0) {
 3525                         BUF_UNLOCK(bp);
 3526                         continue;
 3527                 }
 3528                 if (bp->b_flags & B_INVAL) {
 3529                         bremfreef(bp);
 3530                         brelse(bp);
 3531                         flushed++;
 3532                         continue;
 3533                 }
 3534 
 3535                 if (!LIST_EMPTY(&bp->b_dep) && buf_countdeps(bp, 0)) {
 3536                         if (flushdeps == 0) {
 3537                                 BUF_UNLOCK(bp);
 3538                                 continue;
 3539                         }
 3540                         hasdeps = 1;
 3541                 } else
 3542                         hasdeps = 0;
 3543                 /*
 3544                  * We must hold the lock on a vnode before writing
 3545                  * one of its buffers. Otherwise we may confuse, or
 3546                  * in the case of a snapshot vnode, deadlock the
 3547                  * system.
 3548                  *
 3549                  * The lock order here is the reverse of the normal
 3550                  * of vnode followed by buf lock.  This is ok because
 3551                  * the NOWAIT will prevent deadlock.
 3552                  */
 3553                 vp = bp->b_vp;
 3554                 if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
 3555                         BUF_UNLOCK(bp);
 3556                         continue;
 3557                 }
 3558                 if (lvp == NULL) {
 3559                         unlock = true;
 3560                         error = vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT);
 3561                 } else {
 3562                         ASSERT_VOP_LOCKED(vp, "getbuf");
 3563                         unlock = false;
 3564                         error = VOP_ISLOCKED(vp) == LK_EXCLUSIVE ? 0 :
 3565                             vn_lock(vp, LK_TRYUPGRADE);
 3566                 }
 3567                 if (error == 0) {
 3568                         CTR3(KTR_BUF, "flushbufqueue(%p) vp %p flags %X",
 3569                             bp, bp->b_vp, bp->b_flags);
 3570                         if (curproc == bufdaemonproc) {
 3571                                 vfs_bio_awrite(bp);
 3572                         } else {
 3573                                 bremfree(bp);
 3574                                 bwrite(bp);
 3575                                 counter_u64_add(notbufdflushes, 1);
 3576                         }
 3577                         vn_finished_write(mp);
 3578                         if (unlock)
 3579                                 VOP_UNLOCK(vp, 0);
 3580                         flushwithdeps += hasdeps;
 3581                         flushed++;
 3582 
 3583                         /*
 3584                          * Sleeping on runningbufspace while holding
 3585                          * vnode lock leads to deadlock.
 3586                          */
 3587                         if (curproc == bufdaemonproc &&
 3588                             runningbufspace > hirunningspace)
 3589                                 waitrunningbufspace();
 3590                         continue;
 3591                 }
 3592                 vn_finished_write(mp);
 3593                 BUF_UNLOCK(bp);
 3594         }
 3595         BQ_LOCK(bq);
 3596         TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
 3597         BQ_UNLOCK(bq);
 3598         free(sentinel, M_TEMP);
 3599         return (flushed);
 3600 }
 3601 
 3602 /*
 3603  * Check to see if a block is currently memory resident.
 3604  */
 3605 struct buf *
 3606 incore(struct bufobj *bo, daddr_t blkno)
 3607 {
 3608         struct buf *bp;
 3609 
 3610         BO_RLOCK(bo);
 3611         bp = gbincore(bo, blkno);
 3612         BO_RUNLOCK(bo);
 3613         return (bp);
 3614 }
 3615 
 3616 /*
 3617  * Returns true if no I/O is needed to access the
 3618  * associated VM object.  This is like incore except
 3619  * it also hunts around in the VM system for the data.
 3620  */
 3621 
 3622 static int
 3623 inmem(struct vnode * vp, daddr_t blkno)
 3624 {
 3625         vm_object_t obj;
 3626         vm_offset_t toff, tinc, size;
 3627         vm_page_t m;
 3628         vm_ooffset_t off;
 3629 
 3630         ASSERT_VOP_LOCKED(vp, "inmem");
 3631 
 3632         if (incore(&vp->v_bufobj, blkno))
 3633                 return 1;
 3634         if (vp->v_mount == NULL)
 3635                 return 0;
 3636         obj = vp->v_object;
 3637         if (obj == NULL)
 3638                 return (0);
 3639 
 3640         size = PAGE_SIZE;
 3641         if (size > vp->v_mount->mnt_stat.f_iosize)
 3642                 size = vp->v_mount->mnt_stat.f_iosize;
 3643         off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
 3644 
 3645         VM_OBJECT_RLOCK(obj);
 3646         for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
 3647                 m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
 3648                 if (!m)
 3649                         goto notinmem;
 3650                 tinc = size;
 3651                 if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
 3652                         tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
 3653                 if (vm_page_is_valid(m,
 3654                     (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
 3655                         goto notinmem;
 3656         }
 3657         VM_OBJECT_RUNLOCK(obj);
 3658         return 1;
 3659 
 3660 notinmem:
 3661         VM_OBJECT_RUNLOCK(obj);
 3662         return (0);
 3663 }
 3664 
 3665 /*
 3666  * Set the dirty range for a buffer based on the status of the dirty
 3667  * bits in the pages comprising the buffer.  The range is limited
 3668  * to the size of the buffer.
 3669  *
 3670  * Tell the VM system that the pages associated with this buffer
 3671  * are clean.  This is used for delayed writes where the data is
 3672  * going to go to disk eventually without additional VM intevention.
 3673  *
 3674  * Note that while we only really need to clean through to b_bcount, we
 3675  * just go ahead and clean through to b_bufsize.
 3676  */
 3677 static void
 3678 vfs_clean_pages_dirty_buf(struct buf *bp)
 3679 {
 3680         vm_ooffset_t foff, noff, eoff;
 3681         vm_page_t m;
 3682         int i;
 3683 
 3684         if ((bp->b_flags & B_VMIO) == 0 || bp->b_bufsize == 0)
 3685                 return;
 3686 
 3687         foff = bp->b_offset;
 3688         KASSERT(bp->b_offset != NOOFFSET,
 3689             ("vfs_clean_pages_dirty_buf: no buffer offset"));
 3690 
 3691         VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
 3692         vfs_drain_busy_pages(bp);
 3693         vfs_setdirty_locked_object(bp);
 3694         for (i = 0; i < bp->b_npages; i++) {
 3695                 noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
 3696                 eoff = noff;
 3697                 if (eoff > bp->b_offset + bp->b_bufsize)
 3698                         eoff = bp->b_offset + bp->b_bufsize;
 3699                 m = bp->b_pages[i];
 3700                 vfs_page_set_validclean(bp, foff, m);
 3701                 /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
 3702                 foff = noff;
 3703         }
 3704         VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
 3705 }
 3706 
 3707 static void
 3708 vfs_setdirty_locked_object(struct buf *bp)
 3709 {
 3710         vm_object_t object;
 3711         int i;
 3712 
 3713         object = bp->b_bufobj->bo_object;
 3714         VM_OBJECT_ASSERT_WLOCKED(object);
 3715 
 3716         /*
 3717          * We qualify the scan for modified pages on whether the
 3718          * object has been flushed yet.
 3719          */
 3720         if ((object->flags & OBJ_MIGHTBEDIRTY) != 0) {
 3721                 vm_offset_t boffset;
 3722                 vm_offset_t eoffset;
 3723 
 3724                 /*
 3725                  * test the pages to see if they have been modified directly
 3726                  * by users through the VM system.
 3727                  */
 3728                 for (i = 0; i < bp->b_npages; i++)
 3729                         vm_page_test_dirty(bp->b_pages[i]);
 3730 
 3731                 /*
 3732                  * Calculate the encompassing dirty range, boffset and eoffset,
 3733                  * (eoffset - boffset) bytes.
 3734                  */
 3735 
 3736                 for (i = 0; i < bp->b_npages; i++) {
 3737                         if (bp->b_pages[i]->dirty)
 3738                                 break;
 3739                 }
 3740                 boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
 3741 
 3742                 for (i = bp->b_npages - 1; i >= 0; --i) {
 3743                         if (bp->b_pages[i]->dirty) {
 3744                                 break;
 3745                         }
 3746                 }
 3747                 eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
 3748 
 3749                 /*
 3750                  * Fit it to the buffer.
 3751                  */
 3752 
 3753                 if (eoffset > bp->b_bcount)
 3754                         eoffset = bp->b_bcount;
 3755 
 3756                 /*
 3757                  * If we have a good dirty range, merge with the existing
 3758                  * dirty range.
 3759                  */
 3760 
 3761                 if (boffset < eoffset) {
 3762                         if (bp->b_dirtyoff > boffset)
 3763                                 bp->b_dirtyoff = boffset;
 3764                         if (bp->b_dirtyend < eoffset)
 3765                                 bp->b_dirtyend = eoffset;
 3766                 }
 3767         }
 3768 }
 3769 
 3770 /*
 3771  * Allocate the KVA mapping for an existing buffer.
 3772  * If an unmapped buffer is provided but a mapped buffer is requested, take
 3773  * also care to properly setup mappings between pages and KVA.
 3774  */
 3775 static void
 3776 bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
 3777 {
 3778         int bsize, maxsize, need_mapping, need_kva;
 3779         off_t offset;
 3780 
 3781         need_mapping = bp->b_data == unmapped_buf &&
 3782             (gbflags & GB_UNMAPPED) == 0;
 3783         need_kva = bp->b_kvabase == unmapped_buf &&
 3784             bp->b_data == unmapped_buf &&
 3785             (gbflags & GB_KVAALLOC) != 0;
 3786         if (!need_mapping && !need_kva)
 3787                 return;
 3788 
 3789         BUF_CHECK_UNMAPPED(bp);
 3790 
 3791         if (need_mapping && bp->b_kvabase != unmapped_buf) {
 3792                 /*
 3793                  * Buffer is not mapped, but the KVA was already
 3794                  * reserved at the time of the instantiation.  Use the
 3795                  * allocated space.
 3796                  */
 3797                 goto has_addr;
 3798         }
 3799 
 3800         /*
 3801          * Calculate the amount of the address space we would reserve
 3802          * if the buffer was mapped.
 3803          */
 3804         bsize = vn_isdisk(bp->b_vp, NULL) ? DEV_BSIZE : bp->b_bufobj->bo_bsize;
 3805         KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
 3806         offset = blkno * bsize;
 3807         maxsize = size + (offset & PAGE_MASK);
 3808         maxsize = imax(maxsize, bsize);
 3809 
 3810         while (bufkva_alloc(bp, maxsize, gbflags) != 0) {
 3811                 if ((gbflags & GB_NOWAIT_BD) != 0) {
 3812                         /*
 3813                          * XXXKIB: defragmentation cannot
 3814                          * succeed, not sure what else to do.
 3815                          */
 3816                         panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
 3817                 }
 3818                 counter_u64_add(mappingrestarts, 1);
 3819                 bufspace_wait(bufdomain(bp), bp->b_vp, gbflags, 0, 0);
 3820         }
 3821 has_addr:
 3822         if (need_mapping) {
 3823                 /* b_offset is handled by bpmap_qenter. */
 3824                 bp->b_data = bp->b_kvabase;
 3825                 BUF_CHECK_MAPPED(bp);
 3826                 bpmap_qenter(bp);
 3827         }
 3828 }
 3829 
 3830 struct buf *
 3831 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo,
 3832     int flags)
 3833 {
 3834         struct buf *bp;
 3835         int error;
 3836 
 3837         error = getblkx(vp, blkno, size, slpflag, slptimeo, flags, &bp);
 3838         if (error != 0)
 3839                 return (NULL);
 3840         return (bp);
 3841 }
 3842 
 3843 /*
 3844  *      getblkx:
 3845  *
 3846  *      Get a block given a specified block and offset into a file/device.
 3847  *      The buffers B_DONE bit will be cleared on return, making it almost
 3848  *      ready for an I/O initiation.  B_INVAL may or may not be set on 
 3849  *      return.  The caller should clear B_INVAL prior to initiating a
 3850  *      READ.
 3851  *
 3852  *      For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
 3853  *      an existing buffer.
 3854  *
 3855  *      For a VMIO buffer, B_CACHE is modified according to the backing VM.
 3856  *      If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
 3857  *      and then cleared based on the backing VM.  If the previous buffer is
 3858  *      non-0-sized but invalid, B_CACHE will be cleared.
 3859  *
 3860  *      If getblk() must create a new buffer, the new buffer is returned with
 3861  *      both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
 3862  *      case it is returned with B_INVAL clear and B_CACHE set based on the
 3863  *      backing VM.
 3864  *
 3865  *      getblk() also forces a bwrite() for any B_DELWRI buffer whos
 3866  *      B_CACHE bit is clear.
 3867  *      
 3868  *      What this means, basically, is that the caller should use B_CACHE to
 3869  *      determine whether the buffer is fully valid or not and should clear
 3870  *      B_INVAL prior to issuing a read.  If the caller intends to validate
 3871  *      the buffer by loading its data area with something, the caller needs
 3872  *      to clear B_INVAL.  If the caller does this without issuing an I/O, 
 3873  *      the caller should set B_CACHE ( as an optimization ), else the caller
 3874  *      should issue the I/O and biodone() will set B_CACHE if the I/O was
 3875  *      a write attempt or if it was a successful read.  If the caller 
 3876  *      intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
 3877  *      prior to issuing the READ.  biodone() will *not* clear B_INVAL.
 3878  */
 3879 int
 3880 getblkx(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo,
 3881     int flags, struct buf **bpp)
 3882 {
 3883         struct buf *bp;
 3884         struct bufobj *bo;
 3885         daddr_t d_blkno;
 3886         int bsize, error, maxsize, vmio;
 3887         off_t offset;
 3888 
 3889         CTR3(KTR_BUF, "getblk(%p, %ld, %d)", vp, (long)blkno, size);
 3890         KASSERT((flags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
 3891             ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
 3892         ASSERT_VOP_LOCKED(vp, "getblk");
 3893         if (size > maxbcachebuf)
 3894                 panic("getblk: size(%d) > maxbcachebuf(%d)\n", size,
 3895                     maxbcachebuf);
 3896         if (!unmapped_buf_allowed)
 3897                 flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
 3898 
 3899         bo = &vp->v_bufobj;
 3900         d_blkno = blkno;
 3901 loop:
 3902         BO_RLOCK(bo);
 3903         bp = gbincore(bo, blkno);
 3904         if (bp != NULL) {
 3905                 int lockflags;
 3906                 /*
 3907                  * Buffer is in-core.  If the buffer is not busy nor managed,
 3908                  * it must be on a queue.
 3909                  */
 3910                 lockflags = LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK;
 3911 
 3912                 if ((flags & GB_LOCK_NOWAIT) != 0)
 3913                         lockflags |= LK_NOWAIT;
 3914 
 3915                 error = BUF_TIMELOCK(bp, lockflags,
 3916                     BO_LOCKPTR(bo), "getblk", slpflag, slptimeo);
 3917 
 3918                 /*
 3919                  * If we slept and got the lock we have to restart in case
 3920                  * the buffer changed identities.
 3921                  */
 3922                 if (error == ENOLCK)
 3923                         goto loop;
 3924                 /* We timed out or were interrupted. */
 3925                 else if (error != 0)
 3926                         return (error);
 3927                 /* If recursed, assume caller knows the rules. */
 3928                 else if (BUF_LOCKRECURSED(bp))
 3929                         goto end;
 3930 
 3931                 /*
 3932                  * The buffer is locked.  B_CACHE is cleared if the buffer is 
 3933                  * invalid.  Otherwise, for a non-VMIO buffer, B_CACHE is set
 3934                  * and for a VMIO buffer B_CACHE is adjusted according to the
 3935                  * backing VM cache.
 3936                  */
 3937                 if (bp->b_flags & B_INVAL)
 3938                         bp->b_flags &= ~B_CACHE;
 3939                 else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
 3940                         bp->b_flags |= B_CACHE;
 3941                 if (bp->b_flags & B_MANAGED)
 3942                         MPASS(bp->b_qindex == QUEUE_NONE);
 3943                 else
 3944                         bremfree(bp);
 3945 
 3946                 /*
 3947                  * check for size inconsistencies for non-VMIO case.
 3948                  */
 3949                 if (bp->b_bcount != size) {
 3950                         if ((bp->b_flags & B_VMIO) == 0 ||
 3951                             (size > bp->b_kvasize)) {
 3952                                 if (bp->b_flags & B_DELWRI) {
 3953                                         bp->b_flags |= B_NOCACHE;
 3954                                         bwrite(bp);
 3955                                 } else {
 3956                                         if (LIST_EMPTY(&bp->b_dep)) {
 3957                                                 bp->b_flags |= B_RELBUF;
 3958                                                 brelse(bp);
 3959                                         } else {
 3960                                                 bp->b_flags |= B_NOCACHE;
 3961                                                 bwrite(bp);
 3962                                         }
 3963                                 }
 3964                                 goto loop;
 3965                         }
 3966                 }
 3967 
 3968                 /*
 3969                  * Handle the case of unmapped buffer which should
 3970                  * become mapped, or the buffer for which KVA
 3971                  * reservation is requested.
 3972                  */
 3973                 bp_unmapped_get_kva(bp, blkno, size, flags);
 3974 
 3975                 /*
 3976                  * If the size is inconsistent in the VMIO case, we can resize
 3977                  * the buffer.  This might lead to B_CACHE getting set or
 3978                  * cleared.  If the size has not changed, B_CACHE remains
 3979                  * unchanged from its previous state.
 3980                  */
 3981                 allocbuf(bp, size);
 3982 
 3983                 KASSERT(bp->b_offset != NOOFFSET, 
 3984                     ("getblk: no buffer offset"));
 3985 
 3986                 /*
 3987                  * A buffer with B_DELWRI set and B_CACHE clear must
 3988                  * be committed before we can return the buffer in
 3989                  * order to prevent the caller from issuing a read
 3990                  * ( due to B_CACHE not being set ) and overwriting
 3991                  * it.
 3992                  *
 3993                  * Most callers, including NFS and FFS, need this to
 3994                  * operate properly either because they assume they
 3995                  * can issue a read if B_CACHE is not set, or because
 3996                  * ( for example ) an uncached B_DELWRI might loop due 
 3997                  * to softupdates re-dirtying the buffer.  In the latter
 3998                  * case, B_CACHE is set after the first write completes,
 3999                  * preventing further loops.
 4000                  * NOTE!  b*write() sets B_CACHE.  If we cleared B_CACHE
 4001                  * above while extending the buffer, we cannot allow the
 4002                  * buffer to remain with B_CACHE set after the write
 4003                  * completes or it will represent a corrupt state.  To
 4004                  * deal with this we set B_NOCACHE to scrap the buffer
 4005                  * after the write.
 4006                  *
 4007                  * We might be able to do something fancy, like setting
 4008                  * B_CACHE in bwrite() except if B_DELWRI is already set,
 4009                  * so the below call doesn't set B_CACHE, but that gets real
 4010                  * confusing.  This is much easier.
 4011                  */
 4012 
 4013                 if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
 4014                         bp->b_flags |= B_NOCACHE;
 4015                         bwrite(bp);
 4016                         goto loop;
 4017                 }
 4018                 bp->b_flags &= ~B_DONE;
 4019         } else {
 4020                 /*
 4021                  * Buffer is not in-core, create new buffer.  The buffer
 4022                  * returned by getnewbuf() is locked.  Note that the returned
 4023                  * buffer is also considered valid (not marked B_INVAL).
 4024                  */
 4025                 BO_RUNLOCK(bo);
 4026                 /*
 4027                  * If the user does not want us to create the buffer, bail out
 4028                  * here.
 4029                  */
 4030                 if (flags & GB_NOCREAT)
 4031                         return (EEXIST);
 4032                 if (bdomain[bo->bo_domain].bd_freebuffers == 0 &&
 4033                     TD_IS_IDLETHREAD(curthread))
 4034                         return (EBUSY);
 4035 
 4036                 bsize = vn_isdisk(vp, NULL) ? DEV_BSIZE : bo->bo_bsize;
 4037                 KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
 4038                 offset = blkno * bsize;
 4039                 vmio = vp->v_object != NULL;
 4040                 if (vmio) {
 4041                         maxsize = size + (offset & PAGE_MASK);
 4042                 } else {
 4043                         maxsize = size;
 4044                         /* Do not allow non-VMIO notmapped buffers. */
 4045                         flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
 4046                 }
 4047                 maxsize = imax(maxsize, bsize);
 4048                 if ((flags & GB_NOSPARSE) != 0 && vmio &&
 4049                     !vn_isdisk(vp, NULL)) {
 4050                         error = VOP_BMAP(vp, blkno, NULL, &d_blkno, 0, 0);
 4051                         KASSERT(error != EOPNOTSUPP,
 4052                             ("GB_NOSPARSE from fs not supporting bmap, vp %p",
 4053                             vp));
 4054                         if (error != 0)
 4055                                 return (error);
 4056                         if (d_blkno == -1)
 4057                                 return (EJUSTRETURN);
 4058                 }
 4059 
 4060                 bp = getnewbuf(vp, slpflag, slptimeo, maxsize, flags);
 4061                 if (bp == NULL) {
 4062                         if (slpflag || slptimeo)
 4063                                 return (ETIMEDOUT);
 4064                         /*
 4065                          * XXX This is here until the sleep path is diagnosed
 4066                          * enough to work under very low memory conditions.
 4067                          *
 4068                          * There's an issue on low memory, 4BSD+non-preempt
 4069                          * systems (eg MIPS routers with 32MB RAM) where buffer
 4070                          * exhaustion occurs without sleeping for buffer
 4071                          * reclaimation.  This just sticks in a loop and
 4072                          * constantly attempts to allocate a buffer, which
 4073                          * hits exhaustion and tries to wakeup bufdaemon.
 4074                          * This never happens because we never yield.
 4075                          *
 4076                          * The real solution is to identify and fix these cases
 4077                          * so we aren't effectively busy-waiting in a loop
 4078                          * until the reclaimation path has cycles to run.
 4079                          */
 4080                         kern_yield(PRI_USER);
 4081                         goto loop;
 4082                 }
 4083 
 4084                 /*
 4085                  * This code is used to make sure that a buffer is not
 4086                  * created while the getnewbuf routine is blocked.
 4087                  * This can be a problem whether the vnode is locked or not.
 4088                  * If the buffer is created out from under us, we have to
 4089                  * throw away the one we just created.
 4090                  *
 4091                  * Note: this must occur before we associate the buffer
 4092                  * with the vp especially considering limitations in
 4093                  * the splay tree implementation when dealing with duplicate
 4094                  * lblkno's.
 4095                  */
 4096                 BO_LOCK(bo);
 4097                 if (gbincore(bo, blkno)) {
 4098                         BO_UNLOCK(bo);
 4099                         bp->b_flags |= B_INVAL;
 4100                         bufspace_release(bufdomain(bp), maxsize);
 4101                         brelse(bp);
 4102                         goto loop;
 4103                 }
 4104 
 4105                 /*
 4106                  * Insert the buffer into the hash, so that it can
 4107                  * be found by incore.
 4108                  */
 4109                 bp->b_lblkno = blkno;
 4110                 bp->b_blkno = d_blkno;
 4111                 bp->b_offset = offset;
 4112                 bgetvp(vp, bp);
 4113                 BO_UNLOCK(bo);
 4114 
 4115                 /*
 4116                  * set B_VMIO bit.  allocbuf() the buffer bigger.  Since the
 4117                  * buffer size starts out as 0, B_CACHE will be set by
 4118                  * allocbuf() for the VMIO case prior to it testing the
 4119                  * backing store for validity.
 4120                  */
 4121 
 4122                 if (vmio) {
 4123                         bp->b_flags |= B_VMIO;
 4124                         KASSERT(vp->v_object == bp->b_bufobj->bo_object,
 4125                             ("ARGH! different b_bufobj->bo_object %p %p %p\n",
 4126                             bp, vp->v_object, bp->b_bufobj->bo_object));
 4127                 } else {
 4128                         bp->b_flags &= ~B_VMIO;
 4129                         KASSERT(bp->b_bufobj->bo_object == NULL,
 4130                             ("ARGH! has b_bufobj->bo_object %p %p\n",
 4131                             bp, bp->b_bufobj->bo_object));
 4132                         BUF_CHECK_MAPPED(bp);
 4133                 }
 4134 
 4135                 allocbuf(bp, size);
 4136                 bufspace_release(bufdomain(bp), maxsize);
 4137                 bp->b_flags &= ~B_DONE;
 4138         }
 4139         CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
 4140         BUF_ASSERT_HELD(bp);
 4141 end:
 4142         buf_track(bp, __func__);
 4143         KASSERT(bp->b_bufobj == bo,
 4144             ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
 4145         *bpp = bp;
 4146         return (0);
 4147 }
 4148 
 4149 /*
 4150  * Get an empty, disassociated buffer of given size.  The buffer is initially
 4151  * set to B_INVAL.
 4152  */
 4153 struct buf *
 4154 geteblk(int size, int flags)
 4155 {
 4156         struct buf *bp;
 4157         int maxsize;
 4158 
 4159         maxsize = (size + BKVAMASK) & ~BKVAMASK;
 4160         while ((bp = getnewbuf(NULL, 0, 0, maxsize, flags)) == NULL) {
 4161                 if ((flags & GB_NOWAIT_BD) &&
 4162                     (curthread->td_pflags & TDP_BUFNEED) != 0)
 4163                         return (NULL);
 4164         }
 4165         allocbuf(bp, size);
 4166         bufspace_release(bufdomain(bp), maxsize);
 4167         bp->b_flags |= B_INVAL; /* b_dep cleared by getnewbuf() */
 4168         BUF_ASSERT_HELD(bp);
 4169         return (bp);
 4170 }
 4171 
 4172 /*
 4173  * Truncate the backing store for a non-vmio buffer.
 4174  */
 4175 static void
 4176 vfs_nonvmio_truncate(struct buf *bp, int newbsize)
 4177 {
 4178 
 4179         if (bp->b_flags & B_MALLOC) {
 4180                 /*
 4181                  * malloced buffers are not shrunk
 4182                  */
 4183                 if (newbsize == 0) {
 4184                         bufmallocadjust(bp, 0);
 4185                         free(bp->b_data, M_BIOBUF);
 4186                         bp->b_data = bp->b_kvabase;
 4187                         bp->b_flags &= ~B_MALLOC;
 4188                 }
 4189                 return;
 4190         }
 4191         vm_hold_free_pages(bp, newbsize);
 4192         bufspace_adjust(bp, newbsize);
 4193 }
 4194 
 4195 /*
 4196  * Extend the backing for a non-VMIO buffer.
 4197  */
 4198 static void
 4199 vfs_nonvmio_extend(struct buf *bp, int newbsize)
 4200 {
 4201         caddr_t origbuf;
 4202         int origbufsize;
 4203 
 4204         /*
 4205          * We only use malloced memory on the first allocation.
 4206          * and revert to page-allocated memory when the buffer
 4207          * grows.
 4208          *
 4209          * There is a potential smp race here that could lead
 4210          * to bufmallocspace slightly passing the max.  It
 4211          * is probably extremely rare and not worth worrying
 4212          * over.
 4213          */
 4214         if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
 4215             bufmallocspace < maxbufmallocspace) {
 4216                 bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
 4217                 bp->b_flags |= B_MALLOC;
 4218                 bufmallocadjust(bp, newbsize);
 4219                 return;
 4220         }
 4221 
 4222         /*
 4223          * If the buffer is growing on its other-than-first
 4224          * allocation then we revert to the page-allocation
 4225          * scheme.
 4226          */
 4227         origbuf = NULL;
 4228         origbufsize = 0;
 4229         if (bp->b_flags & B_MALLOC) {
 4230                 origbuf = bp->b_data;
 4231                 origbufsize = bp->b_bufsize;
 4232                 bp->b_data = bp->b_kvabase;
 4233                 bufmallocadjust(bp, 0);
 4234                 bp->b_flags &= ~B_MALLOC;
 4235                 newbsize = round_page(newbsize);
 4236         }
 4237         vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
 4238             (vm_offset_t) bp->b_data + newbsize);
 4239         if (origbuf != NULL) {
 4240                 bcopy(origbuf, bp->b_data, origbufsize);
 4241                 free(origbuf, M_BIOBUF);
 4242         }
 4243         bufspace_adjust(bp, newbsize);
 4244 }
 4245 
 4246 /*
 4247  * This code constitutes the buffer memory from either anonymous system
 4248  * memory (in the case of non-VMIO operations) or from an associated
 4249  * VM object (in the case of VMIO operations).  This code is able to
 4250  * resize a buffer up or down.
 4251  *
 4252  * Note that this code is tricky, and has many complications to resolve
 4253  * deadlock or inconsistent data situations.  Tread lightly!!! 
 4254  * There are B_CACHE and B_DELWRI interactions that must be dealt with by 
 4255  * the caller.  Calling this code willy nilly can result in the loss of data.
 4256  *
 4257  * allocbuf() only adjusts B_CACHE for VMIO buffers.  getblk() deals with
 4258  * B_CACHE for the non-VMIO case.
 4259  */
 4260 int
 4261 allocbuf(struct buf *bp, int size)
 4262 {
 4263         int newbsize;
 4264 
 4265         BUF_ASSERT_HELD(bp);
 4266 
 4267         if (bp->b_bcount == size)
 4268                 return (1);
 4269 
 4270         if (bp->b_kvasize != 0 && bp->b_kvasize < size)
 4271                 panic("allocbuf: buffer too small");
 4272 
 4273         newbsize = roundup2(size, DEV_BSIZE);
 4274         if ((bp->b_flags & B_VMIO) == 0) {
 4275                 if ((bp->b_flags & B_MALLOC) == 0)
 4276                         newbsize = round_page(newbsize);
 4277                 /*
 4278                  * Just get anonymous memory from the kernel.  Don't
 4279                  * mess with B_CACHE.
 4280                  */
 4281                 if (newbsize < bp->b_bufsize)
 4282                         vfs_nonvmio_truncate(bp, newbsize);
 4283                 else if (newbsize > bp->b_bufsize)
 4284                         vfs_nonvmio_extend(bp, newbsize);
 4285         } else {
 4286                 int desiredpages;
 4287 
 4288                 desiredpages = (size == 0) ? 0 :
 4289                     num_pages((bp->b_offset & PAGE_MASK) + newbsize);
 4290 
 4291                 if (bp->b_flags & B_MALLOC)
 4292                         panic("allocbuf: VMIO buffer can't be malloced");
 4293                 /*
 4294                  * Set B_CACHE initially if buffer is 0 length or will become
 4295                  * 0-length.
 4296                  */
 4297                 if (size == 0 || bp->b_bufsize == 0)
 4298                         bp->b_flags |= B_CACHE;
 4299 
 4300                 if (newbsize < bp->b_bufsize)
 4301                         vfs_vmio_truncate(bp, desiredpages);
 4302                 /* XXX This looks as if it should be newbsize > b_bufsize */
 4303                 else if (size > bp->b_bcount)
 4304                         vfs_vmio_extend(bp, desiredpages, size);
 4305                 bufspace_adjust(bp, newbsize);
 4306         }
 4307         bp->b_bcount = size;            /* requested buffer size. */
 4308         return (1);
 4309 }
 4310 
 4311 extern int inflight_transient_maps;
 4312 
 4313 static struct bio_queue nondump_bios;
 4314 
 4315 void
 4316 biodone(struct bio *bp)
 4317 {
 4318         struct mtx *mtxp;
 4319         void (*done)(struct bio *);
 4320         vm_offset_t start, end;
 4321 
 4322         biotrack(bp, __func__);
 4323 
 4324         /*
 4325          * Avoid completing I/O when dumping after a panic since that may
 4326          * result in a deadlock in the filesystem or pager code.  Note that
 4327          * this doesn't affect dumps that were started manually since we aim
 4328          * to keep the system usable after it has been resumed.
 4329          */
 4330         if (__predict_false(dumping && SCHEDULER_STOPPED())) {
 4331                 TAILQ_INSERT_HEAD(&nondump_bios, bp, bio_queue);
 4332                 return;
 4333         }
 4334         if ((bp->bio_flags & BIO_TRANSIENT_MAPPING) != 0) {
 4335                 bp->bio_flags &= ~BIO_TRANSIENT_MAPPING;
 4336                 bp->bio_flags |= BIO_UNMAPPED;
 4337                 start = trunc_page((vm_offset_t)bp->bio_data);
 4338                 end = round_page((vm_offset_t)bp->bio_data + bp->bio_length);
 4339                 bp->bio_data = unmapped_buf;
 4340                 pmap_qremove(start, atop(end - start));
 4341                 vmem_free(transient_arena, start, end - start);
 4342                 atomic_add_int(&inflight_transient_maps, -1);
 4343         }
 4344         done = bp->bio_done;
 4345         if (done == NULL) {
 4346                 mtxp = mtx_pool_find(mtxpool_sleep, bp);
 4347                 mtx_lock(mtxp);
 4348                 bp->bio_flags |= BIO_DONE;
 4349                 wakeup(bp);
 4350                 mtx_unlock(mtxp);
 4351         } else
 4352                 done(bp);
 4353 }
 4354 
 4355 /*
 4356  * Wait for a BIO to finish.
 4357  */
 4358 int
 4359 biowait(struct bio *bp, const char *wchan)
 4360 {
 4361         struct mtx *mtxp;
 4362 
 4363         mtxp = mtx_pool_find(mtxpool_sleep, bp);
 4364         mtx_lock(mtxp);
 4365         while ((bp->bio_flags & BIO_DONE) == 0)
 4366                 msleep(bp, mtxp, PRIBIO, wchan, 0);
 4367         mtx_unlock(mtxp);
 4368         if (bp->bio_error != 0)
 4369                 return (bp->bio_error);
 4370         if (!(bp->bio_flags & BIO_ERROR))
 4371                 return (0);
 4372         return (EIO);
 4373 }
 4374 
 4375 void
 4376 biofinish(struct bio *bp, struct devstat *stat, int error)
 4377 {
 4378         
 4379         if (error) {
 4380                 bp->bio_error = error;
 4381                 bp->bio_flags |= BIO_ERROR;
 4382         }
 4383         if (stat != NULL)
 4384                 devstat_end_transaction_bio(stat, bp);
 4385         biodone(bp);
 4386 }
 4387 
 4388 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
 4389 void
 4390 biotrack_buf(struct bio *bp, const char *location)
 4391 {
 4392 
 4393         buf_track(bp->bio_track_bp, location);
 4394 }
 4395 #endif
 4396 
 4397 /*
 4398  *      bufwait:
 4399  *
 4400  *      Wait for buffer I/O completion, returning error status.  The buffer
 4401  *      is left locked and B_DONE on return.  B_EINTR is converted into an EINTR
 4402  *      error and cleared.
 4403  */
 4404 int
 4405 bufwait(struct buf *bp)
 4406 {
 4407         if (bp->b_iocmd == BIO_READ)
 4408                 bwait(bp, PRIBIO, "biord");
 4409         else
 4410                 bwait(bp, PRIBIO, "biowr");
 4411         if (bp->b_flags & B_EINTR) {
 4412                 bp->b_flags &= ~B_EINTR;
 4413                 return (EINTR);
 4414         }
 4415         if (bp->b_ioflags & BIO_ERROR) {
 4416                 return (bp->b_error ? bp->b_error : EIO);
 4417         } else {
 4418                 return (0);
 4419         }
 4420 }
 4421 
 4422 /*
 4423  *      bufdone:
 4424  *
 4425  *      Finish I/O on a buffer, optionally calling a completion function.
 4426  *      This is usually called from an interrupt so process blocking is
 4427  *      not allowed.
 4428  *
 4429  *      biodone is also responsible for setting B_CACHE in a B_VMIO bp.
 4430  *      In a non-VMIO bp, B_CACHE will be set on the next getblk() 
 4431  *      assuming B_INVAL is clear.
 4432  *
 4433  *      For the VMIO case, we set B_CACHE if the op was a read and no
 4434  *      read error occurred, or if the op was a write.  B_CACHE is never
 4435  *      set if the buffer is invalid or otherwise uncacheable.
 4436  *
 4437  *      biodone does not mess with B_INVAL, allowing the I/O routine or the
 4438  *      initiator to leave B_INVAL set to brelse the buffer out of existence
 4439  *      in the biodone routine.
 4440  */
 4441 void
 4442 bufdone(struct buf *bp)
 4443 {
 4444         struct bufobj *dropobj;
 4445         void    (*biodone)(struct buf *);
 4446 
 4447         buf_track(bp, __func__);
 4448         CTR3(KTR_BUF, "bufdone(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
 4449         dropobj = NULL;
 4450 
 4451         KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
 4452         BUF_ASSERT_HELD(bp);
 4453 
 4454         runningbufwakeup(bp);
 4455         if (bp->b_iocmd == BIO_WRITE)
 4456                 dropobj = bp->b_bufobj;
 4457         /* call optional completion function if requested */
 4458         if (bp->b_iodone != NULL) {
 4459                 biodone = bp->b_iodone;
 4460                 bp->b_iodone = NULL;
 4461                 (*biodone) (bp);
 4462                 if (dropobj)
 4463                         bufobj_wdrop(dropobj);
 4464                 return;
 4465         }
 4466         if (bp->b_flags & B_VMIO) {
 4467                 /*
 4468                  * Set B_CACHE if the op was a normal read and no error
 4469                  * occurred.  B_CACHE is set for writes in the b*write()
 4470                  * routines.
 4471                  */
 4472                 if (bp->b_iocmd == BIO_READ &&
 4473                     !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
 4474                     !(bp->b_ioflags & BIO_ERROR))
 4475                         bp->b_flags |= B_CACHE;
 4476                 vfs_vmio_iodone(bp);
 4477         }
 4478         if (!LIST_EMPTY(&bp->b_dep))
 4479                 buf_complete(bp);
 4480         if ((bp->b_flags & B_CKHASH) != 0) {
 4481                 KASSERT(bp->b_iocmd == BIO_READ,
 4482                     ("bufdone: b_iocmd %d not BIO_READ", bp->b_iocmd));
 4483                 KASSERT(buf_mapped(bp), ("bufdone: bp %p not mapped", bp));
 4484                 (*bp->b_ckhashcalc)(bp);
 4485         }
 4486         /*
 4487          * For asynchronous completions, release the buffer now. The brelse
 4488          * will do a wakeup there if necessary - so no need to do a wakeup
 4489          * here in the async case. The sync case always needs to do a wakeup.
 4490          */
 4491         if (bp->b_flags & B_ASYNC) {
 4492                 if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) ||
 4493                     (bp->b_ioflags & BIO_ERROR))
 4494                         brelse(bp);
 4495                 else
 4496                         bqrelse(bp);
 4497         } else
 4498                 bdone(bp);
 4499         if (dropobj)
 4500                 bufobj_wdrop(dropobj);
 4501 }
 4502 
 4503 /*
 4504  * This routine is called in lieu of iodone in the case of
 4505  * incomplete I/O.  This keeps the busy status for pages
 4506  * consistent.
 4507  */
 4508 void
 4509 vfs_unbusy_pages(struct buf *bp)
 4510 {
 4511         int i;
 4512         vm_object_t obj;
 4513         vm_page_t m;
 4514 
 4515         runningbufwakeup(bp);
 4516         if (!(bp->b_flags & B_VMIO))
 4517                 return;
 4518 
 4519         obj = bp->b_bufobj->bo_object;
 4520         VM_OBJECT_WLOCK(obj);
 4521         for (i = 0; i < bp->b_npages; i++) {
 4522                 m = bp->b_pages[i];
 4523                 if (m == bogus_page) {
 4524                         m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
 4525                         if (!m)
 4526                                 panic("vfs_unbusy_pages: page missing\n");
 4527                         bp->b_pages[i] = m;
 4528                         if (buf_mapped(bp)) {
 4529                                 BUF_CHECK_MAPPED(bp);
 4530                                 pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
 4531                                     bp->b_pages, bp->b_npages);
 4532                         } else
 4533                                 BUF_CHECK_UNMAPPED(bp);
 4534                 }
 4535                 vm_page_sunbusy(m);
 4536         }
 4537         vm_object_pip_wakeupn(obj, bp->b_npages);
 4538         VM_OBJECT_WUNLOCK(obj);
 4539 }
 4540 
 4541 /*
 4542  * vfs_page_set_valid:
 4543  *
 4544  *      Set the valid bits in a page based on the supplied offset.   The
 4545  *      range is restricted to the buffer's size.
 4546  *
 4547  *      This routine is typically called after a read completes.
 4548  */
 4549 static void
 4550 vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m)
 4551 {
 4552         vm_ooffset_t eoff;
 4553 
 4554         /*
 4555          * Compute the end offset, eoff, such that [off, eoff) does not span a
 4556          * page boundary and eoff is not greater than the end of the buffer.
 4557          * The end of the buffer, in this case, is our file EOF, not the
 4558          * allocation size of the buffer.
 4559          */
 4560         eoff = (off + PAGE_SIZE) & ~(vm_ooffset_t)PAGE_MASK;
 4561         if (eoff > bp->b_offset + bp->b_bcount)
 4562                 eoff = bp->b_offset + bp->b_bcount;
 4563 
 4564         /*
 4565          * Set valid range.  This is typically the entire buffer and thus the
 4566          * entire page.
 4567          */
 4568         if (eoff > off)
 4569                 vm_page_set_valid_range(m, off & PAGE_MASK, eoff - off);
 4570 }
 4571 
 4572 /*
 4573  * vfs_page_set_validclean:
 4574  *
 4575  *      Set the valid bits and clear the dirty bits in a page based on the
 4576  *      supplied offset.   The range is restricted to the buffer's size.
 4577  */
 4578 static void
 4579 vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m)
 4580 {
 4581         vm_ooffset_t soff, eoff;
 4582 
 4583         /*
 4584          * Start and end offsets in buffer.  eoff - soff may not cross a
 4585          * page boundary or cross the end of the buffer.  The end of the
 4586          * buffer, in this case, is our file EOF, not the allocation size
 4587          * of the buffer.
 4588          */
 4589         soff = off;
 4590         eoff = (off + PAGE_SIZE) & ~(off_t)PAGE_MASK;
 4591         if (eoff > bp->b_offset + bp->b_bcount)
 4592                 eoff = bp->b_offset + bp->b_bcount;
 4593 
 4594         /*
 4595          * Set valid range.  This is typically the entire buffer and thus the
 4596          * entire page.
 4597          */
 4598         if (eoff > soff) {
 4599                 vm_page_set_validclean(
 4600                     m,
 4601                    (vm_offset_t) (soff & PAGE_MASK),
 4602                    (vm_offset_t) (eoff - soff)
 4603                 );
 4604         }
 4605 }
 4606 
 4607 /*
 4608  * Ensure that all buffer pages are not exclusive busied.  If any page is
 4609  * exclusive busy, drain it.
 4610  */
 4611 void
 4612 vfs_drain_busy_pages(struct buf *bp)
 4613 {
 4614         vm_page_t m;
 4615         int i, last_busied;
 4616 
 4617         VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object);
 4618         last_busied = 0;
 4619         for (i = 0; i < bp->b_npages; i++) {
 4620                 m = bp->b_pages[i];
 4621                 if (vm_page_xbusied(m)) {
 4622                         for (; last_busied < i; last_busied++)
 4623                                 vm_page_sbusy(bp->b_pages[last_busied]);
 4624                         while (vm_page_xbusied(m)) {
 4625                                 vm_page_lock(m);
 4626                                 VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
 4627                                 vm_page_busy_sleep(m, "vbpage", true);
 4628                                 VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
 4629                         }
 4630                 }
 4631         }
 4632         for (i = 0; i < last_busied; i++)
 4633                 vm_page_sunbusy(bp->b_pages[i]);
 4634 }
 4635 
 4636 /*
 4637  * This routine is called before a device strategy routine.
 4638  * It is used to tell the VM system that paging I/O is in
 4639  * progress, and treat the pages associated with the buffer
 4640  * almost as being exclusive busy.  Also the object paging_in_progress
 4641  * flag is handled to make sure that the object doesn't become
 4642  * inconsistent.
 4643  *
 4644  * Since I/O has not been initiated yet, certain buffer flags
 4645  * such as BIO_ERROR or B_INVAL may be in an inconsistent state
 4646  * and should be ignored.
 4647  */
 4648 void
 4649 vfs_busy_pages(struct buf *bp, int clear_modify)
 4650 {
 4651         vm_object_t obj;
 4652         vm_ooffset_t foff;
 4653         vm_page_t m;
 4654         int i;
 4655         bool bogus;
 4656 
 4657         if (!(bp->b_flags & B_VMIO))
 4658                 return;
 4659 
 4660         obj = bp->b_bufobj->bo_object;
 4661         foff = bp->b_offset;
 4662         KASSERT(bp->b_offset != NOOFFSET,
 4663             ("vfs_busy_pages: no buffer offset"));
 4664         VM_OBJECT_WLOCK(obj);
 4665         vfs_drain_busy_pages(bp);
 4666         if (bp->b_bufsize != 0)
 4667                 vfs_setdirty_locked_object(bp);
 4668         bogus = false;
 4669         for (i = 0; i < bp->b_npages; i++) {
 4670                 m = bp->b_pages[i];
 4671 
 4672                 if ((bp->b_flags & B_CLUSTER) == 0) {
 4673                         vm_object_pip_add(obj, 1);
 4674                         vm_page_sbusy(m);
 4675                 }
 4676                 /*
 4677                  * When readying a buffer for a read ( i.e
 4678                  * clear_modify == 0 ), it is important to do
 4679                  * bogus_page replacement for valid pages in 
 4680                  * partially instantiated buffers.  Partially 
 4681                  * instantiated buffers can, in turn, occur when
 4682                  * reconstituting a buffer from its VM backing store
 4683                  * base.  We only have to do this if B_CACHE is
 4684                  * clear ( which causes the I/O to occur in the
 4685                  * first place ).  The replacement prevents the read
 4686                  * I/O from overwriting potentially dirty VM-backed
 4687                  * pages.  XXX bogus page replacement is, uh, bogus.
 4688                  * It may not work properly with small-block devices.
 4689                  * We need to find a better way.
 4690                  */
 4691                 if (clear_modify) {
 4692                         pmap_remove_write(m);
 4693                         vfs_page_set_validclean(bp, foff, m);
 4694                 } else if (m->valid == VM_PAGE_BITS_ALL &&
 4695                     (bp->b_flags & B_CACHE) == 0) {
 4696                         bp->b_pages[i] = bogus_page;
 4697                         bogus = true;
 4698                 }
 4699                 foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
 4700         }
 4701         VM_OBJECT_WUNLOCK(obj);
 4702         if (bogus && buf_mapped(bp)) {
 4703                 BUF_CHECK_MAPPED(bp);
 4704                 pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
 4705                     bp->b_pages, bp->b_npages);
 4706         }
 4707 }
 4708 
 4709 /*
 4710  *      vfs_bio_set_valid:
 4711  *
 4712  *      Set the range within the buffer to valid.  The range is
 4713  *      relative to the beginning of the buffer, b_offset.  Note that
 4714  *      b_offset itself may be offset from the beginning of the first
 4715  *      page.
 4716  */
 4717 void   
 4718 vfs_bio_set_valid(struct buf *bp, int base, int size)
 4719 {
 4720         int i, n;
 4721         vm_page_t m;
 4722 
 4723         if (!(bp->b_flags & B_VMIO))
 4724                 return;
 4725 
 4726         /*
 4727          * Fixup base to be relative to beginning of first page.
 4728          * Set initial n to be the maximum number of bytes in the
 4729          * first page that can be validated.
 4730          */
 4731         base += (bp->b_offset & PAGE_MASK);
 4732         n = PAGE_SIZE - (base & PAGE_MASK);
 4733 
 4734         VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
 4735         for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
 4736                 m = bp->b_pages[i];
 4737                 if (n > size)
 4738                         n = size;
 4739                 vm_page_set_valid_range(m, base & PAGE_MASK, n);
 4740                 base += n;
 4741                 size -= n;
 4742                 n = PAGE_SIZE;
 4743         }
 4744         VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
 4745 }
 4746 
 4747 /*
 4748  *      vfs_bio_clrbuf:
 4749  *
 4750  *      If the specified buffer is a non-VMIO buffer, clear the entire
 4751  *      buffer.  If the specified buffer is a VMIO buffer, clear and
 4752  *      validate only the previously invalid portions of the buffer.
 4753  *      This routine essentially fakes an I/O, so we need to clear
 4754  *      BIO_ERROR and B_INVAL.
 4755  *
 4756  *      Note that while we only theoretically need to clear through b_bcount,
 4757  *      we go ahead and clear through b_bufsize.
 4758  */
 4759 void
 4760 vfs_bio_clrbuf(struct buf *bp) 
 4761 {
 4762         int i, j, mask, sa, ea, slide;
 4763 
 4764         if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) {
 4765                 clrbuf(bp);
 4766                 return;
 4767         }
 4768         bp->b_flags &= ~B_INVAL;
 4769         bp->b_ioflags &= ~BIO_ERROR;
 4770         VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
 4771         if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
 4772             (bp->b_offset & PAGE_MASK) == 0) {
 4773                 if (bp->b_pages[0] == bogus_page)
 4774                         goto unlock;
 4775                 mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
 4776                 VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object);
 4777                 if ((bp->b_pages[0]->valid & mask) == mask)
 4778                         goto unlock;
 4779                 if ((bp->b_pages[0]->valid & mask) == 0) {
 4780                         pmap_zero_page_area(bp->b_pages[0], 0, bp->b_bufsize);
 4781                         bp->b_pages[0]->valid |= mask;
 4782                         goto unlock;
 4783                 }
 4784         }
 4785         sa = bp->b_offset & PAGE_MASK;
 4786         slide = 0;
 4787         for (i = 0; i < bp->b_npages; i++, sa = 0) {
 4788                 slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize);
 4789                 ea = slide & PAGE_MASK;
 4790                 if (ea == 0)
 4791                         ea = PAGE_SIZE;
 4792                 if (bp->b_pages[i] == bogus_page)
 4793                         continue;
 4794                 j = sa / DEV_BSIZE;
 4795                 mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
 4796                 VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[i]->object);
 4797                 if ((bp->b_pages[i]->valid & mask) == mask)
 4798                         continue;
 4799                 if ((bp->b_pages[i]->valid & mask) == 0)
 4800                         pmap_zero_page_area(bp->b_pages[i], sa, ea - sa);
 4801                 else {
 4802                         for (; sa < ea; sa += DEV_BSIZE, j++) {
 4803                                 if ((bp->b_pages[i]->valid & (1 << j)) == 0) {
 4804                                         pmap_zero_page_area(bp->b_pages[i],
 4805                                             sa, DEV_BSIZE);
 4806                                 }
 4807                         }
 4808                 }
 4809                 bp->b_pages[i]->valid |= mask;
 4810         }
 4811 unlock:
 4812         VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
 4813         bp->b_resid = 0;
 4814 }
 4815 
 4816 void
 4817 vfs_bio_bzero_buf(struct buf *bp, int base, int size)
 4818 {
 4819         vm_page_t m;
 4820         int i, n;
 4821 
 4822         if (buf_mapped(bp)) {
 4823                 BUF_CHECK_MAPPED(bp);
 4824                 bzero(bp->b_data + base, size);
 4825         } else {
 4826                 BUF_CHECK_UNMAPPED(bp);
 4827                 n = PAGE_SIZE - (base & PAGE_MASK);
 4828                 for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
 4829                         m = bp->b_pages[i];
 4830                         if (n > size)
 4831                                 n = size;
 4832                         pmap_zero_page_area(m, base & PAGE_MASK, n);
 4833                         base += n;
 4834                         size -= n;
 4835                         n = PAGE_SIZE;
 4836                 }
 4837         }
 4838 }
 4839 
 4840 /*
 4841  * Update buffer flags based on I/O request parameters, optionally releasing the
 4842  * buffer.  If it's VMIO or direct I/O, the buffer pages are released to the VM,
 4843  * where they may be placed on a page queue (VMIO) or freed immediately (direct
 4844  * I/O).  Otherwise the buffer is released to the cache.
 4845  */
 4846 static void
 4847 b_io_dismiss(struct buf *bp, int ioflag, bool release)
 4848 {
 4849 
 4850         KASSERT((ioflag & IO_NOREUSE) == 0 || (ioflag & IO_VMIO) != 0,
 4851             ("buf %p non-VMIO noreuse", bp));
 4852 
 4853         if ((ioflag & IO_DIRECT) != 0)
 4854                 bp->b_flags |= B_DIRECT;
 4855         if ((ioflag & (IO_VMIO | IO_DIRECT)) != 0 && LIST_EMPTY(&bp->b_dep)) {
 4856                 bp->b_flags |= B_RELBUF;
 4857                 if ((ioflag & IO_NOREUSE) != 0)
 4858                         bp->b_flags |= B_NOREUSE;
 4859                 if (release)
 4860                         brelse(bp);
 4861         } else if (release)
 4862                 bqrelse(bp);
 4863 }
 4864 
 4865 void
 4866 vfs_bio_brelse(struct buf *bp, int ioflag)
 4867 {
 4868 
 4869         b_io_dismiss(bp, ioflag, true);
 4870 }
 4871 
 4872 void
 4873 vfs_bio_set_flags(struct buf *bp, int ioflag)
 4874 {
 4875 
 4876         b_io_dismiss(bp, ioflag, false);
 4877 }
 4878 
 4879 /*
 4880  * vm_hold_load_pages and vm_hold_free_pages get pages into
 4881  * a buffers address space.  The pages are anonymous and are
 4882  * not associated with a file object.
 4883  */
 4884 static void
 4885 vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
 4886 {
 4887         vm_offset_t pg;
 4888         vm_page_t p;
 4889         int index;
 4890 
 4891         BUF_CHECK_MAPPED(bp);
 4892 
 4893         to = round_page(to);
 4894         from = round_page(from);
 4895         index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
 4896 
 4897         for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
 4898                 /*
 4899                  * note: must allocate system pages since blocking here
 4900                  * could interfere with paging I/O, no matter which
 4901                  * process we are.
 4902                  */
 4903                 p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
 4904                     VM_ALLOC_WIRED | VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT) |
 4905                     VM_ALLOC_WAITOK);
 4906                 pmap_qenter(pg, &p, 1);
 4907                 bp->b_pages[index] = p;
 4908         }
 4909         bp->b_npages = index;
 4910 }
 4911 
 4912 /* Return pages associated with this buf to the vm system */
 4913 static void
 4914 vm_hold_free_pages(struct buf *bp, int newbsize)
 4915 {
 4916         vm_offset_t from;
 4917         vm_page_t p;
 4918         int index, newnpages;
 4919 
 4920         BUF_CHECK_MAPPED(bp);
 4921 
 4922         from = round_page((vm_offset_t)bp->b_data + newbsize);
 4923         newnpages = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
 4924         if (bp->b_npages > newnpages)
 4925                 pmap_qremove(from, bp->b_npages - newnpages);
 4926         for (index = newnpages; index < bp->b_npages; index++) {
 4927                 p = bp->b_pages[index];
 4928                 bp->b_pages[index] = NULL;
 4929                 p->wire_count--;
 4930                 vm_page_free(p);
 4931         }
 4932         vm_wire_sub(bp->b_npages - newnpages);
 4933         bp->b_npages = newnpages;
 4934 }
 4935 
 4936 /*
 4937  * Map an IO request into kernel virtual address space.
 4938  *
 4939  * All requests are (re)mapped into kernel VA space.
 4940  * Notice that we use b_bufsize for the size of the buffer
 4941  * to be mapped.  b_bcount might be modified by the driver.
 4942  *
 4943  * Note that even if the caller determines that the address space should
 4944  * be valid, a race or a smaller-file mapped into a larger space may
 4945  * actually cause vmapbuf() to fail, so all callers of vmapbuf() MUST
 4946  * check the return value.
 4947  *
 4948  * This function only works with pager buffers.
 4949  */
 4950 int
 4951 vmapbuf(struct buf *bp, int mapbuf)
 4952 {
 4953         vm_prot_t prot;
 4954         int pidx;
 4955 
 4956         if (bp->b_bufsize < 0)
 4957                 return (-1);
 4958         prot = VM_PROT_READ;
 4959         if (bp->b_iocmd == BIO_READ)
 4960                 prot |= VM_PROT_WRITE;  /* Less backwards than it looks */
 4961         if ((pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
 4962             (vm_offset_t)bp->b_data, bp->b_bufsize, prot, bp->b_pages,
 4963             btoc(MAXPHYS))) < 0)
 4964                 return (-1);
 4965         bp->b_npages = pidx;
 4966         bp->b_offset = ((vm_offset_t)bp->b_data) & PAGE_MASK;
 4967         if (mapbuf || !unmapped_buf_allowed) {
 4968                 pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
 4969                 bp->b_data = bp->b_kvabase + bp->b_offset;
 4970         } else
 4971                 bp->b_data = unmapped_buf;
 4972         return(0);
 4973 }
 4974 
 4975 /*
 4976  * Free the io map PTEs associated with this IO operation.
 4977  * We also invalidate the TLB entries and restore the original b_addr.
 4978  *
 4979  * This function only works with pager buffers.
 4980  */
 4981 void
 4982 vunmapbuf(struct buf *bp)
 4983 {
 4984         int npages;
 4985 
 4986         npages = bp->b_npages;
 4987         if (buf_mapped(bp))
 4988                 pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
 4989         vm_page_unhold_pages(bp->b_pages, npages);
 4990 
 4991         bp->b_data = unmapped_buf;
 4992 }
 4993 
 4994 void
 4995 bdone(struct buf *bp)
 4996 {
 4997         struct mtx *mtxp;
 4998 
 4999         mtxp = mtx_pool_find(mtxpool_sleep, bp);
 5000         mtx_lock(mtxp);
 5001         bp->b_flags |= B_DONE;
 5002         wakeup(bp);
 5003         mtx_unlock(mtxp);
 5004 }
 5005 
 5006 void
 5007 bwait(struct buf *bp, u_char pri, const char *wchan)
 5008 {
 5009         struct mtx *mtxp;
 5010 
 5011         mtxp = mtx_pool_find(mtxpool_sleep, bp);
 5012         mtx_lock(mtxp);
 5013         while ((bp->b_flags & B_DONE) == 0)
 5014                 msleep(bp, mtxp, pri, wchan, 0);
 5015         mtx_unlock(mtxp);
 5016 }
 5017 
 5018 int
 5019 bufsync(struct bufobj *bo, int waitfor)
 5020 {
 5021 
 5022         return (VOP_FSYNC(bo2vnode(bo), waitfor, curthread));
 5023 }
 5024 
 5025 void
 5026 bufstrategy(struct bufobj *bo, struct buf *bp)
 5027 {
 5028         int i __unused;
 5029         struct vnode *vp;
 5030 
 5031         vp = bp->b_vp;
 5032         KASSERT(vp == bo->bo_private, ("Inconsistent vnode bufstrategy"));
 5033         KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
 5034             ("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp));
 5035         i = VOP_STRATEGY(vp, bp);
 5036         KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp));
 5037 }
 5038 
 5039 /*
 5040  * Initialize a struct bufobj before use.  Memory is assumed zero filled.
 5041  */
 5042 void
 5043 bufobj_init(struct bufobj *bo, void *private)
 5044 {
 5045         static volatile int bufobj_cleanq;
 5046 
 5047         bo->bo_domain =
 5048             atomic_fetchadd_int(&bufobj_cleanq, 1) % buf_domains;
 5049         rw_init(BO_LOCKPTR(bo), "bufobj interlock");
 5050         bo->bo_private = private;
 5051         TAILQ_INIT(&bo->bo_clean.bv_hd);
 5052         TAILQ_INIT(&bo->bo_dirty.bv_hd);
 5053 }
 5054 
 5055 void
 5056 bufobj_wrefl(struct bufobj *bo)
 5057 {
 5058 
 5059         KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
 5060         ASSERT_BO_WLOCKED(bo);
 5061         bo->bo_numoutput++;
 5062 }
 5063 
 5064 void
 5065 bufobj_wref(struct bufobj *bo)
 5066 {
 5067 
 5068         KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
 5069         BO_LOCK(bo);
 5070         bo->bo_numoutput++;
 5071         BO_UNLOCK(bo);
 5072 }
 5073 
 5074 void
 5075 bufobj_wdrop(struct bufobj *bo)
 5076 {
 5077 
 5078         KASSERT(bo != NULL, ("NULL bo in bufobj_wdrop"));
 5079         BO_LOCK(bo);
 5080         KASSERT(bo->bo_numoutput > 0, ("bufobj_wdrop non-positive count"));
 5081         if ((--bo->bo_numoutput == 0) && (bo->bo_flag & BO_WWAIT)) {
 5082                 bo->bo_flag &= ~BO_WWAIT;
 5083                 wakeup(&bo->bo_numoutput);
 5084         }
 5085         BO_UNLOCK(bo);
 5086 }
 5087 
 5088 int
 5089 bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
 5090 {
 5091         int error;
 5092 
 5093         KASSERT(bo != NULL, ("NULL bo in bufobj_wwait"));
 5094         ASSERT_BO_WLOCKED(bo);
 5095         error = 0;
 5096         while (bo->bo_numoutput) {
 5097                 bo->bo_flag |= BO_WWAIT;
 5098                 error = msleep(&bo->bo_numoutput, BO_LOCKPTR(bo),
 5099                     slpflag | (PRIBIO + 1), "bo_wwait", timeo);
 5100                 if (error)
 5101                         break;
 5102         }
 5103         return (error);
 5104 }
 5105 
 5106 /*
 5107  * Set bio_data or bio_ma for struct bio from the struct buf.
 5108  */
 5109 void
 5110 bdata2bio(struct buf *bp, struct bio *bip)
 5111 {
 5112 
 5113         if (!buf_mapped(bp)) {
 5114                 KASSERT(unmapped_buf_allowed, ("unmapped"));
 5115                 bip->bio_ma = bp->b_pages;
 5116                 bip->bio_ma_n = bp->b_npages;
 5117                 bip->bio_data = unmapped_buf;
 5118                 bip->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
 5119                 bip->bio_flags |= BIO_UNMAPPED;
 5120                 KASSERT(round_page(bip->bio_ma_offset + bip->bio_length) /
 5121                     PAGE_SIZE == bp->b_npages,
 5122                     ("Buffer %p too short: %d %lld %d", bp, bip->bio_ma_offset,
 5123                     (long long)bip->bio_length, bip->bio_ma_n));
 5124         } else {
 5125                 bip->bio_data = bp->b_data;
 5126                 bip->bio_ma = NULL;
 5127         }
 5128 }
 5129 
 5130 /*
 5131  * The MIPS pmap code currently doesn't handle aliased pages.
 5132  * The VIPT caches may not handle page aliasing themselves, leading
 5133  * to data corruption.
 5134  *
 5135  * As such, this code makes a system extremely unhappy if said
 5136  * system doesn't support unaliasing the above situation in hardware.
 5137  * Some "recent" systems (eg some mips24k/mips74k cores) don't enable
 5138  * this feature at build time, so it has to be handled in software.
 5139  *
 5140  * Once the MIPS pmap/cache code grows to support this function on
 5141  * earlier chips, it should be flipped back off.
 5142  */
 5143 #ifdef  __mips__
 5144 static int buf_pager_relbuf = 1;
 5145 #else
 5146 static int buf_pager_relbuf = 0;
 5147 #endif
 5148 SYSCTL_INT(_vfs, OID_AUTO, buf_pager_relbuf, CTLFLAG_RWTUN,
 5149     &buf_pager_relbuf, 0,
 5150     "Make buffer pager release buffers after reading");
 5151 
 5152 /*
 5153  * The buffer pager.  It uses buffer reads to validate pages.
 5154  *
 5155  * In contrast to the generic local pager from vm/vnode_pager.c, this
 5156  * pager correctly and easily handles volumes where the underlying
 5157  * device block size is greater than the machine page size.  The
 5158  * buffer cache transparently extends the requested page run to be
 5159  * aligned at the block boundary, and does the necessary bogus page
 5160  * replacements in the addends to avoid obliterating already valid
 5161  * pages.
 5162  *
 5163  * The only non-trivial issue is that the exclusive busy state for
 5164  * pages, which is assumed by the vm_pager_getpages() interface, is
 5165  * incompatible with the VMIO buffer cache's desire to share-busy the
 5166  * pages.  This function performs a trivial downgrade of the pages'
 5167  * state before reading buffers, and a less trivial upgrade from the
 5168  * shared-busy to excl-busy state after the read.
 5169  */
 5170 int
 5171 vfs_bio_getpages(struct vnode *vp, vm_page_t *ma, int count,
 5172     int *rbehind, int *rahead, vbg_get_lblkno_t get_lblkno,
 5173     vbg_get_blksize_t get_blksize)
 5174 {
 5175         vm_page_t m;
 5176         vm_object_t object;
 5177         struct buf *bp;
 5178         struct mount *mp;
 5179         daddr_t lbn, lbnp;
 5180         vm_ooffset_t la, lb, poff, poffe;
 5181         long bsize;
 5182         int bo_bs, br_flags, error, i, pgsin, pgsin_a, pgsin_b;
 5183         bool redo, lpart;
 5184 
 5185         object = vp->v_object;
 5186         mp = vp->v_mount;
 5187         error = 0;
 5188         la = IDX_TO_OFF(ma[count - 1]->pindex);
 5189         if (la >= object->un_pager.vnp.vnp_size)
 5190                 return (VM_PAGER_BAD);
 5191 
 5192         /*
 5193          * Change the meaning of la from where the last requested page starts
 5194          * to where it ends, because that's the end of the requested region
 5195          * and the start of the potential read-ahead region.
 5196          */
 5197         la += PAGE_SIZE;
 5198         lpart = la > object->un_pager.vnp.vnp_size;
 5199         bo_bs = get_blksize(vp, get_lblkno(vp, IDX_TO_OFF(ma[0]->pindex)));
 5200 
 5201         /*
 5202          * Calculate read-ahead, behind and total pages.
 5203          */
 5204         pgsin = count;
 5205         lb = IDX_TO_OFF(ma[0]->pindex);
 5206         pgsin_b = OFF_TO_IDX(lb - rounddown2(lb, bo_bs));
 5207         pgsin += pgsin_b;
 5208         if (rbehind != NULL)
 5209                 *rbehind = pgsin_b;
 5210         pgsin_a = OFF_TO_IDX(roundup2(la, bo_bs) - la);
 5211         if (la + IDX_TO_OFF(pgsin_a) >= object->un_pager.vnp.vnp_size)
 5212                 pgsin_a = OFF_TO_IDX(roundup2(object->un_pager.vnp.vnp_size,
 5213                     PAGE_SIZE) - la);
 5214         pgsin += pgsin_a;
 5215         if (rahead != NULL)
 5216                 *rahead = pgsin_a;
 5217         VM_CNT_INC(v_vnodein);
 5218         VM_CNT_ADD(v_vnodepgsin, pgsin);
 5219 
 5220         br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS)
 5221             != 0) ? GB_UNMAPPED : 0;
 5222         VM_OBJECT_WLOCK(object);
 5223 again:
 5224         for (i = 0; i < count; i++)
 5225                 vm_page_busy_downgrade(ma[i]);
 5226         VM_OBJECT_WUNLOCK(object);
 5227 
 5228         lbnp = -1;
 5229         for (i = 0; i < count; i++) {
 5230                 m = ma[i];
 5231 
 5232                 /*
 5233                  * Pages are shared busy and the object lock is not
 5234                  * owned, which together allow for the pages'
 5235                  * invalidation.  The racy test for validity avoids
 5236                  * useless creation of the buffer for the most typical
 5237                  * case when invalidation is not used in redo or for
 5238                  * parallel read.  The shared->excl upgrade loop at
 5239                  * the end of the function catches the race in a
 5240                  * reliable way (protected by the object lock).
 5241                  */
 5242                 if (m->valid == VM_PAGE_BITS_ALL)
 5243                         continue;
 5244 
 5245                 poff = IDX_TO_OFF(m->pindex);
 5246                 poffe = MIN(poff + PAGE_SIZE, object->un_pager.vnp.vnp_size);
 5247                 for (; poff < poffe; poff += bsize) {
 5248                         lbn = get_lblkno(vp, poff);
 5249                         if (lbn == lbnp)
 5250                                 goto next_page;
 5251                         lbnp = lbn;
 5252 
 5253                         bsize = get_blksize(vp, lbn);
 5254                         error = bread_gb(vp, lbn, bsize, curthread->td_ucred,
 5255                             br_flags, &bp);
 5256                         if (error != 0)
 5257                                 goto end_pages;
 5258                         if (LIST_EMPTY(&bp->b_dep)) {
 5259                                 /*
 5260                                  * Invalidation clears m->valid, but
 5261                                  * may leave B_CACHE flag if the
 5262                                  * buffer existed at the invalidation
 5263                                  * time.  In this case, recycle the
 5264                                  * buffer to do real read on next
 5265                                  * bread() after redo.
 5266                                  *
 5267                                  * Otherwise B_RELBUF is not strictly
 5268                                  * necessary, enable to reduce buf
 5269                                  * cache pressure.
 5270                                  */
 5271                                 if (buf_pager_relbuf ||
 5272                                     m->valid != VM_PAGE_BITS_ALL)
 5273                                         bp->b_flags |= B_RELBUF;
 5274 
 5275                                 bp->b_flags &= ~B_NOCACHE;
 5276                                 brelse(bp);
 5277                         } else {
 5278                                 bqrelse(bp);
 5279                         }
 5280                 }
 5281                 KASSERT(1 /* racy, enable for debugging */ ||
 5282                     m->valid == VM_PAGE_BITS_ALL || i == count - 1,
 5283                     ("buf %d %p invalid", i, m));
 5284                 if (i == count - 1 && lpart) {
 5285                         VM_OBJECT_WLOCK(object);
 5286                         if (m->valid != 0 &&
 5287                             m->valid != VM_PAGE_BITS_ALL)
 5288                                 vm_page_zero_invalid(m, TRUE);
 5289                         VM_OBJECT_WUNLOCK(object);
 5290                 }
 5291 next_page:;
 5292         }
 5293 end_pages:
 5294 
 5295         VM_OBJECT_WLOCK(object);
 5296         redo = false;
 5297         for (i = 0; i < count; i++) {
 5298                 vm_page_sunbusy(ma[i]);
 5299                 ma[i] = vm_page_grab(object, ma[i]->pindex, VM_ALLOC_NORMAL);
 5300 
 5301                 /*
 5302                  * Since the pages were only sbusy while neither the
 5303                  * buffer nor the object lock was held by us, or
 5304                  * reallocated while vm_page_grab() slept for busy
 5305                  * relinguish, they could have been invalidated.
 5306                  * Recheck the valid bits and re-read as needed.
 5307                  *
 5308                  * Note that the last page is made fully valid in the
 5309                  * read loop, and partial validity for the page at
 5310                  * index count - 1 could mean that the page was
 5311                  * invalidated or removed, so we must restart for
 5312                  * safety as well.
 5313                  */
 5314                 if (ma[i]->valid != VM_PAGE_BITS_ALL)
 5315                         redo = true;
 5316         }
 5317         if (redo && error == 0)
 5318                 goto again;
 5319         VM_OBJECT_WUNLOCK(object);
 5320         return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
 5321 }
 5322 
 5323 #include "opt_ddb.h"
 5324 #ifdef DDB
 5325 #include <ddb/ddb.h>
 5326 
 5327 /* DDB command to show buffer data */
 5328 DB_SHOW_COMMAND(buffer, db_show_buffer)
 5329 {
 5330         /* get args */
 5331         struct buf *bp = (struct buf *)addr;
 5332 #ifdef FULL_BUF_TRACKING
 5333         uint32_t i, j;
 5334 #endif
 5335 
 5336         if (!have_addr) {
 5337                 db_printf("usage: show buffer <addr>\n");
 5338                 return;
 5339         }
 5340 
 5341         db_printf("buf at %p\n", bp);
 5342         db_printf("b_flags = 0x%b, b_xflags=0x%b, b_vflags=0x%b\n",
 5343             (u_int)bp->b_flags, PRINT_BUF_FLAGS, (u_int)bp->b_xflags,
 5344             PRINT_BUF_XFLAGS, (u_int)bp->b_vflags, PRINT_BUF_VFLAGS);
 5345         db_printf(
 5346             "b_error = %d, b_bufsize = %ld, b_bcount = %ld, b_resid = %ld\n"
 5347             "b_bufobj = (%p), b_data = %p, b_blkno = %jd, b_lblkno = %jd, "
 5348             "b_dep = %p\n",
 5349             bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
 5350             bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
 5351             (intmax_t)bp->b_lblkno, bp->b_dep.lh_first);
 5352         db_printf("b_kvabase = %p, b_kvasize = %d\n",
 5353             bp->b_kvabase, bp->b_kvasize);
 5354         if (bp->b_npages) {
 5355                 int i;
 5356                 db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
 5357                 for (i = 0; i < bp->b_npages; i++) {
 5358                         vm_page_t m;
 5359                         m = bp->b_pages[i];
 5360                         if (m != NULL)
 5361                                 db_printf("(%p, 0x%lx, 0x%lx)", m->object,
 5362                                     (u_long)m->pindex,
 5363                                     (u_long)VM_PAGE_TO_PHYS(m));
 5364                         else
 5365                                 db_printf("( ??? )");
 5366                         if ((i + 1) < bp->b_npages)
 5367                                 db_printf(",");
 5368                 }
 5369                 db_printf("\n");
 5370         }
 5371         BUF_LOCKPRINTINFO(bp);
 5372 #if defined(FULL_BUF_TRACKING)
 5373         db_printf("b_io_tracking: b_io_tcnt = %u\n", bp->b_io_tcnt);
 5374 
 5375         i = bp->b_io_tcnt % BUF_TRACKING_SIZE;
 5376         for (j = 1; j <= BUF_TRACKING_SIZE; j++) {
 5377                 if (bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)] == NULL)
 5378                         continue;
 5379                 db_printf(" %2u: %s\n", j,
 5380                     bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)]);
 5381         }
 5382 #elif defined(BUF_TRACKING)
 5383         db_printf("b_io_tracking: %s\n", bp->b_io_tracking);
 5384 #endif
 5385         db_printf(" ");
 5386 }
 5387 
 5388 DB_SHOW_COMMAND(bufqueues, bufqueues)
 5389 {
 5390         struct bufdomain *bd;
 5391         struct buf *bp;
 5392         long total;
 5393         int i, j, cnt;
 5394 
 5395         db_printf("bqempty: %d\n", bqempty.bq_len);
 5396 
 5397         for (i = 0; i < buf_domains; i++) {
 5398                 bd = &bdomain[i];
 5399                 db_printf("Buf domain %d\n", i);
 5400                 db_printf("\tfreebufs\t%d\n", bd->bd_freebuffers);
 5401                 db_printf("\tlofreebufs\t%d\n", bd->bd_lofreebuffers);
 5402                 db_printf("\thifreebufs\t%d\n", bd->bd_hifreebuffers);
 5403                 db_printf("\n");
 5404                 db_printf("\tbufspace\t%ld\n", bd->bd_bufspace);
 5405                 db_printf("\tmaxbufspace\t%ld\n", bd->bd_maxbufspace);
 5406                 db_printf("\thibufspace\t%ld\n", bd->bd_hibufspace);
 5407                 db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace);
 5408                 db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh);
 5409                 db_printf("\n");
 5410                 db_printf("\tnumdirtybuffers\t%d\n", bd->bd_numdirtybuffers);
 5411                 db_printf("\tlodirtybuffers\t%d\n", bd->bd_lodirtybuffers);
 5412                 db_printf("\thidirtybuffers\t%d\n", bd->bd_hidirtybuffers);
 5413                 db_printf("\tdirtybufthresh\t%d\n", bd->bd_dirtybufthresh);
 5414                 db_printf("\n");
 5415                 total = 0;
 5416                 TAILQ_FOREACH(bp, &bd->bd_cleanq->bq_queue, b_freelist)
 5417                         total += bp->b_bufsize;
 5418                 db_printf("\tcleanq count\t%d (%ld)\n",
 5419                     bd->bd_cleanq->bq_len, total);
 5420                 total = 0;
 5421                 TAILQ_FOREACH(bp, &bd->bd_dirtyq.bq_queue, b_freelist)
 5422                         total += bp->b_bufsize;
 5423                 db_printf("\tdirtyq count\t%d (%ld)\n",
 5424                     bd->bd_dirtyq.bq_len, total);
 5425                 db_printf("\twakeup\t\t%d\n", bd->bd_wanted);
 5426                 db_printf("\tlim\t\t%d\n", bd->bd_lim);
 5427                 db_printf("\tCPU ");
 5428                 for (j = 0; j <= mp_maxid; j++)
 5429                         db_printf("%d, ", bd->bd_subq[j].bq_len);
 5430                 db_printf("\n");
 5431                 cnt = 0;
 5432                 total = 0;
 5433                 for (j = 0; j < nbuf; j++)
 5434                         if (buf[j].b_domain == i && BUF_ISLOCKED(&buf[j])) {
 5435                                 cnt++;
 5436                                 total += buf[j].b_bufsize;
 5437                         }
 5438                 db_printf("\tLocked buffers: %d space %ld\n", cnt, total);
 5439                 cnt = 0;
 5440                 total = 0;
 5441                 for (j = 0; j < nbuf; j++)
 5442                         if (buf[j].b_domain == i) {
 5443                                 cnt++;
 5444                                 total += buf[j].b_bufsize;
 5445                         }
 5446                 db_printf("\tTotal buffers: %d space %ld\n", cnt, total);
 5447         }
 5448 }
 5449 
 5450 DB_SHOW_COMMAND(lockedbufs, lockedbufs)
 5451 {
 5452         struct buf *bp;
 5453         int i;
 5454 
 5455         for (i = 0; i < nbuf; i++) {
 5456                 bp = &buf[i];
 5457                 if (BUF_ISLOCKED(bp)) {
 5458                         db_show_buffer((uintptr_t)bp, 1, 0, NULL);
 5459                         db_printf("\n");
 5460                         if (db_pager_quit)
 5461                                 break;
 5462                 }
 5463         }
 5464 }
 5465 
 5466 DB_SHOW_COMMAND(vnodebufs, db_show_vnodebufs)
 5467 {
 5468         struct vnode *vp;
 5469         struct buf *bp;
 5470 
 5471         if (!have_addr) {
 5472                 db_printf("usage: show vnodebufs <addr>\n");
 5473                 return;
 5474         }
 5475         vp = (struct vnode *)addr;
 5476         db_printf("Clean buffers:\n");
 5477         TAILQ_FOREACH(bp, &vp->v_bufobj.bo_clean.bv_hd, b_bobufs) {
 5478                 db_show_buffer((uintptr_t)bp, 1, 0, NULL);
 5479                 db_printf("\n");
 5480         }
 5481         db_printf("Dirty buffers:\n");
 5482         TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
 5483                 db_show_buffer((uintptr_t)bp, 1, 0, NULL);
 5484                 db_printf("\n");
 5485         }
 5486 }
 5487 
 5488 DB_COMMAND(countfreebufs, db_coundfreebufs)
 5489 {
 5490         struct buf *bp;
 5491         int i, used = 0, nfree = 0;
 5492 
 5493         if (have_addr) {
 5494                 db_printf("usage: countfreebufs\n");
 5495                 return;
 5496         }
 5497 
 5498         for (i = 0; i < nbuf; i++) {
 5499                 bp = &buf[i];
 5500                 if (bp->b_qindex == QUEUE_EMPTY)
 5501                         nfree++;
 5502                 else
 5503                         used++;
 5504         }
 5505 
 5506         db_printf("Counted %d free, %d used (%d tot)\n", nfree, used,
 5507             nfree + used);
 5508         db_printf("numfreebuffers is %d\n", numfreebuffers);
 5509 }
 5510 #endif /* DDB */

Cache object: 5916c49dfec5b838a83031b547dd01f8


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.