The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_bio.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: vfs_bio.c,v 1.122.2.4 2004/10/08 03:25:15 jmc Exp $    */
    2 
    3 /*-
    4  * Copyright (c) 1982, 1986, 1989, 1993
    5  *      The Regents of the University of California.  All rights reserved.
    6  * (c) UNIX System Laboratories, Inc.
    7  * All or some portions of this file are derived from material licensed
    8  * to the University of California by American Telephone and Telegraph
    9  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
   10  * the permission of UNIX System Laboratories, Inc.
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  * 3. Neither the name of the University nor the names of its contributors
   21  *    may be used to endorse or promote products derived from this software
   22  *    without specific prior written permission.
   23  *
   24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   34  * SUCH DAMAGE.
   35  *
   36  *      @(#)vfs_bio.c   8.6 (Berkeley) 1/11/94
   37  */
   38 
   39 /*-
   40  * Copyright (c) 1994 Christopher G. Demetriou
   41  *
   42  * Redistribution and use in source and binary forms, with or without
   43  * modification, are permitted provided that the following conditions
   44  * are met:
   45  * 1. Redistributions of source code must retain the above copyright
   46  *    notice, this list of conditions and the following disclaimer.
   47  * 2. Redistributions in binary form must reproduce the above copyright
   48  *    notice, this list of conditions and the following disclaimer in the
   49  *    documentation and/or other materials provided with the distribution.
   50  * 3. All advertising materials mentioning features or use of this software
   51  *    must display the following acknowledgement:
   52  *      This product includes software developed by the University of
   53  *      California, Berkeley and its contributors.
   54  * 4. Neither the name of the University nor the names of its contributors
   55  *    may be used to endorse or promote products derived from this software
   56  *    without specific prior written permission.
   57  *
   58  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   59  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   60  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   61  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   62  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   63  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   64  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   65  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   66  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   67  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   68  * SUCH DAMAGE.
   69  *
   70  *      @(#)vfs_bio.c   8.6 (Berkeley) 1/11/94
   71  */
   72 
   73 /*
   74  * Some references:
   75  *      Bach: The Design of the UNIX Operating System (Prentice Hall, 1986)
   76  *      Leffler, et al.: The Design and Implementation of the 4.3BSD
   77  *              UNIX Operating System (Addison Welley, 1989)
   78  */
   79 
   80 #include "opt_bufcache.h"
   81 #include "opt_softdep.h"
   82 
   83 #include <sys/cdefs.h>
   84 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.122.2.4 2004/10/08 03:25:15 jmc Exp $");
   85 
   86 #include <sys/param.h>
   87 #include <sys/systm.h>
   88 #include <sys/kernel.h>
   89 #include <sys/proc.h>
   90 #include <sys/buf.h>
   91 #include <sys/vnode.h>
   92 #include <sys/mount.h>
   93 #include <sys/malloc.h>
   94 #include <sys/resourcevar.h>
   95 #include <sys/sysctl.h>
   96 #include <sys/conf.h>
   97 
   98 #include <uvm/uvm.h>
   99 
  100 #include <miscfs/specfs/specdev.h>
  101 
  102 #ifndef BUFPAGES
  103 # define BUFPAGES 0
  104 #endif
  105 
  106 #ifdef BUFCACHE
  107 # if (BUFCACHE < 5) || (BUFCACHE > 95)
  108 #  error BUFCACHE is not between 5 and 95
  109 # endif
  110 #else
  111 # define BUFCACHE 15
  112 #endif
  113 
  114 u_int   nbuf;                   /* XXX - for softdep_lockedbufs */
  115 u_int   bufpages = BUFPAGES;    /* optional hardwired count */
  116 u_int   bufcache = BUFCACHE;    /* max % of RAM to use for buffer cache */
  117 
  118 
  119 /* Macros to clear/set/test flags. */
  120 #define SET(t, f)       (t) |= (f)
  121 #define CLR(t, f)       (t) &= ~(f)
  122 #define ISSET(t, f)     ((t) & (f))
  123 
  124 /*
  125  * Definitions for the buffer hash lists.
  126  */
  127 #define BUFHASH(dvp, lbn)       \
  128         (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash])
  129 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
  130 u_long  bufhash;
  131 #ifndef SOFTDEP
  132 struct bio_ops bioops;  /* I/O operation notification */
  133 #endif
  134 
  135 /*
  136  * Insq/Remq for the buffer hash lists.
  137  */
  138 #define binshash(bp, dp)        LIST_INSERT_HEAD(dp, bp, b_hash)
  139 #define bremhash(bp)            LIST_REMOVE(bp, b_hash)
  140 
  141 /*
  142  * Definitions for the buffer free lists.
  143  */
  144 #define BQUEUES         3               /* number of free buffer queues */
  145 
  146 #define BQ_LOCKED       0               /* super-blocks &c */
  147 #define BQ_LRU          1               /* lru, useful buffers */
  148 #define BQ_AGE          2               /* rubbish */
  149 
  150 TAILQ_HEAD(bqueues, buf) bufqueues[BQUEUES];
  151 int needbuffer;
  152 
  153 /*
  154  * Buffer queue lock.
  155  * Take this lock first if also taking some buffer's b_interlock.
  156  */
  157 struct simplelock bqueue_slock = SIMPLELOCK_INITIALIZER;
  158 
  159 /*
  160  * Buffer pool for I/O buffers.
  161  */
  162 struct pool bufpool;
  163 
  164 /* XXX - somewhat gross.. */
  165 #if MAXBSIZE == 0x2000
  166 #define NMEMPOOLS 4
  167 #elif MAXBSIZE == 0x4000
  168 #define NMEMPOOLS 5
  169 #elif MAXBSIZE == 0x8000
  170 #define NMEMPOOLS 6
  171 #else
  172 #define NMEMPOOLS 7
  173 #endif
  174 
  175 #define MEMPOOL_INDEX_OFFSET 10         /* smallest pool is 1k */
  176 #if (1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) != MAXBSIZE
  177 #error update vfs_bio buffer memory parameters
  178 #endif
  179 
  180 /* Buffer memory pools */
  181 static struct pool bmempools[NMEMPOOLS];
  182 
  183 struct vm_map *buf_map;
  184 
  185 /*
  186  * Buffer memory pool allocator.
  187  */
  188 static void *
  189 bufpool_page_alloc(struct pool *pp, int flags)
  190 {
  191 
  192         return (void *)uvm_km_kmemalloc1(buf_map,
  193             uvm.kernel_object, MAXBSIZE, MAXBSIZE, UVM_UNKNOWN_OFFSET,
  194             (flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK);
  195 }
  196 
  197 static void
  198 bufpool_page_free(struct pool *pp, void *v)
  199 {
  200         uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE);
  201 }
  202 
  203 static struct pool_allocator bufmempool_allocator = {
  204         bufpool_page_alloc, bufpool_page_free, MAXBSIZE,
  205 };
  206 
  207 /* Buffer memory management variables */
  208 u_long bufmem_valimit;
  209 u_long bufmem_hiwater;
  210 u_long bufmem_lowater;
  211 u_long bufmem;
  212 
  213 /*
  214  * MD code can call this to set a hard limit on the amount
  215  * of virtual memory used by the buffer cache.
  216  */
  217 int
  218 buf_setvalimit(vsize_t sz)
  219 {
  220 
  221         /* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */
  222         if (sz < NMEMPOOLS * MAXBSIZE)
  223                 return EINVAL;
  224 
  225         bufmem_valimit = sz;
  226         return 0;
  227 }
  228 
  229 static void buf_setwm(void);
  230 static int buf_trim(void);
  231 
  232 /*
  233  * bread()/breadn() helper.
  234  */
  235 static __inline struct buf *bio_doread(struct vnode *, daddr_t, int,
  236                                         struct ucred *, int);
  237 int count_lock_queue(void);
  238 
  239 /*
  240  * Insq/Remq for the buffer free lists.
  241  * Call with buffer queue locked.
  242  */
  243 #define binsheadfree(bp, dp)    TAILQ_INSERT_HEAD(dp, bp, b_freelist)
  244 #define binstailfree(bp, dp)    TAILQ_INSERT_TAIL(dp, bp, b_freelist)
  245 
  246 static void
  247 buf_setwm(void)
  248 {
  249 
  250         bufmem_hiwater = buf_memcalc();
  251         /* lowater is approx. 2% of memory (with bufcache = 15) */
  252 #define BUFMEM_WMSHIFT  3
  253 #define BUFMEM_HIWMMIN  (64 * 1024 << BUFMEM_WMSHIFT)
  254         if (bufmem_hiwater < BUFMEM_HIWMMIN)
  255                 /* Ensure a reasonable minimum value */
  256                 bufmem_hiwater = BUFMEM_HIWMMIN;
  257         bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT;
  258 }
  259 
  260 #ifdef DEBUG
  261 int debug_verify_freelist = 0;
  262 static int checkfreelist(struct buf *bp, struct bqueues *dp)
  263 {
  264         struct buf *b;
  265         TAILQ_FOREACH(b, dp, b_freelist) {
  266                 if (b == bp)
  267                         return 1;
  268         }
  269         return 0;
  270 }
  271 #endif
  272 
  273 void
  274 bremfree(struct buf *bp)
  275 {
  276         struct bqueues *dp = NULL;
  277 
  278         LOCK_ASSERT(simple_lock_held(&bqueue_slock));
  279 
  280         KDASSERT(!debug_verify_freelist ||
  281                 checkfreelist(bp, &bufqueues[BQ_AGE]) ||
  282                 checkfreelist(bp, &bufqueues[BQ_LRU]) ||
  283                 checkfreelist(bp, &bufqueues[BQ_LOCKED]) );
  284 
  285         /*
  286          * We only calculate the head of the freelist when removing
  287          * the last element of the list as that is the only time that
  288          * it is needed (e.g. to reset the tail pointer).
  289          *
  290          * NB: This makes an assumption about how tailq's are implemented.
  291          *
  292          * We break the TAILQ abstraction in order to efficiently remove a
  293          * buffer from its freelist without having to know exactly which
  294          * freelist it is on.
  295          */
  296         if (TAILQ_NEXT(bp, b_freelist) == NULL) {
  297                 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
  298                         if (dp->tqh_last == &bp->b_freelist.tqe_next)
  299                                 break;
  300                 if (dp == &bufqueues[BQUEUES])
  301                         panic("bremfree: lost tail");
  302         }
  303         TAILQ_REMOVE(dp, bp, b_freelist);
  304 }
  305 
  306 u_long
  307 buf_memcalc(void)
  308 {
  309         u_long n;
  310 
  311         /*
  312          * Determine the upper bound of memory to use for buffers.
  313          *
  314          *      - If bufpages is specified, use that as the number
  315          *        pages.
  316          *
  317          *      - Otherwise, use bufcache as the percentage of
  318          *        physical memory.
  319          */
  320         if (bufpages != 0) {
  321                 n = bufpages;
  322         } else {
  323                 if (bufcache < 5) {
  324                         printf("forcing bufcache %d -> 5", bufcache);
  325                         bufcache = 5;
  326                 }
  327                 if (bufcache > 95) {
  328                         printf("forcing bufcache %d -> 95", bufcache);
  329                         bufcache = 95;
  330                 }
  331                 n = physmem / 100 * bufcache;
  332         }
  333 
  334         n <<= PAGE_SHIFT;
  335         if (bufmem_valimit != 0 && n > bufmem_valimit)
  336                 n = bufmem_valimit;
  337 
  338         return (n);
  339 }
  340 
  341 /*
  342  * Initialize buffers and hash links for buffers.
  343  */
  344 void
  345 bufinit(void)
  346 {
  347         struct bqueues *dp;
  348         int use_std;
  349         u_int i;
  350 
  351         /*
  352          * Initialize buffer cache memory parameters.
  353          */
  354         bufmem = 0;
  355         buf_setwm();
  356 
  357         if (bufmem_valimit != 0) {
  358                 vaddr_t minaddr = 0, maxaddr;
  359                 buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
  360                                           bufmem_valimit, VM_MAP_PAGEABLE,
  361                                           FALSE, 0);
  362                 if (buf_map == NULL)
  363                         panic("bufinit: cannot allocate submap");
  364         } else
  365                 buf_map = kernel_map;
  366 
  367         /*
  368          * Initialize the buffer pools.
  369          */
  370         pool_init(&bufpool, sizeof(struct buf), 0, 0, 0, "bufpl", NULL);
  371 
  372         /* On "small" machines use small pool page sizes where possible */
  373         use_std = (physmem < atop(16*1024*1024));
  374 
  375         /*
  376          * Also use them on systems that can map the pool pages using
  377          * a direct-mapped segment.
  378          */
  379 #ifdef PMAP_MAP_POOLPAGE
  380         use_std = 1;
  381 #endif
  382 
  383         for (i = 0; i < NMEMPOOLS; i++) {
  384                 struct pool_allocator *pa;
  385                 struct pool *pp = &bmempools[i];
  386                 u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET);
  387                 char *name = malloc(8, M_TEMP, M_WAITOK);
  388                 snprintf(name, 8, "buf%dk", 1 << i);
  389                 pa = (size <= PAGE_SIZE && use_std)
  390                         ? &pool_allocator_nointr
  391                         : &bufmempool_allocator;
  392                 pool_init(pp, size, 0, 0, 0, name, pa);
  393                 pool_setlowat(pp, 1);
  394                 pool_sethiwat(pp, 1);
  395         }
  396 
  397         /* Initialize the buffer queues */
  398         for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
  399                 TAILQ_INIT(dp);
  400 
  401         /*
  402          * Estimate hash table size based on the amount of memory we
  403          * intend to use for the buffer cache. The average buffer
  404          * size is dependent on our clients (i.e. filesystems).
  405          *
  406          * For now, use an empirical 3K per buffer.
  407          */
  408         nbuf = (bufmem_hiwater / 1024) / 3;
  409         bufhashtbl = hashinit(nbuf, HASH_LIST, M_CACHE, M_WAITOK, &bufhash);
  410 }
  411 
  412 static int
  413 buf_lotsfree(void)
  414 {
  415         int try, thresh;
  416 
  417         /* Always allocate if less than the low water mark. */
  418         if (bufmem < bufmem_lowater)
  419                 return 1;
  420         
  421         /* Never allocate if greater than the high water mark. */
  422         if (bufmem > bufmem_hiwater)
  423                 return 0;
  424 
  425         /* If there's anything on the AGE list, it should be eaten. */
  426         if (TAILQ_FIRST(&bufqueues[BQ_AGE]) != NULL)
  427                 return 0;
  428 
  429         /*
  430          * The probabily of getting a new allocation is inversely
  431          * proportional to the current size of the cache, using
  432          * a granularity of 16 steps.
  433          */
  434         try = random() & 0x0000000fL;
  435 
  436         /* Don't use "16 * bufmem" here to avoid a 32-bit overflow. */
  437         thresh = (bufmem - bufmem_lowater) /
  438             ((bufmem_hiwater - bufmem_lowater) / 16);
  439 
  440         if (try >= thresh)
  441                 return 1;
  442 
  443         /* Otherwise don't allocate. */
  444         return 0;
  445 }
  446 
  447 /*
  448  * Return estimate of bytes we think need to be
  449  * released to help resolve low memory conditions.
  450  *
  451  * => called at splbio.
  452  * => called with bqueue_slock held.
  453  */
  454 static int
  455 buf_canrelease(void)
  456 {
  457         int pagedemand, ninvalid = 0;
  458         struct buf *bp;
  459 
  460         LOCK_ASSERT(simple_lock_held(&bqueue_slock));
  461 
  462         if (bufmem < bufmem_lowater)
  463                 return 0;
  464 
  465         TAILQ_FOREACH(bp, &bufqueues[BQ_AGE], b_freelist)
  466                 ninvalid += bp->b_bufsize;
  467 
  468         pagedemand = uvmexp.freetarg - uvmexp.free;
  469         if (pagedemand < 0)
  470                 return ninvalid;
  471         return MAX(ninvalid, MIN(2 * MAXBSIZE,
  472             MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE)));
  473 }
  474 
  475 /*
  476  * Buffer memory allocation helper functions
  477  */
  478 static __inline u_long
  479 buf_mempoolidx(u_long size)
  480 {
  481         u_int n = 0;
  482 
  483         size -= 1;
  484         size >>= MEMPOOL_INDEX_OFFSET;
  485         while (size) {
  486                 size >>= 1;
  487                 n += 1;
  488         }
  489         if (n >= NMEMPOOLS)
  490                 panic("buf mem pool index %d", n);
  491         return n;
  492 }
  493 
  494 static __inline u_long
  495 buf_roundsize(u_long size)
  496 {
  497         /* Round up to nearest power of 2 */
  498         return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET));
  499 }
  500 
  501 static __inline caddr_t
  502 buf_malloc(size_t size)
  503 {
  504         u_int n = buf_mempoolidx(size);
  505         caddr_t addr;
  506         int s;
  507 
  508         while (1) {
  509                 addr = pool_get(&bmempools[n], PR_NOWAIT);
  510                 if (addr != NULL)
  511                         break;
  512 
  513                 /* No memory, see if we can free some. If so, try again */
  514                 if (buf_drain(1) > 0)
  515                         continue;
  516 
  517                 /* Wait for buffers to arrive on the LRU queue */
  518                 s = splbio();
  519                 simple_lock(&bqueue_slock);
  520                 needbuffer = 1;
  521                 ltsleep(&needbuffer, PNORELOCK | (PRIBIO + 1),
  522                         "buf_malloc", 0, &bqueue_slock);
  523                 splx(s);
  524         }
  525 
  526         return addr;
  527 }
  528 
  529 static void
  530 buf_mrelease(caddr_t addr, size_t size)
  531 {
  532 
  533         pool_put(&bmempools[buf_mempoolidx(size)], addr);
  534 }
  535 
  536 
  537 static __inline struct buf *
  538 bio_doread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred,
  539     int async)
  540 {
  541         struct buf *bp;
  542         struct lwp *l  = (curlwp != NULL ? curlwp : &lwp0);     /* XXX */
  543         struct proc *p = l->l_proc;
  544 
  545         bp = getblk(vp, blkno, size, 0, 0);
  546 
  547 #ifdef DIAGNOSTIC
  548         if (bp == NULL) {
  549                 panic("bio_doread: no such buf");
  550         }
  551 #endif
  552 
  553         /*
  554          * If buffer does not have data valid, start a read.
  555          * Note that if buffer is B_INVAL, getblk() won't return it.
  556          * Therefore, it's valid if its I/O has completed or been delayed.
  557          */
  558         if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) {
  559                 /* Start I/O for the buffer. */
  560                 SET(bp->b_flags, B_READ | async);
  561                 if (async)
  562                         BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
  563                 else
  564                         BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
  565                 VOP_STRATEGY(vp, bp);
  566 
  567                 /* Pay for the read. */
  568                 p->p_stats->p_ru.ru_inblock++;
  569         } else if (async) {
  570                 brelse(bp);
  571         }
  572 
  573         return (bp);
  574 }
  575 
  576 /*
  577  * Read a disk block.
  578  * This algorithm described in Bach (p.54).
  579  */
  580 int
  581 bread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred,
  582     struct buf **bpp)
  583 {
  584         struct buf *bp;
  585 
  586         /* Get buffer for block. */
  587         bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
  588 
  589         /* Wait for the read to complete, and return result. */
  590         return (biowait(bp));
  591 }
  592 
  593 /*
  594  * Read-ahead multiple disk blocks. The first is sync, the rest async.
  595  * Trivial modification to the breada algorithm presented in Bach (p.55).
  596  */
  597 int
  598 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks,
  599     int *rasizes, int nrablks, struct ucred *cred, struct buf **bpp)
  600 {
  601         struct buf *bp;
  602         int i;
  603 
  604         bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
  605 
  606         /*
  607          * For each of the read-ahead blocks, start a read, if necessary.
  608          */
  609         for (i = 0; i < nrablks; i++) {
  610                 /* If it's in the cache, just go on to next one. */
  611                 if (incore(vp, rablks[i]))
  612                         continue;
  613 
  614                 /* Get a buffer for the read-ahead block */
  615                 (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC);
  616         }
  617 
  618         /* Otherwise, we had to start a read for it; wait until it's valid. */
  619         return (biowait(bp));
  620 }
  621 
  622 /*
  623  * Read with single-block read-ahead.  Defined in Bach (p.55), but
  624  * implemented as a call to breadn().
  625  * XXX for compatibility with old file systems.
  626  */
  627 int
  628 breada(struct vnode *vp, daddr_t blkno, int size, daddr_t rablkno,
  629     int rabsize, struct ucred *cred, struct buf **bpp)
  630 {
  631 
  632         return (breadn(vp, blkno, size, &rablkno, &rabsize, 1, cred, bpp));     
  633 }
  634 
  635 /*
  636  * Block write.  Described in Bach (p.56)
  637  */
  638 int
  639 bwrite(struct buf *bp)
  640 {
  641         int rv, sync, wasdelayed, s;
  642         struct lwp *l  = (curlwp != NULL ? curlwp : &lwp0);     /* XXX */
  643         struct proc *p = l->l_proc;
  644         struct vnode *vp;
  645         struct mount *mp;
  646 
  647         KASSERT(ISSET(bp->b_flags, B_BUSY));
  648 
  649         vp = bp->b_vp;
  650         if (vp != NULL) {
  651                 if (vp->v_type == VBLK)
  652                         mp = vp->v_specmountpoint;
  653                 else
  654                         mp = vp->v_mount;
  655         } else {
  656                 mp = NULL;
  657         }
  658 
  659         /*
  660          * Remember buffer type, to switch on it later.  If the write was
  661          * synchronous, but the file system was mounted with MNT_ASYNC,
  662          * convert it to a delayed write.  
  663          * XXX note that this relies on delayed tape writes being converted
  664          * to async, not sync writes (which is safe, but ugly).
  665          */
  666         sync = !ISSET(bp->b_flags, B_ASYNC);
  667         if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) {
  668                 bdwrite(bp);
  669                 return (0);
  670         }
  671 
  672         /*
  673          * Collect statistics on synchronous and asynchronous writes.
  674          * Writes to block devices are charged to their associated
  675          * filesystem (if any).
  676          */
  677         if (mp != NULL) {
  678                 if (sync)
  679                         mp->mnt_stat.f_syncwrites++;
  680                 else
  681                         mp->mnt_stat.f_asyncwrites++;
  682         }
  683 
  684         s = splbio();
  685         simple_lock(&bp->b_interlock);
  686 
  687         wasdelayed = ISSET(bp->b_flags, B_DELWRI);
  688 
  689         CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI));
  690 
  691         /*
  692          * Pay for the I/O operation and make sure the buf is on the correct
  693          * vnode queue.
  694          */
  695         if (wasdelayed)
  696                 reassignbuf(bp, bp->b_vp);
  697         else
  698                 p->p_stats->p_ru.ru_oublock++;
  699 
  700         /* Initiate disk write.  Make sure the appropriate party is charged. */
  701         V_INCR_NUMOUTPUT(bp->b_vp);
  702         simple_unlock(&bp->b_interlock);
  703         splx(s);
  704 
  705         if (sync)
  706                 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
  707         else
  708                 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
  709 
  710         VOP_STRATEGY(vp, bp);
  711 
  712         if (sync) {
  713                 /* If I/O was synchronous, wait for it to complete. */
  714                 rv = biowait(bp);
  715 
  716                 /* Release the buffer. */
  717                 brelse(bp);
  718 
  719                 return (rv);
  720         } else {
  721                 return (0);
  722         }
  723 }
  724 
  725 int
  726 vn_bwrite(void *v)
  727 {
  728         struct vop_bwrite_args *ap = v;
  729 
  730         return (bwrite(ap->a_bp));
  731 }
  732 
  733 /*
  734  * Delayed write.
  735  *
  736  * The buffer is marked dirty, but is not queued for I/O.
  737  * This routine should be used when the buffer is expected
  738  * to be modified again soon, typically a small write that
  739  * partially fills a buffer.
  740  *
  741  * NB: magnetic tapes cannot be delayed; they must be
  742  * written in the order that the writes are requested.
  743  *
  744  * Described in Leffler, et al. (pp. 208-213).
  745  */
  746 void
  747 bdwrite(struct buf *bp)
  748 {
  749         struct lwp *l  = (curlwp != NULL ? curlwp : &lwp0);     /* XXX */
  750         struct proc *p = l->l_proc;
  751         const struct bdevsw *bdev;
  752         int s;
  753 
  754         /* If this is a tape block, write the block now. */
  755         bdev = bdevsw_lookup(bp->b_dev);
  756         if (bdev != NULL && bdev->d_type == D_TAPE) {
  757                 bawrite(bp);
  758                 return;
  759         }
  760 
  761         /*
  762          * If the block hasn't been seen before:
  763          *      (1) Mark it as having been seen,
  764          *      (2) Charge for the write,
  765          *      (3) Make sure it's on its vnode's correct block list.
  766          */
  767         s = splbio();
  768         simple_lock(&bp->b_interlock);
  769 
  770         KASSERT(ISSET(bp->b_flags, B_BUSY));
  771 
  772         if (!ISSET(bp->b_flags, B_DELWRI)) {
  773                 SET(bp->b_flags, B_DELWRI);
  774                 p->p_stats->p_ru.ru_oublock++;
  775                 reassignbuf(bp, bp->b_vp);
  776         }
  777 
  778         /* Otherwise, the "write" is done, so mark and release the buffer. */
  779         CLR(bp->b_flags, B_DONE);
  780         simple_unlock(&bp->b_interlock);
  781         splx(s);
  782 
  783         brelse(bp);
  784 }
  785 
  786 /*
  787  * Asynchronous block write; just an asynchronous bwrite().
  788  */
  789 void
  790 bawrite(struct buf *bp)
  791 {
  792         int s;
  793 
  794         s = splbio();
  795         simple_lock(&bp->b_interlock);
  796 
  797         KASSERT(ISSET(bp->b_flags, B_BUSY));
  798 
  799         SET(bp->b_flags, B_ASYNC);
  800         simple_unlock(&bp->b_interlock);
  801         splx(s);
  802         VOP_BWRITE(bp);
  803 }
  804 
  805 /*
  806  * Same as first half of bdwrite, mark buffer dirty, but do not release it.
  807  * Call at splbio() and with the buffer interlock locked.
  808  * Note: called only from biodone() through ffs softdep's bioops.io_complete()
  809  */
  810 void
  811 bdirty(struct buf *bp)
  812 {
  813         struct lwp *l  = (curlwp != NULL ? curlwp : &lwp0);     /* XXX */
  814         struct proc *p = l->l_proc;
  815 
  816         LOCK_ASSERT(simple_lock_held(&bp->b_interlock));
  817         KASSERT(ISSET(bp->b_flags, B_BUSY));
  818 
  819         CLR(bp->b_flags, B_AGE);
  820 
  821         if (!ISSET(bp->b_flags, B_DELWRI)) {
  822                 SET(bp->b_flags, B_DELWRI);
  823                 p->p_stats->p_ru.ru_oublock++;
  824                 reassignbuf(bp, bp->b_vp);
  825         }
  826 }
  827 
  828 /*
  829  * Release a buffer on to the free lists.
  830  * Described in Bach (p. 46).
  831  */
  832 void
  833 brelse(struct buf *bp)
  834 {
  835         struct bqueues *bufq;
  836         int s;
  837 
  838         /* Block disk interrupts. */
  839         s = splbio();
  840         simple_lock(&bqueue_slock);
  841         simple_lock(&bp->b_interlock);
  842 
  843         KASSERT(ISSET(bp->b_flags, B_BUSY));
  844         KASSERT(!ISSET(bp->b_flags, B_CALL));
  845 
  846         /* Wake up any processes waiting for any buffer to become free. */
  847         if (needbuffer) {
  848                 needbuffer = 0;
  849                 wakeup(&needbuffer);
  850         }
  851 
  852         /* Wake up any proceeses waiting for _this_ buffer to become free. */
  853         if (ISSET(bp->b_flags, B_WANTED)) {
  854                 CLR(bp->b_flags, B_WANTED|B_AGE);
  855                 wakeup(bp);
  856         }
  857 
  858         /*
  859          * Determine which queue the buffer should be on, then put it there.
  860          */
  861 
  862         /* If it's locked, don't report an error; try again later. */
  863         if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR))
  864                 CLR(bp->b_flags, B_ERROR);
  865 
  866         /* If it's not cacheable, or an error, mark it invalid. */
  867         if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR)))
  868                 SET(bp->b_flags, B_INVAL);
  869 
  870         if (ISSET(bp->b_flags, B_VFLUSH)) {
  871                 /*
  872                  * This is a delayed write buffer that was just flushed to
  873                  * disk.  It is still on the LRU queue.  If it's become
  874                  * invalid, then we need to move it to a different queue;
  875                  * otherwise leave it in its current position.
  876                  */
  877                 CLR(bp->b_flags, B_VFLUSH);
  878                 if (!ISSET(bp->b_flags, B_ERROR|B_INVAL|B_LOCKED|B_AGE)) {
  879                         KDASSERT(!debug_verify_freelist || checkfreelist(bp, &bufqueues[BQ_LRU]));
  880                         goto already_queued;
  881                 } else {
  882                         bremfree(bp);
  883                 }
  884         }
  885 
  886   KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_AGE]));
  887   KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LRU]));
  888   KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LOCKED]));
  889 
  890         if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) {
  891                 /*
  892                  * If it's invalid or empty, dissociate it from its vnode
  893                  * and put on the head of the appropriate queue.
  894                  */
  895                 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
  896                         (*bioops.io_deallocate)(bp);
  897                 CLR(bp->b_flags, B_DONE|B_DELWRI);
  898                 if (bp->b_vp) {
  899                         reassignbuf(bp, bp->b_vp);
  900                         brelvp(bp);
  901                 }
  902                 if (bp->b_bufsize <= 0)
  903                         /* no data */
  904                         goto already_queued;
  905                 else
  906                         /* invalid data */
  907                         bufq = &bufqueues[BQ_AGE];
  908                 binsheadfree(bp, bufq);
  909         } else {
  910                 /*
  911                  * It has valid data.  Put it on the end of the appropriate
  912                  * queue, so that it'll stick around for as long as possible.
  913                  * If buf is AGE, but has dependencies, must put it on last
  914                  * bufqueue to be scanned, ie LRU. This protects against the
  915                  * livelock where BQ_AGE only has buffers with dependencies,
  916                  * and we thus never get to the dependent buffers in BQ_LRU.
  917                  */
  918                 if (ISSET(bp->b_flags, B_LOCKED))
  919                         /* locked in core */
  920                         bufq = &bufqueues[BQ_LOCKED];
  921                 else if (!ISSET(bp->b_flags, B_AGE))
  922                         /* valid data */
  923                         bufq = &bufqueues[BQ_LRU];
  924                 else {
  925                         /* stale but valid data */
  926                         int has_deps;
  927 
  928                         if (LIST_FIRST(&bp->b_dep) != NULL &&
  929                             bioops.io_countdeps)
  930                                 has_deps = (*bioops.io_countdeps)(bp, 0);
  931                         else
  932                                 has_deps = 0;
  933                         bufq = has_deps ? &bufqueues[BQ_LRU] :
  934                             &bufqueues[BQ_AGE];
  935                 }
  936                 binstailfree(bp, bufq);
  937         }
  938 
  939 already_queued:
  940         /* Unlock the buffer. */
  941         CLR(bp->b_flags, B_AGE|B_ASYNC|B_BUSY|B_NOCACHE);
  942         SET(bp->b_flags, B_CACHE);
  943 
  944         /* Allow disk interrupts. */
  945         simple_unlock(&bp->b_interlock);
  946         simple_unlock(&bqueue_slock);
  947         if (bp->b_bufsize <= 0) {
  948 #ifdef DEBUG
  949                 memset((char *)bp, 0, sizeof(*bp));
  950 #endif
  951                 pool_put(&bufpool, bp);
  952         }
  953         splx(s);
  954 }
  955 
  956 /*
  957  * Determine if a block is in the cache.
  958  * Just look on what would be its hash chain.  If it's there, return
  959  * a pointer to it, unless it's marked invalid.  If it's marked invalid,
  960  * we normally don't return the buffer, unless the caller explicitly
  961  * wants us to.
  962  */
  963 struct buf *
  964 incore(struct vnode *vp, daddr_t blkno)
  965 {
  966         struct buf *bp;
  967 
  968         /* Search hash chain */
  969         LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) {
  970                 if (bp->b_lblkno == blkno && bp->b_vp == vp &&
  971                     !ISSET(bp->b_flags, B_INVAL))
  972                 return (bp);
  973         }
  974 
  975         return (NULL);
  976 }
  977 
  978 /*
  979  * Get a block of requested size that is associated with
  980  * a given vnode and block offset. If it is found in the
  981  * block cache, mark it as having been found, make it busy
  982  * and return it. Otherwise, return an empty block of the
  983  * correct size. It is up to the caller to insure that the
  984  * cached blocks be of the correct size.
  985  */
  986 struct buf *
  987 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo)
  988 {
  989         struct buf *bp;
  990         int s, err;
  991         int preserve;
  992 
  993 start:
  994         s = splbio();
  995         simple_lock(&bqueue_slock);
  996         bp = incore(vp, blkno);
  997         if (bp != NULL) {
  998                 simple_lock(&bp->b_interlock);
  999                 if (ISSET(bp->b_flags, B_BUSY)) {
 1000                         simple_unlock(&bqueue_slock);
 1001                         if (curproc == uvm.pagedaemon_proc) {
 1002                                 simple_unlock(&bp->b_interlock);
 1003                                 splx(s);
 1004                                 return NULL;
 1005                         }
 1006                         SET(bp->b_flags, B_WANTED);
 1007                         err = ltsleep(bp, slpflag | (PRIBIO + 1) | PNORELOCK,
 1008                                         "getblk", slptimeo, &bp->b_interlock);
 1009                         splx(s);
 1010                         if (err)
 1011                                 return (NULL);
 1012                         goto start;
 1013                 }
 1014 #ifdef DIAGNOSTIC
 1015                 if (ISSET(bp->b_flags, B_DONE|B_DELWRI) &&
 1016                     bp->b_bcount < size && vp->v_type != VBLK)
 1017                         panic("getblk: block size invariant failed");
 1018 #endif
 1019                 SET(bp->b_flags, B_BUSY);
 1020                 bremfree(bp);
 1021                 preserve = 1;
 1022         } else {
 1023                 if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL) {
 1024                         simple_unlock(&bqueue_slock);
 1025                         splx(s);
 1026                         goto start;
 1027                 }
 1028 
 1029                 binshash(bp, BUFHASH(vp, blkno));
 1030                 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno;
 1031                 bgetvp(vp, bp);
 1032                 preserve = 0;
 1033         }
 1034         simple_unlock(&bp->b_interlock);
 1035         simple_unlock(&bqueue_slock);
 1036         splx(s);
 1037         /*
 1038          * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes)
 1039          * if we re-size buffers here.
 1040          */
 1041         if (ISSET(bp->b_flags, B_LOCKED)) {
 1042                 KASSERT(bp->b_bufsize >= size);
 1043         } else {
 1044                 allocbuf(bp, size, preserve);
 1045         }
 1046         BIO_SETPRIO(bp, BPRIO_DEFAULT);
 1047         return (bp);
 1048 }
 1049 
 1050 /*
 1051  * Get an empty, disassociated buffer of given size.
 1052  */
 1053 struct buf *
 1054 geteblk(int size)
 1055 {
 1056         struct buf *bp; 
 1057         int s;
 1058 
 1059         s = splbio();
 1060         simple_lock(&bqueue_slock);
 1061         while ((bp = getnewbuf(0, 0, 0)) == 0)
 1062                 ;
 1063 
 1064         SET(bp->b_flags, B_INVAL);
 1065         binshash(bp, &invalhash);
 1066         simple_unlock(&bqueue_slock);
 1067         simple_unlock(&bp->b_interlock);
 1068         splx(s);
 1069         BIO_SETPRIO(bp, BPRIO_DEFAULT);
 1070         allocbuf(bp, size, 0);
 1071         return (bp);
 1072 }
 1073 
 1074 /*
 1075  * Expand or contract the actual memory allocated to a buffer.
 1076  *
 1077  * If the buffer shrinks, data is lost, so it's up to the
 1078  * caller to have written it out *first*; this routine will not
 1079  * start a write.  If the buffer grows, it's the callers
 1080  * responsibility to fill out the buffer's additional contents.
 1081  */
 1082 void
 1083 allocbuf(struct buf *bp, int size, int preserve)
 1084 {
 1085         vsize_t oldsize, desired_size;
 1086         caddr_t addr;
 1087         int s, delta;
 1088 
 1089         desired_size = buf_roundsize(size);
 1090         if (desired_size > MAXBSIZE)
 1091                 printf("allocbuf: buffer larger than MAXBSIZE requested");
 1092 
 1093         bp->b_bcount = size;
 1094 
 1095         oldsize = bp->b_bufsize;
 1096         if (oldsize == desired_size)
 1097                 return;
 1098 
 1099         /*
 1100          * If we want a buffer of a different size, re-allocate the
 1101          * buffer's memory; copy old content only if needed.
 1102          */
 1103         addr = buf_malloc(desired_size);
 1104         if (preserve)
 1105                 memcpy(addr, bp->b_data, MIN(oldsize,desired_size));
 1106         if (bp->b_data != NULL)
 1107                 buf_mrelease(bp->b_data, oldsize);
 1108         bp->b_data = addr;
 1109         bp->b_bufsize = desired_size;
 1110 
 1111         /*
 1112          * Update overall buffer memory counter (protected by bqueue_slock)
 1113          */
 1114         delta = (long)desired_size - (long)oldsize;
 1115 
 1116         s = splbio();
 1117         simple_lock(&bqueue_slock);
 1118         if ((bufmem += delta) > bufmem_hiwater) {
 1119                 /*
 1120                  * Need to trim overall memory usage.
 1121                  */
 1122                 while (buf_canrelease()) {
 1123                         if (buf_trim() == 0)
 1124                                 break;
 1125                 }
 1126         }
 1127 
 1128         simple_unlock(&bqueue_slock);
 1129         splx(s);
 1130 }
 1131 
 1132 /*
 1133  * Find a buffer which is available for use.
 1134  * Select something from a free list.
 1135  * Preference is to AGE list, then LRU list.    
 1136  *
 1137  * Called at splbio and with buffer queues locked.
 1138  * Return buffer locked.
 1139  */
 1140 struct buf *
 1141 getnewbuf(int slpflag, int slptimeo, int from_bufq)
 1142 {
 1143         struct buf *bp;
 1144 
 1145 start:
 1146         LOCK_ASSERT(simple_lock_held(&bqueue_slock));
 1147 
 1148         /*
 1149          * Get a new buffer from the pool; but use NOWAIT because
 1150          * we have the buffer queues locked.
 1151          */
 1152         if (buf_lotsfree() && !from_bufq &&
 1153             (bp = pool_get(&bufpool, PR_NOWAIT)) != NULL) {
 1154                 memset((char *)bp, 0, sizeof(*bp));
 1155                 BUF_INIT(bp);
 1156                 bp->b_dev = NODEV;
 1157                 bp->b_vnbufs.le_next = NOLIST;
 1158                 bp->b_flags = B_BUSY;
 1159                 simple_lock(&bp->b_interlock);
 1160                 return (bp);
 1161         }
 1162 
 1163         if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE])) != NULL ||
 1164             (bp = TAILQ_FIRST(&bufqueues[BQ_LRU])) != NULL) {
 1165                 simple_lock(&bp->b_interlock);
 1166                 bremfree(bp);
 1167         } else {
 1168                 /*
 1169                  * XXX: !from_bufq should be removed.
 1170                  */
 1171                 if (!from_bufq || curproc != uvm.pagedaemon_proc) {
 1172                         /* wait for a free buffer of any kind */
 1173                         needbuffer = 1;
 1174                         ltsleep(&needbuffer, slpflag|(PRIBIO + 1),
 1175                             "getnewbuf", slptimeo, &bqueue_slock);
 1176                 }
 1177                 return (NULL);
 1178         }
 1179 
 1180 #ifdef DIAGNOSTIC
 1181         if (bp->b_bufsize <= 0)
 1182                 panic("buffer %p: on queue but empty", bp);
 1183 #endif
 1184 
 1185         if (ISSET(bp->b_flags, B_VFLUSH)) {
 1186                 /*
 1187                  * This is a delayed write buffer being flushed to disk.  Make
 1188                  * sure it gets aged out of the queue when it's finished, and
 1189                  * leave it off the LRU queue.
 1190                  */
 1191                 CLR(bp->b_flags, B_VFLUSH);
 1192                 SET(bp->b_flags, B_AGE);
 1193                 simple_unlock(&bp->b_interlock);
 1194                 goto start;
 1195         }
 1196 
 1197         /* Buffer is no longer on free lists. */
 1198         SET(bp->b_flags, B_BUSY);
 1199 
 1200         /*
 1201          * If buffer was a delayed write, start it and return NULL
 1202          * (since we might sleep while starting the write).
 1203          */
 1204         if (ISSET(bp->b_flags, B_DELWRI)) {
 1205                 /*
 1206                  * This buffer has gone through the LRU, so make sure it gets
 1207                  * reused ASAP.
 1208                  */
 1209                 SET(bp->b_flags, B_AGE);
 1210                 simple_unlock(&bp->b_interlock);
 1211                 simple_unlock(&bqueue_slock);
 1212                 bawrite(bp);
 1213                 simple_lock(&bqueue_slock);
 1214                 return (NULL);
 1215         }
 1216 
 1217         /* disassociate us from our vnode, if we had one... */
 1218         if (bp->b_vp)
 1219                 brelvp(bp);
 1220 
 1221         if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
 1222                 (*bioops.io_deallocate)(bp);
 1223 
 1224         /* clear out various other fields */
 1225         bp->b_flags = B_BUSY;
 1226         bp->b_dev = NODEV;
 1227         bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = 0;
 1228         bp->b_iodone = 0;
 1229         bp->b_error = 0;
 1230         bp->b_resid = 0;
 1231         bp->b_bcount = 0;
 1232         
 1233         bremhash(bp);
 1234         return (bp); 
 1235 }
 1236 
 1237 /*
 1238  * Attempt to free an aged buffer off the queues.
 1239  * Called at splbio and with queue lock held.
 1240  * Returns the amount of buffer memory freed.
 1241  */
 1242 int
 1243 buf_trim(void)
 1244 {
 1245         struct buf *bp;
 1246         long size = 0;
 1247         int wanted;
 1248 
 1249         /* Instruct getnewbuf() to get buffers off the queues */
 1250         if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL)
 1251                 return 0;
 1252 
 1253         wanted = ISSET(bp->b_flags, B_WANTED);
 1254         simple_unlock(&bp->b_interlock);
 1255         if (wanted) {
 1256                 printf("buftrim: got WANTED buffer\n");
 1257                 SET(bp->b_flags, B_INVAL);
 1258                 binshash(bp, &invalhash);
 1259                 simple_unlock(&bqueue_slock);
 1260                 goto out;
 1261         }
 1262         size = bp->b_bufsize;
 1263         bufmem -= size;
 1264         simple_unlock(&bqueue_slock);
 1265         if (size > 0) {
 1266                 buf_mrelease(bp->b_data, size);
 1267                 bp->b_bcount = bp->b_bufsize = 0;
 1268         }
 1269 
 1270 out:
 1271         /* brelse() will return the buffer to the global buffer pool */
 1272         brelse(bp);
 1273         simple_lock(&bqueue_slock);
 1274         return size;
 1275 }
 1276 
 1277 int
 1278 buf_drain(int n)
 1279 {
 1280         int s, size = 0, sz;
 1281 
 1282         s = splbio();
 1283         simple_lock(&bqueue_slock);
 1284 
 1285         while (size < n && bufmem > bufmem_lowater) {
 1286                 sz = buf_trim();
 1287                 if (sz <= 0)
 1288                         break;
 1289                 size += sz;
 1290         }
 1291 
 1292         simple_unlock(&bqueue_slock);
 1293         splx(s);
 1294         return size;
 1295 }
 1296 
 1297 /*
 1298  * Wait for operations on the buffer to complete.
 1299  * When they do, extract and return the I/O's error value.
 1300  */
 1301 int
 1302 biowait(struct buf *bp)
 1303 {
 1304         int s, error;
 1305         
 1306         s = splbio();
 1307         simple_lock(&bp->b_interlock);
 1308         while (!ISSET(bp->b_flags, B_DONE | B_DELWRI))
 1309                 ltsleep(bp, PRIBIO + 1, "biowait", 0, &bp->b_interlock);
 1310 
 1311         /* check for interruption of I/O (e.g. via NFS), then errors. */
 1312         if (ISSET(bp->b_flags, B_EINTR)) {
 1313                 CLR(bp->b_flags, B_EINTR);
 1314                 error = EINTR;
 1315         } else if (ISSET(bp->b_flags, B_ERROR))
 1316                 error = bp->b_error ? bp->b_error : EIO;
 1317         else
 1318                 error = 0;
 1319 
 1320         simple_unlock(&bp->b_interlock);
 1321         splx(s);
 1322         return (error);
 1323 }
 1324 
 1325 /*
 1326  * Mark I/O complete on a buffer.
 1327  *
 1328  * If a callback has been requested, e.g. the pageout
 1329  * daemon, do so. Otherwise, awaken waiting processes.
 1330  *
 1331  * [ Leffler, et al., says on p.247:
 1332  *      "This routine wakes up the blocked process, frees the buffer
 1333  *      for an asynchronous write, or, for a request by the pagedaemon
 1334  *      process, invokes a procedure specified in the buffer structure" ]
 1335  *
 1336  * In real life, the pagedaemon (or other system processes) wants
 1337  * to do async stuff to, and doesn't want the buffer brelse()'d.
 1338  * (for swap pager, that puts swap buffers on the free lists (!!!),
 1339  * for the vn device, that puts malloc'd buffers on the free lists!)
 1340  */
 1341 void
 1342 biodone(struct buf *bp)
 1343 {
 1344         int s = splbio();
 1345 
 1346         simple_lock(&bp->b_interlock);
 1347         if (ISSET(bp->b_flags, B_DONE))
 1348                 panic("biodone already");
 1349         SET(bp->b_flags, B_DONE);               /* note that it's done */
 1350         BIO_SETPRIO(bp, BPRIO_DEFAULT);
 1351 
 1352         if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
 1353                 (*bioops.io_complete)(bp);
 1354 
 1355         if (!ISSET(bp->b_flags, B_READ))        /* wake up reader */
 1356                 vwakeup(bp);
 1357 
 1358         /*
 1359          * If necessary, call out.  Unlock the buffer before calling
 1360          * iodone() as the buffer isn't valid any more when it return.
 1361          */
 1362         if (ISSET(bp->b_flags, B_CALL)) {
 1363                 CLR(bp->b_flags, B_CALL);       /* but note callout done */
 1364                 simple_unlock(&bp->b_interlock);
 1365                 (*bp->b_iodone)(bp);
 1366         } else {
 1367                 if (ISSET(bp->b_flags, B_ASYNC)) {      /* if async, release */
 1368                         simple_unlock(&bp->b_interlock);
 1369                         brelse(bp);
 1370                 } else {                        /* or just wakeup the buffer */
 1371                         CLR(bp->b_flags, B_WANTED);
 1372                         wakeup(bp);
 1373                         simple_unlock(&bp->b_interlock);
 1374                 }
 1375         }
 1376 
 1377         splx(s);
 1378 }
 1379 
 1380 /*
 1381  * Return a count of buffers on the "locked" queue.
 1382  */
 1383 int
 1384 count_lock_queue(void)
 1385 {
 1386         struct buf *bp;
 1387         int n = 0;
 1388 
 1389         simple_lock(&bqueue_slock);
 1390         TAILQ_FOREACH(bp, &bufqueues[BQ_LOCKED], b_freelist)
 1391                 n++;
 1392         simple_unlock(&bqueue_slock);
 1393         return (n);
 1394 }
 1395 
 1396 /*
 1397  * Wait for all buffers to complete I/O
 1398  * Return the number of "stuck" buffers.
 1399  */
 1400 int
 1401 buf_syncwait(void)
 1402 {
 1403         struct buf *bp;
 1404         int iter, nbusy, nbusy_prev = 0, dcount, s, ihash;
 1405 
 1406         dcount = 10000;
 1407         for (iter = 0; iter < 20;) {
 1408                 s = splbio();
 1409                 simple_lock(&bqueue_slock);
 1410                 nbusy = 0;
 1411                 for (ihash = 0; ihash < bufhash+1; ihash++) {
 1412                     LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
 1413                         if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY)
 1414                                 nbusy++;
 1415                         /*
 1416                          * With soft updates, some buffers that are
 1417                          * written will be remarked as dirty until other
 1418                          * buffers are written.
 1419                          */
 1420                         if (bp->b_vp && bp->b_vp->v_mount
 1421                             && (bp->b_vp->v_mount->mnt_flag & MNT_SOFTDEP)
 1422                             && (bp->b_flags & B_DELWRI)) {
 1423                                 simple_lock(&bp->b_interlock);
 1424                                 bremfree(bp);
 1425                                 bp->b_flags |= B_BUSY;
 1426                                 nbusy++;
 1427                                 simple_unlock(&bp->b_interlock);
 1428                                 simple_unlock(&bqueue_slock);
 1429                                 bawrite(bp);
 1430                                 if (dcount-- <= 0) {
 1431                                         printf("softdep ");
 1432                                         goto fail;
 1433                                 }
 1434                                 simple_lock(&bqueue_slock);
 1435                         }
 1436                     }
 1437                 }
 1438 
 1439                 simple_unlock(&bqueue_slock);
 1440                 splx(s);
 1441 
 1442                 if (nbusy == 0)
 1443                         break;
 1444                 if (nbusy_prev == 0)
 1445                         nbusy_prev = nbusy;
 1446                 printf("%d ", nbusy);
 1447                 tsleep(&nbusy, PRIBIO, "bflush",
 1448                     (iter == 0) ? 1 : hz / 25 * iter);
 1449                 if (nbusy >= nbusy_prev) /* we didn't flush anything */
 1450                         iter++;
 1451                 else
 1452                         nbusy_prev = nbusy;
 1453         }
 1454 
 1455         if (nbusy) {
 1456 fail:;
 1457 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY)
 1458                 printf("giving up\nPrinting vnodes for busy buffers\n");
 1459                 for (ihash = 0; ihash < bufhash+1; ihash++) {
 1460                     LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
 1461                         if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY)
 1462                                 vprint(NULL, bp->b_vp);
 1463                     }
 1464                 }
 1465 #endif
 1466         }
 1467 
 1468         return nbusy;
 1469 }
 1470 
 1471 static void
 1472 sysctl_fillbuf(struct buf *i, struct buf_sysctl *o)
 1473 {
 1474 
 1475         o->b_flags = i->b_flags;
 1476         o->b_error = i->b_error;
 1477         o->b_prio = i->b_prio;
 1478         o->b_dev = i->b_dev;
 1479         o->b_bufsize = i->b_bufsize;
 1480         o->b_bcount = i->b_bcount;
 1481         o->b_resid = i->b_resid;
 1482         o->b_addr = PTRTOUINT64(i->b_un.b_addr);
 1483         o->b_blkno = i->b_blkno;
 1484         o->b_rawblkno = i->b_rawblkno;
 1485         o->b_iodone = PTRTOUINT64(i->b_iodone);
 1486         o->b_proc = PTRTOUINT64(i->b_proc);
 1487         o->b_vp = PTRTOUINT64(i->b_vp);
 1488         o->b_saveaddr = PTRTOUINT64(i->b_saveaddr);
 1489         o->b_lblkno = i->b_lblkno;
 1490 }
 1491 
 1492 #define KERN_BUFSLOP 20
 1493 static int
 1494 sysctl_dobuf(SYSCTLFN_ARGS)
 1495 {
 1496         struct buf *bp;
 1497         struct buf_sysctl bs;
 1498         char *dp;
 1499         u_int i, op, arg;
 1500         size_t len, needed, elem_size, out_size;
 1501         int error, s, elem_count;
 1502 
 1503         if (namelen == 1 && name[0] == CTL_QUERY)
 1504                 return (sysctl_query(SYSCTLFN_CALL(rnode)));
 1505 
 1506         if (namelen != 4)
 1507                 return (EINVAL);
 1508 
 1509         dp = oldp;
 1510         len = (oldp != NULL) ? *oldlenp : 0;
 1511         op = name[0];
 1512         arg = name[1];
 1513         elem_size = name[2];
 1514         elem_count = name[3];
 1515         out_size = MIN(sizeof(bs), elem_size);
 1516 
 1517         /*
 1518          * at the moment, these are just "placeholders" to make the
 1519          * API for retrieving kern.buf data more extensible in the
 1520          * future.
 1521          *
 1522          * XXX kern.buf currently has "netbsd32" issues.  hopefully
 1523          * these will be resolved at a later point.
 1524          */
 1525         if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL ||
 1526             elem_size < 1 || elem_count < 0)
 1527                 return (EINVAL);
 1528 
 1529         error = 0;
 1530         needed = 0;
 1531         s = splbio();
 1532         simple_lock(&bqueue_slock);
 1533         for (i = 0; i < BQUEUES; i++) {
 1534                 TAILQ_FOREACH(bp, &bufqueues[i], b_freelist) {
 1535                         if (len >= elem_size && elem_count > 0) {
 1536                                 sysctl_fillbuf(bp, &bs);
 1537                                 error = copyout(&bs, dp, out_size);
 1538                                 if (error)
 1539                                         goto cleanup;
 1540                                 dp += elem_size;
 1541                                 len -= elem_size;
 1542                         }
 1543                         if (elem_count > 0) {
 1544                                 needed += elem_size;
 1545                                 if (elem_count != INT_MAX)
 1546                                         elem_count--;
 1547                         }
 1548                 }
 1549         }
 1550 cleanup:
 1551         simple_unlock(&bqueue_slock);
 1552         splx(s);
 1553 
 1554         *oldlenp = needed;
 1555         if (oldp == NULL)
 1556                 *oldlenp += KERN_BUFSLOP * sizeof(struct buf);
 1557 
 1558         return (error);
 1559 }
 1560 
 1561 static int
 1562 sysctl_bufvm_update(SYSCTLFN_ARGS)
 1563 {
 1564         int t, error;
 1565         struct sysctlnode node;
 1566 
 1567         node = *rnode;
 1568         node.sysctl_data = &t;
 1569         t = *(int*)rnode->sysctl_data;
 1570         error = sysctl_lookup(SYSCTLFN_CALL(&node));
 1571         if (error || newp == NULL)
 1572                 return (error);
 1573 
 1574         if (rnode->sysctl_data == &bufcache) {
 1575                 if (t < 0 || t > 100)
 1576                         return (EINVAL);
 1577                 bufcache = t;
 1578                 buf_setwm();
 1579         } else if (rnode->sysctl_data == &bufmem_lowater) {
 1580                 if (bufmem_hiwater - bufmem_lowater < 16)
 1581                         return (EINVAL);
 1582                 bufmem_lowater = t;
 1583         } else if (rnode->sysctl_data == &bufmem_hiwater) {
 1584                 if (bufmem_hiwater - bufmem_lowater < 16)
 1585                         return (EINVAL);
 1586                 bufmem_hiwater = t;
 1587         } else
 1588                 return (EINVAL);
 1589 
 1590         /* Drain until below new high water mark */
 1591         while ((t = bufmem - bufmem_hiwater) >= 0) {
 1592                 if (buf_drain(t / (2*1024)) <= 0)
 1593                         break;
 1594         }
 1595 
 1596         return 0;
 1597 }
 1598 
 1599 SYSCTL_SETUP(sysctl_kern_buf_setup, "sysctl kern.buf subtree setup")
 1600 {
 1601 
 1602         sysctl_createv(clog, 0, NULL, NULL,
 1603                        CTLFLAG_PERMANENT,
 1604                        CTLTYPE_NODE, "kern", NULL,
 1605                        NULL, 0, NULL, 0,
 1606                        CTL_KERN, CTL_EOL);
 1607         sysctl_createv(clog, 0, NULL, NULL,
 1608                        CTLFLAG_PERMANENT,
 1609                        CTLTYPE_NODE, "buf",
 1610                        SYSCTL_DESCR("Kernel buffer cache information"),
 1611                        sysctl_dobuf, 0, NULL, 0,
 1612                        CTL_KERN, KERN_BUF, CTL_EOL);
 1613 }
 1614 
 1615 SYSCTL_SETUP(sysctl_vm_buf_setup, "sysctl vm.buf* subtree setup")
 1616 {
 1617 
 1618         sysctl_createv(clog, 0, NULL, NULL,
 1619                        CTLFLAG_PERMANENT,
 1620                        CTLTYPE_NODE, "vm", NULL,
 1621                        NULL, 0, NULL, 0,
 1622                        CTL_VM, CTL_EOL);
 1623 
 1624         sysctl_createv(clog, 0, NULL, NULL,
 1625                        CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
 1626                        CTLTYPE_INT, "bufcache",
 1627                        SYSCTL_DESCR("Percentage of kernel memory to use for "
 1628                                     "buffer cache"),
 1629                        sysctl_bufvm_update, 0, &bufcache, 0,
 1630                        CTL_VM, CTL_CREATE, CTL_EOL);
 1631         sysctl_createv(clog, 0, NULL, NULL,
 1632                        CTLFLAG_PERMANENT|CTLFLAG_READONLY,
 1633                        CTLTYPE_INT, "bufmem",
 1634                        SYSCTL_DESCR("Amount of kernel memory used by buffer "
 1635                                     "cache"),
 1636                        NULL, 0, &bufmem, 0,
 1637                        CTL_VM, CTL_CREATE, CTL_EOL);
 1638         sysctl_createv(clog, 0, NULL, NULL,
 1639                        CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
 1640                        CTLTYPE_INT, "bufmem_lowater",
 1641                        SYSCTL_DESCR("Minimum amount of kernel memory to "
 1642                                     "reserve for buffer cache"),
 1643                        sysctl_bufvm_update, 0, &bufmem_lowater, 0,
 1644                        CTL_VM, CTL_CREATE, CTL_EOL);
 1645         sysctl_createv(clog, 0, NULL, NULL,
 1646                        CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
 1647                        CTLTYPE_INT, "bufmem_hiwater",
 1648                        SYSCTL_DESCR("Maximum amount of kernel memory to use "
 1649                                     "for buffer cache"),
 1650                        sysctl_bufvm_update, 0, &bufmem_hiwater, 0,
 1651                        CTL_VM, CTL_CREATE, CTL_EOL);
 1652 }
 1653 
 1654 #ifdef DEBUG
 1655 /*
 1656  * Print out statistics on the current allocation of the buffer pool.
 1657  * Can be enabled to print out on every ``sync'' by setting "syncprt"
 1658  * in vfs_syscalls.c using sysctl.
 1659  */
 1660 void
 1661 vfs_bufstats(void)
 1662 {
 1663         int s, i, j, count;
 1664         struct buf *bp;
 1665         struct bqueues *dp;
 1666         int counts[(MAXBSIZE / PAGE_SIZE) + 1];
 1667         static char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" };
 1668 
 1669         for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
 1670                 count = 0;
 1671                 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
 1672                         counts[j] = 0;
 1673                 s = splbio();
 1674                 TAILQ_FOREACH(bp, dp, b_freelist) {
 1675                         counts[bp->b_bufsize/PAGE_SIZE]++;
 1676                         count++;
 1677                 }
 1678                 splx(s);
 1679                 printf("%s: total-%d", bname[i], count);
 1680                 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
 1681                         if (counts[j] != 0)
 1682                                 printf(", %d-%d", j * PAGE_SIZE, counts[j]);
 1683                 printf("\n");
 1684         }
 1685 }
 1686 #endif /* DEBUG */

Cache object: 7049c2d4d8debeff5e24af0caac37884


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.