FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_bio.c
1 /* $NetBSD: vfs_bio.c,v 1.122.2.4 2004/10/08 03:25:15 jmc Exp $ */
2
3 /*-
4 * Copyright (c) 1982, 1986, 1989, 1993
5 * The Regents of the University of California. All rights reserved.
6 * (c) UNIX System Laboratories, Inc.
7 * All or some portions of this file are derived from material licensed
8 * to the University of California by American Telephone and Telegraph
9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10 * the permission of UNIX System Laboratories, Inc.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94
37 */
38
39 /*-
40 * Copyright (c) 1994 Christopher G. Demetriou
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. All advertising materials mentioning features or use of this software
51 * must display the following acknowledgement:
52 * This product includes software developed by the University of
53 * California, Berkeley and its contributors.
54 * 4. Neither the name of the University nor the names of its contributors
55 * may be used to endorse or promote products derived from this software
56 * without specific prior written permission.
57 *
58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
68 * SUCH DAMAGE.
69 *
70 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94
71 */
72
73 /*
74 * Some references:
75 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986)
76 * Leffler, et al.: The Design and Implementation of the 4.3BSD
77 * UNIX Operating System (Addison Welley, 1989)
78 */
79
80 #include "opt_bufcache.h"
81 #include "opt_softdep.h"
82
83 #include <sys/cdefs.h>
84 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.122.2.4 2004/10/08 03:25:15 jmc Exp $");
85
86 #include <sys/param.h>
87 #include <sys/systm.h>
88 #include <sys/kernel.h>
89 #include <sys/proc.h>
90 #include <sys/buf.h>
91 #include <sys/vnode.h>
92 #include <sys/mount.h>
93 #include <sys/malloc.h>
94 #include <sys/resourcevar.h>
95 #include <sys/sysctl.h>
96 #include <sys/conf.h>
97
98 #include <uvm/uvm.h>
99
100 #include <miscfs/specfs/specdev.h>
101
102 #ifndef BUFPAGES
103 # define BUFPAGES 0
104 #endif
105
106 #ifdef BUFCACHE
107 # if (BUFCACHE < 5) || (BUFCACHE > 95)
108 # error BUFCACHE is not between 5 and 95
109 # endif
110 #else
111 # define BUFCACHE 15
112 #endif
113
114 u_int nbuf; /* XXX - for softdep_lockedbufs */
115 u_int bufpages = BUFPAGES; /* optional hardwired count */
116 u_int bufcache = BUFCACHE; /* max % of RAM to use for buffer cache */
117
118
119 /* Macros to clear/set/test flags. */
120 #define SET(t, f) (t) |= (f)
121 #define CLR(t, f) (t) &= ~(f)
122 #define ISSET(t, f) ((t) & (f))
123
124 /*
125 * Definitions for the buffer hash lists.
126 */
127 #define BUFHASH(dvp, lbn) \
128 (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash])
129 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
130 u_long bufhash;
131 #ifndef SOFTDEP
132 struct bio_ops bioops; /* I/O operation notification */
133 #endif
134
135 /*
136 * Insq/Remq for the buffer hash lists.
137 */
138 #define binshash(bp, dp) LIST_INSERT_HEAD(dp, bp, b_hash)
139 #define bremhash(bp) LIST_REMOVE(bp, b_hash)
140
141 /*
142 * Definitions for the buffer free lists.
143 */
144 #define BQUEUES 3 /* number of free buffer queues */
145
146 #define BQ_LOCKED 0 /* super-blocks &c */
147 #define BQ_LRU 1 /* lru, useful buffers */
148 #define BQ_AGE 2 /* rubbish */
149
150 TAILQ_HEAD(bqueues, buf) bufqueues[BQUEUES];
151 int needbuffer;
152
153 /*
154 * Buffer queue lock.
155 * Take this lock first if also taking some buffer's b_interlock.
156 */
157 struct simplelock bqueue_slock = SIMPLELOCK_INITIALIZER;
158
159 /*
160 * Buffer pool for I/O buffers.
161 */
162 struct pool bufpool;
163
164 /* XXX - somewhat gross.. */
165 #if MAXBSIZE == 0x2000
166 #define NMEMPOOLS 4
167 #elif MAXBSIZE == 0x4000
168 #define NMEMPOOLS 5
169 #elif MAXBSIZE == 0x8000
170 #define NMEMPOOLS 6
171 #else
172 #define NMEMPOOLS 7
173 #endif
174
175 #define MEMPOOL_INDEX_OFFSET 10 /* smallest pool is 1k */
176 #if (1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) != MAXBSIZE
177 #error update vfs_bio buffer memory parameters
178 #endif
179
180 /* Buffer memory pools */
181 static struct pool bmempools[NMEMPOOLS];
182
183 struct vm_map *buf_map;
184
185 /*
186 * Buffer memory pool allocator.
187 */
188 static void *
189 bufpool_page_alloc(struct pool *pp, int flags)
190 {
191
192 return (void *)uvm_km_kmemalloc1(buf_map,
193 uvm.kernel_object, MAXBSIZE, MAXBSIZE, UVM_UNKNOWN_OFFSET,
194 (flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK);
195 }
196
197 static void
198 bufpool_page_free(struct pool *pp, void *v)
199 {
200 uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE);
201 }
202
203 static struct pool_allocator bufmempool_allocator = {
204 bufpool_page_alloc, bufpool_page_free, MAXBSIZE,
205 };
206
207 /* Buffer memory management variables */
208 u_long bufmem_valimit;
209 u_long bufmem_hiwater;
210 u_long bufmem_lowater;
211 u_long bufmem;
212
213 /*
214 * MD code can call this to set a hard limit on the amount
215 * of virtual memory used by the buffer cache.
216 */
217 int
218 buf_setvalimit(vsize_t sz)
219 {
220
221 /* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */
222 if (sz < NMEMPOOLS * MAXBSIZE)
223 return EINVAL;
224
225 bufmem_valimit = sz;
226 return 0;
227 }
228
229 static void buf_setwm(void);
230 static int buf_trim(void);
231
232 /*
233 * bread()/breadn() helper.
234 */
235 static __inline struct buf *bio_doread(struct vnode *, daddr_t, int,
236 struct ucred *, int);
237 int count_lock_queue(void);
238
239 /*
240 * Insq/Remq for the buffer free lists.
241 * Call with buffer queue locked.
242 */
243 #define binsheadfree(bp, dp) TAILQ_INSERT_HEAD(dp, bp, b_freelist)
244 #define binstailfree(bp, dp) TAILQ_INSERT_TAIL(dp, bp, b_freelist)
245
246 static void
247 buf_setwm(void)
248 {
249
250 bufmem_hiwater = buf_memcalc();
251 /* lowater is approx. 2% of memory (with bufcache = 15) */
252 #define BUFMEM_WMSHIFT 3
253 #define BUFMEM_HIWMMIN (64 * 1024 << BUFMEM_WMSHIFT)
254 if (bufmem_hiwater < BUFMEM_HIWMMIN)
255 /* Ensure a reasonable minimum value */
256 bufmem_hiwater = BUFMEM_HIWMMIN;
257 bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT;
258 }
259
260 #ifdef DEBUG
261 int debug_verify_freelist = 0;
262 static int checkfreelist(struct buf *bp, struct bqueues *dp)
263 {
264 struct buf *b;
265 TAILQ_FOREACH(b, dp, b_freelist) {
266 if (b == bp)
267 return 1;
268 }
269 return 0;
270 }
271 #endif
272
273 void
274 bremfree(struct buf *bp)
275 {
276 struct bqueues *dp = NULL;
277
278 LOCK_ASSERT(simple_lock_held(&bqueue_slock));
279
280 KDASSERT(!debug_verify_freelist ||
281 checkfreelist(bp, &bufqueues[BQ_AGE]) ||
282 checkfreelist(bp, &bufqueues[BQ_LRU]) ||
283 checkfreelist(bp, &bufqueues[BQ_LOCKED]) );
284
285 /*
286 * We only calculate the head of the freelist when removing
287 * the last element of the list as that is the only time that
288 * it is needed (e.g. to reset the tail pointer).
289 *
290 * NB: This makes an assumption about how tailq's are implemented.
291 *
292 * We break the TAILQ abstraction in order to efficiently remove a
293 * buffer from its freelist without having to know exactly which
294 * freelist it is on.
295 */
296 if (TAILQ_NEXT(bp, b_freelist) == NULL) {
297 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
298 if (dp->tqh_last == &bp->b_freelist.tqe_next)
299 break;
300 if (dp == &bufqueues[BQUEUES])
301 panic("bremfree: lost tail");
302 }
303 TAILQ_REMOVE(dp, bp, b_freelist);
304 }
305
306 u_long
307 buf_memcalc(void)
308 {
309 u_long n;
310
311 /*
312 * Determine the upper bound of memory to use for buffers.
313 *
314 * - If bufpages is specified, use that as the number
315 * pages.
316 *
317 * - Otherwise, use bufcache as the percentage of
318 * physical memory.
319 */
320 if (bufpages != 0) {
321 n = bufpages;
322 } else {
323 if (bufcache < 5) {
324 printf("forcing bufcache %d -> 5", bufcache);
325 bufcache = 5;
326 }
327 if (bufcache > 95) {
328 printf("forcing bufcache %d -> 95", bufcache);
329 bufcache = 95;
330 }
331 n = physmem / 100 * bufcache;
332 }
333
334 n <<= PAGE_SHIFT;
335 if (bufmem_valimit != 0 && n > bufmem_valimit)
336 n = bufmem_valimit;
337
338 return (n);
339 }
340
341 /*
342 * Initialize buffers and hash links for buffers.
343 */
344 void
345 bufinit(void)
346 {
347 struct bqueues *dp;
348 int use_std;
349 u_int i;
350
351 /*
352 * Initialize buffer cache memory parameters.
353 */
354 bufmem = 0;
355 buf_setwm();
356
357 if (bufmem_valimit != 0) {
358 vaddr_t minaddr = 0, maxaddr;
359 buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
360 bufmem_valimit, VM_MAP_PAGEABLE,
361 FALSE, 0);
362 if (buf_map == NULL)
363 panic("bufinit: cannot allocate submap");
364 } else
365 buf_map = kernel_map;
366
367 /*
368 * Initialize the buffer pools.
369 */
370 pool_init(&bufpool, sizeof(struct buf), 0, 0, 0, "bufpl", NULL);
371
372 /* On "small" machines use small pool page sizes where possible */
373 use_std = (physmem < atop(16*1024*1024));
374
375 /*
376 * Also use them on systems that can map the pool pages using
377 * a direct-mapped segment.
378 */
379 #ifdef PMAP_MAP_POOLPAGE
380 use_std = 1;
381 #endif
382
383 for (i = 0; i < NMEMPOOLS; i++) {
384 struct pool_allocator *pa;
385 struct pool *pp = &bmempools[i];
386 u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET);
387 char *name = malloc(8, M_TEMP, M_WAITOK);
388 snprintf(name, 8, "buf%dk", 1 << i);
389 pa = (size <= PAGE_SIZE && use_std)
390 ? &pool_allocator_nointr
391 : &bufmempool_allocator;
392 pool_init(pp, size, 0, 0, 0, name, pa);
393 pool_setlowat(pp, 1);
394 pool_sethiwat(pp, 1);
395 }
396
397 /* Initialize the buffer queues */
398 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++)
399 TAILQ_INIT(dp);
400
401 /*
402 * Estimate hash table size based on the amount of memory we
403 * intend to use for the buffer cache. The average buffer
404 * size is dependent on our clients (i.e. filesystems).
405 *
406 * For now, use an empirical 3K per buffer.
407 */
408 nbuf = (bufmem_hiwater / 1024) / 3;
409 bufhashtbl = hashinit(nbuf, HASH_LIST, M_CACHE, M_WAITOK, &bufhash);
410 }
411
412 static int
413 buf_lotsfree(void)
414 {
415 int try, thresh;
416
417 /* Always allocate if less than the low water mark. */
418 if (bufmem < bufmem_lowater)
419 return 1;
420
421 /* Never allocate if greater than the high water mark. */
422 if (bufmem > bufmem_hiwater)
423 return 0;
424
425 /* If there's anything on the AGE list, it should be eaten. */
426 if (TAILQ_FIRST(&bufqueues[BQ_AGE]) != NULL)
427 return 0;
428
429 /*
430 * The probabily of getting a new allocation is inversely
431 * proportional to the current size of the cache, using
432 * a granularity of 16 steps.
433 */
434 try = random() & 0x0000000fL;
435
436 /* Don't use "16 * bufmem" here to avoid a 32-bit overflow. */
437 thresh = (bufmem - bufmem_lowater) /
438 ((bufmem_hiwater - bufmem_lowater) / 16);
439
440 if (try >= thresh)
441 return 1;
442
443 /* Otherwise don't allocate. */
444 return 0;
445 }
446
447 /*
448 * Return estimate of bytes we think need to be
449 * released to help resolve low memory conditions.
450 *
451 * => called at splbio.
452 * => called with bqueue_slock held.
453 */
454 static int
455 buf_canrelease(void)
456 {
457 int pagedemand, ninvalid = 0;
458 struct buf *bp;
459
460 LOCK_ASSERT(simple_lock_held(&bqueue_slock));
461
462 if (bufmem < bufmem_lowater)
463 return 0;
464
465 TAILQ_FOREACH(bp, &bufqueues[BQ_AGE], b_freelist)
466 ninvalid += bp->b_bufsize;
467
468 pagedemand = uvmexp.freetarg - uvmexp.free;
469 if (pagedemand < 0)
470 return ninvalid;
471 return MAX(ninvalid, MIN(2 * MAXBSIZE,
472 MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE)));
473 }
474
475 /*
476 * Buffer memory allocation helper functions
477 */
478 static __inline u_long
479 buf_mempoolidx(u_long size)
480 {
481 u_int n = 0;
482
483 size -= 1;
484 size >>= MEMPOOL_INDEX_OFFSET;
485 while (size) {
486 size >>= 1;
487 n += 1;
488 }
489 if (n >= NMEMPOOLS)
490 panic("buf mem pool index %d", n);
491 return n;
492 }
493
494 static __inline u_long
495 buf_roundsize(u_long size)
496 {
497 /* Round up to nearest power of 2 */
498 return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET));
499 }
500
501 static __inline caddr_t
502 buf_malloc(size_t size)
503 {
504 u_int n = buf_mempoolidx(size);
505 caddr_t addr;
506 int s;
507
508 while (1) {
509 addr = pool_get(&bmempools[n], PR_NOWAIT);
510 if (addr != NULL)
511 break;
512
513 /* No memory, see if we can free some. If so, try again */
514 if (buf_drain(1) > 0)
515 continue;
516
517 /* Wait for buffers to arrive on the LRU queue */
518 s = splbio();
519 simple_lock(&bqueue_slock);
520 needbuffer = 1;
521 ltsleep(&needbuffer, PNORELOCK | (PRIBIO + 1),
522 "buf_malloc", 0, &bqueue_slock);
523 splx(s);
524 }
525
526 return addr;
527 }
528
529 static void
530 buf_mrelease(caddr_t addr, size_t size)
531 {
532
533 pool_put(&bmempools[buf_mempoolidx(size)], addr);
534 }
535
536
537 static __inline struct buf *
538 bio_doread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred,
539 int async)
540 {
541 struct buf *bp;
542 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */
543 struct proc *p = l->l_proc;
544
545 bp = getblk(vp, blkno, size, 0, 0);
546
547 #ifdef DIAGNOSTIC
548 if (bp == NULL) {
549 panic("bio_doread: no such buf");
550 }
551 #endif
552
553 /*
554 * If buffer does not have data valid, start a read.
555 * Note that if buffer is B_INVAL, getblk() won't return it.
556 * Therefore, it's valid if its I/O has completed or been delayed.
557 */
558 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) {
559 /* Start I/O for the buffer. */
560 SET(bp->b_flags, B_READ | async);
561 if (async)
562 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
563 else
564 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
565 VOP_STRATEGY(vp, bp);
566
567 /* Pay for the read. */
568 p->p_stats->p_ru.ru_inblock++;
569 } else if (async) {
570 brelse(bp);
571 }
572
573 return (bp);
574 }
575
576 /*
577 * Read a disk block.
578 * This algorithm described in Bach (p.54).
579 */
580 int
581 bread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred,
582 struct buf **bpp)
583 {
584 struct buf *bp;
585
586 /* Get buffer for block. */
587 bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
588
589 /* Wait for the read to complete, and return result. */
590 return (biowait(bp));
591 }
592
593 /*
594 * Read-ahead multiple disk blocks. The first is sync, the rest async.
595 * Trivial modification to the breada algorithm presented in Bach (p.55).
596 */
597 int
598 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks,
599 int *rasizes, int nrablks, struct ucred *cred, struct buf **bpp)
600 {
601 struct buf *bp;
602 int i;
603
604 bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
605
606 /*
607 * For each of the read-ahead blocks, start a read, if necessary.
608 */
609 for (i = 0; i < nrablks; i++) {
610 /* If it's in the cache, just go on to next one. */
611 if (incore(vp, rablks[i]))
612 continue;
613
614 /* Get a buffer for the read-ahead block */
615 (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC);
616 }
617
618 /* Otherwise, we had to start a read for it; wait until it's valid. */
619 return (biowait(bp));
620 }
621
622 /*
623 * Read with single-block read-ahead. Defined in Bach (p.55), but
624 * implemented as a call to breadn().
625 * XXX for compatibility with old file systems.
626 */
627 int
628 breada(struct vnode *vp, daddr_t blkno, int size, daddr_t rablkno,
629 int rabsize, struct ucred *cred, struct buf **bpp)
630 {
631
632 return (breadn(vp, blkno, size, &rablkno, &rabsize, 1, cred, bpp));
633 }
634
635 /*
636 * Block write. Described in Bach (p.56)
637 */
638 int
639 bwrite(struct buf *bp)
640 {
641 int rv, sync, wasdelayed, s;
642 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */
643 struct proc *p = l->l_proc;
644 struct vnode *vp;
645 struct mount *mp;
646
647 KASSERT(ISSET(bp->b_flags, B_BUSY));
648
649 vp = bp->b_vp;
650 if (vp != NULL) {
651 if (vp->v_type == VBLK)
652 mp = vp->v_specmountpoint;
653 else
654 mp = vp->v_mount;
655 } else {
656 mp = NULL;
657 }
658
659 /*
660 * Remember buffer type, to switch on it later. If the write was
661 * synchronous, but the file system was mounted with MNT_ASYNC,
662 * convert it to a delayed write.
663 * XXX note that this relies on delayed tape writes being converted
664 * to async, not sync writes (which is safe, but ugly).
665 */
666 sync = !ISSET(bp->b_flags, B_ASYNC);
667 if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) {
668 bdwrite(bp);
669 return (0);
670 }
671
672 /*
673 * Collect statistics on synchronous and asynchronous writes.
674 * Writes to block devices are charged to their associated
675 * filesystem (if any).
676 */
677 if (mp != NULL) {
678 if (sync)
679 mp->mnt_stat.f_syncwrites++;
680 else
681 mp->mnt_stat.f_asyncwrites++;
682 }
683
684 s = splbio();
685 simple_lock(&bp->b_interlock);
686
687 wasdelayed = ISSET(bp->b_flags, B_DELWRI);
688
689 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI));
690
691 /*
692 * Pay for the I/O operation and make sure the buf is on the correct
693 * vnode queue.
694 */
695 if (wasdelayed)
696 reassignbuf(bp, bp->b_vp);
697 else
698 p->p_stats->p_ru.ru_oublock++;
699
700 /* Initiate disk write. Make sure the appropriate party is charged. */
701 V_INCR_NUMOUTPUT(bp->b_vp);
702 simple_unlock(&bp->b_interlock);
703 splx(s);
704
705 if (sync)
706 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
707 else
708 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
709
710 VOP_STRATEGY(vp, bp);
711
712 if (sync) {
713 /* If I/O was synchronous, wait for it to complete. */
714 rv = biowait(bp);
715
716 /* Release the buffer. */
717 brelse(bp);
718
719 return (rv);
720 } else {
721 return (0);
722 }
723 }
724
725 int
726 vn_bwrite(void *v)
727 {
728 struct vop_bwrite_args *ap = v;
729
730 return (bwrite(ap->a_bp));
731 }
732
733 /*
734 * Delayed write.
735 *
736 * The buffer is marked dirty, but is not queued for I/O.
737 * This routine should be used when the buffer is expected
738 * to be modified again soon, typically a small write that
739 * partially fills a buffer.
740 *
741 * NB: magnetic tapes cannot be delayed; they must be
742 * written in the order that the writes are requested.
743 *
744 * Described in Leffler, et al. (pp. 208-213).
745 */
746 void
747 bdwrite(struct buf *bp)
748 {
749 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */
750 struct proc *p = l->l_proc;
751 const struct bdevsw *bdev;
752 int s;
753
754 /* If this is a tape block, write the block now. */
755 bdev = bdevsw_lookup(bp->b_dev);
756 if (bdev != NULL && bdev->d_type == D_TAPE) {
757 bawrite(bp);
758 return;
759 }
760
761 /*
762 * If the block hasn't been seen before:
763 * (1) Mark it as having been seen,
764 * (2) Charge for the write,
765 * (3) Make sure it's on its vnode's correct block list.
766 */
767 s = splbio();
768 simple_lock(&bp->b_interlock);
769
770 KASSERT(ISSET(bp->b_flags, B_BUSY));
771
772 if (!ISSET(bp->b_flags, B_DELWRI)) {
773 SET(bp->b_flags, B_DELWRI);
774 p->p_stats->p_ru.ru_oublock++;
775 reassignbuf(bp, bp->b_vp);
776 }
777
778 /* Otherwise, the "write" is done, so mark and release the buffer. */
779 CLR(bp->b_flags, B_DONE);
780 simple_unlock(&bp->b_interlock);
781 splx(s);
782
783 brelse(bp);
784 }
785
786 /*
787 * Asynchronous block write; just an asynchronous bwrite().
788 */
789 void
790 bawrite(struct buf *bp)
791 {
792 int s;
793
794 s = splbio();
795 simple_lock(&bp->b_interlock);
796
797 KASSERT(ISSET(bp->b_flags, B_BUSY));
798
799 SET(bp->b_flags, B_ASYNC);
800 simple_unlock(&bp->b_interlock);
801 splx(s);
802 VOP_BWRITE(bp);
803 }
804
805 /*
806 * Same as first half of bdwrite, mark buffer dirty, but do not release it.
807 * Call at splbio() and with the buffer interlock locked.
808 * Note: called only from biodone() through ffs softdep's bioops.io_complete()
809 */
810 void
811 bdirty(struct buf *bp)
812 {
813 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */
814 struct proc *p = l->l_proc;
815
816 LOCK_ASSERT(simple_lock_held(&bp->b_interlock));
817 KASSERT(ISSET(bp->b_flags, B_BUSY));
818
819 CLR(bp->b_flags, B_AGE);
820
821 if (!ISSET(bp->b_flags, B_DELWRI)) {
822 SET(bp->b_flags, B_DELWRI);
823 p->p_stats->p_ru.ru_oublock++;
824 reassignbuf(bp, bp->b_vp);
825 }
826 }
827
828 /*
829 * Release a buffer on to the free lists.
830 * Described in Bach (p. 46).
831 */
832 void
833 brelse(struct buf *bp)
834 {
835 struct bqueues *bufq;
836 int s;
837
838 /* Block disk interrupts. */
839 s = splbio();
840 simple_lock(&bqueue_slock);
841 simple_lock(&bp->b_interlock);
842
843 KASSERT(ISSET(bp->b_flags, B_BUSY));
844 KASSERT(!ISSET(bp->b_flags, B_CALL));
845
846 /* Wake up any processes waiting for any buffer to become free. */
847 if (needbuffer) {
848 needbuffer = 0;
849 wakeup(&needbuffer);
850 }
851
852 /* Wake up any proceeses waiting for _this_ buffer to become free. */
853 if (ISSET(bp->b_flags, B_WANTED)) {
854 CLR(bp->b_flags, B_WANTED|B_AGE);
855 wakeup(bp);
856 }
857
858 /*
859 * Determine which queue the buffer should be on, then put it there.
860 */
861
862 /* If it's locked, don't report an error; try again later. */
863 if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR))
864 CLR(bp->b_flags, B_ERROR);
865
866 /* If it's not cacheable, or an error, mark it invalid. */
867 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR)))
868 SET(bp->b_flags, B_INVAL);
869
870 if (ISSET(bp->b_flags, B_VFLUSH)) {
871 /*
872 * This is a delayed write buffer that was just flushed to
873 * disk. It is still on the LRU queue. If it's become
874 * invalid, then we need to move it to a different queue;
875 * otherwise leave it in its current position.
876 */
877 CLR(bp->b_flags, B_VFLUSH);
878 if (!ISSET(bp->b_flags, B_ERROR|B_INVAL|B_LOCKED|B_AGE)) {
879 KDASSERT(!debug_verify_freelist || checkfreelist(bp, &bufqueues[BQ_LRU]));
880 goto already_queued;
881 } else {
882 bremfree(bp);
883 }
884 }
885
886 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_AGE]));
887 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LRU]));
888 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LOCKED]));
889
890 if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) {
891 /*
892 * If it's invalid or empty, dissociate it from its vnode
893 * and put on the head of the appropriate queue.
894 */
895 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
896 (*bioops.io_deallocate)(bp);
897 CLR(bp->b_flags, B_DONE|B_DELWRI);
898 if (bp->b_vp) {
899 reassignbuf(bp, bp->b_vp);
900 brelvp(bp);
901 }
902 if (bp->b_bufsize <= 0)
903 /* no data */
904 goto already_queued;
905 else
906 /* invalid data */
907 bufq = &bufqueues[BQ_AGE];
908 binsheadfree(bp, bufq);
909 } else {
910 /*
911 * It has valid data. Put it on the end of the appropriate
912 * queue, so that it'll stick around for as long as possible.
913 * If buf is AGE, but has dependencies, must put it on last
914 * bufqueue to be scanned, ie LRU. This protects against the
915 * livelock where BQ_AGE only has buffers with dependencies,
916 * and we thus never get to the dependent buffers in BQ_LRU.
917 */
918 if (ISSET(bp->b_flags, B_LOCKED))
919 /* locked in core */
920 bufq = &bufqueues[BQ_LOCKED];
921 else if (!ISSET(bp->b_flags, B_AGE))
922 /* valid data */
923 bufq = &bufqueues[BQ_LRU];
924 else {
925 /* stale but valid data */
926 int has_deps;
927
928 if (LIST_FIRST(&bp->b_dep) != NULL &&
929 bioops.io_countdeps)
930 has_deps = (*bioops.io_countdeps)(bp, 0);
931 else
932 has_deps = 0;
933 bufq = has_deps ? &bufqueues[BQ_LRU] :
934 &bufqueues[BQ_AGE];
935 }
936 binstailfree(bp, bufq);
937 }
938
939 already_queued:
940 /* Unlock the buffer. */
941 CLR(bp->b_flags, B_AGE|B_ASYNC|B_BUSY|B_NOCACHE);
942 SET(bp->b_flags, B_CACHE);
943
944 /* Allow disk interrupts. */
945 simple_unlock(&bp->b_interlock);
946 simple_unlock(&bqueue_slock);
947 if (bp->b_bufsize <= 0) {
948 #ifdef DEBUG
949 memset((char *)bp, 0, sizeof(*bp));
950 #endif
951 pool_put(&bufpool, bp);
952 }
953 splx(s);
954 }
955
956 /*
957 * Determine if a block is in the cache.
958 * Just look on what would be its hash chain. If it's there, return
959 * a pointer to it, unless it's marked invalid. If it's marked invalid,
960 * we normally don't return the buffer, unless the caller explicitly
961 * wants us to.
962 */
963 struct buf *
964 incore(struct vnode *vp, daddr_t blkno)
965 {
966 struct buf *bp;
967
968 /* Search hash chain */
969 LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) {
970 if (bp->b_lblkno == blkno && bp->b_vp == vp &&
971 !ISSET(bp->b_flags, B_INVAL))
972 return (bp);
973 }
974
975 return (NULL);
976 }
977
978 /*
979 * Get a block of requested size that is associated with
980 * a given vnode and block offset. If it is found in the
981 * block cache, mark it as having been found, make it busy
982 * and return it. Otherwise, return an empty block of the
983 * correct size. It is up to the caller to insure that the
984 * cached blocks be of the correct size.
985 */
986 struct buf *
987 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo)
988 {
989 struct buf *bp;
990 int s, err;
991 int preserve;
992
993 start:
994 s = splbio();
995 simple_lock(&bqueue_slock);
996 bp = incore(vp, blkno);
997 if (bp != NULL) {
998 simple_lock(&bp->b_interlock);
999 if (ISSET(bp->b_flags, B_BUSY)) {
1000 simple_unlock(&bqueue_slock);
1001 if (curproc == uvm.pagedaemon_proc) {
1002 simple_unlock(&bp->b_interlock);
1003 splx(s);
1004 return NULL;
1005 }
1006 SET(bp->b_flags, B_WANTED);
1007 err = ltsleep(bp, slpflag | (PRIBIO + 1) | PNORELOCK,
1008 "getblk", slptimeo, &bp->b_interlock);
1009 splx(s);
1010 if (err)
1011 return (NULL);
1012 goto start;
1013 }
1014 #ifdef DIAGNOSTIC
1015 if (ISSET(bp->b_flags, B_DONE|B_DELWRI) &&
1016 bp->b_bcount < size && vp->v_type != VBLK)
1017 panic("getblk: block size invariant failed");
1018 #endif
1019 SET(bp->b_flags, B_BUSY);
1020 bremfree(bp);
1021 preserve = 1;
1022 } else {
1023 if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL) {
1024 simple_unlock(&bqueue_slock);
1025 splx(s);
1026 goto start;
1027 }
1028
1029 binshash(bp, BUFHASH(vp, blkno));
1030 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno;
1031 bgetvp(vp, bp);
1032 preserve = 0;
1033 }
1034 simple_unlock(&bp->b_interlock);
1035 simple_unlock(&bqueue_slock);
1036 splx(s);
1037 /*
1038 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes)
1039 * if we re-size buffers here.
1040 */
1041 if (ISSET(bp->b_flags, B_LOCKED)) {
1042 KASSERT(bp->b_bufsize >= size);
1043 } else {
1044 allocbuf(bp, size, preserve);
1045 }
1046 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1047 return (bp);
1048 }
1049
1050 /*
1051 * Get an empty, disassociated buffer of given size.
1052 */
1053 struct buf *
1054 geteblk(int size)
1055 {
1056 struct buf *bp;
1057 int s;
1058
1059 s = splbio();
1060 simple_lock(&bqueue_slock);
1061 while ((bp = getnewbuf(0, 0, 0)) == 0)
1062 ;
1063
1064 SET(bp->b_flags, B_INVAL);
1065 binshash(bp, &invalhash);
1066 simple_unlock(&bqueue_slock);
1067 simple_unlock(&bp->b_interlock);
1068 splx(s);
1069 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1070 allocbuf(bp, size, 0);
1071 return (bp);
1072 }
1073
1074 /*
1075 * Expand or contract the actual memory allocated to a buffer.
1076 *
1077 * If the buffer shrinks, data is lost, so it's up to the
1078 * caller to have written it out *first*; this routine will not
1079 * start a write. If the buffer grows, it's the callers
1080 * responsibility to fill out the buffer's additional contents.
1081 */
1082 void
1083 allocbuf(struct buf *bp, int size, int preserve)
1084 {
1085 vsize_t oldsize, desired_size;
1086 caddr_t addr;
1087 int s, delta;
1088
1089 desired_size = buf_roundsize(size);
1090 if (desired_size > MAXBSIZE)
1091 printf("allocbuf: buffer larger than MAXBSIZE requested");
1092
1093 bp->b_bcount = size;
1094
1095 oldsize = bp->b_bufsize;
1096 if (oldsize == desired_size)
1097 return;
1098
1099 /*
1100 * If we want a buffer of a different size, re-allocate the
1101 * buffer's memory; copy old content only if needed.
1102 */
1103 addr = buf_malloc(desired_size);
1104 if (preserve)
1105 memcpy(addr, bp->b_data, MIN(oldsize,desired_size));
1106 if (bp->b_data != NULL)
1107 buf_mrelease(bp->b_data, oldsize);
1108 bp->b_data = addr;
1109 bp->b_bufsize = desired_size;
1110
1111 /*
1112 * Update overall buffer memory counter (protected by bqueue_slock)
1113 */
1114 delta = (long)desired_size - (long)oldsize;
1115
1116 s = splbio();
1117 simple_lock(&bqueue_slock);
1118 if ((bufmem += delta) > bufmem_hiwater) {
1119 /*
1120 * Need to trim overall memory usage.
1121 */
1122 while (buf_canrelease()) {
1123 if (buf_trim() == 0)
1124 break;
1125 }
1126 }
1127
1128 simple_unlock(&bqueue_slock);
1129 splx(s);
1130 }
1131
1132 /*
1133 * Find a buffer which is available for use.
1134 * Select something from a free list.
1135 * Preference is to AGE list, then LRU list.
1136 *
1137 * Called at splbio and with buffer queues locked.
1138 * Return buffer locked.
1139 */
1140 struct buf *
1141 getnewbuf(int slpflag, int slptimeo, int from_bufq)
1142 {
1143 struct buf *bp;
1144
1145 start:
1146 LOCK_ASSERT(simple_lock_held(&bqueue_slock));
1147
1148 /*
1149 * Get a new buffer from the pool; but use NOWAIT because
1150 * we have the buffer queues locked.
1151 */
1152 if (buf_lotsfree() && !from_bufq &&
1153 (bp = pool_get(&bufpool, PR_NOWAIT)) != NULL) {
1154 memset((char *)bp, 0, sizeof(*bp));
1155 BUF_INIT(bp);
1156 bp->b_dev = NODEV;
1157 bp->b_vnbufs.le_next = NOLIST;
1158 bp->b_flags = B_BUSY;
1159 simple_lock(&bp->b_interlock);
1160 return (bp);
1161 }
1162
1163 if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE])) != NULL ||
1164 (bp = TAILQ_FIRST(&bufqueues[BQ_LRU])) != NULL) {
1165 simple_lock(&bp->b_interlock);
1166 bremfree(bp);
1167 } else {
1168 /*
1169 * XXX: !from_bufq should be removed.
1170 */
1171 if (!from_bufq || curproc != uvm.pagedaemon_proc) {
1172 /* wait for a free buffer of any kind */
1173 needbuffer = 1;
1174 ltsleep(&needbuffer, slpflag|(PRIBIO + 1),
1175 "getnewbuf", slptimeo, &bqueue_slock);
1176 }
1177 return (NULL);
1178 }
1179
1180 #ifdef DIAGNOSTIC
1181 if (bp->b_bufsize <= 0)
1182 panic("buffer %p: on queue but empty", bp);
1183 #endif
1184
1185 if (ISSET(bp->b_flags, B_VFLUSH)) {
1186 /*
1187 * This is a delayed write buffer being flushed to disk. Make
1188 * sure it gets aged out of the queue when it's finished, and
1189 * leave it off the LRU queue.
1190 */
1191 CLR(bp->b_flags, B_VFLUSH);
1192 SET(bp->b_flags, B_AGE);
1193 simple_unlock(&bp->b_interlock);
1194 goto start;
1195 }
1196
1197 /* Buffer is no longer on free lists. */
1198 SET(bp->b_flags, B_BUSY);
1199
1200 /*
1201 * If buffer was a delayed write, start it and return NULL
1202 * (since we might sleep while starting the write).
1203 */
1204 if (ISSET(bp->b_flags, B_DELWRI)) {
1205 /*
1206 * This buffer has gone through the LRU, so make sure it gets
1207 * reused ASAP.
1208 */
1209 SET(bp->b_flags, B_AGE);
1210 simple_unlock(&bp->b_interlock);
1211 simple_unlock(&bqueue_slock);
1212 bawrite(bp);
1213 simple_lock(&bqueue_slock);
1214 return (NULL);
1215 }
1216
1217 /* disassociate us from our vnode, if we had one... */
1218 if (bp->b_vp)
1219 brelvp(bp);
1220
1221 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate)
1222 (*bioops.io_deallocate)(bp);
1223
1224 /* clear out various other fields */
1225 bp->b_flags = B_BUSY;
1226 bp->b_dev = NODEV;
1227 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = 0;
1228 bp->b_iodone = 0;
1229 bp->b_error = 0;
1230 bp->b_resid = 0;
1231 bp->b_bcount = 0;
1232
1233 bremhash(bp);
1234 return (bp);
1235 }
1236
1237 /*
1238 * Attempt to free an aged buffer off the queues.
1239 * Called at splbio and with queue lock held.
1240 * Returns the amount of buffer memory freed.
1241 */
1242 int
1243 buf_trim(void)
1244 {
1245 struct buf *bp;
1246 long size = 0;
1247 int wanted;
1248
1249 /* Instruct getnewbuf() to get buffers off the queues */
1250 if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL)
1251 return 0;
1252
1253 wanted = ISSET(bp->b_flags, B_WANTED);
1254 simple_unlock(&bp->b_interlock);
1255 if (wanted) {
1256 printf("buftrim: got WANTED buffer\n");
1257 SET(bp->b_flags, B_INVAL);
1258 binshash(bp, &invalhash);
1259 simple_unlock(&bqueue_slock);
1260 goto out;
1261 }
1262 size = bp->b_bufsize;
1263 bufmem -= size;
1264 simple_unlock(&bqueue_slock);
1265 if (size > 0) {
1266 buf_mrelease(bp->b_data, size);
1267 bp->b_bcount = bp->b_bufsize = 0;
1268 }
1269
1270 out:
1271 /* brelse() will return the buffer to the global buffer pool */
1272 brelse(bp);
1273 simple_lock(&bqueue_slock);
1274 return size;
1275 }
1276
1277 int
1278 buf_drain(int n)
1279 {
1280 int s, size = 0, sz;
1281
1282 s = splbio();
1283 simple_lock(&bqueue_slock);
1284
1285 while (size < n && bufmem > bufmem_lowater) {
1286 sz = buf_trim();
1287 if (sz <= 0)
1288 break;
1289 size += sz;
1290 }
1291
1292 simple_unlock(&bqueue_slock);
1293 splx(s);
1294 return size;
1295 }
1296
1297 /*
1298 * Wait for operations on the buffer to complete.
1299 * When they do, extract and return the I/O's error value.
1300 */
1301 int
1302 biowait(struct buf *bp)
1303 {
1304 int s, error;
1305
1306 s = splbio();
1307 simple_lock(&bp->b_interlock);
1308 while (!ISSET(bp->b_flags, B_DONE | B_DELWRI))
1309 ltsleep(bp, PRIBIO + 1, "biowait", 0, &bp->b_interlock);
1310
1311 /* check for interruption of I/O (e.g. via NFS), then errors. */
1312 if (ISSET(bp->b_flags, B_EINTR)) {
1313 CLR(bp->b_flags, B_EINTR);
1314 error = EINTR;
1315 } else if (ISSET(bp->b_flags, B_ERROR))
1316 error = bp->b_error ? bp->b_error : EIO;
1317 else
1318 error = 0;
1319
1320 simple_unlock(&bp->b_interlock);
1321 splx(s);
1322 return (error);
1323 }
1324
1325 /*
1326 * Mark I/O complete on a buffer.
1327 *
1328 * If a callback has been requested, e.g. the pageout
1329 * daemon, do so. Otherwise, awaken waiting processes.
1330 *
1331 * [ Leffler, et al., says on p.247:
1332 * "This routine wakes up the blocked process, frees the buffer
1333 * for an asynchronous write, or, for a request by the pagedaemon
1334 * process, invokes a procedure specified in the buffer structure" ]
1335 *
1336 * In real life, the pagedaemon (or other system processes) wants
1337 * to do async stuff to, and doesn't want the buffer brelse()'d.
1338 * (for swap pager, that puts swap buffers on the free lists (!!!),
1339 * for the vn device, that puts malloc'd buffers on the free lists!)
1340 */
1341 void
1342 biodone(struct buf *bp)
1343 {
1344 int s = splbio();
1345
1346 simple_lock(&bp->b_interlock);
1347 if (ISSET(bp->b_flags, B_DONE))
1348 panic("biodone already");
1349 SET(bp->b_flags, B_DONE); /* note that it's done */
1350 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1351
1352 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete)
1353 (*bioops.io_complete)(bp);
1354
1355 if (!ISSET(bp->b_flags, B_READ)) /* wake up reader */
1356 vwakeup(bp);
1357
1358 /*
1359 * If necessary, call out. Unlock the buffer before calling
1360 * iodone() as the buffer isn't valid any more when it return.
1361 */
1362 if (ISSET(bp->b_flags, B_CALL)) {
1363 CLR(bp->b_flags, B_CALL); /* but note callout done */
1364 simple_unlock(&bp->b_interlock);
1365 (*bp->b_iodone)(bp);
1366 } else {
1367 if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release */
1368 simple_unlock(&bp->b_interlock);
1369 brelse(bp);
1370 } else { /* or just wakeup the buffer */
1371 CLR(bp->b_flags, B_WANTED);
1372 wakeup(bp);
1373 simple_unlock(&bp->b_interlock);
1374 }
1375 }
1376
1377 splx(s);
1378 }
1379
1380 /*
1381 * Return a count of buffers on the "locked" queue.
1382 */
1383 int
1384 count_lock_queue(void)
1385 {
1386 struct buf *bp;
1387 int n = 0;
1388
1389 simple_lock(&bqueue_slock);
1390 TAILQ_FOREACH(bp, &bufqueues[BQ_LOCKED], b_freelist)
1391 n++;
1392 simple_unlock(&bqueue_slock);
1393 return (n);
1394 }
1395
1396 /*
1397 * Wait for all buffers to complete I/O
1398 * Return the number of "stuck" buffers.
1399 */
1400 int
1401 buf_syncwait(void)
1402 {
1403 struct buf *bp;
1404 int iter, nbusy, nbusy_prev = 0, dcount, s, ihash;
1405
1406 dcount = 10000;
1407 for (iter = 0; iter < 20;) {
1408 s = splbio();
1409 simple_lock(&bqueue_slock);
1410 nbusy = 0;
1411 for (ihash = 0; ihash < bufhash+1; ihash++) {
1412 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1413 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY)
1414 nbusy++;
1415 /*
1416 * With soft updates, some buffers that are
1417 * written will be remarked as dirty until other
1418 * buffers are written.
1419 */
1420 if (bp->b_vp && bp->b_vp->v_mount
1421 && (bp->b_vp->v_mount->mnt_flag & MNT_SOFTDEP)
1422 && (bp->b_flags & B_DELWRI)) {
1423 simple_lock(&bp->b_interlock);
1424 bremfree(bp);
1425 bp->b_flags |= B_BUSY;
1426 nbusy++;
1427 simple_unlock(&bp->b_interlock);
1428 simple_unlock(&bqueue_slock);
1429 bawrite(bp);
1430 if (dcount-- <= 0) {
1431 printf("softdep ");
1432 goto fail;
1433 }
1434 simple_lock(&bqueue_slock);
1435 }
1436 }
1437 }
1438
1439 simple_unlock(&bqueue_slock);
1440 splx(s);
1441
1442 if (nbusy == 0)
1443 break;
1444 if (nbusy_prev == 0)
1445 nbusy_prev = nbusy;
1446 printf("%d ", nbusy);
1447 tsleep(&nbusy, PRIBIO, "bflush",
1448 (iter == 0) ? 1 : hz / 25 * iter);
1449 if (nbusy >= nbusy_prev) /* we didn't flush anything */
1450 iter++;
1451 else
1452 nbusy_prev = nbusy;
1453 }
1454
1455 if (nbusy) {
1456 fail:;
1457 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY)
1458 printf("giving up\nPrinting vnodes for busy buffers\n");
1459 for (ihash = 0; ihash < bufhash+1; ihash++) {
1460 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1461 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY)
1462 vprint(NULL, bp->b_vp);
1463 }
1464 }
1465 #endif
1466 }
1467
1468 return nbusy;
1469 }
1470
1471 static void
1472 sysctl_fillbuf(struct buf *i, struct buf_sysctl *o)
1473 {
1474
1475 o->b_flags = i->b_flags;
1476 o->b_error = i->b_error;
1477 o->b_prio = i->b_prio;
1478 o->b_dev = i->b_dev;
1479 o->b_bufsize = i->b_bufsize;
1480 o->b_bcount = i->b_bcount;
1481 o->b_resid = i->b_resid;
1482 o->b_addr = PTRTOUINT64(i->b_un.b_addr);
1483 o->b_blkno = i->b_blkno;
1484 o->b_rawblkno = i->b_rawblkno;
1485 o->b_iodone = PTRTOUINT64(i->b_iodone);
1486 o->b_proc = PTRTOUINT64(i->b_proc);
1487 o->b_vp = PTRTOUINT64(i->b_vp);
1488 o->b_saveaddr = PTRTOUINT64(i->b_saveaddr);
1489 o->b_lblkno = i->b_lblkno;
1490 }
1491
1492 #define KERN_BUFSLOP 20
1493 static int
1494 sysctl_dobuf(SYSCTLFN_ARGS)
1495 {
1496 struct buf *bp;
1497 struct buf_sysctl bs;
1498 char *dp;
1499 u_int i, op, arg;
1500 size_t len, needed, elem_size, out_size;
1501 int error, s, elem_count;
1502
1503 if (namelen == 1 && name[0] == CTL_QUERY)
1504 return (sysctl_query(SYSCTLFN_CALL(rnode)));
1505
1506 if (namelen != 4)
1507 return (EINVAL);
1508
1509 dp = oldp;
1510 len = (oldp != NULL) ? *oldlenp : 0;
1511 op = name[0];
1512 arg = name[1];
1513 elem_size = name[2];
1514 elem_count = name[3];
1515 out_size = MIN(sizeof(bs), elem_size);
1516
1517 /*
1518 * at the moment, these are just "placeholders" to make the
1519 * API for retrieving kern.buf data more extensible in the
1520 * future.
1521 *
1522 * XXX kern.buf currently has "netbsd32" issues. hopefully
1523 * these will be resolved at a later point.
1524 */
1525 if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL ||
1526 elem_size < 1 || elem_count < 0)
1527 return (EINVAL);
1528
1529 error = 0;
1530 needed = 0;
1531 s = splbio();
1532 simple_lock(&bqueue_slock);
1533 for (i = 0; i < BQUEUES; i++) {
1534 TAILQ_FOREACH(bp, &bufqueues[i], b_freelist) {
1535 if (len >= elem_size && elem_count > 0) {
1536 sysctl_fillbuf(bp, &bs);
1537 error = copyout(&bs, dp, out_size);
1538 if (error)
1539 goto cleanup;
1540 dp += elem_size;
1541 len -= elem_size;
1542 }
1543 if (elem_count > 0) {
1544 needed += elem_size;
1545 if (elem_count != INT_MAX)
1546 elem_count--;
1547 }
1548 }
1549 }
1550 cleanup:
1551 simple_unlock(&bqueue_slock);
1552 splx(s);
1553
1554 *oldlenp = needed;
1555 if (oldp == NULL)
1556 *oldlenp += KERN_BUFSLOP * sizeof(struct buf);
1557
1558 return (error);
1559 }
1560
1561 static int
1562 sysctl_bufvm_update(SYSCTLFN_ARGS)
1563 {
1564 int t, error;
1565 struct sysctlnode node;
1566
1567 node = *rnode;
1568 node.sysctl_data = &t;
1569 t = *(int*)rnode->sysctl_data;
1570 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1571 if (error || newp == NULL)
1572 return (error);
1573
1574 if (rnode->sysctl_data == &bufcache) {
1575 if (t < 0 || t > 100)
1576 return (EINVAL);
1577 bufcache = t;
1578 buf_setwm();
1579 } else if (rnode->sysctl_data == &bufmem_lowater) {
1580 if (bufmem_hiwater - bufmem_lowater < 16)
1581 return (EINVAL);
1582 bufmem_lowater = t;
1583 } else if (rnode->sysctl_data == &bufmem_hiwater) {
1584 if (bufmem_hiwater - bufmem_lowater < 16)
1585 return (EINVAL);
1586 bufmem_hiwater = t;
1587 } else
1588 return (EINVAL);
1589
1590 /* Drain until below new high water mark */
1591 while ((t = bufmem - bufmem_hiwater) >= 0) {
1592 if (buf_drain(t / (2*1024)) <= 0)
1593 break;
1594 }
1595
1596 return 0;
1597 }
1598
1599 SYSCTL_SETUP(sysctl_kern_buf_setup, "sysctl kern.buf subtree setup")
1600 {
1601
1602 sysctl_createv(clog, 0, NULL, NULL,
1603 CTLFLAG_PERMANENT,
1604 CTLTYPE_NODE, "kern", NULL,
1605 NULL, 0, NULL, 0,
1606 CTL_KERN, CTL_EOL);
1607 sysctl_createv(clog, 0, NULL, NULL,
1608 CTLFLAG_PERMANENT,
1609 CTLTYPE_NODE, "buf",
1610 SYSCTL_DESCR("Kernel buffer cache information"),
1611 sysctl_dobuf, 0, NULL, 0,
1612 CTL_KERN, KERN_BUF, CTL_EOL);
1613 }
1614
1615 SYSCTL_SETUP(sysctl_vm_buf_setup, "sysctl vm.buf* subtree setup")
1616 {
1617
1618 sysctl_createv(clog, 0, NULL, NULL,
1619 CTLFLAG_PERMANENT,
1620 CTLTYPE_NODE, "vm", NULL,
1621 NULL, 0, NULL, 0,
1622 CTL_VM, CTL_EOL);
1623
1624 sysctl_createv(clog, 0, NULL, NULL,
1625 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1626 CTLTYPE_INT, "bufcache",
1627 SYSCTL_DESCR("Percentage of kernel memory to use for "
1628 "buffer cache"),
1629 sysctl_bufvm_update, 0, &bufcache, 0,
1630 CTL_VM, CTL_CREATE, CTL_EOL);
1631 sysctl_createv(clog, 0, NULL, NULL,
1632 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
1633 CTLTYPE_INT, "bufmem",
1634 SYSCTL_DESCR("Amount of kernel memory used by buffer "
1635 "cache"),
1636 NULL, 0, &bufmem, 0,
1637 CTL_VM, CTL_CREATE, CTL_EOL);
1638 sysctl_createv(clog, 0, NULL, NULL,
1639 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1640 CTLTYPE_INT, "bufmem_lowater",
1641 SYSCTL_DESCR("Minimum amount of kernel memory to "
1642 "reserve for buffer cache"),
1643 sysctl_bufvm_update, 0, &bufmem_lowater, 0,
1644 CTL_VM, CTL_CREATE, CTL_EOL);
1645 sysctl_createv(clog, 0, NULL, NULL,
1646 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1647 CTLTYPE_INT, "bufmem_hiwater",
1648 SYSCTL_DESCR("Maximum amount of kernel memory to use "
1649 "for buffer cache"),
1650 sysctl_bufvm_update, 0, &bufmem_hiwater, 0,
1651 CTL_VM, CTL_CREATE, CTL_EOL);
1652 }
1653
1654 #ifdef DEBUG
1655 /*
1656 * Print out statistics on the current allocation of the buffer pool.
1657 * Can be enabled to print out on every ``sync'' by setting "syncprt"
1658 * in vfs_syscalls.c using sysctl.
1659 */
1660 void
1661 vfs_bufstats(void)
1662 {
1663 int s, i, j, count;
1664 struct buf *bp;
1665 struct bqueues *dp;
1666 int counts[(MAXBSIZE / PAGE_SIZE) + 1];
1667 static char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" };
1668
1669 for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
1670 count = 0;
1671 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1672 counts[j] = 0;
1673 s = splbio();
1674 TAILQ_FOREACH(bp, dp, b_freelist) {
1675 counts[bp->b_bufsize/PAGE_SIZE]++;
1676 count++;
1677 }
1678 splx(s);
1679 printf("%s: total-%d", bname[i], count);
1680 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1681 if (counts[j] != 0)
1682 printf(", %d-%d", j * PAGE_SIZE, counts[j]);
1683 printf("\n");
1684 }
1685 }
1686 #endif /* DEBUG */
Cache object: 7049c2d4d8debeff5e24af0caac37884
|