FreeBSD/Linux Kernel Cross Reference
sys/kern/vfs_bio.c
1 /* $NetBSD: vfs_bio.c,v 1.210 2008/09/11 09:14:46 hannken Exp $ */
2
3 /*-
4 * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 * This code is derived from software contributed to The NetBSD Foundation
10 * by Wasabi Systems, Inc.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 /*-
35 * Copyright (c) 1982, 1986, 1989, 1993
36 * The Regents of the University of California. All rights reserved.
37 * (c) UNIX System Laboratories, Inc.
38 * All or some portions of this file are derived from material licensed
39 * to the University of California by American Telephone and Telegraph
40 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
41 * the permission of UNIX System Laboratories, Inc.
42 *
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
45 * are met:
46 * 1. Redistributions of source code must retain the above copyright
47 * notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 * 3. Neither the name of the University nor the names of its contributors
52 * may be used to endorse or promote products derived from this software
53 * without specific prior written permission.
54 *
55 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
56 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
58 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
65 * SUCH DAMAGE.
66 *
67 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94
68 */
69
70 /*-
71 * Copyright (c) 1994 Christopher G. Demetriou
72 *
73 * Redistribution and use in source and binary forms, with or without
74 * modification, are permitted provided that the following conditions
75 * are met:
76 * 1. Redistributions of source code must retain the above copyright
77 * notice, this list of conditions and the following disclaimer.
78 * 2. Redistributions in binary form must reproduce the above copyright
79 * notice, this list of conditions and the following disclaimer in the
80 * documentation and/or other materials provided with the distribution.
81 * 3. All advertising materials mentioning features or use of this software
82 * must display the following acknowledgement:
83 * This product includes software developed by the University of
84 * California, Berkeley and its contributors.
85 * 4. Neither the name of the University nor the names of its contributors
86 * may be used to endorse or promote products derived from this software
87 * without specific prior written permission.
88 *
89 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
90 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
91 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
92 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
93 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
94 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
95 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
96 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
97 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
98 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
99 * SUCH DAMAGE.
100 *
101 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94
102 */
103
104 /*
105 * Some references:
106 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986)
107 * Leffler, et al.: The Design and Implementation of the 4.3BSD
108 * UNIX Operating System (Addison Welley, 1989)
109 */
110
111 #include <sys/cdefs.h>
112 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.210 2008/09/11 09:14:46 hannken Exp $");
113
114 #include "fs_ffs.h"
115 #include "opt_bufcache.h"
116
117 #include <sys/param.h>
118 #include <sys/systm.h>
119 #include <sys/kernel.h>
120 #include <sys/proc.h>
121 #include <sys/buf.h>
122 #include <sys/vnode.h>
123 #include <sys/mount.h>
124 #include <sys/resourcevar.h>
125 #include <sys/sysctl.h>
126 #include <sys/conf.h>
127 #include <sys/kauth.h>
128 #include <sys/fstrans.h>
129 #include <sys/intr.h>
130 #include <sys/cpu.h>
131 #include <sys/wapbl.h>
132
133 #include <uvm/uvm.h>
134
135 #include <miscfs/specfs/specdev.h>
136
137 #ifndef BUFPAGES
138 # define BUFPAGES 0
139 #endif
140
141 #ifdef BUFCACHE
142 # if (BUFCACHE < 5) || (BUFCACHE > 95)
143 # error BUFCACHE is not between 5 and 95
144 # endif
145 #else
146 # define BUFCACHE 15
147 #endif
148
149 u_int nbuf; /* XXX - for softdep_lockedbufs */
150 u_int bufpages = BUFPAGES; /* optional hardwired count */
151 u_int bufcache = BUFCACHE; /* max % of RAM to use for buffer cache */
152
153 /* Function prototypes */
154 struct bqueue;
155
156 static void buf_setwm(void);
157 static int buf_trim(void);
158 static void *bufpool_page_alloc(struct pool *, int);
159 static void bufpool_page_free(struct pool *, void *);
160 static buf_t *bio_doread(struct vnode *, daddr_t, int,
161 kauth_cred_t, int);
162 static buf_t *getnewbuf(int, int, int);
163 static int buf_lotsfree(void);
164 static int buf_canrelease(void);
165 static u_long buf_mempoolidx(u_long);
166 static u_long buf_roundsize(u_long);
167 static void *buf_malloc(size_t);
168 static void buf_mrelease(void *, size_t);
169 static void binsheadfree(buf_t *, struct bqueue *);
170 static void binstailfree(buf_t *, struct bqueue *);
171 int count_lock_queue(void); /* XXX */
172 #ifdef DEBUG
173 static int checkfreelist(buf_t *, struct bqueue *, int);
174 #endif
175 static void biointr(void *);
176 static void biodone2(buf_t *);
177 static void bref(buf_t *);
178 static void brele(buf_t *);
179
180 /*
181 * Definitions for the buffer hash lists.
182 */
183 #define BUFHASH(dvp, lbn) \
184 (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash])
185 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash;
186 u_long bufhash;
187 struct bqueue bufqueues[BQUEUES];
188 const struct bio_ops *bioopsp; /* I/O operation notification */
189
190 static kcondvar_t needbuffer_cv;
191
192 /*
193 * Buffer queue lock.
194 */
195 kmutex_t bufcache_lock;
196 kmutex_t buffer_lock;
197
198 /* Software ISR for completed transfers. */
199 static void *biodone_sih;
200
201 /* Buffer pool for I/O buffers. */
202 static pool_cache_t buf_cache;
203 static pool_cache_t bufio_cache;
204
205 /* XXX - somewhat gross.. */
206 #if MAXBSIZE == 0x2000
207 #define NMEMPOOLS 5
208 #elif MAXBSIZE == 0x4000
209 #define NMEMPOOLS 6
210 #elif MAXBSIZE == 0x8000
211 #define NMEMPOOLS 7
212 #else
213 #define NMEMPOOLS 8
214 #endif
215
216 #define MEMPOOL_INDEX_OFFSET 9 /* smallest pool is 512 bytes */
217 #if (1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) != MAXBSIZE
218 #error update vfs_bio buffer memory parameters
219 #endif
220
221 /* Buffer memory pools */
222 static struct pool bmempools[NMEMPOOLS];
223
224 static struct vm_map *buf_map;
225
226 /*
227 * Buffer memory pool allocator.
228 */
229 static void *
230 bufpool_page_alloc(struct pool *pp, int flags)
231 {
232
233 return (void *)uvm_km_alloc(buf_map,
234 MAXBSIZE, MAXBSIZE,
235 ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK)
236 | UVM_KMF_WIRED);
237 }
238
239 static void
240 bufpool_page_free(struct pool *pp, void *v)
241 {
242
243 uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED);
244 }
245
246 static struct pool_allocator bufmempool_allocator = {
247 .pa_alloc = bufpool_page_alloc,
248 .pa_free = bufpool_page_free,
249 .pa_pagesz = MAXBSIZE,
250 };
251
252 /* Buffer memory management variables */
253 u_long bufmem_valimit;
254 u_long bufmem_hiwater;
255 u_long bufmem_lowater;
256 u_long bufmem;
257
258 /*
259 * MD code can call this to set a hard limit on the amount
260 * of virtual memory used by the buffer cache.
261 */
262 int
263 buf_setvalimit(vsize_t sz)
264 {
265
266 /* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */
267 if (sz < NMEMPOOLS * MAXBSIZE)
268 return EINVAL;
269
270 bufmem_valimit = sz;
271 return 0;
272 }
273
274 static void
275 buf_setwm(void)
276 {
277
278 bufmem_hiwater = buf_memcalc();
279 /* lowater is approx. 2% of memory (with bufcache = 15) */
280 #define BUFMEM_WMSHIFT 3
281 #define BUFMEM_HIWMMIN (64 * 1024 << BUFMEM_WMSHIFT)
282 if (bufmem_hiwater < BUFMEM_HIWMMIN)
283 /* Ensure a reasonable minimum value */
284 bufmem_hiwater = BUFMEM_HIWMMIN;
285 bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT;
286 }
287
288 #ifdef DEBUG
289 int debug_verify_freelist = 0;
290 static int
291 checkfreelist(buf_t *bp, struct bqueue *dp, int ison)
292 {
293 buf_t *b;
294
295 if (!debug_verify_freelist)
296 return 1;
297
298 TAILQ_FOREACH(b, &dp->bq_queue, b_freelist) {
299 if (b == bp)
300 return ison ? 1 : 0;
301 }
302
303 return ison ? 0 : 1;
304 }
305 #endif
306
307 /*
308 * Insq/Remq for the buffer hash lists.
309 * Call with buffer queue locked.
310 */
311 static void
312 binsheadfree(buf_t *bp, struct bqueue *dp)
313 {
314
315 KASSERT(mutex_owned(&bufcache_lock));
316 KASSERT(bp->b_freelistindex == -1);
317 TAILQ_INSERT_HEAD(&dp->bq_queue, bp, b_freelist);
318 dp->bq_bytes += bp->b_bufsize;
319 bp->b_freelistindex = dp - bufqueues;
320 }
321
322 static void
323 binstailfree(buf_t *bp, struct bqueue *dp)
324 {
325
326 KASSERT(mutex_owned(&bufcache_lock));
327 KASSERT(bp->b_freelistindex == -1);
328 TAILQ_INSERT_TAIL(&dp->bq_queue, bp, b_freelist);
329 dp->bq_bytes += bp->b_bufsize;
330 bp->b_freelistindex = dp - bufqueues;
331 }
332
333 void
334 bremfree(buf_t *bp)
335 {
336 struct bqueue *dp;
337 int bqidx = bp->b_freelistindex;
338
339 KASSERT(mutex_owned(&bufcache_lock));
340
341 KASSERT(bqidx != -1);
342 dp = &bufqueues[bqidx];
343 KDASSERT(checkfreelist(bp, dp, 1));
344 KASSERT(dp->bq_bytes >= bp->b_bufsize);
345 TAILQ_REMOVE(&dp->bq_queue, bp, b_freelist);
346 dp->bq_bytes -= bp->b_bufsize;
347
348 /* For the sysctl helper. */
349 if (bp == dp->bq_marker)
350 dp->bq_marker = NULL;
351
352 #if defined(DIAGNOSTIC)
353 bp->b_freelistindex = -1;
354 #endif /* defined(DIAGNOSTIC) */
355 }
356
357 /*
358 * Add a reference to an buffer structure that came from buf_cache.
359 */
360 static inline void
361 bref(buf_t *bp)
362 {
363
364 KASSERT(mutex_owned(&bufcache_lock));
365 KASSERT(bp->b_refcnt > 0);
366
367 bp->b_refcnt++;
368 }
369
370 /*
371 * Free an unused buffer structure that came from buf_cache.
372 */
373 static inline void
374 brele(buf_t *bp)
375 {
376
377 KASSERT(mutex_owned(&bufcache_lock));
378 KASSERT(bp->b_refcnt > 0);
379
380 if (bp->b_refcnt-- == 1) {
381 buf_destroy(bp);
382 #ifdef DEBUG
383 memset((char *)bp, 0, sizeof(*bp));
384 #endif
385 pool_cache_put(buf_cache, bp);
386 }
387 }
388
389 /*
390 * note that for some ports this is used by pmap bootstrap code to
391 * determine kva size.
392 */
393 u_long
394 buf_memcalc(void)
395 {
396 u_long n;
397
398 /*
399 * Determine the upper bound of memory to use for buffers.
400 *
401 * - If bufpages is specified, use that as the number
402 * pages.
403 *
404 * - Otherwise, use bufcache as the percentage of
405 * physical memory.
406 */
407 if (bufpages != 0) {
408 n = bufpages;
409 } else {
410 if (bufcache < 5) {
411 printf("forcing bufcache %d -> 5", bufcache);
412 bufcache = 5;
413 }
414 if (bufcache > 95) {
415 printf("forcing bufcache %d -> 95", bufcache);
416 bufcache = 95;
417 }
418 n = calc_cache_size(buf_map, bufcache,
419 (buf_map != kernel_map) ? 100 : BUFCACHE_VA_MAXPCT)
420 / PAGE_SIZE;
421 }
422
423 n <<= PAGE_SHIFT;
424 if (bufmem_valimit != 0 && n > bufmem_valimit)
425 n = bufmem_valimit;
426
427 return (n);
428 }
429
430 /*
431 * Initialize buffers and hash links for buffers.
432 */
433 void
434 bufinit(void)
435 {
436 struct bqueue *dp;
437 int use_std;
438 u_int i;
439
440 mutex_init(&bufcache_lock, MUTEX_DEFAULT, IPL_NONE);
441 mutex_init(&buffer_lock, MUTEX_DEFAULT, IPL_NONE);
442 cv_init(&needbuffer_cv, "needbuf");
443
444 if (bufmem_valimit != 0) {
445 vaddr_t minaddr = 0, maxaddr;
446 buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
447 bufmem_valimit, 0, false, 0);
448 if (buf_map == NULL)
449 panic("bufinit: cannot allocate submap");
450 } else
451 buf_map = kernel_map;
452
453 /*
454 * Initialize buffer cache memory parameters.
455 */
456 bufmem = 0;
457 buf_setwm();
458
459 /* On "small" machines use small pool page sizes where possible */
460 use_std = (physmem < atop(16*1024*1024));
461
462 /*
463 * Also use them on systems that can map the pool pages using
464 * a direct-mapped segment.
465 */
466 #ifdef PMAP_MAP_POOLPAGE
467 use_std = 1;
468 #endif
469
470 buf_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
471 "bufpl", NULL, IPL_SOFTBIO, NULL, NULL, NULL);
472 bufio_cache = pool_cache_init(sizeof(buf_t), 0, 0, 0,
473 "biopl", NULL, IPL_BIO, NULL, NULL, NULL);
474
475 bufmempool_allocator.pa_backingmap = buf_map;
476 for (i = 0; i < NMEMPOOLS; i++) {
477 struct pool_allocator *pa;
478 struct pool *pp = &bmempools[i];
479 u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET);
480 char *name = kmem_alloc(8, KM_SLEEP);
481 if (__predict_true(size >= 1024))
482 (void)snprintf(name, 8, "buf%dk", size / 1024);
483 else
484 (void)snprintf(name, 8, "buf%db", size);
485 pa = (size <= PAGE_SIZE && use_std)
486 ? &pool_allocator_nointr
487 : &bufmempool_allocator;
488 pool_init(pp, size, 0, 0, 0, name, pa, IPL_NONE);
489 pool_setlowat(pp, 1);
490 pool_sethiwat(pp, 1);
491 }
492
493 /* Initialize the buffer queues */
494 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) {
495 TAILQ_INIT(&dp->bq_queue);
496 dp->bq_bytes = 0;
497 }
498
499 /*
500 * Estimate hash table size based on the amount of memory we
501 * intend to use for the buffer cache. The average buffer
502 * size is dependent on our clients (i.e. filesystems).
503 *
504 * For now, use an empirical 3K per buffer.
505 */
506 nbuf = (bufmem_hiwater / 1024) / 3;
507 bufhashtbl = hashinit(nbuf, HASH_LIST, true, &bufhash);
508 }
509
510 void
511 bufinit2(void)
512 {
513
514 biodone_sih = softint_establish(SOFTINT_BIO | SOFTINT_MPSAFE, biointr,
515 NULL);
516 if (biodone_sih == NULL)
517 panic("bufinit2: can't establish soft interrupt");
518 }
519
520 static int
521 buf_lotsfree(void)
522 {
523 int try, thresh;
524
525 /* Always allocate if less than the low water mark. */
526 if (bufmem < bufmem_lowater)
527 return 1;
528
529 /* Never allocate if greater than the high water mark. */
530 if (bufmem > bufmem_hiwater)
531 return 0;
532
533 /* If there's anything on the AGE list, it should be eaten. */
534 if (TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue) != NULL)
535 return 0;
536
537 /*
538 * The probabily of getting a new allocation is inversely
539 * proportional to the current size of the cache, using
540 * a granularity of 16 steps.
541 */
542 try = random() & 0x0000000fL;
543
544 /* Don't use "16 * bufmem" here to avoid a 32-bit overflow. */
545 thresh = (bufmem - bufmem_lowater) /
546 ((bufmem_hiwater - bufmem_lowater) / 16);
547
548 if (try >= thresh)
549 return 1;
550
551 /* Otherwise don't allocate. */
552 return 0;
553 }
554
555 /*
556 * Return estimate of bytes we think need to be
557 * released to help resolve low memory conditions.
558 *
559 * => called with bufcache_lock held.
560 */
561 static int
562 buf_canrelease(void)
563 {
564 int pagedemand, ninvalid = 0;
565
566 KASSERT(mutex_owned(&bufcache_lock));
567
568 if (bufmem < bufmem_lowater)
569 return 0;
570
571 if (bufmem > bufmem_hiwater)
572 return bufmem - bufmem_hiwater;
573
574 ninvalid += bufqueues[BQ_AGE].bq_bytes;
575
576 pagedemand = uvmexp.freetarg - uvmexp.free;
577 if (pagedemand < 0)
578 return ninvalid;
579 return MAX(ninvalid, MIN(2 * MAXBSIZE,
580 MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE)));
581 }
582
583 /*
584 * Buffer memory allocation helper functions
585 */
586 static u_long
587 buf_mempoolidx(u_long size)
588 {
589 u_int n = 0;
590
591 size -= 1;
592 size >>= MEMPOOL_INDEX_OFFSET;
593 while (size) {
594 size >>= 1;
595 n += 1;
596 }
597 if (n >= NMEMPOOLS)
598 panic("buf mem pool index %d", n);
599 return n;
600 }
601
602 static u_long
603 buf_roundsize(u_long size)
604 {
605 /* Round up to nearest power of 2 */
606 return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET));
607 }
608
609 static void *
610 buf_malloc(size_t size)
611 {
612 u_int n = buf_mempoolidx(size);
613 void *addr;
614
615 while (1) {
616 addr = pool_get(&bmempools[n], PR_NOWAIT);
617 if (addr != NULL)
618 break;
619
620 /* No memory, see if we can free some. If so, try again */
621 mutex_enter(&bufcache_lock);
622 if (buf_drain(1) > 0) {
623 mutex_exit(&bufcache_lock);
624 continue;
625 }
626
627 if (curlwp == uvm.pagedaemon_lwp) {
628 mutex_exit(&bufcache_lock);
629 return NULL;
630 }
631
632 /* Wait for buffers to arrive on the LRU queue */
633 cv_timedwait(&needbuffer_cv, &bufcache_lock, hz / 4);
634 mutex_exit(&bufcache_lock);
635 }
636
637 return addr;
638 }
639
640 static void
641 buf_mrelease(void *addr, size_t size)
642 {
643
644 pool_put(&bmempools[buf_mempoolidx(size)], addr);
645 }
646
647 /*
648 * bread()/breadn() helper.
649 */
650 static buf_t *
651 bio_doread(struct vnode *vp, daddr_t blkno, int size, kauth_cred_t cred,
652 int async)
653 {
654 buf_t *bp;
655 struct mount *mp;
656
657 bp = getblk(vp, blkno, size, 0, 0);
658
659 #ifdef DIAGNOSTIC
660 if (bp == NULL) {
661 panic("bio_doread: no such buf");
662 }
663 #endif
664
665 /*
666 * If buffer does not have data valid, start a read.
667 * Note that if buffer is BC_INVAL, getblk() won't return it.
668 * Therefore, it's valid if its I/O has completed or been delayed.
669 */
670 if (!ISSET(bp->b_oflags, (BO_DONE | BO_DELWRI))) {
671 /* Start I/O for the buffer. */
672 SET(bp->b_flags, B_READ | async);
673 if (async)
674 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
675 else
676 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
677 VOP_STRATEGY(vp, bp);
678
679 /* Pay for the read. */
680 curlwp->l_ru.ru_inblock++;
681 } else if (async)
682 brelse(bp, 0);
683
684 if (vp->v_type == VBLK)
685 mp = vp->v_specmountpoint;
686 else
687 mp = vp->v_mount;
688
689 /*
690 * Collect statistics on synchronous and asynchronous reads.
691 * Reads from block devices are charged to their associated
692 * filesystem (if any).
693 */
694 if (mp != NULL) {
695 if (async == 0)
696 mp->mnt_stat.f_syncreads++;
697 else
698 mp->mnt_stat.f_asyncreads++;
699 }
700
701 return (bp);
702 }
703
704 /*
705 * Read a disk block.
706 * This algorithm described in Bach (p.54).
707 */
708 int
709 bread(struct vnode *vp, daddr_t blkno, int size, kauth_cred_t cred,
710 int flags, buf_t **bpp)
711 {
712 buf_t *bp;
713 int error;
714
715 /* Get buffer for block. */
716 bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
717
718 /* Wait for the read to complete, and return result. */
719 error = biowait(bp);
720 if (error == 0 && (flags & B_MODIFY) != 0) /* XXXX before the next code block or after? */
721 error = fscow_run(bp, true);
722
723 if (!error) {
724 struct mount *mp = wapbl_vptomp(vp);
725
726 if (mp && mp->mnt_wapbl_replay &&
727 WAPBL_REPLAY_ISOPEN(mp)) {
728 error = WAPBL_REPLAY_READ(mp, bp->b_data, bp->b_blkno,
729 bp->b_bcount);
730 if (error) {
731 mutex_enter(&bufcache_lock);
732 SET(bp->b_cflags, BC_INVAL);
733 mutex_exit(&bufcache_lock);
734 }
735 }
736 }
737 return error;
738 }
739
740 /*
741 * Read-ahead multiple disk blocks. The first is sync, the rest async.
742 * Trivial modification to the breada algorithm presented in Bach (p.55).
743 */
744 int
745 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks,
746 int *rasizes, int nrablks, kauth_cred_t cred, int flags, buf_t **bpp)
747 {
748 buf_t *bp;
749 int error, i;
750
751 bp = *bpp = bio_doread(vp, blkno, size, cred, 0);
752
753 /*
754 * For each of the read-ahead blocks, start a read, if necessary.
755 */
756 mutex_enter(&bufcache_lock);
757 for (i = 0; i < nrablks; i++) {
758 /* If it's in the cache, just go on to next one. */
759 if (incore(vp, rablks[i]))
760 continue;
761
762 /* Get a buffer for the read-ahead block */
763 mutex_exit(&bufcache_lock);
764 (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC);
765 mutex_enter(&bufcache_lock);
766 }
767 mutex_exit(&bufcache_lock);
768
769 /* Otherwise, we had to start a read for it; wait until it's valid. */
770 error = biowait(bp);
771 if (error == 0 && (flags & B_MODIFY) != 0)
772 error = fscow_run(bp, true);
773 return error;
774 }
775
776 /*
777 * Read with single-block read-ahead. Defined in Bach (p.55), but
778 * implemented as a call to breadn().
779 * XXX for compatibility with old file systems.
780 */
781 int
782 breada(struct vnode *vp, daddr_t blkno, int size, daddr_t rablkno,
783 int rabsize, kauth_cred_t cred, int flags, buf_t **bpp)
784 {
785
786 return (breadn(vp, blkno, size, &rablkno, &rabsize, 1,
787 cred, flags, bpp));
788 }
789
790 /*
791 * Block write. Described in Bach (p.56)
792 */
793 int
794 bwrite(buf_t *bp)
795 {
796 int rv, sync, wasdelayed;
797 struct vnode *vp;
798 struct mount *mp;
799
800 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
801 KASSERT(!cv_has_waiters(&bp->b_done));
802
803 vp = bp->b_vp;
804 if (vp != NULL) {
805 KASSERT(bp->b_objlock == &vp->v_interlock);
806 if (vp->v_type == VBLK)
807 mp = vp->v_specmountpoint;
808 else
809 mp = vp->v_mount;
810 } else {
811 mp = NULL;
812 }
813
814 if (mp && mp->mnt_wapbl) {
815 if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
816 bdwrite(bp);
817 return 0;
818 }
819 }
820
821 /*
822 * Remember buffer type, to switch on it later. If the write was
823 * synchronous, but the file system was mounted with MNT_ASYNC,
824 * convert it to a delayed write.
825 * XXX note that this relies on delayed tape writes being converted
826 * to async, not sync writes (which is safe, but ugly).
827 */
828 sync = !ISSET(bp->b_flags, B_ASYNC);
829 if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) {
830 bdwrite(bp);
831 return (0);
832 }
833
834 /*
835 * Collect statistics on synchronous and asynchronous writes.
836 * Writes to block devices are charged to their associated
837 * filesystem (if any).
838 */
839 if (mp != NULL) {
840 if (sync)
841 mp->mnt_stat.f_syncwrites++;
842 else
843 mp->mnt_stat.f_asyncwrites++;
844 }
845
846 /*
847 * Pay for the I/O operation and make sure the buf is on the correct
848 * vnode queue.
849 */
850 bp->b_error = 0;
851 wasdelayed = ISSET(bp->b_oflags, BO_DELWRI);
852 CLR(bp->b_flags, B_READ);
853 if (wasdelayed) {
854 mutex_enter(&bufcache_lock);
855 mutex_enter(bp->b_objlock);
856 CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
857 reassignbuf(bp, bp->b_vp);
858 mutex_exit(&bufcache_lock);
859 } else {
860 curlwp->l_ru.ru_oublock++;
861 mutex_enter(bp->b_objlock);
862 CLR(bp->b_oflags, BO_DONE | BO_DELWRI);
863 }
864 if (vp != NULL)
865 vp->v_numoutput++;
866 mutex_exit(bp->b_objlock);
867
868 /* Initiate disk write. */
869 if (sync)
870 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
871 else
872 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
873
874 VOP_STRATEGY(vp, bp);
875
876 if (sync) {
877 /* If I/O was synchronous, wait for it to complete. */
878 rv = biowait(bp);
879
880 /* Release the buffer. */
881 brelse(bp, 0);
882
883 return (rv);
884 } else {
885 return (0);
886 }
887 }
888
889 int
890 vn_bwrite(void *v)
891 {
892 struct vop_bwrite_args *ap = v;
893
894 return (bwrite(ap->a_bp));
895 }
896
897 /*
898 * Delayed write.
899 *
900 * The buffer is marked dirty, but is not queued for I/O.
901 * This routine should be used when the buffer is expected
902 * to be modified again soon, typically a small write that
903 * partially fills a buffer.
904 *
905 * NB: magnetic tapes cannot be delayed; they must be
906 * written in the order that the writes are requested.
907 *
908 * Described in Leffler, et al. (pp. 208-213).
909 */
910 void
911 bdwrite(buf_t *bp)
912 {
913
914 KASSERT(bp->b_vp == NULL || bp->b_vp->v_tag != VT_UFS ||
915 bp->b_vp->v_type == VBLK || ISSET(bp->b_flags, B_COWDONE));
916 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
917 KASSERT(!cv_has_waiters(&bp->b_done));
918
919 /* If this is a tape block, write the block now. */
920 if (bdev_type(bp->b_dev) == D_TAPE) {
921 bawrite(bp);
922 return;
923 }
924
925 if (wapbl_vphaswapbl(bp->b_vp)) {
926 struct mount *mp = wapbl_vptomp(bp->b_vp);
927
928 if (bp->b_iodone != mp->mnt_wapbl_op->wo_wapbl_biodone) {
929 WAPBL_ADD_BUF(mp, bp);
930 }
931 }
932
933 /*
934 * If the block hasn't been seen before:
935 * (1) Mark it as having been seen,
936 * (2) Charge for the write,
937 * (3) Make sure it's on its vnode's correct block list.
938 */
939 KASSERT(bp->b_vp == NULL || bp->b_objlock == &bp->b_vp->v_interlock);
940
941 if (!ISSET(bp->b_oflags, BO_DELWRI)) {
942 mutex_enter(&bufcache_lock);
943 mutex_enter(bp->b_objlock);
944 SET(bp->b_oflags, BO_DELWRI);
945 curlwp->l_ru.ru_oublock++;
946 reassignbuf(bp, bp->b_vp);
947 mutex_exit(&bufcache_lock);
948 } else {
949 mutex_enter(bp->b_objlock);
950 }
951 /* Otherwise, the "write" is done, so mark and release the buffer. */
952 CLR(bp->b_oflags, BO_DONE);
953 mutex_exit(bp->b_objlock);
954
955 brelse(bp, 0);
956 }
957
958 /*
959 * Asynchronous block write; just an asynchronous bwrite().
960 */
961 void
962 bawrite(buf_t *bp)
963 {
964
965 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
966
967 SET(bp->b_flags, B_ASYNC);
968 VOP_BWRITE(bp);
969 }
970
971 /*
972 * Same as first half of bdwrite, mark buffer dirty, but do not release it.
973 * Call with the buffer interlock held.
974 *
975 * Note: called only from biodone() through ffs softdep's io_complete()
976 * Note2: smbfs also learned about bdirty().
977 */
978 void
979 bdirty(buf_t *bp)
980 {
981
982 KASSERT(mutex_owned(&bufcache_lock));
983 KASSERT(bp->b_objlock == &bp->b_vp->v_interlock);
984 KASSERT(mutex_owned(bp->b_objlock));
985 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
986
987 CLR(bp->b_cflags, BC_AGE);
988
989 if (!ISSET(bp->b_oflags, BO_DELWRI)) {
990 SET(bp->b_oflags, BO_DELWRI);
991 curlwp->l_ru.ru_oublock++;
992 reassignbuf(bp, bp->b_vp);
993 }
994 }
995
996
997 /*
998 * Release a buffer on to the free lists.
999 * Described in Bach (p. 46).
1000 */
1001 void
1002 brelsel(buf_t *bp, int set)
1003 {
1004 struct bqueue *bufq;
1005 struct vnode *vp;
1006
1007 KASSERT(mutex_owned(&bufcache_lock));
1008 KASSERT(!cv_has_waiters(&bp->b_done));
1009 KASSERT(bp->b_refcnt > 0);
1010
1011 SET(bp->b_cflags, set);
1012
1013 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1014 KASSERT(bp->b_iodone == NULL);
1015
1016 /* Wake up any processes waiting for any buffer to become free. */
1017 cv_signal(&needbuffer_cv);
1018
1019 /* Wake up any proceeses waiting for _this_ buffer to become */
1020 if (ISSET(bp->b_cflags, BC_WANTED))
1021 CLR(bp->b_cflags, BC_WANTED|BC_AGE);
1022
1023 /*
1024 * Determine which queue the buffer should be on, then put it there.
1025 */
1026
1027 /* If it's locked, don't report an error; try again later. */
1028 if (ISSET(bp->b_flags, B_LOCKED))
1029 bp->b_error = 0;
1030
1031 /* If it's not cacheable, or an error, mark it invalid. */
1032 if (ISSET(bp->b_cflags, BC_NOCACHE) || bp->b_error != 0)
1033 SET(bp->b_cflags, BC_INVAL);
1034
1035 if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1036 /*
1037 * This is a delayed write buffer that was just flushed to
1038 * disk. It is still on the LRU queue. If it's become
1039 * invalid, then we need to move it to a different queue;
1040 * otherwise leave it in its current position.
1041 */
1042 CLR(bp->b_cflags, BC_VFLUSH);
1043 if (!ISSET(bp->b_cflags, BC_INVAL|BC_AGE) &&
1044 !ISSET(bp->b_flags, B_LOCKED) && bp->b_error == 0) {
1045 KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 1));
1046 goto already_queued;
1047 } else {
1048 bremfree(bp);
1049 }
1050 }
1051
1052 KDASSERT(checkfreelist(bp, &bufqueues[BQ_AGE], 0));
1053 KDASSERT(checkfreelist(bp, &bufqueues[BQ_LRU], 0));
1054 KDASSERT(checkfreelist(bp, &bufqueues[BQ_LOCKED], 0));
1055
1056 if ((bp->b_bufsize <= 0) || ISSET(bp->b_cflags, BC_INVAL)) {
1057 /*
1058 * If it's invalid or empty, dissociate it from its vnode
1059 * and put on the head of the appropriate queue.
1060 */
1061 if (bioopsp != NULL)
1062 (*bioopsp->io_deallocate)(bp);
1063
1064 if (ISSET(bp->b_flags, B_LOCKED)) {
1065 if (wapbl_vphaswapbl(vp = bp->b_vp)) {
1066 struct mount *mp = wapbl_vptomp(vp);
1067
1068 KASSERT(bp->b_iodone
1069 != mp->mnt_wapbl_op->wo_wapbl_biodone);
1070 WAPBL_REMOVE_BUF(mp, bp);
1071 }
1072 }
1073
1074 mutex_enter(bp->b_objlock);
1075 CLR(bp->b_oflags, BO_DONE|BO_DELWRI);
1076 if ((vp = bp->b_vp) != NULL) {
1077 KASSERT(bp->b_objlock == &vp->v_interlock);
1078 reassignbuf(bp, bp->b_vp);
1079 brelvp(bp);
1080 mutex_exit(&vp->v_interlock);
1081 } else {
1082 KASSERT(bp->b_objlock == &buffer_lock);
1083 mutex_exit(bp->b_objlock);
1084 }
1085
1086 if (bp->b_bufsize <= 0)
1087 /* no data */
1088 goto already_queued;
1089 else
1090 /* invalid data */
1091 bufq = &bufqueues[BQ_AGE];
1092 binsheadfree(bp, bufq);
1093 } else {
1094 /*
1095 * It has valid data. Put it on the end of the appropriate
1096 * queue, so that it'll stick around for as long as possible.
1097 * If buf is AGE, but has dependencies, must put it on last
1098 * bufqueue to be scanned, ie LRU. This protects against the
1099 * livelock where BQ_AGE only has buffers with dependencies,
1100 * and we thus never get to the dependent buffers in BQ_LRU.
1101 */
1102 if (ISSET(bp->b_flags, B_LOCKED)) {
1103 /* locked in core */
1104 bufq = &bufqueues[BQ_LOCKED];
1105 } else if (!ISSET(bp->b_cflags, BC_AGE)) {
1106 /* valid data */
1107 bufq = &bufqueues[BQ_LRU];
1108 } else {
1109 /* stale but valid data */
1110 int has_deps;
1111
1112 if (bioopsp != NULL)
1113 has_deps = (*bioopsp->io_countdeps)(bp, 0);
1114 else
1115 has_deps = 0;
1116 bufq = has_deps ? &bufqueues[BQ_LRU] :
1117 &bufqueues[BQ_AGE];
1118 }
1119 binstailfree(bp, bufq);
1120 }
1121 already_queued:
1122 /* Unlock the buffer. */
1123 CLR(bp->b_cflags, BC_AGE|BC_BUSY|BC_NOCACHE);
1124 CLR(bp->b_flags, B_ASYNC);
1125 cv_broadcast(&bp->b_busy);
1126
1127 if (bp->b_bufsize <= 0)
1128 brele(bp);
1129 }
1130
1131 void
1132 brelse(buf_t *bp, int set)
1133 {
1134
1135 mutex_enter(&bufcache_lock);
1136 brelsel(bp, set);
1137 mutex_exit(&bufcache_lock);
1138 }
1139
1140 /*
1141 * Determine if a block is in the cache.
1142 * Just look on what would be its hash chain. If it's there, return
1143 * a pointer to it, unless it's marked invalid. If it's marked invalid,
1144 * we normally don't return the buffer, unless the caller explicitly
1145 * wants us to.
1146 */
1147 buf_t *
1148 incore(struct vnode *vp, daddr_t blkno)
1149 {
1150 buf_t *bp;
1151
1152 KASSERT(mutex_owned(&bufcache_lock));
1153
1154 /* Search hash chain */
1155 LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) {
1156 if (bp->b_lblkno == blkno && bp->b_vp == vp &&
1157 !ISSET(bp->b_cflags, BC_INVAL)) {
1158 KASSERT(bp->b_objlock == &vp->v_interlock);
1159 return (bp);
1160 }
1161 }
1162
1163 return (NULL);
1164 }
1165
1166 /*
1167 * Get a block of requested size that is associated with
1168 * a given vnode and block offset. If it is found in the
1169 * block cache, mark it as having been found, make it busy
1170 * and return it. Otherwise, return an empty block of the
1171 * correct size. It is up to the caller to insure that the
1172 * cached blocks be of the correct size.
1173 */
1174 buf_t *
1175 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo)
1176 {
1177 int err, preserve;
1178 buf_t *bp;
1179
1180 mutex_enter(&bufcache_lock);
1181 loop:
1182 bp = incore(vp, blkno);
1183 if (bp != NULL) {
1184 err = bbusy(bp, ((slpflag & PCATCH) != 0), slptimeo, NULL);
1185 if (err != 0) {
1186 if (err == EPASSTHROUGH)
1187 goto loop;
1188 mutex_exit(&bufcache_lock);
1189 return (NULL);
1190 }
1191 KASSERT(!cv_has_waiters(&bp->b_done));
1192 #ifdef DIAGNOSTIC
1193 if (ISSET(bp->b_oflags, BO_DONE|BO_DELWRI) &&
1194 bp->b_bcount < size && vp->v_type != VBLK)
1195 panic("getblk: block size invariant failed");
1196 #endif
1197 bremfree(bp);
1198 preserve = 1;
1199 } else {
1200 if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL)
1201 goto loop;
1202
1203 if (incore(vp, blkno) != NULL) {
1204 /* The block has come into memory in the meantime. */
1205 brelsel(bp, 0);
1206 goto loop;
1207 }
1208
1209 LIST_INSERT_HEAD(BUFHASH(vp, blkno), bp, b_hash);
1210 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno;
1211 mutex_enter(&vp->v_interlock);
1212 bgetvp(vp, bp);
1213 mutex_exit(&vp->v_interlock);
1214 preserve = 0;
1215 }
1216 mutex_exit(&bufcache_lock);
1217
1218 /*
1219 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes)
1220 * if we re-size buffers here.
1221 */
1222 if (ISSET(bp->b_flags, B_LOCKED)) {
1223 KASSERT(bp->b_bufsize >= size);
1224 } else {
1225 if (allocbuf(bp, size, preserve)) {
1226 mutex_enter(&bufcache_lock);
1227 LIST_REMOVE(bp, b_hash);
1228 mutex_exit(&bufcache_lock);
1229 brelse(bp, BC_INVAL);
1230 return NULL;
1231 }
1232 }
1233 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1234 return (bp);
1235 }
1236
1237 /*
1238 * Get an empty, disassociated buffer of given size.
1239 */
1240 buf_t *
1241 geteblk(int size)
1242 {
1243 buf_t *bp;
1244 int error;
1245
1246 mutex_enter(&bufcache_lock);
1247 while ((bp = getnewbuf(0, 0, 0)) == NULL)
1248 ;
1249
1250 SET(bp->b_cflags, BC_INVAL);
1251 LIST_INSERT_HEAD(&invalhash, bp, b_hash);
1252 mutex_exit(&bufcache_lock);
1253 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1254 error = allocbuf(bp, size, 0);
1255 KASSERT(error == 0);
1256 return (bp);
1257 }
1258
1259 /*
1260 * Expand or contract the actual memory allocated to a buffer.
1261 *
1262 * If the buffer shrinks, data is lost, so it's up to the
1263 * caller to have written it out *first*; this routine will not
1264 * start a write. If the buffer grows, it's the callers
1265 * responsibility to fill out the buffer's additional contents.
1266 */
1267 int
1268 allocbuf(buf_t *bp, int size, int preserve)
1269 {
1270 void *addr;
1271 vsize_t oldsize, desired_size;
1272 int oldcount;
1273 int delta;
1274
1275 desired_size = buf_roundsize(size);
1276 if (desired_size > MAXBSIZE)
1277 printf("allocbuf: buffer larger than MAXBSIZE requested");
1278
1279 oldcount = bp->b_bcount;
1280
1281 bp->b_bcount = size;
1282
1283 oldsize = bp->b_bufsize;
1284 if (oldsize == desired_size)
1285 goto out;
1286
1287 /*
1288 * If we want a buffer of a different size, re-allocate the
1289 * buffer's memory; copy old content only if needed.
1290 */
1291 addr = buf_malloc(desired_size);
1292 if (addr == NULL)
1293 return ENOMEM;
1294 if (preserve)
1295 memcpy(addr, bp->b_data, MIN(oldsize,desired_size));
1296 if (bp->b_data != NULL)
1297 buf_mrelease(bp->b_data, oldsize);
1298 bp->b_data = addr;
1299 bp->b_bufsize = desired_size;
1300
1301 /*
1302 * Update overall buffer memory counter (protected by bufcache_lock)
1303 */
1304 delta = (long)desired_size - (long)oldsize;
1305
1306 mutex_enter(&bufcache_lock);
1307 if ((bufmem += delta) > bufmem_hiwater) {
1308 /*
1309 * Need to trim overall memory usage.
1310 */
1311 while (buf_canrelease()) {
1312 if (curcpu()->ci_schedstate.spc_flags &
1313 SPCF_SHOULDYIELD) {
1314 mutex_exit(&bufcache_lock);
1315 preempt();
1316 mutex_enter(&bufcache_lock);
1317 }
1318 if (buf_trim() == 0)
1319 break;
1320 }
1321 }
1322 mutex_exit(&bufcache_lock);
1323
1324 out:
1325 if (wapbl_vphaswapbl(bp->b_vp))
1326 WAPBL_RESIZE_BUF(wapbl_vptomp(bp->b_vp), bp, oldsize, oldcount);
1327
1328 return 0;
1329 }
1330
1331 /*
1332 * Find a buffer which is available for use.
1333 * Select something from a free list.
1334 * Preference is to AGE list, then LRU list.
1335 *
1336 * Called with the buffer queues locked.
1337 * Return buffer locked.
1338 */
1339 buf_t *
1340 getnewbuf(int slpflag, int slptimeo, int from_bufq)
1341 {
1342 buf_t *bp;
1343 struct vnode *vp;
1344
1345 start:
1346 KASSERT(mutex_owned(&bufcache_lock));
1347
1348 /*
1349 * Get a new buffer from the pool.
1350 */
1351 if (!from_bufq && buf_lotsfree()) {
1352 mutex_exit(&bufcache_lock);
1353 bp = pool_cache_get(buf_cache, PR_NOWAIT);
1354 if (bp != NULL) {
1355 memset((char *)bp, 0, sizeof(*bp));
1356 buf_init(bp);
1357 SET(bp->b_cflags, BC_BUSY); /* mark buffer busy */
1358 mutex_enter(&bufcache_lock);
1359 #if defined(DIAGNOSTIC)
1360 bp->b_freelistindex = -1;
1361 #endif /* defined(DIAGNOSTIC) */
1362 return (bp);
1363 }
1364 mutex_enter(&bufcache_lock);
1365 }
1366
1367 KASSERT(mutex_owned(&bufcache_lock));
1368 if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue)) != NULL ||
1369 (bp = TAILQ_FIRST(&bufqueues[BQ_LRU].bq_queue)) != NULL) {
1370 KASSERT(!ISSET(bp->b_cflags, BC_BUSY) || ISSET(bp->b_cflags, BC_VFLUSH));
1371 bremfree(bp);
1372
1373 /* Buffer is no longer on free lists. */
1374 SET(bp->b_cflags, BC_BUSY);
1375 } else {
1376 /*
1377 * XXX: !from_bufq should be removed.
1378 */
1379 if (!from_bufq || curlwp != uvm.pagedaemon_lwp) {
1380 /* wait for a free buffer of any kind */
1381 if ((slpflag & PCATCH) != 0)
1382 (void)cv_timedwait_sig(&needbuffer_cv,
1383 &bufcache_lock, slptimeo);
1384 else
1385 (void)cv_timedwait(&needbuffer_cv,
1386 &bufcache_lock, slptimeo);
1387 }
1388 return (NULL);
1389 }
1390
1391 #ifdef DIAGNOSTIC
1392 if (bp->b_bufsize <= 0)
1393 panic("buffer %p: on queue but empty", bp);
1394 #endif
1395
1396 if (ISSET(bp->b_cflags, BC_VFLUSH)) {
1397 /*
1398 * This is a delayed write buffer being flushed to disk. Make
1399 * sure it gets aged out of the queue when it's finished, and
1400 * leave it off the LRU queue.
1401 */
1402 CLR(bp->b_cflags, BC_VFLUSH);
1403 SET(bp->b_cflags, BC_AGE);
1404 goto start;
1405 }
1406
1407 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1408 KASSERT(bp->b_refcnt > 0);
1409 KASSERT(!cv_has_waiters(&bp->b_done));
1410
1411 /*
1412 * If buffer was a delayed write, start it and return NULL
1413 * (since we might sleep while starting the write).
1414 */
1415 if (ISSET(bp->b_oflags, BO_DELWRI)) {
1416 /*
1417 * This buffer has gone through the LRU, so make sure it gets
1418 * reused ASAP.
1419 */
1420 SET(bp->b_cflags, BC_AGE);
1421 mutex_exit(&bufcache_lock);
1422 bawrite(bp);
1423 mutex_enter(&bufcache_lock);
1424 return (NULL);
1425 }
1426
1427 vp = bp->b_vp;
1428 if (bioopsp != NULL)
1429 (*bioopsp->io_deallocate)(bp);
1430
1431 /* clear out various other fields */
1432 bp->b_cflags = BC_BUSY;
1433 bp->b_oflags = 0;
1434 bp->b_flags = 0;
1435 bp->b_dev = NODEV;
1436 bp->b_blkno = 0;
1437 bp->b_lblkno = 0;
1438 bp->b_rawblkno = 0;
1439 bp->b_iodone = 0;
1440 bp->b_error = 0;
1441 bp->b_resid = 0;
1442 bp->b_bcount = 0;
1443
1444 LIST_REMOVE(bp, b_hash);
1445
1446 /* Disassociate us from our vnode, if we had one... */
1447 if (vp != NULL) {
1448 mutex_enter(&vp->v_interlock);
1449 brelvp(bp);
1450 mutex_exit(&vp->v_interlock);
1451 }
1452
1453 return (bp);
1454 }
1455
1456 /*
1457 * Attempt to free an aged buffer off the queues.
1458 * Called with queue lock held.
1459 * Returns the amount of buffer memory freed.
1460 */
1461 static int
1462 buf_trim(void)
1463 {
1464 buf_t *bp;
1465 long size = 0;
1466
1467 KASSERT(mutex_owned(&bufcache_lock));
1468
1469 /* Instruct getnewbuf() to get buffers off the queues */
1470 if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL)
1471 return 0;
1472
1473 KASSERT((bp->b_cflags & BC_WANTED) == 0);
1474 size = bp->b_bufsize;
1475 bufmem -= size;
1476 if (size > 0) {
1477 buf_mrelease(bp->b_data, size);
1478 bp->b_bcount = bp->b_bufsize = 0;
1479 }
1480 /* brelse() will return the buffer to the global buffer pool */
1481 brelsel(bp, 0);
1482 return size;
1483 }
1484
1485 int
1486 buf_drain(int n)
1487 {
1488 int size = 0, sz;
1489
1490 KASSERT(mutex_owned(&bufcache_lock));
1491
1492 while (size < n && bufmem > bufmem_lowater) {
1493 sz = buf_trim();
1494 if (sz <= 0)
1495 break;
1496 size += sz;
1497 }
1498
1499 return size;
1500 }
1501
1502 /*
1503 * Wait for operations on the buffer to complete.
1504 * When they do, extract and return the I/O's error value.
1505 */
1506 int
1507 biowait(buf_t *bp)
1508 {
1509
1510 KASSERT(ISSET(bp->b_cflags, BC_BUSY));
1511 KASSERT(bp->b_refcnt > 0);
1512
1513 mutex_enter(bp->b_objlock);
1514 while (!ISSET(bp->b_oflags, BO_DONE | BO_DELWRI))
1515 cv_wait(&bp->b_done, bp->b_objlock);
1516 mutex_exit(bp->b_objlock);
1517
1518 return bp->b_error;
1519 }
1520
1521 /*
1522 * Mark I/O complete on a buffer.
1523 *
1524 * If a callback has been requested, e.g. the pageout
1525 * daemon, do so. Otherwise, awaken waiting processes.
1526 *
1527 * [ Leffler, et al., says on p.247:
1528 * "This routine wakes up the blocked process, frees the buffer
1529 * for an asynchronous write, or, for a request by the pagedaemon
1530 * process, invokes a procedure specified in the buffer structure" ]
1531 *
1532 * In real life, the pagedaemon (or other system processes) wants
1533 * to do async stuff to, and doesn't want the buffer brelse()'d.
1534 * (for swap pager, that puts swap buffers on the free lists (!!!),
1535 * for the vn device, that puts malloc'd buffers on the free lists!)
1536 */
1537 void
1538 biodone(buf_t *bp)
1539 {
1540 int s;
1541
1542 KASSERT(!ISSET(bp->b_oflags, BO_DONE));
1543
1544 if (cpu_intr_p()) {
1545 /* From interrupt mode: defer to a soft interrupt. */
1546 s = splvm();
1547 TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_biodone, bp, b_actq);
1548 softint_schedule(biodone_sih);
1549 splx(s);
1550 } else {
1551 /* Process now - the buffer may be freed soon. */
1552 biodone2(bp);
1553 }
1554 }
1555
1556 static void
1557 biodone2(buf_t *bp)
1558 {
1559 void (*callout)(buf_t *);
1560
1561 if (bioopsp != NULL)
1562 (*bioopsp->io_complete)(bp);
1563
1564 mutex_enter(bp->b_objlock);
1565 /* Note that the transfer is done. */
1566 if (ISSET(bp->b_oflags, BO_DONE))
1567 panic("biodone2 already");
1568 CLR(bp->b_flags, B_COWDONE);
1569 SET(bp->b_oflags, BO_DONE);
1570 BIO_SETPRIO(bp, BPRIO_DEFAULT);
1571
1572 /* Wake up waiting writers. */
1573 if (!ISSET(bp->b_flags, B_READ))
1574 vwakeup(bp);
1575
1576 if ((callout = bp->b_iodone) != NULL) {
1577 /* Note callout done, then call out. */
1578 KASSERT(!cv_has_waiters(&bp->b_done));
1579 KERNEL_LOCK(1, NULL); /* XXXSMP */
1580 bp->b_iodone = NULL;
1581 mutex_exit(bp->b_objlock);
1582 (*callout)(bp);
1583 KERNEL_UNLOCK_ONE(NULL); /* XXXSMP */
1584 } else if (ISSET(bp->b_flags, B_ASYNC)) {
1585 /* If async, release. */
1586 KASSERT(!cv_has_waiters(&bp->b_done));
1587 mutex_exit(bp->b_objlock);
1588 brelse(bp, 0);
1589 } else {
1590 /* Otherwise just wake up waiters in biowait(). */
1591 cv_broadcast(&bp->b_done);
1592 mutex_exit(bp->b_objlock);
1593 }
1594 }
1595
1596 static void
1597 biointr(void *cookie)
1598 {
1599 struct cpu_info *ci;
1600 buf_t *bp;
1601 int s;
1602
1603 ci = curcpu();
1604
1605 while (!TAILQ_EMPTY(&ci->ci_data.cpu_biodone)) {
1606 KASSERT(curcpu() == ci);
1607
1608 s = splvm();
1609 bp = TAILQ_FIRST(&ci->ci_data.cpu_biodone);
1610 TAILQ_REMOVE(&ci->ci_data.cpu_biodone, bp, b_actq);
1611 splx(s);
1612
1613 biodone2(bp);
1614 }
1615 }
1616
1617 /*
1618 * Return a count of buffers on the "locked" queue.
1619 */
1620 int
1621 count_lock_queue(void)
1622 {
1623 buf_t *bp;
1624 int n = 0;
1625
1626 mutex_enter(&bufcache_lock);
1627 TAILQ_FOREACH(bp, &bufqueues[BQ_LOCKED].bq_queue, b_freelist)
1628 n++;
1629 mutex_exit(&bufcache_lock);
1630 return (n);
1631 }
1632
1633 /*
1634 * Wait for all buffers to complete I/O
1635 * Return the number of "stuck" buffers.
1636 */
1637 int
1638 buf_syncwait(void)
1639 {
1640 buf_t *bp;
1641 int iter, nbusy, nbusy_prev = 0, dcount, ihash;
1642
1643 dcount = 10000;
1644 for (iter = 0; iter < 20;) {
1645 mutex_enter(&bufcache_lock);
1646 nbusy = 0;
1647 for (ihash = 0; ihash < bufhash+1; ihash++) {
1648 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1649 if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY)
1650 nbusy += ((bp->b_flags & B_READ) == 0);
1651 /*
1652 * With soft updates, some buffers that are
1653 * written will be remarked as dirty until other
1654 * buffers are written.
1655 */
1656 if (bp->b_vp && bp->b_vp->v_mount
1657 && (bp->b_vp->v_mount->mnt_flag & MNT_SOFTDEP)
1658 && (bp->b_oflags & BO_DELWRI)) {
1659 bremfree(bp);
1660 bp->b_cflags |= BC_BUSY;
1661 nbusy++;
1662 mutex_exit(&bufcache_lock);
1663 bawrite(bp);
1664 if (dcount-- <= 0) {
1665 printf("softdep ");
1666 goto fail;
1667 }
1668 mutex_enter(&bufcache_lock);
1669 }
1670 }
1671 }
1672 mutex_exit(&bufcache_lock);
1673
1674 if (nbusy == 0)
1675 break;
1676 if (nbusy_prev == 0)
1677 nbusy_prev = nbusy;
1678 printf("%d ", nbusy);
1679 kpause("bflush", false, (iter == 0) ? 1 : hz / 25 * iter, NULL);
1680 if (nbusy >= nbusy_prev) /* we didn't flush anything */
1681 iter++;
1682 else
1683 nbusy_prev = nbusy;
1684 }
1685
1686 if (nbusy) {
1687 fail:;
1688 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY)
1689 printf("giving up\nPrinting vnodes for busy buffers\n");
1690 for (ihash = 0; ihash < bufhash+1; ihash++) {
1691 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) {
1692 if ((bp->b_cflags & (BC_BUSY|BC_INVAL)) == BC_BUSY &&
1693 (bp->b_flags & B_READ) == 0)
1694 vprint(NULL, bp->b_vp);
1695 }
1696 }
1697 #endif
1698 }
1699
1700 return nbusy;
1701 }
1702
1703 static void
1704 sysctl_fillbuf(buf_t *i, struct buf_sysctl *o)
1705 {
1706
1707 o->b_flags = i->b_flags | i->b_cflags | i->b_oflags;
1708 o->b_error = i->b_error;
1709 o->b_prio = i->b_prio;
1710 o->b_dev = i->b_dev;
1711 o->b_bufsize = i->b_bufsize;
1712 o->b_bcount = i->b_bcount;
1713 o->b_resid = i->b_resid;
1714 o->b_addr = PTRTOUINT64(i->b_data);
1715 o->b_blkno = i->b_blkno;
1716 o->b_rawblkno = i->b_rawblkno;
1717 o->b_iodone = PTRTOUINT64(i->b_iodone);
1718 o->b_proc = PTRTOUINT64(i->b_proc);
1719 o->b_vp = PTRTOUINT64(i->b_vp);
1720 o->b_saveaddr = PTRTOUINT64(i->b_saveaddr);
1721 o->b_lblkno = i->b_lblkno;
1722 }
1723
1724 #define KERN_BUFSLOP 20
1725 static int
1726 sysctl_dobuf(SYSCTLFN_ARGS)
1727 {
1728 buf_t *bp;
1729 struct buf_sysctl bs;
1730 struct bqueue *bq;
1731 char *dp;
1732 u_int i, op, arg;
1733 size_t len, needed, elem_size, out_size;
1734 int error, elem_count, retries;
1735
1736 if (namelen == 1 && name[0] == CTL_QUERY)
1737 return (sysctl_query(SYSCTLFN_CALL(rnode)));
1738
1739 if (namelen != 4)
1740 return (EINVAL);
1741
1742 retries = 100;
1743 retry:
1744 dp = oldp;
1745 len = (oldp != NULL) ? *oldlenp : 0;
1746 op = name[0];
1747 arg = name[1];
1748 elem_size = name[2];
1749 elem_count = name[3];
1750 out_size = MIN(sizeof(bs), elem_size);
1751
1752 /*
1753 * at the moment, these are just "placeholders" to make the
1754 * API for retrieving kern.buf data more extensible in the
1755 * future.
1756 *
1757 * XXX kern.buf currently has "netbsd32" issues. hopefully
1758 * these will be resolved at a later point.
1759 */
1760 if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL ||
1761 elem_size < 1 || elem_count < 0)
1762 return (EINVAL);
1763
1764 error = 0;
1765 needed = 0;
1766 sysctl_unlock();
1767 mutex_enter(&bufcache_lock);
1768 for (i = 0; i < BQUEUES; i++) {
1769 bq = &bufqueues[i];
1770 TAILQ_FOREACH(bp, &bq->bq_queue, b_freelist) {
1771 bq->bq_marker = bp;
1772 if (len >= elem_size && elem_count > 0) {
1773 sysctl_fillbuf(bp, &bs);
1774 mutex_exit(&bufcache_lock);
1775 error = copyout(&bs, dp, out_size);
1776 mutex_enter(&bufcache_lock);
1777 if (error)
1778 break;
1779 if (bq->bq_marker != bp) {
1780 /*
1781 * This sysctl node is only for
1782 * statistics. Retry; if the
1783 * queue keeps changing, then
1784 * bail out.
1785 */
1786 if (retries-- == 0) {
1787 error = EAGAIN;
1788 break;
1789 }
1790 mutex_exit(&bufcache_lock);
1791 goto retry;
1792 }
1793 dp += elem_size;
1794 len -= elem_size;
1795 }
1796 if (elem_count > 0) {
1797 needed += elem_size;
1798 if (elem_count != INT_MAX)
1799 elem_count--;
1800 }
1801 }
1802 if (error != 0)
1803 break;
1804 }
1805 mutex_exit(&bufcache_lock);
1806 sysctl_relock();
1807
1808 *oldlenp = needed;
1809 if (oldp == NULL)
1810 *oldlenp += KERN_BUFSLOP * sizeof(buf_t);
1811
1812 return (error);
1813 }
1814
1815 static int
1816 sysctl_bufvm_update(SYSCTLFN_ARGS)
1817 {
1818 int t, error, rv;
1819 struct sysctlnode node;
1820
1821 node = *rnode;
1822 node.sysctl_data = &t;
1823 t = *(int *)rnode->sysctl_data;
1824 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1825 if (error || newp == NULL)
1826 return (error);
1827
1828 if (t < 0)
1829 return EINVAL;
1830 if (rnode->sysctl_data == &bufcache) {
1831 if (t > 100)
1832 return (EINVAL);
1833 bufcache = t;
1834 buf_setwm();
1835 } else if (rnode->sysctl_data == &bufmem_lowater) {
1836 if (bufmem_hiwater - t < 16)
1837 return (EINVAL);
1838 bufmem_lowater = t;
1839 } else if (rnode->sysctl_data == &bufmem_hiwater) {
1840 if (t - bufmem_lowater < 16)
1841 return (EINVAL);
1842 bufmem_hiwater = t;
1843 } else
1844 return (EINVAL);
1845
1846 /* Drain until below new high water mark */
1847 sysctl_unlock();
1848 mutex_enter(&bufcache_lock);
1849 while ((t = bufmem - bufmem_hiwater) >= 0) {
1850 rv = buf_drain(t / (2 * 1024));
1851 if (rv <= 0)
1852 break;
1853 }
1854 mutex_exit(&bufcache_lock);
1855 sysctl_relock();
1856
1857 return 0;
1858 }
1859
1860 SYSCTL_SETUP(sysctl_kern_buf_setup, "sysctl kern.buf subtree setup")
1861 {
1862
1863 sysctl_createv(clog, 0, NULL, NULL,
1864 CTLFLAG_PERMANENT,
1865 CTLTYPE_NODE, "kern", NULL,
1866 NULL, 0, NULL, 0,
1867 CTL_KERN, CTL_EOL);
1868 sysctl_createv(clog, 0, NULL, NULL,
1869 CTLFLAG_PERMANENT,
1870 CTLTYPE_NODE, "buf",
1871 SYSCTL_DESCR("Kernel buffer cache information"),
1872 sysctl_dobuf, 0, NULL, 0,
1873 CTL_KERN, KERN_BUF, CTL_EOL);
1874 }
1875
1876 SYSCTL_SETUP(sysctl_vm_buf_setup, "sysctl vm.buf* subtree setup")
1877 {
1878
1879 sysctl_createv(clog, 0, NULL, NULL,
1880 CTLFLAG_PERMANENT,
1881 CTLTYPE_NODE, "vm", NULL,
1882 NULL, 0, NULL, 0,
1883 CTL_VM, CTL_EOL);
1884
1885 sysctl_createv(clog, 0, NULL, NULL,
1886 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1887 CTLTYPE_INT, "bufcache",
1888 SYSCTL_DESCR("Percentage of physical memory to use for "
1889 "buffer cache"),
1890 sysctl_bufvm_update, 0, &bufcache, 0,
1891 CTL_VM, CTL_CREATE, CTL_EOL);
1892 sysctl_createv(clog, 0, NULL, NULL,
1893 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
1894 CTLTYPE_INT, "bufmem",
1895 SYSCTL_DESCR("Amount of kernel memory used by buffer "
1896 "cache"),
1897 NULL, 0, &bufmem, 0,
1898 CTL_VM, CTL_CREATE, CTL_EOL);
1899 sysctl_createv(clog, 0, NULL, NULL,
1900 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1901 CTLTYPE_INT, "bufmem_lowater",
1902 SYSCTL_DESCR("Minimum amount of kernel memory to "
1903 "reserve for buffer cache"),
1904 sysctl_bufvm_update, 0, &bufmem_lowater, 0,
1905 CTL_VM, CTL_CREATE, CTL_EOL);
1906 sysctl_createv(clog, 0, NULL, NULL,
1907 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
1908 CTLTYPE_INT, "bufmem_hiwater",
1909 SYSCTL_DESCR("Maximum amount of kernel memory to use "
1910 "for buffer cache"),
1911 sysctl_bufvm_update, 0, &bufmem_hiwater, 0,
1912 CTL_VM, CTL_CREATE, CTL_EOL);
1913 }
1914
1915 #ifdef DEBUG
1916 /*
1917 * Print out statistics on the current allocation of the buffer pool.
1918 * Can be enabled to print out on every ``sync'' by setting "syncprt"
1919 * in vfs_syscalls.c using sysctl.
1920 */
1921 void
1922 vfs_bufstats(void)
1923 {
1924 int i, j, count;
1925 buf_t *bp;
1926 struct bqueue *dp;
1927 int counts[(MAXBSIZE / PAGE_SIZE) + 1];
1928 static const char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" };
1929
1930 for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) {
1931 count = 0;
1932 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1933 counts[j] = 0;
1934 TAILQ_FOREACH(bp, &dp->bq_queue, b_freelist) {
1935 counts[bp->b_bufsize/PAGE_SIZE]++;
1936 count++;
1937 }
1938 printf("%s: total-%d", bname[i], count);
1939 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++)
1940 if (counts[j] != 0)
1941 printf(", %d-%d", j * PAGE_SIZE, counts[j]);
1942 printf("\n");
1943 }
1944 }
1945 #endif /* DEBUG */
1946
1947 /* ------------------------------ */
1948
1949 buf_t *
1950 getiobuf(struct vnode *vp, bool waitok)
1951 {
1952 buf_t *bp;
1953
1954 bp = pool_cache_get(bufio_cache, (waitok ? PR_WAITOK : PR_NOWAIT));
1955 if (bp == NULL)
1956 return bp;
1957
1958 buf_init(bp);
1959
1960 if ((bp->b_vp = vp) == NULL)
1961 bp->b_objlock = &buffer_lock;
1962 else
1963 bp->b_objlock = &vp->v_interlock;
1964
1965 return bp;
1966 }
1967
1968 void
1969 putiobuf(buf_t *bp)
1970 {
1971
1972 buf_destroy(bp);
1973 pool_cache_put(bufio_cache, bp);
1974 }
1975
1976 /*
1977 * nestiobuf_iodone: b_iodone callback for nested buffers.
1978 */
1979
1980 void
1981 nestiobuf_iodone(buf_t *bp)
1982 {
1983 buf_t *mbp = bp->b_private;
1984 int error;
1985 int donebytes;
1986
1987 KASSERT(bp->b_bcount <= bp->b_bufsize);
1988 KASSERT(mbp != bp);
1989
1990 error = bp->b_error;
1991 if (bp->b_error == 0 &&
1992 (bp->b_bcount < bp->b_bufsize || bp->b_resid > 0)) {
1993 /*
1994 * Not all got transfered, raise an error. We have no way to
1995 * propagate these conditions to mbp.
1996 */
1997 error = EIO;
1998 }
1999
2000 donebytes = bp->b_bufsize;
2001
2002 putiobuf(bp);
2003 nestiobuf_done(mbp, donebytes, error);
2004 }
2005
2006 /*
2007 * nestiobuf_setup: setup a "nested" buffer.
2008 *
2009 * => 'mbp' is a "master" buffer which is being divided into sub pieces.
2010 * => 'bp' should be a buffer allocated by getiobuf.
2011 * => 'offset' is a byte offset in the master buffer.
2012 * => 'size' is a size in bytes of this nested buffer.
2013 */
2014
2015 void
2016 nestiobuf_setup(buf_t *mbp, buf_t *bp, int offset, size_t size)
2017 {
2018 const int b_read = mbp->b_flags & B_READ;
2019 struct vnode *vp = mbp->b_vp;
2020
2021 KASSERT(mbp->b_bcount >= offset + size);
2022 bp->b_vp = vp;
2023 bp->b_dev = mbp->b_dev;
2024 bp->b_objlock = mbp->b_objlock;
2025 bp->b_cflags = BC_BUSY;
2026 bp->b_flags = B_ASYNC | b_read;
2027 bp->b_iodone = nestiobuf_iodone;
2028 bp->b_data = (char *)mbp->b_data + offset;
2029 bp->b_resid = bp->b_bcount = size;
2030 bp->b_bufsize = bp->b_bcount;
2031 bp->b_private = mbp;
2032 BIO_COPYPRIO(bp, mbp);
2033 if (!b_read && vp != NULL) {
2034 mutex_enter(&vp->v_interlock);
2035 vp->v_numoutput++;
2036 mutex_exit(&vp->v_interlock);
2037 }
2038 }
2039
2040 /*
2041 * nestiobuf_done: propagate completion to the master buffer.
2042 *
2043 * => 'donebytes' specifies how many bytes in the 'mbp' is completed.
2044 * => 'error' is an errno(2) that 'donebytes' has been completed with.
2045 */
2046
2047 void
2048 nestiobuf_done(buf_t *mbp, int donebytes, int error)
2049 {
2050
2051 if (donebytes == 0) {
2052 return;
2053 }
2054 mutex_enter(mbp->b_objlock);
2055 KASSERT(mbp->b_resid >= donebytes);
2056 mbp->b_resid -= donebytes;
2057 if (error)
2058 mbp->b_error = error;
2059 if (mbp->b_resid == 0) {
2060 mutex_exit(mbp->b_objlock);
2061 biodone(mbp);
2062 } else
2063 mutex_exit(mbp->b_objlock);
2064 }
2065
2066 void
2067 buf_init(buf_t *bp)
2068 {
2069
2070 LIST_INIT(&bp->b_dep);
2071 cv_init(&bp->b_busy, "biolock");
2072 cv_init(&bp->b_done, "biowait");
2073 bp->b_dev = NODEV;
2074 bp->b_error = 0;
2075 bp->b_flags = 0;
2076 bp->b_cflags = 0;
2077 bp->b_oflags = 0;
2078 bp->b_objlock = &buffer_lock;
2079 bp->b_iodone = NULL;
2080 bp->b_refcnt = 1;
2081 bp->b_dev = NODEV;
2082 bp->b_vnbufs.le_next = NOLIST;
2083 BIO_SETPRIO(bp, BPRIO_DEFAULT);
2084 }
2085
2086 void
2087 buf_destroy(buf_t *bp)
2088 {
2089
2090 cv_destroy(&bp->b_done);
2091 cv_destroy(&bp->b_busy);
2092 }
2093
2094 int
2095 bbusy(buf_t *bp, bool intr, int timo, kmutex_t *interlock)
2096 {
2097 int error;
2098
2099 KASSERT(mutex_owned(&bufcache_lock));
2100
2101 if ((bp->b_cflags & BC_BUSY) != 0) {
2102 if (curlwp == uvm.pagedaemon_lwp)
2103 return EDEADLK;
2104 bp->b_cflags |= BC_WANTED;
2105 bref(bp);
2106 if (interlock != NULL)
2107 mutex_exit(interlock);
2108 if (intr) {
2109 error = cv_timedwait_sig(&bp->b_busy, &bufcache_lock,
2110 timo);
2111 } else {
2112 error = cv_timedwait(&bp->b_busy, &bufcache_lock,
2113 timo);
2114 }
2115 brele(bp);
2116 if (interlock != NULL)
2117 mutex_enter(interlock);
2118 if (error != 0)
2119 return error;
2120 return EPASSTHROUGH;
2121 }
2122 bp->b_cflags |= BC_BUSY;
2123
2124 return 0;
2125 }
Cache object: 2846341b90b3c9d943301474c5e83c6f
|