1 /*-
2 * Copyright (c) 1993
3 * The Regents of the University of California. All rights reserved.
4 * Modifications/enhancements:
5 * Copyright (c) 1995 John S. Dyson. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 4. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * @(#)vfs_cluster.c 8.7 (Berkeley) 2/13/94
32 */
33
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36
37 #include "opt_debug_cluster.h"
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/proc.h>
43 #include <sys/bio.h>
44 #include <sys/buf.h>
45 #include <sys/vnode.h>
46 #include <sys/malloc.h>
47 #include <sys/mount.h>
48 #include <sys/resourcevar.h>
49 #include <sys/vmmeter.h>
50 #include <vm/vm.h>
51 #include <vm/vm_object.h>
52 #include <vm/vm_page.h>
53 #include <sys/sysctl.h>
54
55 #if defined(CLUSTERDEBUG)
56 static int rcluster= 0;
57 SYSCTL_INT(_debug, OID_AUTO, rcluster, CTLFLAG_RW, &rcluster, 0,
58 "Debug VFS clustering code");
59 #endif
60
61 static MALLOC_DEFINE(M_SEGMENT, "cluster_save buffer", "cluster_save buffer");
62
63 static struct cluster_save *
64 cluster_collectbufs(struct vnode *vp, struct buf *last_bp);
65 static struct buf *
66 cluster_rbuild(struct vnode *vp, u_quad_t filesize, daddr_t lbn,
67 daddr_t blkno, long size, int run, struct buf *fbp);
68
69 static int write_behind = 1;
70 SYSCTL_INT(_vfs, OID_AUTO, write_behind, CTLFLAG_RW, &write_behind, 0,
71 "Cluster write-behind; 0: disable, 1: enable, 2: backed off");
72
73 static int read_max = 8;
74 SYSCTL_INT(_vfs, OID_AUTO, read_max, CTLFLAG_RW, &read_max, 0,
75 "Cluster read-ahead max block count");
76
77 /* Page expended to mark partially backed buffers */
78 extern vm_page_t bogus_page;
79
80 /*
81 * Read data to a buf, including read-ahead if we find this to be beneficial.
82 * cluster_read replaces bread.
83 */
84 int
85 cluster_read(vp, filesize, lblkno, size, cred, totread, seqcount, bpp)
86 struct vnode *vp;
87 u_quad_t filesize;
88 daddr_t lblkno;
89 long size;
90 struct ucred *cred;
91 long totread;
92 int seqcount;
93 struct buf **bpp;
94 {
95 struct buf *bp, *rbp, *reqbp;
96 daddr_t blkno, origblkno;
97 int maxra, racluster;
98 int error, ncontig;
99 int i;
100
101 error = 0;
102
103 /*
104 * Try to limit the amount of read-ahead by a few
105 * ad-hoc parameters. This needs work!!!
106 */
107 racluster = vp->v_mount->mnt_iosize_max / size;
108 maxra = seqcount;
109 maxra = min(read_max, maxra);
110 maxra = min(nbuf/8, maxra);
111 if (((u_quad_t)(lblkno + maxra + 1) * size) > filesize)
112 maxra = (filesize / size) - lblkno;
113
114 /*
115 * get the requested block
116 */
117 *bpp = reqbp = bp = getblk(vp, lblkno, size, 0, 0, 0);
118 origblkno = lblkno;
119
120 /*
121 * if it is in the cache, then check to see if the reads have been
122 * sequential. If they have, then try some read-ahead, otherwise
123 * back-off on prospective read-aheads.
124 */
125 if (bp->b_flags & B_CACHE) {
126 if (!seqcount) {
127 return 0;
128 } else if ((bp->b_flags & B_RAM) == 0) {
129 return 0;
130 } else {
131 int s;
132 bp->b_flags &= ~B_RAM;
133 /*
134 * We do the spl here so that there is no window
135 * between the incore and the b_usecount increment
136 * below. We opt to keep the spl out of the loop
137 * for efficiency.
138 */
139 s = splbio();
140 VI_LOCK(vp);
141 for (i = 1; i < maxra; i++) {
142 /*
143 * Stop if the buffer does not exist or it
144 * is invalid (about to go away?)
145 */
146 rbp = gbincore(vp, lblkno+i);
147 if (rbp == NULL || (rbp->b_flags & B_INVAL))
148 break;
149
150 /*
151 * Set another read-ahead mark so we know
152 * to check again.
153 */
154 if (((i % racluster) == (racluster - 1)) ||
155 (i == (maxra - 1)))
156 rbp->b_flags |= B_RAM;
157 }
158 VI_UNLOCK(vp);
159 splx(s);
160 if (i >= maxra) {
161 return 0;
162 }
163 lblkno += i;
164 }
165 reqbp = bp = NULL;
166 /*
167 * If it isn't in the cache, then get a chunk from
168 * disk if sequential, otherwise just get the block.
169 */
170 } else {
171 off_t firstread = bp->b_offset;
172 int nblks;
173
174 KASSERT(bp->b_offset != NOOFFSET,
175 ("cluster_read: no buffer offset"));
176
177 ncontig = 0;
178
179 /*
180 * Compute the total number of blocks that we should read
181 * synchronously.
182 */
183 if (firstread + totread > filesize)
184 totread = filesize - firstread;
185 nblks = howmany(totread, size);
186 if (nblks > racluster)
187 nblks = racluster;
188
189 /*
190 * Now compute the number of contiguous blocks.
191 */
192 if (nblks > 1) {
193 error = VOP_BMAP(vp, lblkno, NULL,
194 &blkno, &ncontig, NULL);
195 /*
196 * If this failed to map just do the original block.
197 */
198 if (error || blkno == -1)
199 ncontig = 0;
200 }
201
202 /*
203 * If we have contiguous data available do a cluster
204 * otherwise just read the requested block.
205 */
206 if (ncontig) {
207 /* Account for our first block. */
208 ncontig = min(ncontig + 1, nblks);
209 if (ncontig < nblks)
210 nblks = ncontig;
211 bp = cluster_rbuild(vp, filesize, lblkno,
212 blkno, size, nblks, bp);
213 lblkno += (bp->b_bufsize / size);
214 } else {
215 bp->b_flags |= B_RAM;
216 bp->b_iocmd = BIO_READ;
217 lblkno += 1;
218 }
219 }
220
221 /*
222 * handle the synchronous read so that it is available ASAP.
223 */
224 if (bp) {
225 if ((bp->b_flags & B_CLUSTER) == 0) {
226 vfs_busy_pages(bp, 0);
227 }
228 bp->b_flags &= ~B_INVAL;
229 bp->b_ioflags &= ~BIO_ERROR;
230 if ((bp->b_flags & B_ASYNC) || bp->b_iodone != NULL)
231 BUF_KERNPROC(bp);
232 bp->b_iooffset = dbtob(bp->b_blkno);
233 error = VOP_STRATEGY(vp, bp);
234 curproc->p_stats->p_ru.ru_inblock++;
235 if (error)
236 return (error);
237 }
238
239 /*
240 * If we have been doing sequential I/O, then do some read-ahead.
241 */
242 while (lblkno < (origblkno + maxra)) {
243 error = VOP_BMAP(vp, lblkno, NULL, &blkno, &ncontig, NULL);
244 if (error)
245 break;
246
247 if (blkno == -1)
248 break;
249
250 /*
251 * We could throttle ncontig here by maxra but we might as
252 * well read the data if it is contiguous. We're throttled
253 * by racluster anyway.
254 */
255 if (ncontig) {
256 ncontig = min(ncontig + 1, racluster);
257 rbp = cluster_rbuild(vp, filesize, lblkno, blkno,
258 size, ncontig, NULL);
259 lblkno += (rbp->b_bufsize / size);
260 if (rbp->b_flags & B_DELWRI) {
261 bqrelse(rbp);
262 continue;
263 }
264 } else {
265 rbp = getblk(vp, lblkno, size, 0, 0, 0);
266 lblkno += 1;
267 if (rbp->b_flags & B_DELWRI) {
268 bqrelse(rbp);
269 continue;
270 }
271 rbp->b_flags |= B_ASYNC | B_RAM;
272 rbp->b_iocmd = BIO_READ;
273 rbp->b_blkno = blkno;
274 }
275 if (rbp->b_flags & B_CACHE) {
276 rbp->b_flags &= ~B_ASYNC;
277 bqrelse(rbp);
278 continue;
279 }
280 if ((rbp->b_flags & B_CLUSTER) == 0) {
281 vfs_busy_pages(rbp, 0);
282 }
283 rbp->b_flags &= ~B_INVAL;
284 rbp->b_ioflags &= ~BIO_ERROR;
285 if ((rbp->b_flags & B_ASYNC) || rbp->b_iodone != NULL)
286 BUF_KERNPROC(rbp);
287 rbp->b_iooffset = dbtob(rbp->b_blkno);
288 (void) VOP_STRATEGY(vp, rbp);
289 curproc->p_stats->p_ru.ru_inblock++;
290 }
291
292 if (reqbp)
293 return (bufwait(reqbp));
294 else
295 return (error);
296 }
297
298 /*
299 * If blocks are contiguous on disk, use this to provide clustered
300 * read ahead. We will read as many blocks as possible sequentially
301 * and then parcel them up into logical blocks in the buffer hash table.
302 */
303 static struct buf *
304 cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp)
305 struct vnode *vp;
306 u_quad_t filesize;
307 daddr_t lbn;
308 daddr_t blkno;
309 long size;
310 int run;
311 struct buf *fbp;
312 {
313 struct buf *bp, *tbp;
314 daddr_t bn;
315 int i, inc, j;
316
317 GIANT_REQUIRED;
318
319 KASSERT(size == vp->v_mount->mnt_stat.f_iosize,
320 ("cluster_rbuild: size %ld != filesize %jd\n",
321 size, (intmax_t)vp->v_mount->mnt_stat.f_iosize));
322
323 /*
324 * avoid a division
325 */
326 while ((u_quad_t) size * (lbn + run) > filesize) {
327 --run;
328 }
329
330 if (fbp) {
331 tbp = fbp;
332 tbp->b_iocmd = BIO_READ;
333 } else {
334 tbp = getblk(vp, lbn, size, 0, 0, 0);
335 if (tbp->b_flags & B_CACHE)
336 return tbp;
337 tbp->b_flags |= B_ASYNC | B_RAM;
338 tbp->b_iocmd = BIO_READ;
339 }
340
341 tbp->b_blkno = blkno;
342 if( (tbp->b_flags & B_MALLOC) ||
343 ((tbp->b_flags & B_VMIO) == 0) || (run <= 1) )
344 return tbp;
345
346 bp = trypbuf(&cluster_pbuf_freecnt);
347 if (bp == 0)
348 return tbp;
349
350 /*
351 * We are synthesizing a buffer out of vm_page_t's, but
352 * if the block size is not page aligned then the starting
353 * address may not be either. Inherit the b_data offset
354 * from the original buffer.
355 */
356 bp->b_data = (char *)((vm_offset_t)bp->b_data |
357 ((vm_offset_t)tbp->b_data & PAGE_MASK));
358 bp->b_flags = B_ASYNC | B_CLUSTER | B_VMIO;
359 bp->b_iocmd = BIO_READ;
360 bp->b_iodone = cluster_callback;
361 bp->b_blkno = blkno;
362 bp->b_lblkno = lbn;
363 bp->b_offset = tbp->b_offset;
364 KASSERT(bp->b_offset != NOOFFSET, ("cluster_rbuild: no buffer offset"));
365 pbgetvp(vp, bp);
366
367 TAILQ_INIT(&bp->b_cluster.cluster_head);
368
369 bp->b_bcount = 0;
370 bp->b_bufsize = 0;
371 bp->b_npages = 0;
372
373 inc = btodb(size);
374 for (bn = blkno, i = 0; i < run; ++i, bn += inc) {
375 if (i != 0) {
376 if ((bp->b_npages * PAGE_SIZE) +
377 round_page(size) > vp->v_mount->mnt_iosize_max) {
378 break;
379 }
380
381 tbp = getblk(vp, lbn + i, size, 0, 0, GB_LOCK_NOWAIT);
382
383 /* Don't wait around for locked bufs. */
384 if (tbp == NULL)
385 break;
386
387 /*
388 * Stop scanning if the buffer is fully valid
389 * (marked B_CACHE), or locked (may be doing a
390 * background write), or if the buffer is not
391 * VMIO backed. The clustering code can only deal
392 * with VMIO-backed buffers.
393 */
394 VI_LOCK(bp->b_vp);
395 if ((tbp->b_vflags & BV_BKGRDINPROG) ||
396 (tbp->b_flags & B_CACHE) ||
397 (tbp->b_flags & B_VMIO) == 0) {
398 VI_UNLOCK(bp->b_vp);
399 bqrelse(tbp);
400 break;
401 }
402 VI_UNLOCK(bp->b_vp);
403
404 /*
405 * The buffer must be completely invalid in order to
406 * take part in the cluster. If it is partially valid
407 * then we stop.
408 */
409 VM_OBJECT_LOCK(tbp->b_object);
410 for (j = 0;j < tbp->b_npages; j++) {
411 VM_OBJECT_LOCK_ASSERT(tbp->b_pages[j]->object,
412 MA_OWNED);
413 if (tbp->b_pages[j]->valid)
414 break;
415 }
416 VM_OBJECT_UNLOCK(tbp->b_object);
417 if (j != tbp->b_npages) {
418 bqrelse(tbp);
419 break;
420 }
421
422 /*
423 * Set a read-ahead mark as appropriate
424 */
425 if ((fbp && (i == 1)) || (i == (run - 1)))
426 tbp->b_flags |= B_RAM;
427
428 /*
429 * Set the buffer up for an async read (XXX should
430 * we do this only if we do not wind up brelse()ing?).
431 * Set the block number if it isn't set, otherwise
432 * if it is make sure it matches the block number we
433 * expect.
434 */
435 tbp->b_flags |= B_ASYNC;
436 tbp->b_iocmd = BIO_READ;
437 if (tbp->b_blkno == tbp->b_lblkno) {
438 tbp->b_blkno = bn;
439 } else if (tbp->b_blkno != bn) {
440 brelse(tbp);
441 break;
442 }
443 }
444 /*
445 * XXX fbp from caller may not be B_ASYNC, but we are going
446 * to biodone() it in cluster_callback() anyway
447 */
448 BUF_KERNPROC(tbp);
449 TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
450 tbp, b_cluster.cluster_entry);
451 VM_OBJECT_LOCK(tbp->b_object);
452 for (j = 0; j < tbp->b_npages; j += 1) {
453 vm_page_t m;
454 m = tbp->b_pages[j];
455 vm_page_io_start(m);
456 vm_object_pip_add(m->object, 1);
457 if ((bp->b_npages == 0) ||
458 (bp->b_pages[bp->b_npages-1] != m)) {
459 bp->b_pages[bp->b_npages] = m;
460 bp->b_npages++;
461 }
462 if ((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL)
463 tbp->b_pages[j] = bogus_page;
464 }
465 VM_OBJECT_UNLOCK(tbp->b_object);
466 /*
467 * XXX shouldn't this be += size for both, like in
468 * cluster_wbuild()?
469 *
470 * Don't inherit tbp->b_bufsize as it may be larger due to
471 * a non-page-aligned size. Instead just aggregate using
472 * 'size'.
473 */
474 if (tbp->b_bcount != size)
475 printf("warning: tbp->b_bcount wrong %ld vs %ld\n", tbp->b_bcount, size);
476 if (tbp->b_bufsize != size)
477 printf("warning: tbp->b_bufsize wrong %ld vs %ld\n", tbp->b_bufsize, size);
478 bp->b_bcount += size;
479 bp->b_bufsize += size;
480 }
481
482 /*
483 * Fully valid pages in the cluster are already good and do not need
484 * to be re-read from disk. Replace the page with bogus_page
485 */
486 VM_OBJECT_LOCK(bp->b_object);
487 for (j = 0; j < bp->b_npages; j++) {
488 VM_OBJECT_LOCK_ASSERT(bp->b_pages[j]->object, MA_OWNED);
489 if ((bp->b_pages[j]->valid & VM_PAGE_BITS_ALL) ==
490 VM_PAGE_BITS_ALL) {
491 bp->b_pages[j] = bogus_page;
492 }
493 }
494 VM_OBJECT_UNLOCK(bp->b_object);
495 if (bp->b_bufsize > bp->b_kvasize)
496 panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
497 bp->b_bufsize, bp->b_kvasize);
498 bp->b_kvasize = bp->b_bufsize;
499
500 pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
501 (vm_page_t *)bp->b_pages, bp->b_npages);
502 return (bp);
503 }
504
505 /*
506 * Cleanup after a clustered read or write.
507 * This is complicated by the fact that any of the buffers might have
508 * extra memory (if there were no empty buffer headers at allocbuf time)
509 * that we will need to shift around.
510 */
511 void
512 cluster_callback(bp)
513 struct buf *bp;
514 {
515 struct buf *nbp, *tbp;
516 int error = 0;
517
518 GIANT_REQUIRED;
519
520 /*
521 * Must propogate errors to all the components.
522 */
523 if (bp->b_ioflags & BIO_ERROR)
524 error = bp->b_error;
525
526 pmap_qremove(trunc_page((vm_offset_t) bp->b_data), bp->b_npages);
527 /*
528 * Move memory from the large cluster buffer into the component
529 * buffers and mark IO as done on these.
530 */
531 for (tbp = TAILQ_FIRST(&bp->b_cluster.cluster_head);
532 tbp; tbp = nbp) {
533 nbp = TAILQ_NEXT(&tbp->b_cluster, cluster_entry);
534 if (error) {
535 tbp->b_ioflags |= BIO_ERROR;
536 tbp->b_error = error;
537 } else {
538 tbp->b_dirtyoff = tbp->b_dirtyend = 0;
539 tbp->b_flags &= ~B_INVAL;
540 tbp->b_ioflags &= ~BIO_ERROR;
541 /*
542 * XXX the bdwrite()/bqrelse() issued during
543 * cluster building clears B_RELBUF (see bqrelse()
544 * comment). If direct I/O was specified, we have
545 * to restore it here to allow the buffer and VM
546 * to be freed.
547 */
548 if (tbp->b_flags & B_DIRECT)
549 tbp->b_flags |= B_RELBUF;
550 }
551 bufdone(tbp);
552 }
553 relpbuf(bp, &cluster_pbuf_freecnt);
554 }
555
556 /*
557 * cluster_wbuild_wb:
558 *
559 * Implement modified write build for cluster.
560 *
561 * write_behind = 0 write behind disabled
562 * write_behind = 1 write behind normal (default)
563 * write_behind = 2 write behind backed-off
564 */
565
566 static __inline int
567 cluster_wbuild_wb(struct vnode *vp, long size, daddr_t start_lbn, int len)
568 {
569 int r = 0;
570
571 switch(write_behind) {
572 case 2:
573 if (start_lbn < len)
574 break;
575 start_lbn -= len;
576 /* FALLTHROUGH */
577 case 1:
578 r = cluster_wbuild(vp, size, start_lbn, len);
579 /* FALLTHROUGH */
580 default:
581 /* FALLTHROUGH */
582 break;
583 }
584 return(r);
585 }
586
587 /*
588 * Do clustered write for FFS.
589 *
590 * Three cases:
591 * 1. Write is not sequential (write asynchronously)
592 * Write is sequential:
593 * 2. beginning of cluster - begin cluster
594 * 3. middle of a cluster - add to cluster
595 * 4. end of a cluster - asynchronously write cluster
596 */
597 void
598 cluster_write(bp, filesize, seqcount)
599 struct buf *bp;
600 u_quad_t filesize;
601 int seqcount;
602 {
603 struct vnode *vp;
604 daddr_t lbn;
605 int maxclen, cursize;
606 int lblocksize;
607 int async;
608
609 vp = bp->b_vp;
610 if (vp->v_type == VREG) {
611 async = vp->v_mount->mnt_flag & MNT_ASYNC;
612 lblocksize = vp->v_mount->mnt_stat.f_iosize;
613 } else {
614 async = 0;
615 lblocksize = bp->b_bufsize;
616 }
617 lbn = bp->b_lblkno;
618 KASSERT(bp->b_offset != NOOFFSET, ("cluster_write: no buffer offset"));
619
620 /* Initialize vnode to beginning of file. */
621 if (lbn == 0)
622 vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0;
623
624 if (vp->v_clen == 0 || lbn != vp->v_lastw + 1 ||
625 (bp->b_blkno != vp->v_lasta + btodb(lblocksize))) {
626 maxclen = vp->v_mount->mnt_iosize_max / lblocksize - 1;
627 if (vp->v_clen != 0) {
628 /*
629 * Next block is not sequential.
630 *
631 * If we are not writing at end of file, the process
632 * seeked to another point in the file since its last
633 * write, or we have reached our maximum cluster size,
634 * then push the previous cluster. Otherwise try
635 * reallocating to make it sequential.
636 *
637 * Change to algorithm: only push previous cluster if
638 * it was sequential from the point of view of the
639 * seqcount heuristic, otherwise leave the buffer
640 * intact so we can potentially optimize the I/O
641 * later on in the buf_daemon or update daemon
642 * flush.
643 */
644 cursize = vp->v_lastw - vp->v_cstart + 1;
645 if (((u_quad_t) bp->b_offset + lblocksize) != filesize ||
646 lbn != vp->v_lastw + 1 || vp->v_clen <= cursize) {
647 if (!async && seqcount > 0) {
648 cluster_wbuild_wb(vp, lblocksize,
649 vp->v_cstart, cursize);
650 }
651 } else {
652 struct buf **bpp, **endbp;
653 struct cluster_save *buflist;
654
655 buflist = cluster_collectbufs(vp, bp);
656 endbp = &buflist->bs_children
657 [buflist->bs_nchildren - 1];
658 if (VOP_REALLOCBLKS(vp, buflist)) {
659 /*
660 * Failed, push the previous cluster
661 * if *really* writing sequentially
662 * in the logical file (seqcount > 1),
663 * otherwise delay it in the hopes that
664 * the low level disk driver can
665 * optimize the write ordering.
666 */
667 for (bpp = buflist->bs_children;
668 bpp < endbp; bpp++)
669 brelse(*bpp);
670 free(buflist, M_SEGMENT);
671 if (seqcount > 1) {
672 cluster_wbuild_wb(vp,
673 lblocksize, vp->v_cstart,
674 cursize);
675 }
676 } else {
677 /*
678 * Succeeded, keep building cluster.
679 */
680 for (bpp = buflist->bs_children;
681 bpp <= endbp; bpp++)
682 bdwrite(*bpp);
683 free(buflist, M_SEGMENT);
684 vp->v_lastw = lbn;
685 vp->v_lasta = bp->b_blkno;
686 return;
687 }
688 }
689 }
690 /*
691 * Consider beginning a cluster. If at end of file, make
692 * cluster as large as possible, otherwise find size of
693 * existing cluster.
694 */
695 if ((vp->v_type == VREG) &&
696 ((u_quad_t) bp->b_offset + lblocksize) != filesize &&
697 (bp->b_blkno == bp->b_lblkno) &&
698 (VOP_BMAP(vp, lbn, NULL, &bp->b_blkno, &maxclen, NULL) ||
699 bp->b_blkno == -1)) {
700 bawrite(bp);
701 vp->v_clen = 0;
702 vp->v_lasta = bp->b_blkno;
703 vp->v_cstart = lbn + 1;
704 vp->v_lastw = lbn;
705 return;
706 }
707 vp->v_clen = maxclen;
708 if (!async && maxclen == 0) { /* I/O not contiguous */
709 vp->v_cstart = lbn + 1;
710 bawrite(bp);
711 } else { /* Wait for rest of cluster */
712 vp->v_cstart = lbn;
713 bdwrite(bp);
714 }
715 } else if (lbn == vp->v_cstart + vp->v_clen) {
716 /*
717 * At end of cluster, write it out if seqcount tells us we
718 * are operating sequentially, otherwise let the buf or
719 * update daemon handle it.
720 */
721 bdwrite(bp);
722 if (seqcount > 1)
723 cluster_wbuild_wb(vp, lblocksize, vp->v_cstart, vp->v_clen + 1);
724 vp->v_clen = 0;
725 vp->v_cstart = lbn + 1;
726 } else if (vm_page_count_severe()) {
727 /*
728 * We are low on memory, get it going NOW
729 */
730 bawrite(bp);
731 } else {
732 /*
733 * In the middle of a cluster, so just delay the I/O for now.
734 */
735 bdwrite(bp);
736 }
737 vp->v_lastw = lbn;
738 vp->v_lasta = bp->b_blkno;
739 }
740
741
742 /*
743 * This is an awful lot like cluster_rbuild...wish they could be combined.
744 * The last lbn argument is the current block on which I/O is being
745 * performed. Check to see that it doesn't fall in the middle of
746 * the current block (if last_bp == NULL).
747 */
748 int
749 cluster_wbuild(vp, size, start_lbn, len)
750 struct vnode *vp;
751 long size;
752 daddr_t start_lbn;
753 int len;
754 {
755 struct buf *bp, *tbp;
756 int i, j, s;
757 int totalwritten = 0;
758 int dbsize = btodb(size);
759
760 GIANT_REQUIRED;
761
762 while (len > 0) {
763 s = splbio();
764 /*
765 * If the buffer is not delayed-write (i.e. dirty), or it
766 * is delayed-write but either locked or inval, it cannot
767 * partake in the clustered write.
768 */
769 VI_LOCK(vp);
770 if ((tbp = gbincore(vp, start_lbn)) == NULL ||
771 (tbp->b_vflags & BV_BKGRDINPROG)) {
772 VI_UNLOCK(vp);
773 ++start_lbn;
774 --len;
775 splx(s);
776 continue;
777 }
778 if (BUF_LOCK(tbp,
779 LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK, VI_MTX(vp))) {
780 ++start_lbn;
781 --len;
782 splx(s);
783 continue;
784 }
785 if ((tbp->b_flags & (B_INVAL | B_DELWRI)) != B_DELWRI) {
786 BUF_UNLOCK(tbp);
787 ++start_lbn;
788 --len;
789 splx(s);
790 continue;
791 }
792 bremfree(tbp);
793 tbp->b_flags &= ~B_DONE;
794 splx(s);
795
796 /*
797 * Extra memory in the buffer, punt on this buffer.
798 * XXX we could handle this in most cases, but we would
799 * have to push the extra memory down to after our max
800 * possible cluster size and then potentially pull it back
801 * up if the cluster was terminated prematurely--too much
802 * hassle.
803 */
804 if (((tbp->b_flags & (B_CLUSTEROK | B_MALLOC | B_VMIO)) !=
805 (B_CLUSTEROK | B_VMIO)) ||
806 (tbp->b_bcount != tbp->b_bufsize) ||
807 (tbp->b_bcount != size) ||
808 (len == 1) ||
809 ((bp = getpbuf(&cluster_pbuf_freecnt)) == NULL)) {
810 totalwritten += tbp->b_bufsize;
811 bawrite(tbp);
812 ++start_lbn;
813 --len;
814 continue;
815 }
816
817 /*
818 * We got a pbuf to make the cluster in.
819 * so initialise it.
820 */
821 TAILQ_INIT(&bp->b_cluster.cluster_head);
822 bp->b_bcount = 0;
823 bp->b_magic = tbp->b_magic;
824 bp->b_op = tbp->b_op;
825 bp->b_bufsize = 0;
826 bp->b_npages = 0;
827 if (tbp->b_wcred != NOCRED)
828 bp->b_wcred = crhold(tbp->b_wcred);
829
830 bp->b_blkno = tbp->b_blkno;
831 bp->b_lblkno = tbp->b_lblkno;
832 bp->b_offset = tbp->b_offset;
833
834 /*
835 * We are synthesizing a buffer out of vm_page_t's, but
836 * if the block size is not page aligned then the starting
837 * address may not be either. Inherit the b_data offset
838 * from the original buffer.
839 */
840 bp->b_data = (char *)((vm_offset_t)bp->b_data |
841 ((vm_offset_t)tbp->b_data & PAGE_MASK));
842 bp->b_flags |= B_CLUSTER |
843 (tbp->b_flags & (B_VMIO | B_NEEDCOMMIT));
844 bp->b_iodone = cluster_callback;
845 pbgetvp(vp, bp);
846 /*
847 * From this location in the file, scan forward to see
848 * if there are buffers with adjacent data that need to
849 * be written as well.
850 */
851 for (i = 0; i < len; ++i, ++start_lbn) {
852 if (i != 0) { /* If not the first buffer */
853 s = splbio();
854 /*
855 * If the adjacent data is not even in core it
856 * can't need to be written.
857 */
858 VI_LOCK(vp);
859 if ((tbp = gbincore(vp, start_lbn)) == NULL ||
860 (tbp->b_vflags & BV_BKGRDINPROG)) {
861 VI_UNLOCK(vp);
862 splx(s);
863 break;
864 }
865
866 /*
867 * If it IS in core, but has different
868 * characteristics, or is locked (which
869 * means it could be undergoing a background
870 * I/O or be in a weird state), then don't
871 * cluster with it.
872 */
873 if (BUF_LOCK(tbp,
874 LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK,
875 VI_MTX(vp))) {
876 splx(s);
877 break;
878 }
879
880 if ((tbp->b_flags & (B_VMIO | B_CLUSTEROK |
881 B_INVAL | B_DELWRI | B_NEEDCOMMIT))
882 != (B_DELWRI | B_CLUSTEROK |
883 (bp->b_flags & (B_VMIO | B_NEEDCOMMIT))) ||
884 tbp->b_wcred != bp->b_wcred) {
885 BUF_UNLOCK(tbp);
886 splx(s);
887 break;
888 }
889
890 /*
891 * Check that the combined cluster
892 * would make sense with regard to pages
893 * and would not be too large
894 */
895 if ((tbp->b_bcount != size) ||
896 ((bp->b_blkno + (dbsize * i)) !=
897 tbp->b_blkno) ||
898 ((tbp->b_npages + bp->b_npages) >
899 (vp->v_mount->mnt_iosize_max / PAGE_SIZE))) {
900 BUF_UNLOCK(tbp);
901 splx(s);
902 break;
903 }
904 /*
905 * Ok, it's passed all the tests,
906 * so remove it from the free list
907 * and mark it busy. We will use it.
908 */
909 bremfree(tbp);
910 tbp->b_flags &= ~B_DONE;
911 splx(s);
912 } /* end of code for non-first buffers only */
913 /* check for latent dependencies to be handled */
914 if ((LIST_FIRST(&tbp->b_dep)) != NULL) {
915 tbp->b_iocmd = BIO_WRITE;
916 buf_start(tbp);
917 }
918 /*
919 * If the IO is via the VM then we do some
920 * special VM hackery (yuck). Since the buffer's
921 * block size may not be page-aligned it is possible
922 * for a page to be shared between two buffers. We
923 * have to get rid of the duplication when building
924 * the cluster.
925 */
926 if (tbp->b_flags & B_VMIO) {
927 vm_page_t m;
928
929 VM_OBJECT_LOCK(tbp->b_object);
930 if (i != 0) { /* if not first buffer */
931 for (j = 0; j < tbp->b_npages; j += 1) {
932 m = tbp->b_pages[j];
933 if (m->flags & PG_BUSY) {
934 VM_OBJECT_UNLOCK(
935 tbp->b_object);
936 bqrelse(tbp);
937 goto finishcluster;
938 }
939 }
940 }
941 for (j = 0; j < tbp->b_npages; j += 1) {
942 m = tbp->b_pages[j];
943 vm_page_io_start(m);
944 vm_object_pip_add(m->object, 1);
945 if ((bp->b_npages == 0) ||
946 (bp->b_pages[bp->b_npages - 1] != m)) {
947 bp->b_pages[bp->b_npages] = m;
948 bp->b_npages++;
949 }
950 }
951 VM_OBJECT_UNLOCK(tbp->b_object);
952 }
953 bp->b_bcount += size;
954 bp->b_bufsize += size;
955
956 s = splbio();
957 bundirty(tbp);
958 tbp->b_flags &= ~B_DONE;
959 tbp->b_ioflags &= ~BIO_ERROR;
960 tbp->b_flags |= B_ASYNC;
961 tbp->b_iocmd = BIO_WRITE;
962 reassignbuf(tbp); /* put on clean list */
963 VI_LOCK(tbp->b_vp);
964 ++tbp->b_vp->v_numoutput;
965 VI_UNLOCK(tbp->b_vp);
966 splx(s);
967 BUF_KERNPROC(tbp);
968 TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
969 tbp, b_cluster.cluster_entry);
970 }
971 finishcluster:
972 pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
973 (vm_page_t *) bp->b_pages, bp->b_npages);
974 if (bp->b_bufsize > bp->b_kvasize)
975 panic(
976 "cluster_wbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
977 bp->b_bufsize, bp->b_kvasize);
978 bp->b_kvasize = bp->b_bufsize;
979 totalwritten += bp->b_bufsize;
980 bp->b_dirtyoff = 0;
981 bp->b_dirtyend = bp->b_bufsize;
982 bawrite(bp);
983
984 len -= i;
985 }
986 return totalwritten;
987 }
988
989 /*
990 * Collect together all the buffers in a cluster.
991 * Plus add one additional buffer.
992 */
993 static struct cluster_save *
994 cluster_collectbufs(vp, last_bp)
995 struct vnode *vp;
996 struct buf *last_bp;
997 {
998 struct cluster_save *buflist;
999 struct buf *bp;
1000 daddr_t lbn;
1001 int i, len;
1002
1003 len = vp->v_lastw - vp->v_cstart + 1;
1004 buflist = malloc(sizeof(struct buf *) * (len + 1) + sizeof(*buflist),
1005 M_SEGMENT, M_WAITOK);
1006 buflist->bs_nchildren = 0;
1007 buflist->bs_children = (struct buf **) (buflist + 1);
1008 for (lbn = vp->v_cstart, i = 0; i < len; lbn++, i++) {
1009 (void) bread(vp, lbn, last_bp->b_bcount, NOCRED, &bp);
1010 buflist->bs_children[i] = bp;
1011 if (bp->b_blkno == bp->b_lblkno)
1012 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno,
1013 NULL, NULL);
1014 }
1015 buflist->bs_children[i] = bp = last_bp;
1016 if (bp->b_blkno == bp->b_lblkno)
1017 VOP_BMAP(bp->b_vp, bp->b_lblkno, NULL, &bp->b_blkno,
1018 NULL, NULL);
1019 buflist->bs_nchildren = i + 1;
1020 return (buflist);
1021 }
Cache object: 503139f8dd78540e3d33c91ec1a0ad94
|