Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/vm/vnode_pager.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1990 University of Utah. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 1993, 1994 John S. Dyson 6 * Copyright (c) 1995, David Greenman 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 41 */ 42 43 /* 44 * Page to/from files (vnodes). 45 */ 46 47 /* 48 * TODO: 49 * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will 50 * greatly re-simplify the vnode_pager. 51 */ 52 53 #include <sys/cdefs.h> 54 __FBSDID("$FreeBSD: releng/7.3/sys/vm/vnode_pager.c 192154 2009-05-15 19:54:19Z jhb $"); 55 56 #include <sys/param.h> 57 #include <sys/systm.h> 58 #include <sys/proc.h> 59 #include <sys/vnode.h> 60 #include <sys/mount.h> 61 #include <sys/bio.h> 62 #include <sys/buf.h> 63 #include <sys/vmmeter.h> 64 #include <sys/limits.h> 65 #include <sys/conf.h> 66 #include <sys/sf_buf.h> 67 68 #include <machine/atomic.h> 69 70 #include <vm/vm.h> 71 #include <vm/vm_object.h> 72 #include <vm/vm_page.h> 73 #include <vm/vm_pager.h> 74 #include <vm/vm_map.h> 75 #include <vm/vnode_pager.h> 76 #include <vm/vm_extern.h> 77 78 static int vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, 79 daddr_t *rtaddress, int *run); 80 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m); 81 static int vnode_pager_input_old(vm_object_t object, vm_page_t m); 82 static void vnode_pager_dealloc(vm_object_t); 83 static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int); 84 static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); 85 static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); 86 static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t); 87 88 struct pagerops vnodepagerops = { 89 .pgo_alloc = vnode_pager_alloc, 90 .pgo_dealloc = vnode_pager_dealloc, 91 .pgo_getpages = vnode_pager_getpages, 92 .pgo_putpages = vnode_pager_putpages, 93 .pgo_haspage = vnode_pager_haspage, 94 }; 95 96 int vnode_pbuf_freecnt; 97 98 /* Create the VM system backing object for this vnode */ 99 int 100 vnode_create_vobject(struct vnode *vp, off_t isize, struct thread *td) 101 { 102 vm_object_t object; 103 vm_ooffset_t size = isize; 104 struct vattr va; 105 106 if (!vn_isdisk(vp, NULL) && vn_canvmio(vp) == FALSE) 107 return (0); 108 109 while ((object = vp->v_object) != NULL) { 110 VM_OBJECT_LOCK(object); 111 if (!(object->flags & OBJ_DEAD)) { 112 VM_OBJECT_UNLOCK(object); 113 return (0); 114 } 115 VOP_UNLOCK(vp, 0, td); 116 vm_object_set_flag(object, OBJ_DISCONNECTWNT); 117 msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vodead", 0); 118 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td); 119 } 120 121 if (size == 0) { 122 if (vn_isdisk(vp, NULL)) { 123 size = IDX_TO_OFF(INT_MAX); 124 } else { 125 if (VOP_GETATTR(vp, &va, td->td_ucred, td) != 0) 126 return (0); 127 size = va.va_size; 128 } 129 } 130 131 object = vnode_pager_alloc(vp, size, 0, 0); 132 /* 133 * Dereference the reference we just created. This assumes 134 * that the object is associated with the vp. 135 */ 136 VM_OBJECT_LOCK(object); 137 object->ref_count--; 138 VM_OBJECT_UNLOCK(object); 139 vrele(vp); 140 141 KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object")); 142 143 return (0); 144 } 145 146 void 147 vnode_destroy_vobject(struct vnode *vp) 148 { 149 struct vm_object *obj; 150 151 obj = vp->v_object; 152 if (obj == NULL) 153 return; 154 ASSERT_VOP_ELOCKED(vp, "vnode_destroy_vobject"); 155 VM_OBJECT_LOCK(obj); 156 if (obj->ref_count == 0) { 157 /* 158 * vclean() may be called twice. The first time 159 * removes the primary reference to the object, 160 * the second time goes one further and is a 161 * special-case to terminate the object. 162 * 163 * don't double-terminate the object 164 */ 165 if ((obj->flags & OBJ_DEAD) == 0) 166 vm_object_terminate(obj); 167 else 168 VM_OBJECT_UNLOCK(obj); 169 } else { 170 /* 171 * Woe to the process that tries to page now :-). 172 */ 173 vm_pager_deallocate(obj); 174 VM_OBJECT_UNLOCK(obj); 175 } 176 vp->v_object = NULL; 177 } 178 179 180 /* 181 * Allocate (or lookup) pager for a vnode. 182 * Handle is a vnode pointer. 183 * 184 * MPSAFE 185 */ 186 vm_object_t 187 vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, 188 vm_ooffset_t offset) 189 { 190 vm_object_t object; 191 struct vnode *vp; 192 193 /* 194 * Pageout to vnode, no can do yet. 195 */ 196 if (handle == NULL) 197 return (NULL); 198 199 vp = (struct vnode *) handle; 200 201 /* 202 * If the object is being terminated, wait for it to 203 * go away. 204 */ 205 retry: 206 while ((object = vp->v_object) != NULL) { 207 VM_OBJECT_LOCK(object); 208 if ((object->flags & OBJ_DEAD) == 0) 209 break; 210 vm_object_set_flag(object, OBJ_DISCONNECTWNT); 211 msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vadead", 0); 212 } 213 214 if (vp->v_usecount == 0) 215 panic("vnode_pager_alloc: no vnode reference"); 216 217 if (object == NULL) { 218 /* 219 * Add an object of the appropriate size 220 */ 221 object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); 222 223 object->un_pager.vnp.vnp_size = size; 224 225 object->handle = handle; 226 if (VFS_NEEDSGIANT(vp->v_mount)) 227 vm_object_set_flag(object, OBJ_NEEDGIANT); 228 VI_LOCK(vp); 229 if (vp->v_object != NULL) { 230 /* 231 * Object has been created while we were sleeping 232 */ 233 VI_UNLOCK(vp); 234 vm_object_destroy(object); 235 goto retry; 236 } 237 vp->v_object = object; 238 VI_UNLOCK(vp); 239 } else { 240 object->ref_count++; 241 VM_OBJECT_UNLOCK(object); 242 } 243 vref(vp); 244 return (object); 245 } 246 247 /* 248 * The object must be locked. 249 */ 250 static void 251 vnode_pager_dealloc(object) 252 vm_object_t object; 253 { 254 struct vnode *vp = object->handle; 255 256 if (vp == NULL) 257 panic("vnode_pager_dealloc: pager already dealloced"); 258 259 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 260 vm_object_pip_wait(object, "vnpdea"); 261 262 object->handle = NULL; 263 object->type = OBJT_DEAD; 264 if (object->flags & OBJ_DISCONNECTWNT) { 265 vm_object_clear_flag(object, OBJ_DISCONNECTWNT); 266 wakeup(object); 267 } 268 ASSERT_VOP_ELOCKED(vp, "vnode_pager_dealloc"); 269 vp->v_object = NULL; 270 vp->v_vflag &= ~VV_TEXT; 271 } 272 273 static boolean_t 274 vnode_pager_haspage(object, pindex, before, after) 275 vm_object_t object; 276 vm_pindex_t pindex; 277 int *before; 278 int *after; 279 { 280 struct vnode *vp = object->handle; 281 daddr_t bn; 282 int err; 283 daddr_t reqblock; 284 int poff; 285 int bsize; 286 int pagesperblock, blocksperpage; 287 int vfslocked; 288 289 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 290 /* 291 * If no vp or vp is doomed or marked transparent to VM, we do not 292 * have the page. 293 */ 294 if (vp == NULL || vp->v_iflag & VI_DOOMED) 295 return FALSE; 296 /* 297 * If the offset is beyond end of file we do 298 * not have the page. 299 */ 300 if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size) 301 return FALSE; 302 303 bsize = vp->v_mount->mnt_stat.f_iosize; 304 pagesperblock = bsize / PAGE_SIZE; 305 blocksperpage = 0; 306 if (pagesperblock > 0) { 307 reqblock = pindex / pagesperblock; 308 } else { 309 blocksperpage = (PAGE_SIZE / bsize); 310 reqblock = pindex * blocksperpage; 311 } 312 VM_OBJECT_UNLOCK(object); 313 vfslocked = VFS_LOCK_GIANT(vp->v_mount); 314 err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before); 315 VFS_UNLOCK_GIANT(vfslocked); 316 VM_OBJECT_LOCK(object); 317 if (err) 318 return TRUE; 319 if (bn == -1) 320 return FALSE; 321 if (pagesperblock > 0) { 322 poff = pindex - (reqblock * pagesperblock); 323 if (before) { 324 *before *= pagesperblock; 325 *before += poff; 326 } 327 if (after) { 328 int numafter; 329 *after *= pagesperblock; 330 numafter = pagesperblock - (poff + 1); 331 if (IDX_TO_OFF(pindex + numafter) > 332 object->un_pager.vnp.vnp_size) { 333 numafter = 334 OFF_TO_IDX(object->un_pager.vnp.vnp_size) - 335 pindex; 336 } 337 *after += numafter; 338 } 339 } else { 340 if (before) { 341 *before /= blocksperpage; 342 } 343 344 if (after) { 345 *after /= blocksperpage; 346 } 347 } 348 return TRUE; 349 } 350 351 /* 352 * Lets the VM system know about a change in size for a file. 353 * We adjust our own internal size and flush any cached pages in 354 * the associated object that are affected by the size change. 355 * 356 * Note: this routine may be invoked as a result of a pager put 357 * operation (possibly at object termination time), so we must be careful. 358 */ 359 void 360 vnode_pager_setsize(vp, nsize) 361 struct vnode *vp; 362 vm_ooffset_t nsize; 363 { 364 vm_object_t object; 365 vm_page_t m; 366 vm_pindex_t nobjsize; 367 368 if ((object = vp->v_object) == NULL) 369 return; 370 VM_OBJECT_LOCK(object); 371 if (nsize == object->un_pager.vnp.vnp_size) { 372 /* 373 * Hasn't changed size 374 */ 375 VM_OBJECT_UNLOCK(object); 376 return; 377 } 378 nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); 379 if (nsize < object->un_pager.vnp.vnp_size) { 380 /* 381 * File has shrunk. Toss any cached pages beyond the new EOF. 382 */ 383 if (nobjsize < object->size) 384 vm_object_page_remove(object, nobjsize, object->size, 385 FALSE); 386 /* 387 * this gets rid of garbage at the end of a page that is now 388 * only partially backed by the vnode. 389 * 390 * XXX for some reason (I don't know yet), if we take a 391 * completely invalid page and mark it partially valid 392 * it can screw up NFS reads, so we don't allow the case. 393 */ 394 if ((nsize & PAGE_MASK) && 395 (m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL && 396 m->valid != 0) { 397 int base = (int)nsize & PAGE_MASK; 398 int size = PAGE_SIZE - base; 399 400 /* 401 * Clear out partial-page garbage in case 402 * the page has been mapped. 403 */ 404 pmap_zero_page_area(m, base, size); 405 406 /* 407 * Clear out partial-page dirty bits. This 408 * has the side effect of setting the valid 409 * bits, but that is ok. There are a bunch 410 * of places in the VM system where we expected 411 * m->dirty == VM_PAGE_BITS_ALL. The file EOF 412 * case is one of them. If the page is still 413 * partially dirty, make it fully dirty. 414 * 415 * note that we do not clear out the valid 416 * bits. This would prevent bogus_page 417 * replacement from working properly. 418 */ 419 vm_page_lock_queues(); 420 vm_page_set_validclean(m, base, size); 421 if (m->dirty != 0) 422 m->dirty = VM_PAGE_BITS_ALL; 423 vm_page_unlock_queues(); 424 } else if ((nsize & PAGE_MASK) && 425 __predict_false(object->cache != NULL)) { 426 vm_page_cache_free(object, OFF_TO_IDX(nsize), 427 nobjsize); 428 } 429 } 430 object->un_pager.vnp.vnp_size = nsize; 431 object->size = nobjsize; 432 VM_OBJECT_UNLOCK(object); 433 } 434 435 /* 436 * calculate the linear (byte) disk address of specified virtual 437 * file address 438 */ 439 static int 440 vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, daddr_t *rtaddress, 441 int *run) 442 { 443 int bsize; 444 int err; 445 daddr_t vblock; 446 daddr_t voffset; 447 448 if (address < 0) 449 return -1; 450 451 if (vp->v_iflag & VI_DOOMED) 452 return -1; 453 454 bsize = vp->v_mount->mnt_stat.f_iosize; 455 vblock = address / bsize; 456 voffset = address % bsize; 457 458 err = VOP_BMAP(vp, vblock, NULL, rtaddress, run, NULL); 459 if (err == 0) { 460 if (*rtaddress != -1) 461 *rtaddress += voffset / DEV_BSIZE; 462 if (run) { 463 *run += 1; 464 *run *= bsize/PAGE_SIZE; 465 *run -= voffset/PAGE_SIZE; 466 } 467 } 468 469 return (err); 470 } 471 472 /* 473 * small block filesystem vnode pager input 474 */ 475 static int 476 vnode_pager_input_smlfs(object, m) 477 vm_object_t object; 478 vm_page_t m; 479 { 480 int i; 481 struct vnode *vp; 482 struct bufobj *bo; 483 struct buf *bp; 484 struct sf_buf *sf; 485 daddr_t fileaddr; 486 vm_offset_t bsize; 487 int error = 0; 488 489 vp = object->handle; 490 if (vp->v_iflag & VI_DOOMED) 491 return VM_PAGER_BAD; 492 493 bsize = vp->v_mount->mnt_stat.f_iosize; 494 495 VOP_BMAP(vp, 0, &bo, 0, NULL, NULL); 496 497 sf = sf_buf_alloc(m, 0); 498 499 for (i = 0; i < PAGE_SIZE / bsize; i++) { 500 vm_ooffset_t address; 501 502 if (vm_page_bits(i * bsize, bsize) & m->valid) 503 continue; 504 505 address = IDX_TO_OFF(m->pindex) + i * bsize; 506 if (address >= object->un_pager.vnp.vnp_size) { 507 fileaddr = -1; 508 } else { 509 error = vnode_pager_addr(vp, address, &fileaddr, NULL); 510 if (error) 511 break; 512 } 513 if (fileaddr != -1) { 514 bp = getpbuf(&vnode_pbuf_freecnt); 515 516 /* build a minimal buffer header */ 517 bp->b_iocmd = BIO_READ; 518 bp->b_iodone = bdone; 519 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 520 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 521 bp->b_rcred = crhold(curthread->td_ucred); 522 bp->b_wcred = crhold(curthread->td_ucred); 523 bp->b_data = (caddr_t)sf_buf_kva(sf) + i * bsize; 524 bp->b_blkno = fileaddr; 525 pbgetbo(bo, bp); 526 bp->b_bcount = bsize; 527 bp->b_bufsize = bsize; 528 bp->b_runningbufspace = bp->b_bufsize; 529 atomic_add_long(&runningbufspace, bp->b_runningbufspace); 530 531 /* do the input */ 532 bp->b_iooffset = dbtob(bp->b_blkno); 533 bstrategy(bp); 534 535 bwait(bp, PVM, "vnsrd"); 536 537 if ((bp->b_ioflags & BIO_ERROR) != 0) 538 error = EIO; 539 540 /* 541 * free the buffer header back to the swap buffer pool 542 */ 543 pbrelbo(bp); 544 relpbuf(bp, &vnode_pbuf_freecnt); 545 if (error) 546 break; 547 548 VM_OBJECT_LOCK(object); 549 vm_page_lock_queues(); 550 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 551 vm_page_unlock_queues(); 552 VM_OBJECT_UNLOCK(object); 553 } else { 554 VM_OBJECT_LOCK(object); 555 vm_page_lock_queues(); 556 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 557 vm_page_unlock_queues(); 558 VM_OBJECT_UNLOCK(object); 559 bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize); 560 } 561 } 562 sf_buf_free(sf); 563 vm_page_lock_queues(); 564 pmap_clear_modify(m); 565 vm_page_unlock_queues(); 566 if (error) { 567 return VM_PAGER_ERROR; 568 } 569 return VM_PAGER_OK; 570 571 } 572 573 574 /* 575 * old style vnode pager input routine 576 */ 577 static int 578 vnode_pager_input_old(object, m) 579 vm_object_t object; 580 vm_page_t m; 581 { 582 struct uio auio; 583 struct iovec aiov; 584 int error; 585 int size; 586 struct sf_buf *sf; 587 struct vnode *vp; 588 589 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 590 error = 0; 591 592 /* 593 * Return failure if beyond current EOF 594 */ 595 if (IDX_TO_OFF(m->pindex) >= object->un_pager.vnp.vnp_size) { 596 return VM_PAGER_BAD; 597 } else { 598 size = PAGE_SIZE; 599 if (IDX_TO_OFF(m->pindex) + size > object->un_pager.vnp.vnp_size) 600 size = object->un_pager.vnp.vnp_size - IDX_TO_OFF(m->pindex); 601 vp = object->handle; 602 VM_OBJECT_UNLOCK(object); 603 604 /* 605 * Allocate a kernel virtual address and initialize so that 606 * we can use VOP_READ/WRITE routines. 607 */ 608 sf = sf_buf_alloc(m, 0); 609 610 aiov.iov_base = (caddr_t)sf_buf_kva(sf); 611 aiov.iov_len = size; 612 auio.uio_iov = &aiov; 613 auio.uio_iovcnt = 1; 614 auio.uio_offset = IDX_TO_OFF(m->pindex); 615 auio.uio_segflg = UIO_SYSSPACE; 616 auio.uio_rw = UIO_READ; 617 auio.uio_resid = size; 618 auio.uio_td = curthread; 619 620 error = VOP_READ(vp, &auio, 0, curthread->td_ucred); 621 if (!error) { 622 int count = size - auio.uio_resid; 623 624 if (count == 0) 625 error = EINVAL; 626 else if (count != PAGE_SIZE) 627 bzero((caddr_t)sf_buf_kva(sf) + count, 628 PAGE_SIZE - count); 629 } 630 sf_buf_free(sf); 631 632 VM_OBJECT_LOCK(object); 633 } 634 vm_page_lock_queues(); 635 pmap_clear_modify(m); 636 vm_page_undirty(m); 637 vm_page_unlock_queues(); 638 if (!error) 639 m->valid = VM_PAGE_BITS_ALL; 640 return error ? VM_PAGER_ERROR : VM_PAGER_OK; 641 } 642 643 /* 644 * generic vnode pager input routine 645 */ 646 647 /* 648 * Local media VFS's that do not implement their own VOP_GETPAGES 649 * should have their VOP_GETPAGES call to vnode_pager_generic_getpages() 650 * to implement the previous behaviour. 651 * 652 * All other FS's should use the bypass to get to the local media 653 * backing vp's VOP_GETPAGES. 654 */ 655 static int 656 vnode_pager_getpages(object, m, count, reqpage) 657 vm_object_t object; 658 vm_page_t *m; 659 int count; 660 int reqpage; 661 { 662 int rtval; 663 struct vnode *vp; 664 int bytes = count * PAGE_SIZE; 665 int vfslocked; 666 667 vp = object->handle; 668 VM_OBJECT_UNLOCK(object); 669 vfslocked = VFS_LOCK_GIANT(vp->v_mount); 670 rtval = VOP_GETPAGES(vp, m, bytes, reqpage, 0); 671 KASSERT(rtval != EOPNOTSUPP, 672 ("vnode_pager: FS getpages not implemented\n")); 673 VFS_UNLOCK_GIANT(vfslocked); 674 VM_OBJECT_LOCK(object); 675 return rtval; 676 } 677 678 /* 679 * This is now called from local media FS's to operate against their 680 * own vnodes if they fail to implement VOP_GETPAGES. 681 */ 682 int 683 vnode_pager_generic_getpages(vp, m, bytecount, reqpage) 684 struct vnode *vp; 685 vm_page_t *m; 686 int bytecount; 687 int reqpage; 688 { 689 vm_object_t object; 690 vm_offset_t kva; 691 off_t foff, tfoff, nextoff; 692 int i, j, size, bsize, first; 693 daddr_t firstaddr, reqblock; 694 struct bufobj *bo; 695 int runpg; 696 int runend; 697 struct buf *bp; 698 int count; 699 int error; 700 701 object = vp->v_object; 702 count = bytecount / PAGE_SIZE; 703 704 KASSERT(vp->v_type != VCHR && vp->v_type != VBLK, 705 ("vnode_pager_generic_getpages does not support devices")); 706 if (vp->v_iflag & VI_DOOMED) 707 return VM_PAGER_BAD; 708 709 bsize = vp->v_mount->mnt_stat.f_iosize; 710 711 /* get the UNDERLYING device for the file with VOP_BMAP() */ 712 713 /* 714 * originally, we did not check for an error return value -- assuming 715 * an fs always has a bmap entry point -- that assumption is wrong!!! 716 */ 717 foff = IDX_TO_OFF(m[reqpage]->pindex); 718 719 /* 720 * if we can't bmap, use old VOP code 721 */ 722 error = VOP_BMAP(vp, foff / bsize, &bo, &reqblock, NULL, NULL); 723 if (error == EOPNOTSUPP) { 724 VM_OBJECT_LOCK(object); 725 vm_page_lock_queues(); 726 for (i = 0; i < count; i++) 727 if (i != reqpage) 728 vm_page_free(m[i]); 729 vm_page_unlock_queues(); 730 PCPU_INC(cnt.v_vnodein); 731 PCPU_INC(cnt.v_vnodepgsin); 732 error = vnode_pager_input_old(object, m[reqpage]); 733 VM_OBJECT_UNLOCK(object); 734 return (error); 735 } else if (error != 0) { 736 VM_OBJECT_LOCK(object); 737 vm_page_lock_queues(); 738 for (i = 0; i < count; i++) 739 if (i != reqpage) 740 vm_page_free(m[i]); 741 vm_page_unlock_queues(); 742 VM_OBJECT_UNLOCK(object); 743 return (VM_PAGER_ERROR); 744 745 /* 746 * if the blocksize is smaller than a page size, then use 747 * special small filesystem code. NFS sometimes has a small 748 * blocksize, but it can handle large reads itself. 749 */ 750 } else if ((PAGE_SIZE / bsize) > 1 && 751 (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { 752 VM_OBJECT_LOCK(object); 753 vm_page_lock_queues(); 754 for (i = 0; i < count; i++) 755 if (i != reqpage) 756 vm_page_free(m[i]); 757 vm_page_unlock_queues(); 758 VM_OBJECT_UNLOCK(object); 759 PCPU_INC(cnt.v_vnodein); 760 PCPU_INC(cnt.v_vnodepgsin); 761 return vnode_pager_input_smlfs(object, m[reqpage]); 762 } 763 764 /* 765 * If we have a completely valid page available to us, we can 766 * clean up and return. Otherwise we have to re-read the 767 * media. 768 */ 769 VM_OBJECT_LOCK(object); 770 if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { 771 vm_page_lock_queues(); 772 for (i = 0; i < count; i++) 773 if (i != reqpage) 774 vm_page_free(m[i]); 775 vm_page_unlock_queues(); 776 VM_OBJECT_UNLOCK(object); 777 return VM_PAGER_OK; 778 } else if (reqblock == -1) { 779 pmap_zero_page(m[reqpage]); 780 vm_page_undirty(m[reqpage]); 781 m[reqpage]->valid = VM_PAGE_BITS_ALL; 782 vm_page_lock_queues(); 783 for (i = 0; i < count; i++) 784 if (i != reqpage) 785 vm_page_free(m[i]); 786 vm_page_unlock_queues(); 787 VM_OBJECT_UNLOCK(object); 788 return (VM_PAGER_OK); 789 } 790 m[reqpage]->valid = 0; 791 VM_OBJECT_UNLOCK(object); 792 793 /* 794 * here on direct device I/O 795 */ 796 firstaddr = -1; 797 798 /* 799 * calculate the run that includes the required page 800 */ 801 for (first = 0, i = 0; i < count; i = runend) { 802 if (vnode_pager_addr(vp, IDX_TO_OFF(m[i]->pindex), &firstaddr, 803 &runpg) != 0) { 804 VM_OBJECT_LOCK(object); 805 vm_page_lock_queues(); 806 for (; i < count; i++) 807 if (i != reqpage) 808 vm_page_free(m[i]); 809 vm_page_unlock_queues(); 810 VM_OBJECT_UNLOCK(object); 811 return (VM_PAGER_ERROR); 812 } 813 if (firstaddr == -1) { 814 VM_OBJECT_LOCK(object); 815 if (i == reqpage && foff < object->un_pager.vnp.vnp_size) { 816 panic("vnode_pager_getpages: unexpected missing page: firstaddr: %jd, foff: 0x%jx%08jx, vnp_size: 0x%jx%08jx", 817 (intmax_t)firstaddr, (uintmax_t)(foff >> 32), 818 (uintmax_t)foff, 819 (uintmax_t) 820 (object->un_pager.vnp.vnp_size >> 32), 821 (uintmax_t)object->un_pager.vnp.vnp_size); 822 } 823 vm_page_lock_queues(); 824 vm_page_free(m[i]); 825 vm_page_unlock_queues(); 826 VM_OBJECT_UNLOCK(object); 827 runend = i + 1; 828 first = runend; 829 continue; 830 } 831 runend = i + runpg; 832 if (runend <= reqpage) { 833 VM_OBJECT_LOCK(object); 834 vm_page_lock_queues(); 835 for (j = i; j < runend; j++) 836 vm_page_free(m[j]); 837 vm_page_unlock_queues(); 838 VM_OBJECT_UNLOCK(object); 839 } else { 840 if (runpg < (count - first)) { 841 VM_OBJECT_LOCK(object); 842 vm_page_lock_queues(); 843 for (i = first + runpg; i < count; i++) 844 vm_page_free(m[i]); 845 vm_page_unlock_queues(); 846 VM_OBJECT_UNLOCK(object); 847 count = first + runpg; 848 } 849 break; 850 } 851 first = runend; 852 } 853 854 /* 855 * the first and last page have been calculated now, move input pages 856 * to be zero based... 857 */ 858 if (first != 0) { 859 m += first; 860 count -= first; 861 reqpage -= first; 862 } 863 864 /* 865 * calculate the file virtual address for the transfer 866 */ 867 foff = IDX_TO_OFF(m[0]->pindex); 868 869 /* 870 * calculate the size of the transfer 871 */ 872 size = count * PAGE_SIZE; 873 KASSERT(count > 0, ("zero count")); 874 if ((foff + size) > object->un_pager.vnp.vnp_size) 875 size = object->un_pager.vnp.vnp_size - foff; 876 KASSERT(size > 0, ("zero size")); 877 878 /* 879 * round up physical size for real devices. 880 */ 881 if (1) { 882 int secmask = bo->bo_bsize - 1; 883 KASSERT(secmask < PAGE_SIZE && secmask > 0, 884 ("vnode_pager_generic_getpages: sector size %d too large", 885 secmask + 1)); 886 size = (size + secmask) & ~secmask; 887 } 888 889 bp = getpbuf(&vnode_pbuf_freecnt); 890 kva = (vm_offset_t) bp->b_data; 891 892 /* 893 * and map the pages to be read into the kva 894 */ 895 pmap_qenter(kva, m, count); 896 897 /* build a minimal buffer header */ 898 bp->b_iocmd = BIO_READ; 899 bp->b_iodone = bdone; 900 KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); 901 KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); 902 bp->b_rcred = crhold(curthread->td_ucred); 903 bp->b_wcred = crhold(curthread->td_ucred); 904 bp->b_blkno = firstaddr; 905 pbgetbo(bo, bp); 906 bp->b_bcount = size; 907 bp->b_bufsize = size; 908 bp->b_runningbufspace = bp->b_bufsize; 909 atomic_add_long(&runningbufspace, bp->b_runningbufspace); 910 911 PCPU_INC(cnt.v_vnodein); 912 PCPU_ADD(cnt.v_vnodepgsin, count); 913 914 /* do the input */ 915 bp->b_iooffset = dbtob(bp->b_blkno); 916 bstrategy(bp); 917 918 bwait(bp, PVM, "vnread"); 919 920 if ((bp->b_ioflags & BIO_ERROR) != 0) 921 error = EIO; 922 923 if (!error) { 924 if (size != count * PAGE_SIZE) 925 bzero((caddr_t) kva + size, PAGE_SIZE * count - size); 926 } 927 pmap_qremove(kva, count); 928 929 /* 930 * free the buffer header back to the swap buffer pool 931 */ 932 pbrelbo(bp); 933 relpbuf(bp, &vnode_pbuf_freecnt); 934 935 VM_OBJECT_LOCK(object); 936 vm_page_lock_queues(); 937 for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { 938 vm_page_t mt; 939 940 nextoff = tfoff + PAGE_SIZE; 941 mt = m[i]; 942 943 if (nextoff <= object->un_pager.vnp.vnp_size) { 944 /* 945 * Read filled up entire page. 946 */ 947 mt->valid = VM_PAGE_BITS_ALL; 948 vm_page_undirty(mt); /* should be an assert? XXX */ 949 pmap_clear_modify(mt); 950 } else { 951 /* 952 * Read did not fill up entire page. Since this 953 * is getpages, the page may be mapped, so we have 954 * to zero the invalid portions of the page even 955 * though we aren't setting them valid. 956 * 957 * Currently we do not set the entire page valid, 958 * we just try to clear the piece that we couldn't 959 * read. 960 */ 961 vm_page_set_validclean(mt, 0, 962 object->un_pager.vnp.vnp_size - tfoff); 963 /* handled by vm_fault now */ 964 /* vm_page_zero_invalid(mt, FALSE); */ 965 } 966 967 if (i != reqpage) { 968 969 /* 970 * whether or not to leave the page activated is up in 971 * the air, but we should put the page on a page queue 972 * somewhere. (it already is in the object). Result: 973 * It appears that empirical results show that 974 * deactivating pages is best. 975 */ 976 977 /* 978 * just in case someone was asking for this page we 979 * now tell them that it is ok to use 980 */ 981 if (!error) { 982 if (mt->oflags & VPO_WANTED) 983 vm_page_activate(mt); 984 else 985 vm_page_deactivate(mt); 986 vm_page_wakeup(mt); 987 } else { 988 vm_page_free(mt); 989 } 990 } 991 } 992 vm_page_unlock_queues(); 993 VM_OBJECT_UNLOCK(object); 994 if (error) { 995 printf("vnode_pager_getpages: I/O read error\n"); 996 } 997 return (error ? VM_PAGER_ERROR : VM_PAGER_OK); 998 } 999 1000 /* 1001 * EOPNOTSUPP is no longer legal. For local media VFS's that do not 1002 * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to 1003 * vnode_pager_generic_putpages() to implement the previous behaviour. 1004 * 1005 * All other FS's should use the bypass to get to the local media 1006 * backing vp's VOP_PUTPAGES. 1007 */ 1008 static void 1009 vnode_pager_putpages(object, m, count, sync, rtvals) 1010 vm_object_t object; 1011 vm_page_t *m; 1012 int count; 1013 boolean_t sync; 1014 int *rtvals; 1015 { 1016 int rtval; 1017 struct vnode *vp; 1018 struct mount *mp; 1019 int bytes = count * PAGE_SIZE; 1020 1021 /* 1022 * Force synchronous operation if we are extremely low on memory 1023 * to prevent a low-memory deadlock. VOP operations often need to 1024 * allocate more memory to initiate the I/O ( i.e. do a BMAP 1025 * operation ). The swapper handles the case by limiting the amount 1026 * of asynchronous I/O, but that sort of solution doesn't scale well 1027 * for the vnode pager without a lot of work. 1028 * 1029 * Also, the backing vnode's iodone routine may not wake the pageout 1030 * daemon up. This should be probably be addressed XXX. 1031 */ 1032 1033 if ((cnt.v_free_count + cnt.v_cache_count) < cnt.v_pageout_free_min) 1034 sync |= OBJPC_SYNC; 1035 1036 /* 1037 * Call device-specific putpages function 1038 */ 1039 vp = object->handle; 1040 VM_OBJECT_UNLOCK(object); 1041 if (vp->v_type != VREG) 1042 mp = NULL; 1043 rtval = VOP_PUTPAGES(vp, m, bytes, sync, rtvals, 0); 1044 KASSERT(rtval != EOPNOTSUPP, 1045 ("vnode_pager: stale FS putpages\n")); 1046 VM_OBJECT_LOCK(object); 1047 } 1048 1049 1050 /* 1051 * This is now called from local media FS's to operate against their 1052 * own vnodes if they fail to implement VOP_PUTPAGES. 1053 * 1054 * This is typically called indirectly via the pageout daemon and 1055 * clustering has already typically occured, so in general we ask the 1056 * underlying filesystem to write the data out asynchronously rather 1057 * then delayed. 1058 */ 1059 int 1060 vnode_pager_generic_putpages(vp, m, bytecount, flags, rtvals) 1061 struct vnode *vp; 1062 vm_page_t *m; 1063 int bytecount; 1064 int flags; 1065 int *rtvals; 1066 { 1067 int i; 1068 vm_object_t object; 1069 int count; 1070 1071 int maxsize, ncount; 1072 vm_ooffset_t poffset; 1073 struct uio auio; 1074 struct iovec aiov; 1075 int error; 1076 int ioflags; 1077 int ppscheck = 0; 1078 static struct timeval lastfail; 1079 static int curfail; 1080 1081 object = vp->v_object; 1082 count = bytecount / PAGE_SIZE; 1083 1084 for (i = 0; i < count; i++) 1085 rtvals[i] = VM_PAGER_AGAIN; 1086 1087 if ((int64_t)m[0]->pindex < 0) { 1088 printf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%lx)\n", 1089 (long)m[0]->pindex, (u_long)m[0]->dirty); 1090 rtvals[0] = VM_PAGER_BAD; 1091 return VM_PAGER_BAD; 1092 } 1093 1094 maxsize = count * PAGE_SIZE; 1095 ncount = count; 1096 1097 poffset = IDX_TO_OFF(m[0]->pindex); 1098 1099 /* 1100 * If the page-aligned write is larger then the actual file we 1101 * have to invalidate pages occuring beyond the file EOF. However, 1102 * there is an edge case where a file may not be page-aligned where 1103 * the last page is partially invalid. In this case the filesystem 1104 * may not properly clear the dirty bits for the entire page (which 1105 * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). 1106 * With the page locked we are free to fix-up the dirty bits here. 1107 * 1108 * We do not under any circumstances truncate the valid bits, as 1109 * this will screw up bogus page replacement. 1110 */ 1111 if (maxsize + poffset > object->un_pager.vnp.vnp_size) { 1112 if (object->un_pager.vnp.vnp_size > poffset) { 1113 int pgoff; 1114 1115 maxsize = object->un_pager.vnp.vnp_size - poffset; 1116 ncount = btoc(maxsize); 1117 if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { 1118 vm_page_lock_queues(); 1119 vm_page_clear_dirty(m[ncount - 1], pgoff, 1120 PAGE_SIZE - pgoff); 1121 vm_page_unlock_queues(); 1122 } 1123 } else { 1124 maxsize = 0; 1125 ncount = 0; 1126 } 1127 if (ncount < count) { 1128 for (i = ncount; i < count; i++) { 1129 rtvals[i] = VM_PAGER_BAD; 1130 } 1131 } 1132 } 1133 1134 /* 1135 * pageouts are already clustered, use IO_ASYNC t o force a bawrite() 1136 * rather then a bdwrite() to prevent paging I/O from saturating 1137 * the buffer cache. Dummy-up the sequential heuristic to cause 1138 * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, 1139 * the system decides how to cluster. 1140 */ 1141 ioflags = IO_VMIO; 1142 if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) 1143 ioflags |= IO_SYNC; 1144 else if ((flags & VM_PAGER_CLUSTER_OK) == 0) 1145 ioflags |= IO_ASYNC; 1146 ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; 1147 ioflags |= IO_SEQMAX << IO_SEQSHIFT; 1148 1149 aiov.iov_base = (caddr_t) 0; 1150 aiov.iov_len = maxsize; 1151 auio.uio_iov = &aiov; 1152 auio.uio_iovcnt = 1; 1153 auio.uio_offset = poffset; 1154 auio.uio_segflg = UIO_NOCOPY; 1155 auio.uio_rw = UIO_WRITE; 1156 auio.uio_resid = maxsize; 1157 auio.uio_td = (struct thread *) 0; 1158 error = VOP_WRITE(vp, &auio, ioflags, curthread->td_ucred); 1159 PCPU_INC(cnt.v_vnodeout); 1160 PCPU_ADD(cnt.v_vnodepgsout, ncount); 1161 1162 if (error) { 1163 if ((ppscheck = ppsratecheck(&lastfail, &curfail, 1))) 1164 printf("vnode_pager_putpages: I/O error %d\n", error); 1165 } 1166 if (auio.uio_resid) { 1167 if (ppscheck || ppsratecheck(&lastfail, &curfail, 1)) 1168 printf("vnode_pager_putpages: residual I/O %d at %lu\n", 1169 auio.uio_resid, (u_long)m[0]->pindex); 1170 } 1171 for (i = 0; i < ncount; i++) { 1172 rtvals[i] = VM_PAGER_OK; 1173 } 1174 return rtvals[0]; 1175 } 1176 1177 struct vnode * 1178 vnode_pager_lock(vm_object_t first_object) 1179 { 1180 struct vnode *vp; 1181 vm_object_t backing_object, object; 1182 1183 VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED); 1184 for (object = first_object; object != NULL; object = backing_object) { 1185 if (object->type != OBJT_VNODE) { 1186 if ((backing_object = object->backing_object) != NULL) 1187 VM_OBJECT_LOCK(backing_object); 1188 if (object != first_object) 1189 VM_OBJECT_UNLOCK(object); 1190 continue; 1191 } 1192 retry: 1193 if (object->flags & OBJ_DEAD) { 1194 if (object != first_object) 1195 VM_OBJECT_UNLOCK(object); 1196 return NULL; 1197 } 1198 vp = object->handle; 1199 VI_LOCK(vp); 1200 VM_OBJECT_UNLOCK(object); 1201 if (first_object != object) 1202 VM_OBJECT_UNLOCK(first_object); 1203 VFS_ASSERT_GIANT(vp->v_mount); 1204 if (vget(vp, LK_CANRECURSE | LK_INTERLOCK | 1205 LK_RETRY | LK_SHARED, curthread)) { 1206 VM_OBJECT_LOCK(first_object); 1207 if (object != first_object) 1208 VM_OBJECT_LOCK(object); 1209 if (object->type != OBJT_VNODE) { 1210 if (object != first_object) 1211 VM_OBJECT_UNLOCK(object); 1212 return NULL; 1213 } 1214 printf("vnode_pager_lock: retrying\n"); 1215 goto retry; 1216 } 1217 VM_OBJECT_LOCK(first_object); 1218 return (vp); 1219 } 1220 return NULL; 1221 }
Cache object: a594bdfc2a6e971c108bcd257afc1f2a
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]