Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_bio.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: uvm_bio.c,v 1.65.10.1 2010/11/21 18:09:00 riz Exp $ */ 2 3 /* 4 * Copyright (c) 1998 Chuck Silvers. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32 /* 33 * uvm_bio.c: buffered i/o object mapping cache 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: uvm_bio.c,v 1.65.10.1 2010/11/21 18:09:00 riz Exp $"); 38 39 #include "opt_uvmhist.h" 40 #include "opt_ubc.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kmem.h> 45 #include <sys/kernel.h> 46 #include <sys/proc.h> 47 48 #include <uvm/uvm.h> 49 50 /* 51 * global data structures 52 */ 53 54 /* 55 * local functions 56 */ 57 58 static int ubc_fault(struct uvm_faultinfo *, vaddr_t, struct vm_page **, 59 int, int, vm_prot_t, int); 60 static struct ubc_map *ubc_find_mapping(struct uvm_object *, voff_t); 61 62 /* 63 * local data structues 64 */ 65 66 #define UBC_HASH(uobj, offset) \ 67 (((((u_long)(uobj)) >> 8) + (((u_long)(offset)) >> PAGE_SHIFT)) & \ 68 ubc_object.hashmask) 69 70 #define UBC_QUEUE(offset) \ 71 (&ubc_object.inactive[(((u_long)(offset)) >> ubc_winshift) & \ 72 (UBC_NQUEUES - 1)]) 73 74 #define UBC_UMAP_ADDR(u) \ 75 (vaddr_t)(ubc_object.kva + (((u) - ubc_object.umap) << ubc_winshift)) 76 77 78 #define UMAP_PAGES_LOCKED 0x0001 79 #define UMAP_MAPPING_CACHED 0x0002 80 81 struct ubc_map 82 { 83 struct uvm_object * uobj; /* mapped object */ 84 voff_t offset; /* offset into uobj */ 85 voff_t writeoff; /* write offset */ 86 vsize_t writelen; /* write len */ 87 int refcount; /* refcount on mapping */ 88 int flags; /* extra state */ 89 int advice; 90 91 LIST_ENTRY(ubc_map) hash; /* hash table */ 92 TAILQ_ENTRY(ubc_map) inactive; /* inactive queue */ 93 }; 94 95 static struct ubc_object 96 { 97 struct uvm_object uobj; /* glue for uvm_map() */ 98 char *kva; /* where ubc_object is mapped */ 99 struct ubc_map *umap; /* array of ubc_map's */ 100 101 LIST_HEAD(, ubc_map) *hash; /* hashtable for cached ubc_map's */ 102 u_long hashmask; /* mask for hashtable */ 103 104 TAILQ_HEAD(ubc_inactive_head, ubc_map) *inactive; 105 /* inactive queues for ubc_map's */ 106 107 } ubc_object; 108 109 const struct uvm_pagerops ubc_pager = { 110 .pgo_fault = ubc_fault, 111 /* ... rest are NULL */ 112 }; 113 114 int ubc_nwins = UBC_NWINS; 115 int ubc_winshift = UBC_WINSHIFT; 116 int ubc_winsize; 117 #if defined(PMAP_PREFER) 118 int ubc_nqueues; 119 #define UBC_NQUEUES ubc_nqueues 120 #else 121 #define UBC_NQUEUES 1 122 #endif 123 124 #if defined(UBC_STATS) 125 126 #define UBC_EVCNT_DEFINE(name) \ 127 struct evcnt ubc_evcnt_##name = \ 128 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "ubc", #name); \ 129 EVCNT_ATTACH_STATIC(ubc_evcnt_##name); 130 #define UBC_EVCNT_INCR(name) ubc_evcnt_##name.ev_count++ 131 132 #else /* defined(UBC_STATS) */ 133 134 #define UBC_EVCNT_DEFINE(name) /* nothing */ 135 #define UBC_EVCNT_INCR(name) /* nothing */ 136 137 #endif /* defined(UBC_STATS) */ 138 139 UBC_EVCNT_DEFINE(wincachehit) 140 UBC_EVCNT_DEFINE(wincachemiss) 141 UBC_EVCNT_DEFINE(faultbusy) 142 143 /* 144 * ubc_init 145 * 146 * init pager private data structures. 147 */ 148 149 void 150 ubc_init(void) 151 { 152 struct ubc_map *umap; 153 vaddr_t va; 154 int i; 155 156 /* 157 * Make sure ubc_winshift is sane. 158 */ 159 if (ubc_winshift < PAGE_SHIFT) 160 ubc_winshift = PAGE_SHIFT; 161 162 /* 163 * init ubc_object. 164 * alloc and init ubc_map's. 165 * init inactive queues. 166 * alloc and init hashtable. 167 * map in ubc_object. 168 */ 169 170 UVM_OBJ_INIT(&ubc_object.uobj, &ubc_pager, UVM_OBJ_KERN); 171 172 ubc_object.umap = kmem_zalloc(ubc_nwins * sizeof(struct ubc_map), 173 KM_SLEEP); 174 if (ubc_object.umap == NULL) 175 panic("ubc_init: failed to allocate ubc_map"); 176 177 if (ubc_winshift < PAGE_SHIFT) { 178 ubc_winshift = PAGE_SHIFT; 179 } 180 va = (vaddr_t)1L; 181 #ifdef PMAP_PREFER 182 PMAP_PREFER(0, &va, 0, 0); /* kernel is never topdown */ 183 ubc_nqueues = va >> ubc_winshift; 184 if (ubc_nqueues == 0) { 185 ubc_nqueues = 1; 186 } 187 #endif 188 ubc_winsize = 1 << ubc_winshift; 189 ubc_object.inactive = kmem_alloc(UBC_NQUEUES * 190 sizeof(struct ubc_inactive_head), KM_SLEEP); 191 if (ubc_object.inactive == NULL) 192 panic("ubc_init: failed to allocate inactive queue heads"); 193 for (i = 0; i < UBC_NQUEUES; i++) { 194 TAILQ_INIT(&ubc_object.inactive[i]); 195 } 196 for (i = 0; i < ubc_nwins; i++) { 197 umap = &ubc_object.umap[i]; 198 TAILQ_INSERT_TAIL(&ubc_object.inactive[i & (UBC_NQUEUES - 1)], 199 umap, inactive); 200 } 201 202 ubc_object.hash = hashinit(ubc_nwins, HASH_LIST, true, 203 &ubc_object.hashmask); 204 for (i = 0; i <= ubc_object.hashmask; i++) { 205 LIST_INIT(&ubc_object.hash[i]); 206 } 207 208 if (uvm_map(kernel_map, (vaddr_t *)&ubc_object.kva, 209 ubc_nwins << ubc_winshift, &ubc_object.uobj, 0, (vsize_t)va, 210 UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE, 211 UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != 0) { 212 panic("ubc_init: failed to map ubc_object"); 213 } 214 UVMHIST_INIT(ubchist, 300); 215 } 216 217 /* 218 * ubc_fault_page: helper of ubc_fault to handle a single page. 219 * 220 * => Caller has UVM object locked. 221 */ 222 223 static inline int 224 ubc_fault_page(const struct uvm_faultinfo *ufi, const struct ubc_map *umap, 225 struct vm_page *pg, vm_prot_t prot, vm_prot_t access_type, vaddr_t va) 226 { 227 struct uvm_object *uobj; 228 vm_prot_t mask; 229 int error; 230 bool rdonly; 231 232 uobj = pg->uobject; 233 KASSERT(mutex_owned(&uobj->vmobjlock)); 234 235 if (pg->flags & PG_WANTED) { 236 wakeup(pg); 237 } 238 KASSERT((pg->flags & PG_FAKE) == 0); 239 if (pg->flags & PG_RELEASED) { 240 mutex_enter(&uvm_pageqlock); 241 uvm_pagefree(pg); 242 mutex_exit(&uvm_pageqlock); 243 return 0; 244 } 245 if (pg->loan_count != 0) { 246 247 /* 248 * Avoid unneeded loan break, if possible. 249 */ 250 251 if ((access_type & VM_PROT_WRITE) == 0) { 252 prot &= ~VM_PROT_WRITE; 253 } 254 if (prot & VM_PROT_WRITE) { 255 struct vm_page *newpg; 256 257 newpg = uvm_loanbreak(pg); 258 if (newpg == NULL) { 259 uvm_page_unbusy(&pg, 1); 260 return ENOMEM; 261 } 262 pg = newpg; 263 } 264 } 265 266 /* 267 * Note that a page whose backing store is partially allocated 268 * is marked as PG_RDONLY. 269 */ 270 271 KASSERT((pg->flags & PG_RDONLY) == 0 || 272 (access_type & VM_PROT_WRITE) == 0 || 273 pg->offset < umap->writeoff || 274 pg->offset + PAGE_SIZE > umap->writeoff + umap->writelen); 275 276 rdonly = ((access_type & VM_PROT_WRITE) == 0 && 277 (pg->flags & PG_RDONLY) != 0) || 278 UVM_OBJ_NEEDS_WRITEFAULT(uobj); 279 mask = rdonly ? ~VM_PROT_WRITE : VM_PROT_ALL; 280 281 error = pmap_enter(ufi->orig_map->pmap, va, VM_PAGE_TO_PHYS(pg), 282 prot & mask, PMAP_CANFAIL | (access_type & mask)); 283 284 mutex_enter(&uvm_pageqlock); 285 uvm_pageactivate(pg); 286 mutex_exit(&uvm_pageqlock); 287 pg->flags &= ~(PG_BUSY|PG_WANTED); 288 UVM_PAGE_OWN(pg, NULL); 289 290 return error; 291 } 292 293 /* 294 * ubc_fault: fault routine for ubc mapping 295 */ 296 297 static int 298 ubc_fault(struct uvm_faultinfo *ufi, vaddr_t ign1, struct vm_page **ign2, 299 int ign3, int ign4, vm_prot_t access_type, int flags) 300 { 301 struct uvm_object *uobj; 302 struct ubc_map *umap; 303 vaddr_t va, eva, ubc_offset, slot_offset; 304 struct vm_page *pgs[ubc_winsize >> PAGE_SHIFT]; 305 int i, error, npages; 306 vm_prot_t prot; 307 308 UVMHIST_FUNC("ubc_fault"); UVMHIST_CALLED(ubchist); 309 310 /* 311 * no need to try with PGO_LOCKED... 312 * we don't need to have the map locked since we know that 313 * no one will mess with it until our reference is released. 314 */ 315 316 if (flags & PGO_LOCKED) { 317 uvmfault_unlockall(ufi, NULL, &ubc_object.uobj, NULL); 318 flags &= ~PGO_LOCKED; 319 } 320 321 va = ufi->orig_rvaddr; 322 ubc_offset = va - (vaddr_t)ubc_object.kva; 323 umap = &ubc_object.umap[ubc_offset >> ubc_winshift]; 324 KASSERT(umap->refcount != 0); 325 KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0); 326 slot_offset = ubc_offset & (ubc_winsize - 1); 327 328 /* 329 * some platforms cannot write to individual bytes atomically, so 330 * software has to do read/modify/write of larger quantities instead. 331 * this means that the access_type for "write" operations 332 * can be VM_PROT_READ, which confuses us mightily. 333 * 334 * deal with this by resetting access_type based on the info 335 * that ubc_alloc() stores for us. 336 */ 337 338 access_type = umap->writelen ? VM_PROT_WRITE : VM_PROT_READ; 339 UVMHIST_LOG(ubchist, "va 0x%lx ubc_offset 0x%lx access_type %d", 340 va, ubc_offset, access_type, 0); 341 342 #ifdef DIAGNOSTIC 343 if ((access_type & VM_PROT_WRITE) != 0) { 344 if (slot_offset < trunc_page(umap->writeoff) || 345 umap->writeoff + umap->writelen <= slot_offset) { 346 panic("ubc_fault: out of range write"); 347 } 348 } 349 #endif 350 351 /* no umap locking needed since we have a ref on the umap */ 352 uobj = umap->uobj; 353 354 if ((access_type & VM_PROT_WRITE) == 0) { 355 npages = (ubc_winsize - slot_offset) >> PAGE_SHIFT; 356 } else { 357 npages = (round_page(umap->offset + umap->writeoff + 358 umap->writelen) - (umap->offset + slot_offset)) 359 >> PAGE_SHIFT; 360 flags |= PGO_PASTEOF; 361 } 362 363 again: 364 memset(pgs, 0, sizeof (pgs)); 365 mutex_enter(&uobj->vmobjlock); 366 367 UVMHIST_LOG(ubchist, "slot_offset 0x%x writeoff 0x%x writelen 0x%x ", 368 slot_offset, umap->writeoff, umap->writelen, 0); 369 UVMHIST_LOG(ubchist, "getpages uobj %p offset 0x%x npages %d", 370 uobj, umap->offset + slot_offset, npages, 0); 371 372 error = (*uobj->pgops->pgo_get)(uobj, umap->offset + slot_offset, pgs, 373 &npages, 0, access_type, umap->advice, flags | PGO_NOBLOCKALLOC | 374 PGO_NOTIMESTAMP); 375 UVMHIST_LOG(ubchist, "getpages error %d npages %d", error, npages, 0, 376 0); 377 378 if (error == EAGAIN) { 379 kpause("ubc_fault", false, hz >> 2, NULL); 380 goto again; 381 } 382 if (error) { 383 return error; 384 } 385 386 /* 387 * For virtually-indexed, virtually-tagged caches we should avoid 388 * creating writable mappings when we do not absolutely need them, 389 * since the "compatible alias" trick does not work on such caches. 390 * Otherwise, we can always map the pages writable. 391 */ 392 393 #ifdef PMAP_CACHE_VIVT 394 prot = VM_PROT_READ | access_type; 395 #else 396 prot = VM_PROT_READ | VM_PROT_WRITE; 397 #endif 398 399 /* 400 * Note: in the common case, all returned pages would have the same 401 * UVM object. However, due to layered file-systems and e.g. tmpfs, 402 * returned pages may have different objects. We "remember" the 403 * last object in the loop to reduce locking overhead and to perform 404 * pmap_update() before object unlock. 405 */ 406 uobj = NULL; 407 408 va = ufi->orig_rvaddr; 409 eva = ufi->orig_rvaddr + (npages << PAGE_SHIFT); 410 411 UVMHIST_LOG(ubchist, "va 0x%lx eva 0x%lx", va, eva, 0, 0); 412 for (i = 0; va < eva; i++, va += PAGE_SIZE) { 413 struct vm_page *pg; 414 415 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i], 0, 0); 416 pg = pgs[i]; 417 418 if (pg == NULL || pg == PGO_DONTCARE) { 419 continue; 420 } 421 if (__predict_false(pg->uobject != uobj)) { 422 /* Check for the first iteration and error cases. */ 423 if (uobj != NULL) { 424 /* Must make VA visible before the unlock. */ 425 pmap_update(ufi->orig_map->pmap); 426 mutex_exit(&uobj->vmobjlock); 427 } 428 uobj = pg->uobject; 429 mutex_enter(&uobj->vmobjlock); 430 } 431 error = ubc_fault_page(ufi, umap, pg, prot, access_type, va); 432 if (error) { 433 /* 434 * Flush (there might be pages entered), drop the lock, 435 * "forget" the object and perform uvm_wait(). 436 * Note: page will re-fault. 437 */ 438 pmap_update(ufi->orig_map->pmap); 439 mutex_exit(&uobj->vmobjlock); 440 uobj = NULL; 441 uvm_wait("ubc_fault"); 442 } 443 } 444 if (__predict_true(uobj != NULL)) { 445 pmap_update(ufi->orig_map->pmap); 446 mutex_exit(&uobj->vmobjlock); 447 } 448 return 0; 449 } 450 451 /* 452 * local functions 453 */ 454 455 static struct ubc_map * 456 ubc_find_mapping(struct uvm_object *uobj, voff_t offset) 457 { 458 struct ubc_map *umap; 459 460 LIST_FOREACH(umap, &ubc_object.hash[UBC_HASH(uobj, offset)], hash) { 461 if (umap->uobj == uobj && umap->offset == offset) { 462 return umap; 463 } 464 } 465 return NULL; 466 } 467 468 469 /* 470 * ubc interface functions 471 */ 472 473 /* 474 * ubc_alloc: allocate a file mapping window 475 */ 476 477 void * 478 ubc_alloc(struct uvm_object *uobj, voff_t offset, vsize_t *lenp, int advice, 479 int flags) 480 { 481 vaddr_t slot_offset, va; 482 struct ubc_map *umap; 483 voff_t umap_offset; 484 int error; 485 UVMHIST_FUNC("ubc_alloc"); UVMHIST_CALLED(ubchist); 486 487 UVMHIST_LOG(ubchist, "uobj %p offset 0x%lx len 0x%lx", 488 uobj, offset, *lenp, 0); 489 490 KASSERT(*lenp > 0); 491 umap_offset = (offset & ~((voff_t)ubc_winsize - 1)); 492 slot_offset = (vaddr_t)(offset & ((voff_t)ubc_winsize - 1)); 493 *lenp = MIN(*lenp, ubc_winsize - slot_offset); 494 495 /* 496 * the object is always locked here, so we don't need to add a ref. 497 */ 498 499 again: 500 mutex_enter(&ubc_object.uobj.vmobjlock); 501 umap = ubc_find_mapping(uobj, umap_offset); 502 if (umap == NULL) { 503 UBC_EVCNT_INCR(wincachemiss); 504 umap = TAILQ_FIRST(UBC_QUEUE(offset)); 505 if (umap == NULL) { 506 mutex_exit(&ubc_object.uobj.vmobjlock); 507 kpause("ubc_alloc", false, hz, NULL); 508 goto again; 509 } 510 511 /* 512 * remove from old hash (if any), add to new hash. 513 */ 514 515 if (umap->uobj != NULL) { 516 LIST_REMOVE(umap, hash); 517 } 518 umap->uobj = uobj; 519 umap->offset = umap_offset; 520 LIST_INSERT_HEAD(&ubc_object.hash[UBC_HASH(uobj, umap_offset)], 521 umap, hash); 522 va = UBC_UMAP_ADDR(umap); 523 if (umap->flags & UMAP_MAPPING_CACHED) { 524 umap->flags &= ~UMAP_MAPPING_CACHED; 525 pmap_remove(pmap_kernel(), va, va + ubc_winsize); 526 pmap_update(pmap_kernel()); 527 } 528 } else { 529 UBC_EVCNT_INCR(wincachehit); 530 va = UBC_UMAP_ADDR(umap); 531 } 532 533 if (umap->refcount == 0) { 534 TAILQ_REMOVE(UBC_QUEUE(offset), umap, inactive); 535 } 536 537 #ifdef DIAGNOSTIC 538 if ((flags & UBC_WRITE) && (umap->writeoff || umap->writelen)) { 539 panic("ubc_alloc: concurrent writes uobj %p", uobj); 540 } 541 #endif 542 if (flags & UBC_WRITE) { 543 umap->writeoff = slot_offset; 544 umap->writelen = *lenp; 545 } 546 547 umap->refcount++; 548 umap->advice = advice; 549 mutex_exit(&ubc_object.uobj.vmobjlock); 550 UVMHIST_LOG(ubchist, "umap %p refs %d va %p flags 0x%x", 551 umap, umap->refcount, va, flags); 552 553 if (flags & UBC_FAULTBUSY) { 554 int npages = (*lenp + PAGE_SIZE - 1) >> PAGE_SHIFT; 555 struct vm_page *pgs[npages]; 556 int gpflags = 557 PGO_SYNCIO|PGO_OVERWRITE|PGO_PASTEOF|PGO_NOBLOCKALLOC| 558 PGO_NOTIMESTAMP; 559 int i; 560 KDASSERT(flags & UBC_WRITE); 561 KASSERT(umap->refcount == 1); 562 563 UBC_EVCNT_INCR(faultbusy); 564 if (umap->flags & UMAP_MAPPING_CACHED) { 565 umap->flags &= ~UMAP_MAPPING_CACHED; 566 pmap_remove(pmap_kernel(), va, va + ubc_winsize); 567 } 568 again_faultbusy: 569 memset(pgs, 0, sizeof(pgs)); 570 mutex_enter(&uobj->vmobjlock); 571 error = (*uobj->pgops->pgo_get)(uobj, trunc_page(offset), pgs, 572 &npages, 0, VM_PROT_READ | VM_PROT_WRITE, advice, gpflags); 573 UVMHIST_LOG(ubchist, "faultbusy getpages %d", error, 0, 0, 0); 574 if (error) { 575 goto out; 576 } 577 for (i = 0; i < npages; i++) { 578 struct vm_page *pg = pgs[i]; 579 580 KASSERT(pg->uobject == uobj); 581 if (pg->loan_count != 0) { 582 mutex_enter(&uobj->vmobjlock); 583 if (pg->loan_count != 0) { 584 pg = uvm_loanbreak(pg); 585 } 586 mutex_exit(&uobj->vmobjlock); 587 if (pg == NULL) { 588 pmap_kremove(va, ubc_winsize); 589 pmap_update(pmap_kernel()); 590 mutex_enter(&uobj->vmobjlock); 591 uvm_page_unbusy(pgs, npages); 592 mutex_exit(&uobj->vmobjlock); 593 uvm_wait("ubc_alloc"); 594 goto again_faultbusy; 595 } 596 pgs[i] = pg; 597 } 598 pmap_kenter_pa(va + slot_offset + (i << PAGE_SHIFT), 599 VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE); 600 } 601 pmap_update(pmap_kernel()); 602 umap->flags |= UMAP_PAGES_LOCKED; 603 } else { 604 KASSERT((umap->flags & UMAP_PAGES_LOCKED) == 0); 605 } 606 607 out: 608 return (void *)(va + slot_offset); 609 } 610 611 /* 612 * ubc_release: free a file mapping window. 613 */ 614 615 void 616 ubc_release(void *va, int flags) 617 { 618 struct ubc_map *umap; 619 struct uvm_object *uobj; 620 vaddr_t umapva; 621 bool unmapped; 622 UVMHIST_FUNC("ubc_release"); UVMHIST_CALLED(ubchist); 623 624 UVMHIST_LOG(ubchist, "va %p", va, 0, 0, 0); 625 umap = &ubc_object.umap[((char *)va - ubc_object.kva) >> ubc_winshift]; 626 umapva = UBC_UMAP_ADDR(umap); 627 uobj = umap->uobj; 628 KASSERT(uobj != NULL); 629 630 if (umap->flags & UMAP_PAGES_LOCKED) { 631 int slot_offset = umap->writeoff; 632 int endoff = umap->writeoff + umap->writelen; 633 int zerolen = round_page(endoff) - endoff; 634 int npages = (int)(round_page(umap->writeoff + umap->writelen) 635 - trunc_page(umap->writeoff)) >> PAGE_SHIFT; 636 struct vm_page *pgs[npages]; 637 paddr_t pa; 638 int i; 639 bool rv; 640 641 KASSERT((umap->flags & UMAP_MAPPING_CACHED) == 0); 642 if (zerolen) { 643 memset((char *)umapva + endoff, 0, zerolen); 644 } 645 umap->flags &= ~UMAP_PAGES_LOCKED; 646 mutex_enter(&uvm_pageqlock); 647 for (i = 0; i < npages; i++) { 648 rv = pmap_extract(pmap_kernel(), 649 umapva + slot_offset + (i << PAGE_SHIFT), &pa); 650 KASSERT(rv); 651 pgs[i] = PHYS_TO_VM_PAGE(pa); 652 pgs[i]->flags &= ~(PG_FAKE|PG_CLEAN); 653 KASSERT(pgs[i]->loan_count == 0); 654 uvm_pageactivate(pgs[i]); 655 } 656 mutex_exit(&uvm_pageqlock); 657 pmap_kremove(umapva, ubc_winsize); 658 pmap_update(pmap_kernel()); 659 mutex_enter(&uobj->vmobjlock); 660 uvm_page_unbusy(pgs, npages); 661 mutex_exit(&uobj->vmobjlock); 662 unmapped = true; 663 } else { 664 unmapped = false; 665 } 666 667 mutex_enter(&ubc_object.uobj.vmobjlock); 668 umap->writeoff = 0; 669 umap->writelen = 0; 670 umap->refcount--; 671 if (umap->refcount == 0) { 672 if (flags & UBC_UNMAP) { 673 674 /* 675 * Invalidate any cached mappings if requested. 676 * This is typically used to avoid leaving 677 * incompatible cache aliases around indefinitely. 678 */ 679 680 pmap_remove(pmap_kernel(), umapva, 681 umapva + ubc_winsize); 682 umap->flags &= ~UMAP_MAPPING_CACHED; 683 pmap_update(pmap_kernel()); 684 LIST_REMOVE(umap, hash); 685 umap->uobj = NULL; 686 TAILQ_INSERT_HEAD(UBC_QUEUE(umap->offset), umap, 687 inactive); 688 } else { 689 if (!unmapped) { 690 umap->flags |= UMAP_MAPPING_CACHED; 691 } 692 TAILQ_INSERT_TAIL(UBC_QUEUE(umap->offset), umap, 693 inactive); 694 } 695 } 696 UVMHIST_LOG(ubchist, "umap %p refs %d", umap, umap->refcount, 0, 0); 697 mutex_exit(&ubc_object.uobj.vmobjlock); 698 } 699 700 /* 701 * ubc_uiomove: move data to/from an object. 702 */ 703 704 int 705 ubc_uiomove(struct uvm_object *uobj, struct uio *uio, vsize_t todo, int advice, 706 int flags) 707 { 708 voff_t off; 709 const bool overwrite = (flags & UBC_FAULTBUSY) != 0; 710 int error; 711 712 KASSERT(todo <= uio->uio_resid); 713 KASSERT(((flags & UBC_WRITE) != 0 && uio->uio_rw == UIO_WRITE) || 714 ((flags & UBC_READ) != 0 && uio->uio_rw == UIO_READ)); 715 716 off = uio->uio_offset; 717 error = 0; 718 while (todo > 0) { 719 vsize_t bytelen = todo; 720 void *win; 721 722 win = ubc_alloc(uobj, off, &bytelen, advice, flags); 723 if (error == 0) { 724 error = uiomove(win, bytelen, uio); 725 } 726 if (error != 0 && overwrite) { 727 /* 728 * if we haven't initialized the pages yet, 729 * do it now. it's safe to use memset here 730 * because we just mapped the pages above. 731 */ 732 printf("%s: error=%d\n", __func__, error); 733 memset(win, 0, bytelen); 734 } 735 ubc_release(win, flags); 736 off += bytelen; 737 todo -= bytelen; 738 if (error != 0 && (flags & UBC_PARTIALOK) != 0) { 739 break; 740 } 741 } 742 743 return error; 744 } 745 746 #if 0 /* notused */ 747 /* 748 * removing a range of mappings from the ubc mapping cache. 749 */ 750 751 void 752 ubc_flush(struct uvm_object *uobj, voff_t start, voff_t end) 753 { 754 struct ubc_map *umap; 755 vaddr_t va; 756 UVMHIST_FUNC("ubc_flush"); UVMHIST_CALLED(ubchist); 757 758 UVMHIST_LOG(ubchist, "uobj %p start 0x%lx end 0x%lx", 759 uobj, start, end, 0); 760 761 mutex_enter(&ubc_object.uobj.vmobjlock); 762 for (umap = ubc_object.umap; 763 umap < &ubc_object.umap[ubc_nwins]; 764 umap++) { 765 766 if (umap->uobj != uobj || umap->offset < start || 767 (umap->offset >= end && end != 0) || 768 umap->refcount > 0) { 769 continue; 770 } 771 772 /* 773 * remove from hash, 774 * move to head of inactive queue. 775 */ 776 777 va = (vaddr_t)(ubc_object.kva + 778 ((umap - ubc_object.umap) << ubc_winshift)); 779 pmap_remove(pmap_kernel(), va, va + ubc_winsize); 780 781 LIST_REMOVE(umap, hash); 782 umap->uobj = NULL; 783 TAILQ_REMOVE(UBC_QUEUE(umap->offset), umap, inactive); 784 TAILQ_INSERT_HEAD(UBC_QUEUE(umap->offset), umap, inactive); 785 } 786 pmap_update(pmap_kernel()); 787 mutex_exit(&ubc_object.uobj.vmobjlock); 788 } 789 #endif /* notused */
Cache object: 0ef229b1d27b76aae5c225f3d231c807
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]