Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_fault.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: uvm_fault.c,v 1.87.2.1 2004/05/10 14:27:00 tron Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 /* 38 * uvm_fault.c: fault handler 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.87.2.1 2004/05/10 14:27:00 tron Exp $"); 43 44 #include "opt_uvmhist.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/proc.h> 50 #include <sys/malloc.h> 51 #include <sys/mman.h> 52 #include <sys/user.h> 53 54 #include <uvm/uvm.h> 55 56 /* 57 * 58 * a word on page faults: 59 * 60 * types of page faults we handle: 61 * 62 * CASE 1: upper layer faults CASE 2: lower layer faults 63 * 64 * CASE 1A CASE 1B CASE 2A CASE 2B 65 * read/write1 write>1 read/write +-cow_write/zero 66 * | | | | 67 * +--|--+ +--|--+ +-----+ + | + | +-----+ 68 * amap | V | | ----------->new| | | | ^ | 69 * +-----+ +-----+ +-----+ + | + | +--|--+ 70 * | | | 71 * +-----+ +-----+ +--|--+ | +--|--+ 72 * uobj | d/c | | d/c | | V | +----| | 73 * +-----+ +-----+ +-----+ +-----+ 74 * 75 * d/c = don't care 76 * 77 * case [0]: layerless fault 78 * no amap or uobj is present. this is an error. 79 * 80 * case [1]: upper layer fault [anon active] 81 * 1A: [read] or [write with anon->an_ref == 1] 82 * I/O takes place in top level anon and uobj is not touched. 83 * 1B: [write with anon->an_ref > 1] 84 * new anon is alloc'd and data is copied off ["COW"] 85 * 86 * case [2]: lower layer fault [uobj] 87 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 88 * I/O takes place directly in object. 89 * 2B: [write to copy_on_write] or [read on NULL uobj] 90 * data is "promoted" from uobj to a new anon. 91 * if uobj is null, then we zero fill. 92 * 93 * we follow the standard UVM locking protocol ordering: 94 * 95 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 96 * we hold a PG_BUSY page if we unlock for I/O 97 * 98 * 99 * the code is structured as follows: 100 * 101 * - init the "IN" params in the ufi structure 102 * ReFault: 103 * - do lookups [locks maps], check protection, handle needs_copy 104 * - check for case 0 fault (error) 105 * - establish "range" of fault 106 * - if we have an amap lock it and extract the anons 107 * - if sequential advice deactivate pages behind us 108 * - at the same time check pmap for unmapped areas and anon for pages 109 * that we could map in (and do map it if found) 110 * - check object for resident pages that we could map in 111 * - if (case 2) goto Case2 112 * - >>> handle case 1 113 * - ensure source anon is resident in RAM 114 * - if case 1B alloc new anon and copy from source 115 * - map the correct page in 116 * Case2: 117 * - >>> handle case 2 118 * - ensure source page is resident (if uobj) 119 * - if case 2B alloc new anon and copy from source (could be zero 120 * fill if uobj == NULL) 121 * - map the correct page in 122 * - done! 123 * 124 * note on paging: 125 * if we have to do I/O we place a PG_BUSY page in the correct object, 126 * unlock everything, and do the I/O. when I/O is done we must reverify 127 * the state of the world before assuming that our data structures are 128 * valid. [because mappings could change while the map is unlocked] 129 * 130 * alternative 1: unbusy the page in question and restart the page fault 131 * from the top (ReFault). this is easy but does not take advantage 132 * of the information that we already have from our previous lookup, 133 * although it is possible that the "hints" in the vm_map will help here. 134 * 135 * alternative 2: the system already keeps track of a "version" number of 136 * a map. [i.e. every time you write-lock a map (e.g. to change a 137 * mapping) you bump the version number up by one...] so, we can save 138 * the version number of the map before we release the lock and start I/O. 139 * then when I/O is done we can relock and check the version numbers 140 * to see if anything changed. this might save us some over 1 because 141 * we don't have to unbusy the page and may be less compares(?). 142 * 143 * alternative 3: put in backpointers or a way to "hold" part of a map 144 * in place while I/O is in progress. this could be complex to 145 * implement (especially with structures like amap that can be referenced 146 * by multiple map entries, and figuring out what should wait could be 147 * complex as well...). 148 * 149 * given that we are not currently multiprocessor or multithreaded we might 150 * as well choose alternative 2 now. maybe alternative 3 would be useful 151 * in the future. XXX keep in mind for future consideration//rechecking. 152 */ 153 154 /* 155 * local data structures 156 */ 157 158 struct uvm_advice { 159 int advice; 160 int nback; 161 int nforw; 162 }; 163 164 /* 165 * page range array: 166 * note: index in array must match "advice" value 167 * XXX: borrowed numbers from freebsd. do they work well for us? 168 */ 169 170 static struct uvm_advice uvmadvice[] = { 171 { MADV_NORMAL, 3, 4 }, 172 { MADV_RANDOM, 0, 0 }, 173 { MADV_SEQUENTIAL, 8, 7}, 174 }; 175 176 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 177 178 /* 179 * private prototypes 180 */ 181 182 static void uvmfault_amapcopy(struct uvm_faultinfo *); 183 static __inline void uvmfault_anonflush(struct vm_anon **, int); 184 185 /* 186 * inline functions 187 */ 188 189 /* 190 * uvmfault_anonflush: try and deactivate pages in specified anons 191 * 192 * => does not have to deactivate page if it is busy 193 */ 194 195 static __inline void 196 uvmfault_anonflush(anons, n) 197 struct vm_anon **anons; 198 int n; 199 { 200 int lcv; 201 struct vm_page *pg; 202 203 for (lcv = 0 ; lcv < n ; lcv++) { 204 if (anons[lcv] == NULL) 205 continue; 206 simple_lock(&anons[lcv]->an_lock); 207 pg = anons[lcv]->u.an_page; 208 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 209 uvm_lock_pageq(); 210 if (pg->wire_count == 0) { 211 pmap_clear_reference(pg); 212 uvm_pagedeactivate(pg); 213 } 214 uvm_unlock_pageq(); 215 } 216 simple_unlock(&anons[lcv]->an_lock); 217 } 218 } 219 220 /* 221 * normal functions 222 */ 223 224 /* 225 * uvmfault_amapcopy: clear "needs_copy" in a map. 226 * 227 * => called with VM data structures unlocked (usually, see below) 228 * => we get a write lock on the maps and clear needs_copy for a VA 229 * => if we are out of RAM we sleep (waiting for more) 230 */ 231 232 static void 233 uvmfault_amapcopy(ufi) 234 struct uvm_faultinfo *ufi; 235 { 236 for (;;) { 237 238 /* 239 * no mapping? give up. 240 */ 241 242 if (uvmfault_lookup(ufi, TRUE) == FALSE) 243 return; 244 245 /* 246 * copy if needed. 247 */ 248 249 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 250 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 251 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 252 253 /* 254 * didn't work? must be out of RAM. unlock and sleep. 255 */ 256 257 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 258 uvmfault_unlockmaps(ufi, TRUE); 259 uvm_wait("fltamapcopy"); 260 continue; 261 } 262 263 /* 264 * got it! unlock and return. 265 */ 266 267 uvmfault_unlockmaps(ufi, TRUE); 268 return; 269 } 270 /*NOTREACHED*/ 271 } 272 273 /* 274 * uvmfault_anonget: get data in an anon into a non-busy, non-released 275 * page in that anon. 276 * 277 * => maps, amap, and anon locked by caller. 278 * => if we fail (result != 0) we unlock everything. 279 * => if we are successful, we return with everything still locked. 280 * => we don't move the page on the queues [gets moved later] 281 * => if we allocate a new page [we_own], it gets put on the queues. 282 * either way, the result is that the page is on the queues at return time 283 * => for pages which are on loan from a uvm_object (and thus are not 284 * owned by the anon): if successful, we return with the owning object 285 * locked. the caller must unlock this object when it unlocks everything 286 * else. 287 */ 288 289 int 290 uvmfault_anonget(ufi, amap, anon) 291 struct uvm_faultinfo *ufi; 292 struct vm_amap *amap; 293 struct vm_anon *anon; 294 { 295 boolean_t we_own; /* we own anon's page? */ 296 boolean_t locked; /* did we relock? */ 297 struct vm_page *pg; 298 int error; 299 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 300 301 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 302 303 error = 0; 304 uvmexp.fltanget++; 305 /* bump rusage counters */ 306 if (anon->u.an_page) 307 curproc->p_stats->p_ru.ru_minflt++; 308 else 309 curproc->p_stats->p_ru.ru_majflt++; 310 311 /* 312 * loop until we get it, or fail. 313 */ 314 315 for (;;) { 316 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 317 pg = anon->u.an_page; 318 319 /* 320 * if there is a resident page and it is loaned, then anon 321 * may not own it. call out to uvm_anon_lockpage() to ensure 322 * the real owner of the page has been identified and locked. 323 */ 324 325 if (pg && pg->loan_count) 326 pg = uvm_anon_lockloanpg(anon); 327 328 /* 329 * page there? make sure it is not busy/released. 330 */ 331 332 if (pg) { 333 334 /* 335 * at this point, if the page has a uobject [meaning 336 * we have it on loan], then that uobject is locked 337 * by us! if the page is busy, we drop all the 338 * locks (including uobject) and try again. 339 */ 340 341 if ((pg->flags & PG_BUSY) == 0) { 342 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 343 return (0); 344 } 345 pg->flags |= PG_WANTED; 346 uvmexp.fltpgwait++; 347 348 /* 349 * the last unlock must be an atomic unlock+wait on 350 * the owner of page 351 */ 352 353 if (pg->uobject) { /* owner is uobject ? */ 354 uvmfault_unlockall(ufi, amap, NULL, anon); 355 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 356 0,0,0); 357 UVM_UNLOCK_AND_WAIT(pg, 358 &pg->uobject->vmobjlock, 359 FALSE, "anonget1",0); 360 } else { 361 /* anon owns page */ 362 uvmfault_unlockall(ufi, amap, NULL, NULL); 363 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 364 0,0,0); 365 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 366 "anonget2",0); 367 } 368 } else { 369 370 /* 371 * no page, we must try and bring it in. 372 */ 373 374 pg = uvm_pagealloc(NULL, 0, anon, 0); 375 if (pg == NULL) { /* out of RAM. */ 376 uvmfault_unlockall(ufi, amap, NULL, anon); 377 uvmexp.fltnoram++; 378 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 379 0,0,0); 380 uvm_wait("flt_noram1"); 381 } else { 382 /* we set the PG_BUSY bit */ 383 we_own = TRUE; 384 uvmfault_unlockall(ufi, amap, NULL, anon); 385 386 /* 387 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 388 * page into the uvm_swap_get function with 389 * all data structures unlocked. note that 390 * it is ok to read an_swslot here because 391 * we hold PG_BUSY on the page. 392 */ 393 uvmexp.pageins++; 394 error = uvm_swap_get(pg, anon->an_swslot, 395 PGO_SYNCIO); 396 397 /* 398 * we clean up after the i/o below in the 399 * "we_own" case 400 */ 401 } 402 } 403 404 /* 405 * now relock and try again 406 */ 407 408 locked = uvmfault_relock(ufi); 409 if (locked && amap != NULL) { 410 amap_lock(amap); 411 } 412 if (locked || we_own) 413 simple_lock(&anon->an_lock); 414 415 /* 416 * if we own the page (i.e. we set PG_BUSY), then we need 417 * to clean up after the I/O. there are three cases to 418 * consider: 419 * [1] page released during I/O: free anon and ReFault. 420 * [2] I/O not OK. free the page and cause the fault 421 * to fail. 422 * [3] I/O OK! activate the page and sync with the 423 * non-we_own case (i.e. drop anon lock if not locked). 424 */ 425 426 if (we_own) { 427 if (pg->flags & PG_WANTED) { 428 wakeup(pg); 429 } 430 if (error) { 431 432 /* 433 * remove the swap slot from the anon 434 * and mark the anon as having no real slot. 435 * don't free the swap slot, thus preventing 436 * it from being used again. 437 */ 438 439 if (anon->an_swslot > 0) 440 uvm_swap_markbad(anon->an_swslot, 1); 441 anon->an_swslot = SWSLOT_BAD; 442 443 if ((pg->flags & PG_RELEASED) != 0) 444 goto released; 445 446 /* 447 * note: page was never !PG_BUSY, so it 448 * can't be mapped and thus no need to 449 * pmap_page_protect it... 450 */ 451 452 uvm_lock_pageq(); 453 uvm_pagefree(pg); 454 uvm_unlock_pageq(); 455 456 if (locked) 457 uvmfault_unlockall(ufi, amap, NULL, 458 anon); 459 else 460 simple_unlock(&anon->an_lock); 461 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 462 return error; 463 } 464 465 if ((pg->flags & PG_RELEASED) != 0) { 466 released: 467 KASSERT(anon->an_ref == 0); 468 469 /* 470 * released while we unlocked amap. 471 */ 472 473 if (locked) 474 uvmfault_unlockall(ufi, amap, NULL, 475 NULL); 476 477 uvm_anon_release(anon); 478 479 if (error) { 480 UVMHIST_LOG(maphist, 481 "<- ERROR/RELEASED", 0,0,0,0); 482 return error; 483 } 484 485 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 486 return ERESTART; 487 } 488 489 /* 490 * we've successfully read the page, activate it. 491 */ 492 493 uvm_lock_pageq(); 494 uvm_pageactivate(pg); 495 uvm_unlock_pageq(); 496 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 497 UVM_PAGE_OWN(pg, NULL); 498 if (!locked) 499 simple_unlock(&anon->an_lock); 500 } 501 502 /* 503 * we were not able to relock. restart fault. 504 */ 505 506 if (!locked) { 507 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 508 return (ERESTART); 509 } 510 511 /* 512 * verify no one has touched the amap and moved the anon on us. 513 */ 514 515 if (ufi != NULL && 516 amap_lookup(&ufi->entry->aref, 517 ufi->orig_rvaddr - ufi->entry->start) != anon) { 518 519 uvmfault_unlockall(ufi, amap, NULL, anon); 520 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 521 return (ERESTART); 522 } 523 524 /* 525 * try it again! 526 */ 527 528 uvmexp.fltanretry++; 529 continue; 530 } 531 /*NOTREACHED*/ 532 } 533 534 /* 535 * F A U L T - m a i n e n t r y p o i n t 536 */ 537 538 /* 539 * uvm_fault: page fault handler 540 * 541 * => called from MD code to resolve a page fault 542 * => VM data structures usually should be unlocked. however, it is 543 * possible to call here with the main map locked if the caller 544 * gets a write lock, sets it recusive, and then calls us (c.f. 545 * uvm_map_pageable). this should be avoided because it keeps 546 * the map locked off during I/O. 547 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 548 */ 549 550 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 551 ~VM_PROT_WRITE : VM_PROT_ALL) 552 553 int 554 uvm_fault(orig_map, vaddr, fault_type, access_type) 555 struct vm_map *orig_map; 556 vaddr_t vaddr; 557 vm_fault_t fault_type; 558 vm_prot_t access_type; 559 { 560 struct uvm_faultinfo ufi; 561 vm_prot_t enter_prot, check_prot; 562 boolean_t wired, narrow, promote, locked, shadowed, wire_fault, cow_now; 563 int npages, nback, nforw, centeridx, error, lcv, gotpages; 564 vaddr_t startva, objaddr, currva; 565 voff_t uoff; 566 paddr_t pa; 567 struct vm_amap *amap; 568 struct uvm_object *uobj; 569 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 570 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 571 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 572 573 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", 574 orig_map, vaddr, fault_type, access_type); 575 576 anon = NULL; 577 pg = NULL; 578 579 uvmexp.faults++; /* XXX: locking? */ 580 581 /* 582 * init the IN parameters in the ufi 583 */ 584 585 ufi.orig_map = orig_map; 586 ufi.orig_rvaddr = trunc_page(vaddr); 587 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 588 wire_fault = fault_type == VM_FAULT_WIRE || 589 fault_type == VM_FAULT_WIREMAX; 590 if (wire_fault) 591 narrow = TRUE; /* don't look for neighborhood 592 * pages on wire */ 593 else 594 narrow = FALSE; /* normal fault */ 595 596 /* 597 * "goto ReFault" means restart the page fault from ground zero. 598 */ 599 ReFault: 600 601 /* 602 * lookup and lock the maps 603 */ 604 605 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 606 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 607 return (EFAULT); 608 } 609 /* locked: maps(read) */ 610 611 #ifdef DIAGNOSTIC 612 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 613 printf("Page fault on non-pageable map:\n"); 614 printf("ufi.map = %p\n", ufi.map); 615 printf("ufi.orig_map = %p\n", ufi.orig_map); 616 printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr); 617 panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0"); 618 } 619 #endif 620 621 /* 622 * check protection 623 */ 624 625 check_prot = fault_type == VM_FAULT_WIREMAX ? 626 ufi.entry->max_protection : ufi.entry->protection; 627 if ((check_prot & access_type) != access_type) { 628 UVMHIST_LOG(maphist, 629 "<- protection failure (prot=0x%x, access=0x%x)", 630 ufi.entry->protection, access_type, 0, 0); 631 uvmfault_unlockmaps(&ufi, FALSE); 632 return EACCES; 633 } 634 635 /* 636 * "enter_prot" is the protection we want to enter the page in at. 637 * for certain pages (e.g. copy-on-write pages) this protection can 638 * be more strict than ufi.entry->protection. "wired" means either 639 * the entry is wired or we are fault-wiring the pg. 640 */ 641 642 enter_prot = ufi.entry->protection; 643 wired = VM_MAPENT_ISWIRED(ufi.entry) || wire_fault; 644 if (wired) { 645 access_type = enter_prot; /* full access for wired */ 646 cow_now = (check_prot & VM_PROT_WRITE) != 0; 647 } else { 648 cow_now = (access_type & VM_PROT_WRITE) != 0; 649 } 650 651 /* 652 * handle "needs_copy" case. if we need to copy the amap we will 653 * have to drop our readlock and relock it with a write lock. (we 654 * need a write lock to change anything in a map entry [e.g. 655 * needs_copy]). 656 */ 657 658 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 659 KASSERT(fault_type != VM_FAULT_WIREMAX); 660 if (cow_now || (ufi.entry->object.uvm_obj == NULL)) { 661 /* need to clear */ 662 UVMHIST_LOG(maphist, 663 " need to clear needs_copy and refault",0,0,0,0); 664 uvmfault_unlockmaps(&ufi, FALSE); 665 uvmfault_amapcopy(&ufi); 666 uvmexp.fltamcopy++; 667 goto ReFault; 668 669 } else { 670 671 /* 672 * ensure that we pmap_enter page R/O since 673 * needs_copy is still true 674 */ 675 676 enter_prot &= ~VM_PROT_WRITE; 677 } 678 } 679 680 /* 681 * identify the players 682 */ 683 684 amap = ufi.entry->aref.ar_amap; /* top layer */ 685 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 686 687 /* 688 * check for a case 0 fault. if nothing backing the entry then 689 * error now. 690 */ 691 692 if (amap == NULL && uobj == NULL) { 693 uvmfault_unlockmaps(&ufi, FALSE); 694 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 695 return (EFAULT); 696 } 697 698 /* 699 * establish range of interest based on advice from mapper 700 * and then clip to fit map entry. note that we only want 701 * to do this the first time through the fault. if we 702 * ReFault we will disable this by setting "narrow" to true. 703 */ 704 705 if (narrow == FALSE) { 706 707 /* wide fault (!narrow) */ 708 KASSERT(uvmadvice[ufi.entry->advice].advice == 709 ufi.entry->advice); 710 nback = MIN(uvmadvice[ufi.entry->advice].nback, 711 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 712 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 713 nforw = MIN(uvmadvice[ufi.entry->advice].nforw, 714 ((ufi.entry->end - ufi.orig_rvaddr) >> 715 PAGE_SHIFT) - 1); 716 /* 717 * note: "-1" because we don't want to count the 718 * faulting page as forw 719 */ 720 npages = nback + nforw + 1; 721 centeridx = nback; 722 723 narrow = TRUE; /* ensure only once per-fault */ 724 725 } else { 726 727 /* narrow fault! */ 728 nback = nforw = 0; 729 startva = ufi.orig_rvaddr; 730 npages = 1; 731 centeridx = 0; 732 733 } 734 735 /* locked: maps(read) */ 736 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 737 narrow, nback, nforw, startva); 738 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 739 amap, uobj, 0); 740 741 /* 742 * if we've got an amap, lock it and extract current anons. 743 */ 744 745 if (amap) { 746 amap_lock(amap); 747 anons = anons_store; 748 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 749 anons, npages); 750 } else { 751 anons = NULL; /* to be safe */ 752 } 753 754 /* locked: maps(read), amap(if there) */ 755 756 /* 757 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 758 * now and then forget about them (for the rest of the fault). 759 */ 760 761 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 762 763 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 764 0,0,0,0); 765 /* flush back-page anons? */ 766 if (amap) 767 uvmfault_anonflush(anons, nback); 768 769 /* flush object? */ 770 if (uobj) { 771 objaddr = 772 (startva - ufi.entry->start) + ufi.entry->offset; 773 simple_lock(&uobj->vmobjlock); 774 (void) (uobj->pgops->pgo_put)(uobj, objaddr, objaddr + 775 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 776 } 777 778 /* now forget about the backpages */ 779 if (amap) 780 anons += nback; 781 startva += (nback << PAGE_SHIFT); 782 npages -= nback; 783 nback = centeridx = 0; 784 } 785 786 /* locked: maps(read), amap(if there) */ 787 788 /* 789 * map in the backpages and frontpages we found in the amap in hopes 790 * of preventing future faults. we also init the pages[] array as 791 * we go. 792 */ 793 794 currva = startva; 795 shadowed = FALSE; 796 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 797 798 /* 799 * dont play with VAs that are already mapped 800 * except for center) 801 */ 802 if (lcv != centeridx && 803 pmap_extract(ufi.orig_map->pmap, currva, &pa)) { 804 pages[lcv] = PGO_DONTCARE; 805 continue; 806 } 807 808 /* 809 * unmapped or center page. check if any anon at this level. 810 */ 811 if (amap == NULL || anons[lcv] == NULL) { 812 pages[lcv] = NULL; 813 continue; 814 } 815 816 /* 817 * check for present page and map if possible. re-activate it. 818 */ 819 820 pages[lcv] = PGO_DONTCARE; 821 if (lcv == centeridx) { /* save center for later! */ 822 shadowed = TRUE; 823 continue; 824 } 825 anon = anons[lcv]; 826 simple_lock(&anon->an_lock); 827 /* ignore loaned pages */ 828 if (anon->u.an_page && anon->u.an_page->loan_count == 0 && 829 (anon->u.an_page->flags & PG_BUSY) == 0) { 830 uvm_lock_pageq(); 831 uvm_pageactivate(anon->u.an_page); 832 uvm_unlock_pageq(); 833 UVMHIST_LOG(maphist, 834 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 835 ufi.orig_map->pmap, currva, anon->u.an_page, 0); 836 uvmexp.fltnamap++; 837 838 /* 839 * Since this isn't the page that's actually faulting, 840 * ignore pmap_enter() failures; it's not critical 841 * that we enter these right now. 842 */ 843 844 (void) pmap_enter(ufi.orig_map->pmap, currva, 845 VM_PAGE_TO_PHYS(anon->u.an_page), 846 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 847 enter_prot, 848 PMAP_CANFAIL | 849 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 850 } 851 simple_unlock(&anon->an_lock); 852 pmap_update(ufi.orig_map->pmap); 853 } 854 855 /* locked: maps(read), amap(if there) */ 856 /* (shadowed == TRUE) if there is an anon at the faulting address */ 857 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 858 (uobj && shadowed == FALSE),0,0); 859 860 /* 861 * note that if we are really short of RAM we could sleep in the above 862 * call to pmap_enter with everything locked. bad? 863 * 864 * XXX Actually, that is bad; pmap_enter() should just fail in that 865 * XXX case. --thorpej 866 */ 867 868 /* 869 * if the desired page is not shadowed by the amap and we have a 870 * backing object, then we check to see if the backing object would 871 * prefer to handle the fault itself (rather than letting us do it 872 * with the usual pgo_get hook). the backing object signals this by 873 * providing a pgo_fault routine. 874 */ 875 876 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 877 simple_lock(&uobj->vmobjlock); 878 879 /* locked: maps(read), amap (if there), uobj */ 880 error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 881 centeridx, fault_type, access_type, PGO_LOCKED|PGO_SYNCIO); 882 883 /* locked: nothing, pgo_fault has unlocked everything */ 884 885 if (error == ERESTART) 886 goto ReFault; /* try again! */ 887 /* 888 * object fault routine responsible for pmap_update(). 889 */ 890 return error; 891 } 892 893 /* 894 * now, if the desired page is not shadowed by the amap and we have 895 * a backing object that does not have a special fault routine, then 896 * we ask (with pgo_get) the object for resident pages that we care 897 * about and attempt to map them in. we do not let pgo_get block 898 * (PGO_LOCKED). 899 */ 900 901 if (uobj && shadowed == FALSE) { 902 simple_lock(&uobj->vmobjlock); 903 904 /* locked (!shadowed): maps(read), amap (if there), uobj */ 905 /* 906 * the following call to pgo_get does _not_ change locking state 907 */ 908 909 uvmexp.fltlget++; 910 gotpages = npages; 911 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 912 (startva - ufi.entry->start), 913 pages, &gotpages, centeridx, 914 access_type & MASK(ufi.entry), 915 ufi.entry->advice, PGO_LOCKED); 916 917 /* 918 * check for pages to map, if we got any 919 */ 920 921 uobjpage = NULL; 922 923 if (gotpages) { 924 currva = startva; 925 for (lcv = 0; lcv < npages; 926 lcv++, currva += PAGE_SIZE) { 927 struct vm_page *curpg; 928 boolean_t readonly; 929 930 curpg = pages[lcv]; 931 if (curpg == NULL || curpg == PGO_DONTCARE) { 932 continue; 933 } 934 935 /* 936 * if center page is resident and not 937 * PG_BUSY|PG_RELEASED then pgo_get 938 * made it PG_BUSY for us and gave 939 * us a handle to it. remember this 940 * page as "uobjpage." (for later use). 941 */ 942 943 if (lcv == centeridx) { 944 uobjpage = curpg; 945 UVMHIST_LOG(maphist, " got uobjpage " 946 "(0x%x) with locked get", 947 uobjpage, 0,0,0); 948 continue; 949 } 950 951 /* 952 * calling pgo_get with PGO_LOCKED returns us 953 * pages which are neither busy nor released, 954 * so we don't need to check for this. 955 * we can just directly enter the pages. 956 */ 957 958 uvm_lock_pageq(); 959 uvm_pageactivate(curpg); 960 uvm_unlock_pageq(); 961 UVMHIST_LOG(maphist, 962 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 963 ufi.orig_map->pmap, currva, curpg, 0); 964 uvmexp.fltnomap++; 965 966 /* 967 * Since this page isn't the page that's 968 * actually faulting, ignore pmap_enter() 969 * failures; it's not critical that we 970 * enter these right now. 971 */ 972 KASSERT((curpg->flags & PG_PAGEOUT) == 0); 973 KASSERT((curpg->flags & PG_RELEASED) == 0); 974 readonly = (curpg->flags & PG_RDONLY) 975 || (curpg->loan_count > 0); 976 977 (void) pmap_enter(ufi.orig_map->pmap, currva, 978 VM_PAGE_TO_PHYS(curpg), 979 readonly ? 980 enter_prot & ~VM_PROT_WRITE : 981 enter_prot & MASK(ufi.entry), 982 PMAP_CANFAIL | 983 (wired ? PMAP_WIRED : 0)); 984 985 /* 986 * NOTE: page can't be PG_WANTED or PG_RELEASED 987 * because we've held the lock the whole time 988 * we've had the handle. 989 */ 990 991 curpg->flags &= ~(PG_BUSY); 992 UVM_PAGE_OWN(curpg, NULL); 993 } 994 pmap_update(ufi.orig_map->pmap); 995 } 996 } else { 997 uobjpage = NULL; 998 } 999 1000 /* locked (shadowed): maps(read), amap */ 1001 /* locked (!shadowed): maps(read), amap(if there), 1002 uobj(if !null), uobjpage(if !null) */ 1003 1004 /* 1005 * note that at this point we are done with any front or back pages. 1006 * we are now going to focus on the center page (i.e. the one we've 1007 * faulted on). if we have faulted on the top (anon) layer 1008 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1009 * not touched it yet). if we have faulted on the bottom (uobj) 1010 * layer [i.e. case 2] and the page was both present and available, 1011 * then we've got a pointer to it as "uobjpage" and we've already 1012 * made it BUSY. 1013 */ 1014 1015 /* 1016 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1017 */ 1018 1019 /* 1020 * redirect case 2: if we are not shadowed, go to case 2. 1021 */ 1022 1023 if (shadowed == FALSE) 1024 goto Case2; 1025 1026 /* locked: maps(read), amap */ 1027 1028 /* 1029 * handle case 1: fault on an anon in our amap 1030 */ 1031 1032 anon = anons[centeridx]; 1033 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1034 simple_lock(&anon->an_lock); 1035 1036 /* locked: maps(read), amap, anon */ 1037 1038 /* 1039 * no matter if we have case 1A or case 1B we are going to need to 1040 * have the anon's memory resident. ensure that now. 1041 */ 1042 1043 /* 1044 * let uvmfault_anonget do the dirty work. 1045 * if it fails (!OK) it will unlock everything for us. 1046 * if it succeeds, locks are still valid and locked. 1047 * also, if it is OK, then the anon's page is on the queues. 1048 * if the page is on loan from a uvm_object, then anonget will 1049 * lock that object for us if it does not fail. 1050 */ 1051 1052 error = uvmfault_anonget(&ufi, amap, anon); 1053 switch (error) { 1054 case 0: 1055 break; 1056 1057 case ERESTART: 1058 goto ReFault; 1059 1060 case EAGAIN: 1061 tsleep(&lbolt, PVM, "fltagain1", 0); 1062 goto ReFault; 1063 1064 default: 1065 return error; 1066 } 1067 1068 /* 1069 * uobj is non null if the page is on loan from an object (i.e. uobj) 1070 */ 1071 1072 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */ 1073 1074 /* locked: maps(read), amap, anon, uobj(if one) */ 1075 1076 /* 1077 * special handling for loaned pages 1078 */ 1079 1080 if (anon->u.an_page->loan_count) { 1081 1082 if (!cow_now) { 1083 1084 /* 1085 * for read faults on loaned pages we just cap the 1086 * protection at read-only. 1087 */ 1088 1089 enter_prot = enter_prot & ~VM_PROT_WRITE; 1090 1091 } else { 1092 /* 1093 * note that we can't allow writes into a loaned page! 1094 * 1095 * if we have a write fault on a loaned page in an 1096 * anon then we need to look at the anon's ref count. 1097 * if it is greater than one then we are going to do 1098 * a normal copy-on-write fault into a new anon (this 1099 * is not a problem). however, if the reference count 1100 * is one (a case where we would normally allow a 1101 * write directly to the page) then we need to kill 1102 * the loan before we continue. 1103 */ 1104 1105 /* >1 case is already ok */ 1106 if (anon->an_ref == 1) { 1107 1108 /* get new un-owned replacement page */ 1109 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1110 if (pg == NULL) { 1111 uvmfault_unlockall(&ufi, amap, uobj, 1112 anon); 1113 uvm_wait("flt_noram2"); 1114 goto ReFault; 1115 } 1116 1117 /* 1118 * copy data, kill loan, and drop uobj lock 1119 * (if any) 1120 */ 1121 /* copy old -> new */ 1122 uvm_pagecopy(anon->u.an_page, pg); 1123 1124 /* force reload */ 1125 pmap_page_protect(anon->u.an_page, 1126 VM_PROT_NONE); 1127 uvm_lock_pageq(); /* KILL loan */ 1128 1129 anon->u.an_page->uanon = NULL; 1130 /* in case we owned */ 1131 anon->u.an_page->pqflags &= ~PQ_ANON; 1132 1133 if (uobj) { 1134 /* if we were receiver of loan */ 1135 anon->u.an_page->loan_count--; 1136 } else { 1137 /* 1138 * we were the lender (A->K); need 1139 * to remove the page from pageq's. 1140 */ 1141 uvm_pagedequeue(anon->u.an_page); 1142 } 1143 1144 uvm_pageactivate(pg); 1145 uvm_unlock_pageq(); 1146 if (uobj) { 1147 simple_unlock(&uobj->vmobjlock); 1148 uobj = NULL; 1149 } 1150 1151 /* install new page in anon */ 1152 anon->u.an_page = pg; 1153 pg->uanon = anon; 1154 pg->pqflags |= PQ_ANON; 1155 pg->flags &= ~(PG_BUSY|PG_FAKE); 1156 UVM_PAGE_OWN(pg, NULL); 1157 1158 /* done! */ 1159 } /* ref == 1 */ 1160 } /* write fault */ 1161 } /* loan count */ 1162 1163 /* 1164 * if we are case 1B then we will need to allocate a new blank 1165 * anon to transfer the data into. note that we have a lock 1166 * on anon, so no one can busy or release the page until we are done. 1167 * also note that the ref count can't drop to zero here because 1168 * it is > 1 and we are only dropping one ref. 1169 * 1170 * in the (hopefully very rare) case that we are out of RAM we 1171 * will unlock, wait for more RAM, and refault. 1172 * 1173 * if we are out of anon VM we kill the process (XXX: could wait?). 1174 */ 1175 1176 if (cow_now && anon->an_ref > 1) { 1177 1178 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1179 uvmexp.flt_acow++; 1180 oanon = anon; /* oanon = old, locked anon */ 1181 anon = uvm_analloc(); 1182 if (anon) { 1183 /* new anon is locked! */ 1184 pg = uvm_pagealloc(NULL, 0, anon, 0); 1185 } 1186 1187 /* check for out of RAM */ 1188 if (anon == NULL || pg == NULL) { 1189 if (anon) { 1190 anon->an_ref--; 1191 simple_unlock(&anon->an_lock); 1192 uvm_anfree(anon); 1193 } 1194 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1195 if (anon == NULL || uvm_swapisfull()) { 1196 UVMHIST_LOG(maphist, 1197 "<- failed. out of VM",0,0,0,0); 1198 uvmexp.fltnoanon++; 1199 return ENOMEM; 1200 } 1201 1202 uvmexp.fltnoram++; 1203 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1204 goto ReFault; 1205 } 1206 1207 /* got all resources, replace anon with nanon */ 1208 uvm_pagecopy(oanon->u.an_page, pg); 1209 uvm_lock_pageq(); 1210 uvm_pageactivate(pg); 1211 pg->flags &= ~(PG_BUSY|PG_FAKE); 1212 uvm_unlock_pageq(); 1213 UVM_PAGE_OWN(pg, NULL); 1214 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1215 anon, 1); 1216 1217 /* deref: can not drop to zero here by defn! */ 1218 oanon->an_ref--; 1219 1220 /* 1221 * note: oanon is still locked, as is the new anon. we 1222 * need to check for this later when we unlock oanon; if 1223 * oanon != anon, we'll have to unlock anon, too. 1224 */ 1225 1226 } else { 1227 1228 uvmexp.flt_anon++; 1229 oanon = anon; /* old, locked anon is same as anon */ 1230 pg = anon->u.an_page; 1231 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1232 enter_prot = enter_prot & ~VM_PROT_WRITE; 1233 1234 } 1235 1236 /* locked: maps(read), amap, oanon, anon (if different from oanon) */ 1237 1238 /* 1239 * now map the page in. 1240 */ 1241 1242 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1243 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1244 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1245 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1246 != 0) { 1247 1248 /* 1249 * No need to undo what we did; we can simply think of 1250 * this as the pmap throwing away the mapping information. 1251 * 1252 * We do, however, have to go through the ReFault path, 1253 * as the map may change while we're asleep. 1254 */ 1255 1256 if (anon != oanon) 1257 simple_unlock(&anon->an_lock); 1258 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1259 if (uvm_swapisfull()) { 1260 UVMHIST_LOG(maphist, 1261 "<- failed. out of VM",0,0,0,0); 1262 /* XXX instrumentation */ 1263 return ENOMEM; 1264 } 1265 /* XXX instrumentation */ 1266 uvm_wait("flt_pmfail1"); 1267 goto ReFault; 1268 } 1269 1270 /* 1271 * ... update the page queues. 1272 */ 1273 1274 uvm_lock_pageq(); 1275 if (wire_fault) { 1276 uvm_pagewire(pg); 1277 1278 /* 1279 * since the now-wired page cannot be paged out, 1280 * release its swap resources for others to use. 1281 * since an anon with no swap cannot be PG_CLEAN, 1282 * clear its clean flag now. 1283 */ 1284 1285 pg->flags &= ~(PG_CLEAN); 1286 uvm_anon_dropswap(anon); 1287 } else { 1288 uvm_pageactivate(pg); 1289 } 1290 uvm_unlock_pageq(); 1291 1292 /* 1293 * done case 1! finish up by unlocking everything and returning success 1294 */ 1295 1296 if (anon != oanon) 1297 simple_unlock(&anon->an_lock); 1298 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1299 pmap_update(ufi.orig_map->pmap); 1300 return 0; 1301 1302 Case2: 1303 /* 1304 * handle case 2: faulting on backing object or zero fill 1305 */ 1306 1307 /* 1308 * locked: 1309 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1310 */ 1311 1312 /* 1313 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1314 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1315 * have a backing object, check and see if we are going to promote 1316 * the data up to an anon during the fault. 1317 */ 1318 1319 if (uobj == NULL) { 1320 uobjpage = PGO_DONTCARE; 1321 promote = TRUE; /* always need anon here */ 1322 } else { 1323 KASSERT(uobjpage != PGO_DONTCARE); 1324 promote = cow_now && UVM_ET_ISCOPYONWRITE(ufi.entry); 1325 } 1326 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1327 promote, (uobj == NULL), 0,0); 1328 1329 /* 1330 * if uobjpage is not null then we do not need to do I/O to get the 1331 * uobjpage. 1332 * 1333 * if uobjpage is null, then we need to unlock and ask the pager to 1334 * get the data for us. once we have the data, we need to reverify 1335 * the state the world. we are currently not holding any resources. 1336 */ 1337 1338 if (uobjpage) { 1339 /* update rusage counters */ 1340 curproc->p_stats->p_ru.ru_minflt++; 1341 } else { 1342 /* update rusage counters */ 1343 curproc->p_stats->p_ru.ru_majflt++; 1344 1345 /* locked: maps(read), amap(if there), uobj */ 1346 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1347 /* locked: uobj */ 1348 1349 uvmexp.fltget++; 1350 gotpages = 1; 1351 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1352 error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1353 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1354 PGO_SYNCIO); 1355 /* locked: uobjpage(if no error) */ 1356 1357 /* 1358 * recover from I/O 1359 */ 1360 1361 if (error) { 1362 if (error == EAGAIN) { 1363 UVMHIST_LOG(maphist, 1364 " pgo_get says TRY AGAIN!",0,0,0,0); 1365 tsleep(&lbolt, PVM, "fltagain2", 0); 1366 goto ReFault; 1367 } 1368 1369 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1370 error, 0,0,0); 1371 return error; 1372 } 1373 1374 /* locked: uobjpage */ 1375 1376 uvm_lock_pageq(); 1377 uvm_pageactivate(uobjpage); 1378 uvm_unlock_pageq(); 1379 1380 /* 1381 * re-verify the state of the world by first trying to relock 1382 * the maps. always relock the object. 1383 */ 1384 1385 locked = uvmfault_relock(&ufi); 1386 if (locked && amap) 1387 amap_lock(amap); 1388 simple_lock(&uobj->vmobjlock); 1389 1390 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1391 /* locked(!locked): uobj, uobjpage */ 1392 1393 /* 1394 * verify that the page has not be released and re-verify 1395 * that amap slot is still free. if there is a problem, 1396 * we unlock and clean up. 1397 */ 1398 1399 if ((uobjpage->flags & PG_RELEASED) != 0 || 1400 (locked && amap && 1401 amap_lookup(&ufi.entry->aref, 1402 ufi.orig_rvaddr - ufi.entry->start))) { 1403 if (locked) 1404 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1405 locked = FALSE; 1406 } 1407 1408 /* 1409 * didn't get the lock? release the page and retry. 1410 */ 1411 1412 if (locked == FALSE) { 1413 UVMHIST_LOG(maphist, 1414 " wasn't able to relock after fault: retry", 1415 0,0,0,0); 1416 if (uobjpage->flags & PG_WANTED) 1417 wakeup(uobjpage); 1418 if (uobjpage->flags & PG_RELEASED) { 1419 uvmexp.fltpgrele++; 1420 uvm_pagefree(uobjpage); 1421 goto ReFault; 1422 } 1423 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1424 UVM_PAGE_OWN(uobjpage, NULL); 1425 simple_unlock(&uobj->vmobjlock); 1426 goto ReFault; 1427 } 1428 1429 /* 1430 * we have the data in uobjpage which is busy and 1431 * not released. we are holding object lock (so the page 1432 * can't be released on us). 1433 */ 1434 1435 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1436 } 1437 1438 /* 1439 * locked: 1440 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1441 */ 1442 1443 /* 1444 * notes: 1445 * - at this point uobjpage can not be NULL 1446 * - at this point uobjpage can not be PG_RELEASED (since we checked 1447 * for it above) 1448 * - at this point uobjpage could be PG_WANTED (handle later) 1449 */ 1450 1451 if (promote == FALSE) { 1452 1453 /* 1454 * we are not promoting. if the mapping is COW ensure that we 1455 * don't give more access than we should (e.g. when doing a read 1456 * fault on a COPYONWRITE mapping we want to map the COW page in 1457 * R/O even though the entry protection could be R/W). 1458 * 1459 * set "pg" to the page we want to map in (uobjpage, usually) 1460 */ 1461 1462 /* no anon in this case. */ 1463 anon = NULL; 1464 1465 uvmexp.flt_obj++; 1466 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1467 enter_prot &= ~VM_PROT_WRITE; 1468 pg = uobjpage; /* map in the actual object */ 1469 1470 /* assert(uobjpage != PGO_DONTCARE) */ 1471 1472 /* 1473 * we are faulting directly on the page. be careful 1474 * about writing to loaned pages... 1475 */ 1476 1477 if (uobjpage->loan_count) { 1478 if (!cow_now) { 1479 /* read fault: cap the protection at readonly */ 1480 /* cap! */ 1481 enter_prot = enter_prot & ~VM_PROT_WRITE; 1482 } else { 1483 /* write fault: must break the loan here */ 1484 1485 pg = uvm_loanbreak(uobjpage); 1486 if (pg == NULL) { 1487 1488 /* 1489 * drop ownership of page, it can't 1490 * be released 1491 */ 1492 1493 if (uobjpage->flags & PG_WANTED) 1494 wakeup(uobjpage); 1495 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1496 UVM_PAGE_OWN(uobjpage, NULL); 1497 1498 uvmfault_unlockall(&ufi, amap, uobj, 1499 NULL); 1500 UVMHIST_LOG(maphist, 1501 " out of RAM breaking loan, waiting", 1502 0,0,0,0); 1503 uvmexp.fltnoram++; 1504 uvm_wait("flt_noram4"); 1505 goto ReFault; 1506 } 1507 uobjpage = pg; 1508 } 1509 } 1510 } else { 1511 1512 /* 1513 * if we are going to promote the data to an anon we 1514 * allocate a blank anon here and plug it into our amap. 1515 */ 1516 #if DIAGNOSTIC 1517 if (amap == NULL) 1518 panic("uvm_fault: want to promote data, but no anon"); 1519 #endif 1520 1521 anon = uvm_analloc(); 1522 if (anon) { 1523 1524 /* 1525 * The new anon is locked. 1526 * 1527 * In `Fill in data...' below, if 1528 * uobjpage == PGO_DONTCARE, we want 1529 * a zero'd, dirty page, so have 1530 * uvm_pagealloc() do that for us. 1531 */ 1532 1533 pg = uvm_pagealloc(NULL, 0, anon, 1534 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1535 } 1536 1537 /* 1538 * out of memory resources? 1539 */ 1540 1541 if (anon == NULL || pg == NULL) { 1542 if (anon != NULL) { 1543 anon->an_ref--; 1544 simple_unlock(&anon->an_lock); 1545 uvm_anfree(anon); 1546 } 1547 1548 /* 1549 * arg! must unbusy our page and fail or sleep. 1550 */ 1551 1552 if (uobjpage != PGO_DONTCARE) { 1553 if (uobjpage->flags & PG_WANTED) 1554 /* still holding object lock */ 1555 wakeup(uobjpage); 1556 1557 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1558 UVM_PAGE_OWN(uobjpage, NULL); 1559 } 1560 1561 /* unlock and fail ... */ 1562 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1563 if (anon == NULL || uvm_swapisfull()) { 1564 UVMHIST_LOG(maphist, " promote: out of VM", 1565 0,0,0,0); 1566 uvmexp.fltnoanon++; 1567 return ENOMEM; 1568 } 1569 1570 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1571 0,0,0,0); 1572 uvmexp.fltnoram++; 1573 uvm_wait("flt_noram5"); 1574 goto ReFault; 1575 } 1576 1577 /* 1578 * fill in the data 1579 */ 1580 1581 if (uobjpage != PGO_DONTCARE) { 1582 uvmexp.flt_prcopy++; 1583 /* copy page [pg now dirty] */ 1584 uvm_pagecopy(uobjpage, pg); 1585 1586 /* 1587 * promote to shared amap? make sure all sharing 1588 * procs see it 1589 */ 1590 1591 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1592 pmap_page_protect(uobjpage, VM_PROT_NONE); 1593 /* 1594 * XXX: PAGE MIGHT BE WIRED! 1595 */ 1596 } 1597 1598 /* 1599 * dispose of uobjpage. it can't be PG_RELEASED 1600 * since we still hold the object lock. 1601 * drop handle to uobj as well. 1602 */ 1603 1604 if (uobjpage->flags & PG_WANTED) 1605 /* still have the obj lock */ 1606 wakeup(uobjpage); 1607 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1608 UVM_PAGE_OWN(uobjpage, NULL); 1609 simple_unlock(&uobj->vmobjlock); 1610 uobj = NULL; 1611 1612 UVMHIST_LOG(maphist, 1613 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1614 uobjpage, anon, pg, 0); 1615 1616 } else { 1617 uvmexp.flt_przero++; 1618 1619 /* 1620 * Page is zero'd and marked dirty by uvm_pagealloc() 1621 * above. 1622 */ 1623 1624 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1625 anon, pg, 0, 0); 1626 } 1627 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1628 anon, 0); 1629 } 1630 1631 /* 1632 * locked: 1633 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj), 1634 * anon(if !null), pg(if anon) 1635 * 1636 * note: pg is either the uobjpage or the new page in the new anon 1637 */ 1638 1639 /* 1640 * all resources are present. we can now map it in and free our 1641 * resources. 1642 */ 1643 1644 UVMHIST_LOG(maphist, 1645 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1646 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1647 KASSERT((access_type & VM_PROT_WRITE) == 0 || 1648 (pg->flags & PG_RDONLY) == 0); 1649 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1650 pg->flags & PG_RDONLY ? enter_prot & ~VM_PROT_WRITE : enter_prot, 1651 access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) { 1652 1653 /* 1654 * No need to undo what we did; we can simply think of 1655 * this as the pmap throwing away the mapping information. 1656 * 1657 * We do, however, have to go through the ReFault path, 1658 * as the map may change while we're asleep. 1659 */ 1660 1661 if (pg->flags & PG_WANTED) 1662 wakeup(pg); 1663 1664 /* 1665 * note that pg can't be PG_RELEASED since we did not drop 1666 * the object lock since the last time we checked. 1667 */ 1668 1669 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1670 UVM_PAGE_OWN(pg, NULL); 1671 uvmfault_unlockall(&ufi, amap, uobj, anon); 1672 if (uvm_swapisfull()) { 1673 UVMHIST_LOG(maphist, 1674 "<- failed. out of VM",0,0,0,0); 1675 /* XXX instrumentation */ 1676 return ENOMEM; 1677 } 1678 /* XXX instrumentation */ 1679 uvm_wait("flt_pmfail2"); 1680 goto ReFault; 1681 } 1682 1683 uvm_lock_pageq(); 1684 if (wire_fault) { 1685 uvm_pagewire(pg); 1686 if (pg->pqflags & PQ_AOBJ) { 1687 1688 /* 1689 * since the now-wired page cannot be paged out, 1690 * release its swap resources for others to use. 1691 * since an aobj page with no swap cannot be PG_CLEAN, 1692 * clear its clean flag now. 1693 */ 1694 1695 pg->flags &= ~(PG_CLEAN); 1696 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1697 } 1698 } else { 1699 uvm_pageactivate(pg); 1700 } 1701 uvm_unlock_pageq(); 1702 if (pg->flags & PG_WANTED) 1703 wakeup(pg); 1704 1705 /* 1706 * note that pg can't be PG_RELEASED since we did not drop the object 1707 * lock since the last time we checked. 1708 */ 1709 1710 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1711 UVM_PAGE_OWN(pg, NULL); 1712 uvmfault_unlockall(&ufi, amap, uobj, anon); 1713 pmap_update(ufi.orig_map->pmap); 1714 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1715 return 0; 1716 } 1717 1718 /* 1719 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1720 * 1721 * => map may be read-locked by caller, but MUST NOT be write-locked. 1722 * => if map is read-locked, any operations which may cause map to 1723 * be write-locked in uvm_fault() must be taken care of by 1724 * the caller. See uvm_map_pageable(). 1725 */ 1726 1727 int 1728 uvm_fault_wire(map, start, end, fault_type, access_type) 1729 struct vm_map *map; 1730 vaddr_t start, end; 1731 vm_fault_t fault_type; 1732 vm_prot_t access_type; 1733 { 1734 vaddr_t va; 1735 int error; 1736 1737 /* 1738 * now fault it in a page at a time. if the fault fails then we have 1739 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1740 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1741 */ 1742 1743 /* 1744 * XXX work around overflowing a vaddr_t. this prevents us from 1745 * wiring the last page in the address space, though. 1746 */ 1747 if (start > end) { 1748 return EFAULT; 1749 } 1750 1751 for (va = start ; va < end ; va += PAGE_SIZE) { 1752 error = uvm_fault(map, va, fault_type, access_type); 1753 if (error) { 1754 if (va != start) { 1755 uvm_fault_unwire(map, start, va); 1756 } 1757 return error; 1758 } 1759 } 1760 return 0; 1761 } 1762 1763 /* 1764 * uvm_fault_unwire(): unwire range of virtual space. 1765 */ 1766 1767 void 1768 uvm_fault_unwire(map, start, end) 1769 struct vm_map *map; 1770 vaddr_t start, end; 1771 { 1772 vm_map_lock_read(map); 1773 uvm_fault_unwire_locked(map, start, end); 1774 vm_map_unlock_read(map); 1775 } 1776 1777 /* 1778 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1779 * 1780 * => map must be at least read-locked. 1781 */ 1782 1783 void 1784 uvm_fault_unwire_locked(map, start, end) 1785 struct vm_map *map; 1786 vaddr_t start, end; 1787 { 1788 struct vm_map_entry *entry; 1789 pmap_t pmap = vm_map_pmap(map); 1790 vaddr_t va; 1791 paddr_t pa; 1792 struct vm_page *pg; 1793 1794 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1795 1796 /* 1797 * we assume that the area we are unwiring has actually been wired 1798 * in the first place. this means that we should be able to extract 1799 * the PAs from the pmap. we also lock out the page daemon so that 1800 * we can call uvm_pageunwire. 1801 */ 1802 1803 uvm_lock_pageq(); 1804 1805 /* 1806 * find the beginning map entry for the region. 1807 */ 1808 1809 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1810 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1811 panic("uvm_fault_unwire_locked: address not in map"); 1812 1813 for (va = start; va < end; va += PAGE_SIZE) { 1814 if (pmap_extract(pmap, va, &pa) == FALSE) 1815 continue; 1816 1817 /* 1818 * find the map entry for the current address. 1819 */ 1820 1821 KASSERT(va >= entry->start); 1822 while (va >= entry->end) { 1823 KASSERT(entry->next != &map->header && 1824 entry->next->start <= entry->end); 1825 entry = entry->next; 1826 } 1827 1828 /* 1829 * if the entry is no longer wired, tell the pmap. 1830 */ 1831 1832 if (VM_MAPENT_ISWIRED(entry) == 0) 1833 pmap_unwire(pmap, va); 1834 1835 pg = PHYS_TO_VM_PAGE(pa); 1836 if (pg) 1837 uvm_pageunwire(pg); 1838 } 1839 1840 uvm_unlock_pageq(); 1841 }
Cache object: 402ceb291afd54fb3d196b864fe7b6f0
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]