Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/uvm/uvm_amap.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* $NetBSD: uvm_amap.c,v 1.76 2006/11/01 10:18:27 yamt Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /* 36 * uvm_amap.c: amap operations 37 */ 38 39 /* 40 * this file contains functions that perform operations on amaps. see 41 * uvm_amap.h for a brief explanation of the role of amaps in uvm. 42 */ 43 44 #include <sys/cdefs.h> 45 __KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.76 2006/11/01 10:18:27 yamt Exp $"); 46 47 #include "opt_uvmhist.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/proc.h> 52 #include <sys/kernel.h> 53 #include <sys/kmem.h> 54 #include <sys/pool.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_swap.h> 58 59 /* 60 * pool for allocation of vm_map structures. note that the pool has 61 * its own simplelock for its protection. also note that in order to 62 * avoid an endless loop, the amap pool's allocator cannot allocate 63 * memory from an amap (it currently goes through the kernel uobj, so 64 * we are ok). 65 */ 66 POOL_INIT(uvm_amap_pool, sizeof(struct vm_amap), 0, 0, 0, "amappl", 67 &pool_allocator_nointr); 68 69 static struct simplelock amap_list_lock = SIMPLELOCK_INITIALIZER; 70 static LIST_HEAD(, vm_amap) amap_list; 71 72 /* 73 * local functions 74 */ 75 76 static inline void 77 amap_list_insert(struct vm_amap *amap) 78 { 79 80 simple_lock(&amap_list_lock); 81 LIST_INSERT_HEAD(&amap_list, amap, am_list); 82 simple_unlock(&amap_list_lock); 83 } 84 85 static inline void 86 amap_list_remove(struct vm_amap *amap) 87 { 88 89 simple_lock(&amap_list_lock); 90 LIST_REMOVE(amap, am_list); 91 simple_unlock(&amap_list_lock); 92 } 93 94 static int 95 amap_roundup_slots(int slots) 96 { 97 98 return kmem_roundup_size(slots * sizeof(int)) / sizeof(int); 99 } 100 101 #ifdef UVM_AMAP_PPREF 102 /* 103 * what is ppref? ppref is an _optional_ amap feature which is used 104 * to keep track of reference counts on a per-page basis. it is enabled 105 * when UVM_AMAP_PPREF is defined. 106 * 107 * when enabled, an array of ints is allocated for the pprefs. this 108 * array is allocated only when a partial reference is added to the 109 * map (either by unmapping part of the amap, or gaining a reference 110 * to only a part of an amap). if the malloc of the array fails 111 * (M_NOWAIT), then we set the array pointer to PPREF_NONE to indicate 112 * that we tried to do ppref's but couldn't alloc the array so just 113 * give up (after all, this is an optional feature!). 114 * 115 * the array is divided into page sized "chunks." for chunks of length 1, 116 * the chunk reference count plus one is stored in that chunk's slot. 117 * for chunks of length > 1 the first slot contains (the reference count 118 * plus one) * -1. [the negative value indicates that the length is 119 * greater than one.] the second slot of the chunk contains the length 120 * of the chunk. here is an example: 121 * 122 * actual REFS: 2 2 2 2 3 1 1 0 0 0 4 4 0 1 1 1 123 * ppref: -3 4 x x 4 -2 2 -1 3 x -5 2 1 -2 3 x 124 * <----------><-><----><-------><----><-><-------> 125 * (x = don't care) 126 * 127 * this allows us to allow one int to contain the ref count for the whole 128 * chunk. note that the "plus one" part is needed because a reference 129 * count of zero is neither positive or negative (need a way to tell 130 * if we've got one zero or a bunch of them). 131 * 132 * here are some in-line functions to help us. 133 */ 134 135 /* 136 * pp_getreflen: get the reference and length for a specific offset 137 * 138 * => ppref's amap must be locked 139 */ 140 static inline void 141 pp_getreflen(int *ppref, int offset, int *refp, int *lenp) 142 { 143 144 if (ppref[offset] > 0) { /* chunk size must be 1 */ 145 *refp = ppref[offset] - 1; /* don't forget to adjust */ 146 *lenp = 1; 147 } else { 148 *refp = (ppref[offset] * -1) - 1; 149 *lenp = ppref[offset+1]; 150 } 151 } 152 153 /* 154 * pp_setreflen: set the reference and length for a specific offset 155 * 156 * => ppref's amap must be locked 157 */ 158 static inline void 159 pp_setreflen(int *ppref, int offset, int ref, int len) 160 { 161 if (len == 0) 162 return; 163 if (len == 1) { 164 ppref[offset] = ref + 1; 165 } else { 166 ppref[offset] = (ref + 1) * -1; 167 ppref[offset+1] = len; 168 } 169 } 170 #endif /* UVM_AMAP_PPREF */ 171 172 /* 173 * amap_alloc1: internal function that allocates an amap, but does not 174 * init the overlay. 175 * 176 * => lock on returned amap is init'd 177 */ 178 static inline struct vm_amap * 179 amap_alloc1(int slots, int padslots, int waitf) 180 { 181 struct vm_amap *amap; 182 int totalslots; 183 km_flag_t kmflags; 184 185 amap = pool_get(&uvm_amap_pool, 186 ((waitf & UVM_FLAG_NOWAIT) != 0) ? PR_NOWAIT : PR_WAITOK); 187 if (amap == NULL) 188 return(NULL); 189 190 kmflags = ((waitf & UVM_FLAG_NOWAIT) != 0) ? KM_NOSLEEP : KM_SLEEP; 191 totalslots = amap_roundup_slots(slots + padslots); 192 simple_lock_init(&amap->am_l); 193 amap->am_ref = 1; 194 amap->am_flags = 0; 195 #ifdef UVM_AMAP_PPREF 196 amap->am_ppref = NULL; 197 #endif 198 amap->am_maxslot = totalslots; 199 amap->am_nslot = slots; 200 amap->am_nused = 0; 201 202 amap->am_slots = kmem_alloc(totalslots * sizeof(int), kmflags); 203 if (amap->am_slots == NULL) 204 goto fail1; 205 206 amap->am_bckptr = kmem_alloc(totalslots * sizeof(int), kmflags); 207 if (amap->am_bckptr == NULL) 208 goto fail2; 209 210 amap->am_anon = kmem_alloc(totalslots * sizeof(struct vm_anon *), 211 kmflags); 212 if (amap->am_anon == NULL) 213 goto fail3; 214 215 return(amap); 216 217 fail3: 218 kmem_free(amap->am_bckptr, totalslots * sizeof(int)); 219 fail2: 220 kmem_free(amap->am_slots, totalslots * sizeof(int)); 221 fail1: 222 pool_put(&uvm_amap_pool, amap); 223 224 /* 225 * XXX hack to tell the pagedaemon how many pages we need, 226 * since we can need more than it would normally free. 227 */ 228 if ((waitf & UVM_FLAG_NOWAIT) != 0) { 229 extern int uvm_extrapages; 230 uvm_extrapages += ((sizeof(int) * 2 + 231 sizeof(struct vm_anon *)) * 232 totalslots) >> PAGE_SHIFT; 233 } 234 return (NULL); 235 } 236 237 /* 238 * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM 239 * 240 * => caller should ensure sz is a multiple of PAGE_SIZE 241 * => reference count to new amap is set to one 242 * => new amap is returned unlocked 243 */ 244 245 struct vm_amap * 246 amap_alloc(vaddr_t sz, vaddr_t padsz, int waitf) 247 { 248 struct vm_amap *amap; 249 int slots, padslots; 250 UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist); 251 252 AMAP_B2SLOT(slots, sz); 253 AMAP_B2SLOT(padslots, padsz); 254 255 amap = amap_alloc1(slots, padslots, waitf); 256 if (amap) { 257 memset(amap->am_anon, 0, 258 amap->am_maxslot * sizeof(struct vm_anon *)); 259 amap_list_insert(amap); 260 } 261 262 UVMHIST_LOG(maphist,"<- done, amap = 0x%x, sz=%d", amap, sz, 0, 0); 263 return(amap); 264 } 265 266 267 /* 268 * amap_free: free an amap 269 * 270 * => the amap must be unlocked 271 * => the amap should have a zero reference count and be empty 272 */ 273 void 274 amap_free(struct vm_amap *amap) 275 { 276 int slots; 277 278 UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist); 279 280 KASSERT(amap->am_ref == 0 && amap->am_nused == 0); 281 KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0); 282 LOCK_ASSERT(!simple_lock_held(&amap->am_l)); 283 slots = amap->am_maxslot; 284 kmem_free(amap->am_slots, slots * sizeof(*amap->am_slots)); 285 kmem_free(amap->am_bckptr, slots * sizeof(*amap->am_bckptr)); 286 kmem_free(amap->am_anon, slots * sizeof(*amap->am_anon)); 287 #ifdef UVM_AMAP_PPREF 288 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) 289 kmem_free(amap->am_ppref, slots * sizeof(*amap->am_ppref)); 290 #endif 291 pool_put(&uvm_amap_pool, amap); 292 UVMHIST_LOG(maphist,"<- done, freed amap = 0x%x", amap, 0, 0, 0); 293 } 294 295 /* 296 * amap_extend: extend the size of an amap (if needed) 297 * 298 * => called from uvm_map when we want to extend an amap to cover 299 * a new mapping (rather than allocate a new one) 300 * => amap should be unlocked (we will lock it) 301 * => to safely extend an amap it should have a reference count of 302 * one (thus it can't be shared) 303 */ 304 int 305 amap_extend(struct vm_map_entry *entry, vsize_t addsize, int flags) 306 { 307 struct vm_amap *amap = entry->aref.ar_amap; 308 int slotoff = entry->aref.ar_pageoff; 309 int slotmapped, slotadd, slotneed, slotadded, slotalloc; 310 int slotadj, slotspace; 311 int oldnslots; 312 #ifdef UVM_AMAP_PPREF 313 int *newppref, *oldppref; 314 #endif 315 int i, *newsl, *newbck, *oldsl, *oldbck; 316 struct vm_anon **newover, **oldover; 317 const km_flag_t kmflags = 318 (flags & AMAP_EXTEND_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 319 320 UVMHIST_FUNC("amap_extend"); UVMHIST_CALLED(maphist); 321 322 UVMHIST_LOG(maphist, " (entry=0x%x, addsize=0x%x, flags=0x%x)", 323 entry, addsize, flags, 0); 324 325 /* 326 * first, determine how many slots we need in the amap. don't 327 * forget that ar_pageoff could be non-zero: this means that 328 * there are some unused slots before us in the amap. 329 */ 330 331 amap_lock(amap); 332 KASSERT(amap_refs(amap) == 1); /* amap can't be shared */ 333 AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */ 334 AMAP_B2SLOT(slotadd, addsize); /* slots to add */ 335 if (flags & AMAP_EXTEND_FORWARDS) { 336 slotneed = slotoff + slotmapped + slotadd; 337 slotadj = 0; 338 slotspace = 0; 339 } 340 else { 341 slotneed = slotadd + slotmapped; 342 slotadj = slotadd - slotoff; 343 slotspace = amap->am_maxslot - slotmapped; 344 } 345 346 /* 347 * case 1: we already have enough slots in the map and thus 348 * only need to bump the reference counts on the slots we are 349 * adding. 350 */ 351 352 if (flags & AMAP_EXTEND_FORWARDS) { 353 if (amap->am_nslot >= slotneed) { 354 #ifdef UVM_AMAP_PPREF 355 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 356 amap_pp_adjref(amap, slotoff + slotmapped, 357 slotadd, 1); 358 } 359 #endif 360 amap_unlock(amap); 361 UVMHIST_LOG(maphist, 362 "<- done (case 1f), amap = 0x%x, sltneed=%d", 363 amap, slotneed, 0, 0); 364 return 0; 365 } 366 } else { 367 if (slotadj <= 0) { 368 slotoff -= slotadd; 369 entry->aref.ar_pageoff = slotoff; 370 #ifdef UVM_AMAP_PPREF 371 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 372 amap_pp_adjref(amap, slotoff, slotadd, 1); 373 } 374 #endif 375 amap_unlock(amap); 376 UVMHIST_LOG(maphist, 377 "<- done (case 1b), amap = 0x%x, sltneed=%d", 378 amap, slotneed, 0, 0); 379 return 0; 380 } 381 } 382 383 /* 384 * case 2: we pre-allocated slots for use and we just need to 385 * bump nslot up to take account for these slots. 386 */ 387 388 if (amap->am_maxslot >= slotneed) { 389 if (flags & AMAP_EXTEND_FORWARDS) { 390 #ifdef UVM_AMAP_PPREF 391 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 392 if ((slotoff + slotmapped) < amap->am_nslot) 393 amap_pp_adjref(amap, 394 slotoff + slotmapped, 395 (amap->am_nslot - 396 (slotoff + slotmapped)), 1); 397 pp_setreflen(amap->am_ppref, amap->am_nslot, 1, 398 slotneed - amap->am_nslot); 399 } 400 #endif 401 amap->am_nslot = slotneed; 402 amap_unlock(amap); 403 404 /* 405 * no need to zero am_anon since that was done at 406 * alloc time and we never shrink an allocation. 407 */ 408 409 UVMHIST_LOG(maphist,"<- done (case 2f), amap = 0x%x, " 410 "slotneed=%d", amap, slotneed, 0, 0); 411 return 0; 412 } else { 413 #ifdef UVM_AMAP_PPREF 414 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 415 /* 416 * Slide up the ref counts on the pages that 417 * are actually in use. 418 */ 419 memmove(amap->am_ppref + slotspace, 420 amap->am_ppref + slotoff, 421 slotmapped * sizeof(int)); 422 /* 423 * Mark the (adjusted) gap at the front as 424 * referenced/not referenced. 425 */ 426 pp_setreflen(amap->am_ppref, 427 0, 0, slotspace - slotadd); 428 pp_setreflen(amap->am_ppref, 429 slotspace - slotadd, 1, slotadd); 430 } 431 #endif 432 433 /* 434 * Slide the anon pointers up and clear out 435 * the space we just made. 436 */ 437 memmove(amap->am_anon + slotspace, 438 amap->am_anon + slotoff, 439 slotmapped * sizeof(struct vm_anon*)); 440 memset(amap->am_anon + slotoff, 0, 441 (slotspace - slotoff) * sizeof(struct vm_anon *)); 442 443 /* 444 * Slide the backpointers up, but don't bother 445 * wiping out the old slots. 446 */ 447 memmove(amap->am_bckptr + slotspace, 448 amap->am_bckptr + slotoff, 449 slotmapped * sizeof(int)); 450 451 /* 452 * Adjust all the useful active slot numbers. 453 */ 454 for (i = 0; i < amap->am_nused; i++) 455 amap->am_slots[i] += (slotspace - slotoff); 456 457 /* 458 * We just filled all the empty space in the 459 * front of the amap by activating a few new 460 * slots. 461 */ 462 amap->am_nslot = amap->am_maxslot; 463 entry->aref.ar_pageoff = slotspace - slotadd; 464 amap_unlock(amap); 465 466 UVMHIST_LOG(maphist,"<- done (case 2b), amap = 0x%x, " 467 "slotneed=%d", amap, slotneed, 0, 0); 468 return 0; 469 } 470 } 471 472 /* 473 * case 3: we need to malloc a new amap and copy all the amap 474 * data over from old amap to the new one. 475 * 476 * note that the use of a kernel realloc() probably would not 477 * help here, since we wish to abort cleanly if one of the 478 * three (or four) mallocs fails. 479 */ 480 481 amap_unlock(amap); /* unlock in case we sleep in malloc */ 482 483 if (slotneed >= UVM_AMAP_LARGE) { 484 return E2BIG; 485 } 486 487 slotalloc = amap_roundup_slots(slotneed); 488 #ifdef UVM_AMAP_PPREF 489 newppref = NULL; 490 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) 491 newppref = kmem_alloc(slotalloc * sizeof(*newppref), kmflags); 492 #endif 493 newsl = kmem_alloc(slotalloc * sizeof(*newsl), kmflags); 494 newbck = kmem_alloc(slotalloc * sizeof(*newbck), kmflags); 495 newover = kmem_alloc(slotalloc * sizeof(*newover), kmflags); 496 if (newsl == NULL || newbck == NULL || newover == NULL) { 497 #ifdef UVM_AMAP_PPREF 498 if (newppref != NULL) { 499 kmem_free(newppref, slotalloc * sizeof(*newppref)); 500 } 501 #endif 502 if (newsl != NULL) { 503 kmem_free(newsl, slotalloc * sizeof(*newsl)); 504 } 505 if (newbck != NULL) { 506 kmem_free(newbck, slotalloc * sizeof(*newbck)); 507 } 508 if (newover != NULL) { 509 kmem_free(newover, slotalloc * sizeof(*newover)); 510 } 511 return ENOMEM; 512 } 513 amap_lock(amap); 514 KASSERT(amap->am_maxslot < slotneed); 515 516 /* 517 * now copy everything over to new malloc'd areas... 518 */ 519 520 slotadded = slotalloc - amap->am_nslot; 521 if (!(flags & AMAP_EXTEND_FORWARDS)) 522 slotspace = slotalloc - slotmapped; 523 524 /* do am_slots */ 525 oldsl = amap->am_slots; 526 if (flags & AMAP_EXTEND_FORWARDS) 527 memcpy(newsl, oldsl, sizeof(int) * amap->am_nused); 528 else 529 for (i = 0; i < amap->am_nused; i++) 530 newsl[i] = oldsl[i] + slotspace - slotoff; 531 amap->am_slots = newsl; 532 533 /* do am_anon */ 534 oldover = amap->am_anon; 535 if (flags & AMAP_EXTEND_FORWARDS) { 536 memcpy(newover, oldover, 537 sizeof(struct vm_anon *) * amap->am_nslot); 538 memset(newover + amap->am_nslot, 0, 539 sizeof(struct vm_anon *) * slotadded); 540 } else { 541 memcpy(newover + slotspace, oldover + slotoff, 542 sizeof(struct vm_anon *) * slotmapped); 543 memset(newover, 0, 544 sizeof(struct vm_anon *) * slotspace); 545 } 546 amap->am_anon = newover; 547 548 /* do am_bckptr */ 549 oldbck = amap->am_bckptr; 550 if (flags & AMAP_EXTEND_FORWARDS) 551 memcpy(newbck, oldbck, sizeof(int) * amap->am_nslot); 552 else 553 memcpy(newbck + slotspace, oldbck + slotoff, 554 sizeof(int) * slotmapped); 555 amap->am_bckptr = newbck; 556 557 #ifdef UVM_AMAP_PPREF 558 /* do ppref */ 559 oldppref = amap->am_ppref; 560 if (newppref) { 561 if (flags & AMAP_EXTEND_FORWARDS) { 562 memcpy(newppref, oldppref, 563 sizeof(int) * amap->am_nslot); 564 memset(newppref + amap->am_nslot, 0, 565 sizeof(int) * slotadded); 566 } else { 567 memcpy(newppref + slotspace, oldppref + slotoff, 568 sizeof(int) * slotmapped); 569 } 570 amap->am_ppref = newppref; 571 if ((flags & AMAP_EXTEND_FORWARDS) && 572 (slotoff + slotmapped) < amap->am_nslot) 573 amap_pp_adjref(amap, slotoff + slotmapped, 574 (amap->am_nslot - (slotoff + slotmapped)), 1); 575 if (flags & AMAP_EXTEND_FORWARDS) 576 pp_setreflen(newppref, amap->am_nslot, 1, 577 slotneed - amap->am_nslot); 578 else { 579 pp_setreflen(newppref, 0, 0, 580 slotalloc - slotneed); 581 pp_setreflen(newppref, slotalloc - slotneed, 1, 582 slotneed - slotmapped); 583 } 584 } else { 585 if (amap->am_ppref) 586 amap->am_ppref = PPREF_NONE; 587 } 588 #endif 589 590 /* update master values */ 591 if (flags & AMAP_EXTEND_FORWARDS) 592 amap->am_nslot = slotneed; 593 else { 594 entry->aref.ar_pageoff = slotspace - slotadd; 595 amap->am_nslot = slotalloc; 596 } 597 oldnslots = amap->am_maxslot; 598 amap->am_maxslot = slotalloc; 599 600 amap_unlock(amap); 601 kmem_free(oldsl, oldnslots * sizeof(*oldsl)); 602 kmem_free(oldbck, oldnslots * sizeof(*oldbck)); 603 kmem_free(oldover, oldnslots * sizeof(*oldover)); 604 #ifdef UVM_AMAP_PPREF 605 if (oldppref && oldppref != PPREF_NONE) 606 kmem_free(oldppref, oldnslots * sizeof(*oldppref)); 607 #endif 608 UVMHIST_LOG(maphist,"<- done (case 3), amap = 0x%x, slotneed=%d", 609 amap, slotneed, 0, 0); 610 return 0; 611 } 612 613 /* 614 * amap_share_protect: change protection of anons in a shared amap 615 * 616 * for shared amaps, given the current data structure layout, it is 617 * not possible for us to directly locate all maps referencing the 618 * shared anon (to change the protection). in order to protect data 619 * in shared maps we use pmap_page_protect(). [this is useful for IPC 620 * mechanisms like map entry passing that may want to write-protect 621 * all mappings of a shared amap.] we traverse am_anon or am_slots 622 * depending on the current state of the amap. 623 * 624 * => entry's map and amap must be locked by the caller 625 */ 626 void 627 amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot) 628 { 629 struct vm_amap *amap = entry->aref.ar_amap; 630 int slots, lcv, slot, stop; 631 632 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 633 634 AMAP_B2SLOT(slots, (entry->end - entry->start)); 635 stop = entry->aref.ar_pageoff + slots; 636 637 if (slots < amap->am_nused) { 638 /* cheaper to traverse am_anon */ 639 for (lcv = entry->aref.ar_pageoff ; lcv < stop ; lcv++) { 640 if (amap->am_anon[lcv] == NULL) 641 continue; 642 if (amap->am_anon[lcv]->an_page != NULL) 643 pmap_page_protect(amap->am_anon[lcv]->an_page, 644 prot); 645 } 646 return; 647 } 648 649 /* cheaper to traverse am_slots */ 650 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 651 slot = amap->am_slots[lcv]; 652 if (slot < entry->aref.ar_pageoff || slot >= stop) 653 continue; 654 if (amap->am_anon[slot]->an_page != NULL) 655 pmap_page_protect(amap->am_anon[slot]->an_page, prot); 656 } 657 } 658 659 /* 660 * amap_wipeout: wipeout all anon's in an amap; then free the amap! 661 * 662 * => called from amap_unref when the final reference to an amap is 663 * discarded (i.e. when reference count drops to 0) 664 * => the amap should be locked (by the caller) 665 */ 666 667 void 668 amap_wipeout(struct vm_amap *amap) 669 { 670 int lcv, slot; 671 struct vm_anon *anon; 672 UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist); 673 UVMHIST_LOG(maphist,"(amap=0x%x)", amap, 0,0,0); 674 675 KASSERT(amap->am_ref == 0); 676 677 if (__predict_false((amap->am_flags & AMAP_SWAPOFF) != 0)) { 678 /* 679 * amap_swap_off will call us again. 680 */ 681 amap_unlock(amap); 682 return; 683 } 684 amap_list_remove(amap); 685 amap_unlock(amap); 686 687 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 688 int refs; 689 690 slot = amap->am_slots[lcv]; 691 anon = amap->am_anon[slot]; 692 693 if (anon == NULL || anon->an_ref == 0) 694 panic("amap_wipeout: corrupt amap"); 695 696 simple_lock(&anon->an_lock); 697 UVMHIST_LOG(maphist," processing anon 0x%x, ref=%d", anon, 698 anon->an_ref, 0, 0); 699 refs = --anon->an_ref; 700 simple_unlock(&anon->an_lock); 701 if (refs == 0) { 702 703 /* 704 * we had the last reference to a vm_anon. free it. 705 */ 706 707 uvm_anfree(anon); 708 } 709 710 if (curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) 711 preempt(1); 712 } 713 714 /* 715 * now we free the map 716 */ 717 718 amap->am_nused = 0; 719 amap_free(amap); /* will unlock and free amap */ 720 UVMHIST_LOG(maphist,"<- done!", 0,0,0,0); 721 } 722 723 /* 724 * amap_copy: ensure that a map entry's "needs_copy" flag is false 725 * by copying the amap if necessary. 726 * 727 * => an entry with a null amap pointer will get a new (blank) one. 728 * => the map that the map entry belongs to must be locked by caller. 729 * => the amap currently attached to "entry" (if any) must be unlocked. 730 * => if canchunk is true, then we may clip the entry into a chunk 731 * => "startva" and "endva" are used only if canchunk is true. they are 732 * used to limit chunking (e.g. if you have a large space that you 733 * know you are going to need to allocate amaps for, there is no point 734 * in allowing that to be chunked) 735 */ 736 737 void 738 amap_copy(struct vm_map *map, struct vm_map_entry *entry, int flags, 739 vaddr_t startva, vaddr_t endva) 740 { 741 struct vm_amap *amap, *srcamap; 742 int slots, lcv; 743 vaddr_t chunksize; 744 const int waitf = (flags & AMAP_COPY_NOWAIT) ? UVM_FLAG_NOWAIT : 0; 745 const boolean_t canchunk = (flags & AMAP_COPY_NOCHUNK) == 0; 746 UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist); 747 UVMHIST_LOG(maphist, " (map=%p, entry=%p, flags=%d)", 748 map, entry, flags, 0); 749 750 KASSERT(map != kernel_map); /* we use nointr pool */ 751 752 /* 753 * is there a map to copy? if not, create one from scratch. 754 */ 755 756 if (entry->aref.ar_amap == NULL) { 757 758 /* 759 * check to see if we have a large amap that we can 760 * chunk. we align startva/endva to chunk-sized 761 * boundaries and then clip to them. 762 */ 763 764 if (canchunk && atop(entry->end - entry->start) >= 765 UVM_AMAP_LARGE) { 766 /* convert slots to bytes */ 767 chunksize = UVM_AMAP_CHUNK << PAGE_SHIFT; 768 startva = (startva / chunksize) * chunksize; 769 endva = roundup(endva, chunksize); 770 UVMHIST_LOG(maphist, " chunk amap ==> clip 0x%x->0x%x" 771 "to 0x%x->0x%x", entry->start, entry->end, startva, 772 endva); 773 UVM_MAP_CLIP_START(map, entry, startva, NULL); 774 /* watch out for endva wrap-around! */ 775 if (endva >= startva) 776 UVM_MAP_CLIP_END(map, entry, endva, NULL); 777 } 778 779 if ((flags & AMAP_COPY_NOMERGE) == 0 && 780 uvm_mapent_trymerge(map, entry, UVM_MERGE_COPYING)) { 781 return; 782 } 783 784 UVMHIST_LOG(maphist, "<- done [creating new amap 0x%x->0x%x]", 785 entry->start, entry->end, 0, 0); 786 entry->aref.ar_pageoff = 0; 787 entry->aref.ar_amap = amap_alloc(entry->end - entry->start, 0, 788 waitf); 789 if (entry->aref.ar_amap != NULL) 790 entry->etype &= ~UVM_ET_NEEDSCOPY; 791 return; 792 } 793 794 /* 795 * first check and see if we are the only map entry 796 * referencing the amap we currently have. if so, then we can 797 * just take it over rather than copying it. note that we are 798 * reading am_ref with the amap unlocked... the value can only 799 * be one if we have the only reference to the amap (via our 800 * locked map). if we are greater than one we fall through to 801 * the next case (where we double check the value). 802 */ 803 804 if (entry->aref.ar_amap->am_ref == 1) { 805 entry->etype &= ~UVM_ET_NEEDSCOPY; 806 UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]", 807 0, 0, 0, 0); 808 return; 809 } 810 811 /* 812 * looks like we need to copy the map. 813 */ 814 815 UVMHIST_LOG(maphist," amap=%p, ref=%d, must copy it", 816 entry->aref.ar_amap, entry->aref.ar_amap->am_ref, 0, 0); 817 AMAP_B2SLOT(slots, entry->end - entry->start); 818 amap = amap_alloc1(slots, 0, waitf); 819 if (amap == NULL) { 820 UVMHIST_LOG(maphist, " amap_alloc1 failed", 0,0,0,0); 821 return; 822 } 823 srcamap = entry->aref.ar_amap; 824 amap_lock(srcamap); 825 826 /* 827 * need to double check reference count now that we've got the 828 * src amap locked down. the reference count could have 829 * changed while we were in malloc. if the reference count 830 * dropped down to one we take over the old map rather than 831 * copying the amap. 832 */ 833 834 if (srcamap->am_ref == 1) { /* take it over? */ 835 entry->etype &= ~UVM_ET_NEEDSCOPY; 836 amap->am_ref--; /* drop final reference to map */ 837 amap_free(amap); /* dispose of new (unused) amap */ 838 amap_unlock(srcamap); 839 return; 840 } 841 842 /* 843 * we must copy it now. 844 */ 845 846 UVMHIST_LOG(maphist, " copying amap now",0, 0, 0, 0); 847 for (lcv = 0 ; lcv < slots; lcv++) { 848 amap->am_anon[lcv] = 849 srcamap->am_anon[entry->aref.ar_pageoff + lcv]; 850 if (amap->am_anon[lcv] == NULL) 851 continue; 852 simple_lock(&amap->am_anon[lcv]->an_lock); 853 amap->am_anon[lcv]->an_ref++; 854 simple_unlock(&amap->am_anon[lcv]->an_lock); 855 amap->am_bckptr[lcv] = amap->am_nused; 856 amap->am_slots[amap->am_nused] = lcv; 857 amap->am_nused++; 858 } 859 memset(&amap->am_anon[lcv], 0, 860 (amap->am_maxslot - lcv) * sizeof(struct vm_anon *)); 861 862 /* 863 * drop our reference to the old amap (srcamap) and unlock. 864 * we know that the reference count on srcamap is greater than 865 * one (we checked above), so there is no way we could drop 866 * the count to zero. [and no need to worry about freeing it] 867 */ 868 869 srcamap->am_ref--; 870 if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) 871 srcamap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ 872 #ifdef UVM_AMAP_PPREF 873 if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) { 874 amap_pp_adjref(srcamap, entry->aref.ar_pageoff, 875 (entry->end - entry->start) >> PAGE_SHIFT, -1); 876 } 877 #endif 878 879 amap_unlock(srcamap); 880 881 amap_list_insert(amap); 882 883 /* 884 * install new amap. 885 */ 886 887 entry->aref.ar_pageoff = 0; 888 entry->aref.ar_amap = amap; 889 entry->etype &= ~UVM_ET_NEEDSCOPY; 890 UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0); 891 } 892 893 /* 894 * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2) 895 * 896 * called during fork(2) when the parent process has a wired map 897 * entry. in that case we want to avoid write-protecting pages 898 * in the parent's map (e.g. like what you'd do for a COW page) 899 * so we resolve the COW here. 900 * 901 * => assume parent's entry was wired, thus all pages are resident. 902 * => assume pages that are loaned out (loan_count) are already mapped 903 * read-only in all maps, and thus no need for us to worry about them 904 * => assume both parent and child vm_map's are locked 905 * => caller passes child's map/entry in to us 906 * => if we run out of memory we will unlock the amap and sleep _with_ the 907 * parent and child vm_map's locked(!). we have to do this since 908 * we are in the middle of a fork(2) and we can't let the parent 909 * map change until we are done copying all the map entrys. 910 * => XXXCDC: out of memory should cause fork to fail, but there is 911 * currently no easy way to do this (needs fix) 912 * => page queues must be unlocked (we may lock them) 913 */ 914 915 void 916 amap_cow_now(struct vm_map *map, struct vm_map_entry *entry) 917 { 918 struct vm_amap *amap = entry->aref.ar_amap; 919 int lcv, slot; 920 struct vm_anon *anon, *nanon; 921 struct vm_page *pg, *npg; 922 923 /* 924 * note that if we unlock the amap then we must ReStart the "lcv" for 925 * loop because some other process could reorder the anon's in the 926 * am_anon[] array on us while the lock is dropped. 927 */ 928 929 ReStart: 930 amap_lock(amap); 931 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 932 933 /* 934 * get the page 935 */ 936 937 slot = amap->am_slots[lcv]; 938 anon = amap->am_anon[slot]; 939 simple_lock(&anon->an_lock); 940 941 /* 942 * If the anon has only one ref, we must have already copied it. 943 * This can happen if we needed to sleep waiting for memory 944 * in a previous run through this loop. The new page might 945 * even have been paged out, since the new page is not wired. 946 */ 947 948 if (anon->an_ref == 1) { 949 KASSERT(anon->an_page != NULL || anon->an_swslot != 0); 950 simple_unlock(&anon->an_lock); 951 continue; 952 } 953 954 /* 955 * The old page must be resident since the parent is wired. 956 */ 957 958 pg = anon->an_page; 959 KASSERT(pg != NULL); 960 KASSERT(pg->wire_count > 0); 961 962 /* 963 * If the page is loaned then it must already be mapped 964 * read-only and we don't need to copy it. 965 */ 966 967 if (pg->loan_count != 0) { 968 simple_unlock(&anon->an_lock); 969 continue; 970 } 971 KASSERT(pg->uanon == anon && pg->uobject == NULL); 972 973 /* 974 * if the page is busy then we have to unlock, wait for 975 * it and then restart. 976 */ 977 978 if (pg->flags & PG_BUSY) { 979 pg->flags |= PG_WANTED; 980 amap_unlock(amap); 981 UVM_UNLOCK_AND_WAIT(pg, &anon->an_lock, FALSE, 982 "cownow", 0); 983 goto ReStart; 984 } 985 986 /* 987 * ok, time to do a copy-on-write to a new anon 988 */ 989 990 nanon = uvm_analloc(); 991 if (nanon) { 992 npg = uvm_pagealloc(NULL, 0, nanon, 0); 993 } else 994 npg = NULL; /* XXX: quiet gcc warning */ 995 if (nanon == NULL || npg == NULL) { 996 997 /* 998 * XXXCDC: we should cause fork to fail, but we can't. 999 */ 1000 1001 if (nanon) { 1002 nanon->an_ref--; 1003 simple_unlock(&nanon->an_lock); 1004 uvm_anfree(nanon); 1005 } 1006 simple_unlock(&anon->an_lock); 1007 amap_unlock(amap); 1008 uvm_wait("cownowpage"); 1009 goto ReStart; 1010 } 1011 1012 /* 1013 * got it... now we can copy the data and replace anon 1014 * with our new one... 1015 */ 1016 1017 uvm_pagecopy(pg, npg); /* old -> new */ 1018 anon->an_ref--; /* can't drop to zero */ 1019 amap->am_anon[slot] = nanon; /* replace */ 1020 1021 /* 1022 * drop PG_BUSY on new page ... since we have had its owner 1023 * locked the whole time it can't be PG_RELEASED or PG_WANTED. 1024 */ 1025 1026 uvm_lock_pageq(); 1027 uvm_pageactivate(npg); 1028 uvm_unlock_pageq(); 1029 npg->flags &= ~(PG_BUSY|PG_FAKE); 1030 UVM_PAGE_OWN(npg, NULL); 1031 simple_unlock(&nanon->an_lock); 1032 simple_unlock(&anon->an_lock); 1033 } 1034 amap_unlock(amap); 1035 } 1036 1037 /* 1038 * amap_splitref: split a single reference into two separate references 1039 * 1040 * => called from uvm_map's clip routines 1041 * => origref's map should be locked 1042 * => origref->ar_amap should be unlocked (we will lock) 1043 */ 1044 void 1045 amap_splitref(struct vm_aref *origref, struct vm_aref *splitref, vaddr_t offset) 1046 { 1047 int leftslots; 1048 struct vm_amap *amap; 1049 1050 KASSERT(splitref->ar_amap == origref->ar_amap); 1051 AMAP_B2SLOT(leftslots, offset); 1052 if (leftslots == 0) 1053 panic("amap_splitref: split at zero offset"); 1054 1055 amap = origref->ar_amap; 1056 amap_lock(amap); 1057 1058 /* 1059 * now: amap is locked and we have a valid am_mapped array. 1060 */ 1061 1062 if (amap->am_nslot - origref->ar_pageoff - leftslots <= 0) 1063 panic("amap_splitref: map size check failed"); 1064 1065 #ifdef UVM_AMAP_PPREF 1066 /* 1067 * establish ppref before we add a duplicate reference to the amap 1068 */ 1069 if (amap->am_ppref == NULL) 1070 amap_pp_establish(amap, origref->ar_pageoff); 1071 #endif 1072 1073 amap->am_ref++; /* not a share reference */ 1074 splitref->ar_pageoff = origref->ar_pageoff + leftslots; 1075 1076 amap_unlock(amap); 1077 } 1078 1079 #ifdef UVM_AMAP_PPREF 1080 1081 /* 1082 * amap_pp_establish: add a ppref array to an amap, if possible 1083 * 1084 * => amap locked by caller 1085 */ 1086 void 1087 amap_pp_establish(struct vm_amap *amap, vaddr_t offset) 1088 { 1089 1090 amap->am_ppref = kmem_alloc(amap->am_maxslot * sizeof(*amap->am_ppref), 1091 KM_NOSLEEP); 1092 1093 /* 1094 * if we fail then we just won't use ppref for this amap 1095 */ 1096 1097 if (amap->am_ppref == NULL) { 1098 amap->am_ppref = PPREF_NONE; /* not using it */ 1099 return; 1100 } 1101 memset(amap->am_ppref, 0, sizeof(int) * amap->am_maxslot); 1102 pp_setreflen(amap->am_ppref, 0, 0, offset); 1103 pp_setreflen(amap->am_ppref, offset, amap->am_ref, 1104 amap->am_nslot - offset); 1105 return; 1106 } 1107 1108 /* 1109 * amap_pp_adjref: adjust reference count to a part of an amap using the 1110 * per-page reference count array. 1111 * 1112 * => map and amap locked by caller 1113 * => caller must check that ppref != PPREF_NONE before calling 1114 */ 1115 void 1116 amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval) 1117 { 1118 int stopslot, *ppref, lcv, prevlcv; 1119 int ref, len, prevref, prevlen; 1120 1121 stopslot = curslot + slotlen; 1122 ppref = amap->am_ppref; 1123 prevlcv = 0; 1124 1125 /* 1126 * first advance to the correct place in the ppref array, 1127 * fragment if needed. 1128 */ 1129 1130 for (lcv = 0 ; lcv < curslot ; lcv += len) { 1131 pp_getreflen(ppref, lcv, &ref, &len); 1132 if (lcv + len > curslot) { /* goes past start? */ 1133 pp_setreflen(ppref, lcv, ref, curslot - lcv); 1134 pp_setreflen(ppref, curslot, ref, len - (curslot -lcv)); 1135 len = curslot - lcv; /* new length of entry @ lcv */ 1136 } 1137 prevlcv = lcv; 1138 } 1139 if (lcv != 0) 1140 pp_getreflen(ppref, prevlcv, &prevref, &prevlen); 1141 else { 1142 /* Ensure that the "prevref == ref" test below always 1143 * fails, since we're starting from the beginning of 1144 * the ppref array; that is, there is no previous 1145 * chunk. 1146 */ 1147 prevref = -1; 1148 prevlen = 0; 1149 } 1150 1151 /* 1152 * now adjust reference counts in range. merge the first 1153 * changed entry with the last unchanged entry if possible. 1154 */ 1155 1156 if (lcv != curslot) 1157 panic("amap_pp_adjref: overshot target"); 1158 1159 for (/* lcv already set */; lcv < stopslot ; lcv += len) { 1160 pp_getreflen(ppref, lcv, &ref, &len); 1161 if (lcv + len > stopslot) { /* goes past end? */ 1162 pp_setreflen(ppref, lcv, ref, stopslot - lcv); 1163 pp_setreflen(ppref, stopslot, ref, 1164 len - (stopslot - lcv)); 1165 len = stopslot - lcv; 1166 } 1167 ref += adjval; 1168 if (ref < 0) 1169 panic("amap_pp_adjref: negative reference count"); 1170 if (lcv == prevlcv + prevlen && ref == prevref) { 1171 pp_setreflen(ppref, prevlcv, ref, prevlen + len); 1172 } else { 1173 pp_setreflen(ppref, lcv, ref, len); 1174 } 1175 if (ref == 0) 1176 amap_wiperange(amap, lcv, len); 1177 } 1178 1179 } 1180 1181 /* 1182 * amap_wiperange: wipe out a range of an amap 1183 * [different from amap_wipeout because the amap is kept intact] 1184 * 1185 * => both map and amap must be locked by caller. 1186 */ 1187 void 1188 amap_wiperange(struct vm_amap *amap, int slotoff, int slots) 1189 { 1190 int byanon, lcv, stop, curslot, ptr, slotend; 1191 struct vm_anon *anon; 1192 1193 /* 1194 * we can either traverse the amap by am_anon or by am_slots depending 1195 * on which is cheaper. decide now. 1196 */ 1197 1198 if (slots < amap->am_nused) { 1199 byanon = TRUE; 1200 lcv = slotoff; 1201 stop = slotoff + slots; 1202 slotend = 0; 1203 } else { 1204 byanon = FALSE; 1205 lcv = 0; 1206 stop = amap->am_nused; 1207 slotend = slotoff + slots; 1208 } 1209 1210 while (lcv < stop) { 1211 int refs; 1212 1213 if (byanon) { 1214 curslot = lcv++; /* lcv advances here */ 1215 if (amap->am_anon[curslot] == NULL) 1216 continue; 1217 } else { 1218 curslot = amap->am_slots[lcv]; 1219 if (curslot < slotoff || curslot >= slotend) { 1220 lcv++; /* lcv advances here */ 1221 continue; 1222 } 1223 stop--; /* drop stop, since anon will be removed */ 1224 } 1225 anon = amap->am_anon[curslot]; 1226 1227 /* 1228 * remove it from the amap 1229 */ 1230 1231 amap->am_anon[curslot] = NULL; 1232 ptr = amap->am_bckptr[curslot]; 1233 if (ptr != (amap->am_nused - 1)) { 1234 amap->am_slots[ptr] = 1235 amap->am_slots[amap->am_nused - 1]; 1236 amap->am_bckptr[amap->am_slots[ptr]] = 1237 ptr; /* back ptr. */ 1238 } 1239 amap->am_nused--; 1240 1241 /* 1242 * drop anon reference count 1243 */ 1244 1245 simple_lock(&anon->an_lock); 1246 refs = --anon->an_ref; 1247 simple_unlock(&anon->an_lock); 1248 if (refs == 0) { 1249 1250 /* 1251 * we just eliminated the last reference to an anon. 1252 * free it. 1253 */ 1254 1255 uvm_anfree(anon); 1256 } 1257 } 1258 } 1259 1260 #endif 1261 1262 #if defined(VMSWAP) 1263 1264 /* 1265 * amap_swap_off: pagein anonymous pages in amaps and drop swap slots. 1266 * 1267 * => called with swap_syscall_lock held. 1268 * => note that we don't always traverse all anons. 1269 * eg. amaps being wiped out, released anons. 1270 * => return TRUE if failed. 1271 */ 1272 1273 boolean_t 1274 amap_swap_off(int startslot, int endslot) 1275 { 1276 struct vm_amap *am; 1277 struct vm_amap *am_next; 1278 struct vm_amap marker_prev; 1279 struct vm_amap marker_next; 1280 struct lwp *l = curlwp; 1281 boolean_t rv = FALSE; 1282 1283 #if defined(DIAGNOSTIC) 1284 memset(&marker_prev, 0, sizeof(marker_prev)); 1285 memset(&marker_next, 0, sizeof(marker_next)); 1286 #endif /* defined(DIAGNOSTIC) */ 1287 1288 PHOLD(l); 1289 simple_lock(&amap_list_lock); 1290 for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) { 1291 int i; 1292 1293 LIST_INSERT_BEFORE(am, &marker_prev, am_list); 1294 LIST_INSERT_AFTER(am, &marker_next, am_list); 1295 1296 if (!amap_lock_try(am)) { 1297 simple_unlock(&amap_list_lock); 1298 preempt(1); 1299 simple_lock(&amap_list_lock); 1300 am_next = LIST_NEXT(&marker_prev, am_list); 1301 if (am_next == &marker_next) { 1302 am_next = LIST_NEXT(am_next, am_list); 1303 } else { 1304 KASSERT(LIST_NEXT(am_next, am_list) == 1305 &marker_next); 1306 } 1307 LIST_REMOVE(&marker_prev, am_list); 1308 LIST_REMOVE(&marker_next, am_list); 1309 continue; 1310 } 1311 1312 simple_unlock(&amap_list_lock); 1313 1314 if (am->am_nused <= 0) { 1315 amap_unlock(am); 1316 goto next; 1317 } 1318 1319 for (i = 0; i < am->am_nused; i++) { 1320 int slot; 1321 int swslot; 1322 struct vm_anon *anon; 1323 1324 slot = am->am_slots[i]; 1325 anon = am->am_anon[slot]; 1326 simple_lock(&anon->an_lock); 1327 1328 swslot = anon->an_swslot; 1329 if (swslot < startslot || endslot <= swslot) { 1330 simple_unlock(&anon->an_lock); 1331 continue; 1332 } 1333 1334 am->am_flags |= AMAP_SWAPOFF; 1335 amap_unlock(am); 1336 1337 rv = uvm_anon_pagein(anon); 1338 1339 amap_lock(am); 1340 am->am_flags &= ~AMAP_SWAPOFF; 1341 if (amap_refs(am) == 0) { 1342 amap_wipeout(am); 1343 am = NULL; 1344 break; 1345 } 1346 if (rv) { 1347 break; 1348 } 1349 i = 0; 1350 } 1351 1352 if (am) { 1353 amap_unlock(am); 1354 } 1355 1356 next: 1357 simple_lock(&amap_list_lock); 1358 KASSERT(LIST_NEXT(&marker_prev, am_list) == &marker_next || 1359 LIST_NEXT(LIST_NEXT(&marker_prev, am_list), am_list) == 1360 &marker_next); 1361 am_next = LIST_NEXT(&marker_next, am_list); 1362 LIST_REMOVE(&marker_prev, am_list); 1363 LIST_REMOVE(&marker_next, am_list); 1364 } 1365 simple_unlock(&amap_list_lock); 1366 PRELE(l); 1367 1368 return rv; 1369 } 1370 1371 #endif /* defined(VMSWAP) */ 1372 1373 /* 1374 * amap_lookup: look up a page in an amap 1375 * 1376 * => amap should be locked by caller. 1377 */ 1378 struct vm_anon * 1379 amap_lookup(struct vm_aref *aref, vaddr_t offset) 1380 { 1381 int slot; 1382 struct vm_amap *amap = aref->ar_amap; 1383 UVMHIST_FUNC("amap_lookup"); UVMHIST_CALLED(maphist); 1384 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1385 1386 AMAP_B2SLOT(slot, offset); 1387 slot += aref->ar_pageoff; 1388 1389 if (slot >= amap->am_nslot) 1390 panic("amap_lookup: offset out of range"); 1391 1392 UVMHIST_LOG(maphist, "<- done (amap=0x%x, offset=0x%x, result=0x%x)", 1393 amap, offset, amap->am_anon[slot], 0); 1394 return(amap->am_anon[slot]); 1395 } 1396 1397 /* 1398 * amap_lookups: look up a range of pages in an amap 1399 * 1400 * => amap should be locked by caller. 1401 * => XXXCDC: this interface is biased toward array-based amaps. fix. 1402 */ 1403 void 1404 amap_lookups(struct vm_aref *aref, vaddr_t offset, struct vm_anon **anons, 1405 int npages) 1406 { 1407 int slot; 1408 struct vm_amap *amap = aref->ar_amap; 1409 UVMHIST_FUNC("amap_lookups"); UVMHIST_CALLED(maphist); 1410 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1411 1412 AMAP_B2SLOT(slot, offset); 1413 slot += aref->ar_pageoff; 1414 1415 UVMHIST_LOG(maphist, " slot=%d, npages=%d, nslot=%d", slot, npages, 1416 amap->am_nslot, 0); 1417 1418 if ((slot + (npages - 1)) >= amap->am_nslot) 1419 panic("amap_lookups: offset out of range"); 1420 1421 memcpy(anons, &amap->am_anon[slot], npages * sizeof(struct vm_anon *)); 1422 1423 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 1424 return; 1425 } 1426 1427 /* 1428 * amap_add: add (or replace) a page to an amap 1429 * 1430 * => caller must lock amap. 1431 * => if (replace) caller must lock anon because we might have to call 1432 * pmap_page_protect on the anon's page. 1433 */ 1434 void 1435 amap_add(struct vm_aref *aref, vaddr_t offset, struct vm_anon *anon, 1436 boolean_t replace) 1437 { 1438 int slot; 1439 struct vm_amap *amap = aref->ar_amap; 1440 UVMHIST_FUNC("amap_add"); UVMHIST_CALLED(maphist); 1441 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1442 1443 AMAP_B2SLOT(slot, offset); 1444 slot += aref->ar_pageoff; 1445 1446 if (slot >= amap->am_nslot) 1447 panic("amap_add: offset out of range"); 1448 1449 if (replace) { 1450 1451 if (amap->am_anon[slot] == NULL) 1452 panic("amap_add: replacing null anon"); 1453 if (amap->am_anon[slot]->an_page != NULL && 1454 (amap->am_flags & AMAP_SHARED) != 0) { 1455 pmap_page_protect(amap->am_anon[slot]->an_page, 1456 VM_PROT_NONE); 1457 /* 1458 * XXX: suppose page is supposed to be wired somewhere? 1459 */ 1460 } 1461 } else { /* !replace */ 1462 if (amap->am_anon[slot] != NULL) 1463 panic("amap_add: slot in use"); 1464 1465 amap->am_bckptr[slot] = amap->am_nused; 1466 amap->am_slots[amap->am_nused] = slot; 1467 amap->am_nused++; 1468 } 1469 amap->am_anon[slot] = anon; 1470 UVMHIST_LOG(maphist, 1471 "<- done (amap=0x%x, offset=0x%x, anon=0x%x, rep=%d)", 1472 amap, offset, anon, replace); 1473 } 1474 1475 /* 1476 * amap_unadd: remove a page from an amap 1477 * 1478 * => caller must lock amap 1479 */ 1480 void 1481 amap_unadd(struct vm_aref *aref, vaddr_t offset) 1482 { 1483 int ptr, slot; 1484 struct vm_amap *amap = aref->ar_amap; 1485 UVMHIST_FUNC("amap_unadd"); UVMHIST_CALLED(maphist); 1486 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1487 1488 AMAP_B2SLOT(slot, offset); 1489 slot += aref->ar_pageoff; 1490 1491 if (slot >= amap->am_nslot) 1492 panic("amap_unadd: offset out of range"); 1493 1494 if (amap->am_anon[slot] == NULL) 1495 panic("amap_unadd: nothing there"); 1496 1497 amap->am_anon[slot] = NULL; 1498 ptr = amap->am_bckptr[slot]; 1499 1500 if (ptr != (amap->am_nused - 1)) { /* swap to keep slots contig? */ 1501 amap->am_slots[ptr] = amap->am_slots[amap->am_nused - 1]; 1502 amap->am_bckptr[amap->am_slots[ptr]] = ptr; /* back link */ 1503 } 1504 amap->am_nused--; 1505 UVMHIST_LOG(maphist, "<- done (amap=0x%x, slot=0x%x)", amap, slot,0, 0); 1506 } 1507 1508 /* 1509 * amap_ref: gain a reference to an amap 1510 * 1511 * => amap must not be locked (we will lock) 1512 * => "offset" and "len" are in units of pages 1513 * => called at fork time to gain the child's reference 1514 */ 1515 void 1516 amap_ref(struct vm_amap *amap, vaddr_t offset, vsize_t len, int flags) 1517 { 1518 UVMHIST_FUNC("amap_ref"); UVMHIST_CALLED(maphist); 1519 1520 amap_lock(amap); 1521 if (flags & AMAP_SHARED) 1522 amap->am_flags |= AMAP_SHARED; 1523 #ifdef UVM_AMAP_PPREF 1524 if (amap->am_ppref == NULL && (flags & AMAP_REFALL) == 0 && 1525 len != amap->am_nslot) 1526 amap_pp_establish(amap, offset); 1527 #endif 1528 amap->am_ref++; 1529 #ifdef UVM_AMAP_PPREF 1530 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 1531 if (flags & AMAP_REFALL) 1532 amap_pp_adjref(amap, 0, amap->am_nslot, 1); 1533 else 1534 amap_pp_adjref(amap, offset, len, 1); 1535 } 1536 #endif 1537 amap_unlock(amap); 1538 UVMHIST_LOG(maphist,"<- done! amap=0x%x", amap, 0, 0, 0); 1539 } 1540 1541 /* 1542 * amap_unref: remove a reference to an amap 1543 * 1544 * => caller must remove all pmap-level references to this amap before 1545 * dropping the reference 1546 * => called from uvm_unmap_detach [only] ... note that entry is no 1547 * longer part of a map and thus has no need for locking 1548 * => amap must be unlocked (we will lock it). 1549 */ 1550 void 1551 amap_unref(struct vm_amap *amap, vaddr_t offset, vsize_t len, boolean_t all) 1552 { 1553 UVMHIST_FUNC("amap_unref"); UVMHIST_CALLED(maphist); 1554 1555 /* 1556 * lock it 1557 */ 1558 amap_lock(amap); 1559 UVMHIST_LOG(maphist," amap=0x%x refs=%d, nused=%d", 1560 amap, amap->am_ref, amap->am_nused, 0); 1561 1562 KASSERT(amap_refs(amap) > 0); 1563 1564 /* 1565 * if we are the last reference, free the amap and return. 1566 */ 1567 1568 amap->am_ref--; 1569 1570 if (amap_refs(amap) == 0) { 1571 amap_wipeout(amap); /* drops final ref and frees */ 1572 UVMHIST_LOG(maphist,"<- done (was last ref)!", 0, 0, 0, 0); 1573 return; /* no need to unlock */ 1574 } 1575 1576 /* 1577 * otherwise just drop the reference count(s) 1578 */ 1579 1580 if (amap_refs(amap) == 1 && (amap->am_flags & AMAP_SHARED) != 0) 1581 amap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ 1582 #ifdef UVM_AMAP_PPREF 1583 if (amap->am_ppref == NULL && all == 0 && len != amap->am_nslot) 1584 amap_pp_establish(amap, offset); 1585 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 1586 if (all) 1587 amap_pp_adjref(amap, 0, amap->am_nslot, -1); 1588 else 1589 amap_pp_adjref(amap, offset, len, -1); 1590 } 1591 #endif 1592 amap_unlock(amap); 1593 1594 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); 1595 } 1596
Cache object: fc028ca74a04a98ccbe6dbc8127b943e
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]