Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/vm/vm_phys.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 2002-2006 Rice University 3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 4 * All rights reserved. 5 * 6 * This software was developed for the FreeBSD Project by Alan L. Cox, 7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 25 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 28 * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_ddb.h" 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/lock.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 #include <sys/mutex.h> 43 #include <sys/queue.h> 44 #include <sys/sbuf.h> 45 #include <sys/sysctl.h> 46 #include <sys/vmmeter.h> 47 #include <sys/vnode.h> 48 49 #include <ddb/ddb.h> 50 51 #include <vm/vm.h> 52 #include <vm/vm_param.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_object.h> 55 #include <vm/vm_page.h> 56 #include <vm/vm_phys.h> 57 58 struct vm_freelist { 59 struct pglist pl; 60 int lcnt; 61 }; 62 63 struct vm_phys_seg { 64 vm_paddr_t start; 65 vm_paddr_t end; 66 vm_page_t first_page; 67 struct vm_freelist (*free_queues)[VM_NFREEPOOL][VM_NFREEORDER]; 68 }; 69 70 static struct vm_phys_seg vm_phys_segs[VM_PHYSSEG_MAX]; 71 72 static int vm_phys_nsegs; 73 74 static struct vm_freelist 75 vm_phys_free_queues[VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER]; 76 77 static int vm_nfreelists = VM_FREELIST_DEFAULT + 1; 78 79 static int cnt_prezero; 80 SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD, 81 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time"); 82 83 static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); 84 SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD, 85 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); 86 87 static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); 88 SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD, 89 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); 90 91 static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind); 92 static int vm_phys_paddr_to_segind(vm_paddr_t pa); 93 static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, 94 int order); 95 96 /* 97 * Outputs the state of the physical memory allocator, specifically, 98 * the amount of physical memory in each free list. 99 */ 100 static int 101 sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS) 102 { 103 struct sbuf sbuf; 104 struct vm_freelist *fl; 105 char *cbuf; 106 const int cbufsize = vm_nfreelists*(VM_NFREEORDER + 1)*81; 107 int error, flind, oind, pind; 108 109 cbuf = malloc(cbufsize, M_TEMP, M_WAITOK | M_ZERO); 110 sbuf_new(&sbuf, cbuf, cbufsize, SBUF_FIXEDLEN); 111 for (flind = 0; flind < vm_nfreelists; flind++) { 112 sbuf_printf(&sbuf, "\nFREE LIST %d:\n" 113 "\n ORDER (SIZE) | NUMBER" 114 "\n ", flind); 115 for (pind = 0; pind < VM_NFREEPOOL; pind++) 116 sbuf_printf(&sbuf, " | POOL %d", pind); 117 sbuf_printf(&sbuf, "\n-- "); 118 for (pind = 0; pind < VM_NFREEPOOL; pind++) 119 sbuf_printf(&sbuf, "-- -- "); 120 sbuf_printf(&sbuf, "--\n"); 121 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 122 sbuf_printf(&sbuf, " %2.2d (%6.6dK)", oind, 123 1 << (PAGE_SHIFT - 10 + oind)); 124 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 125 fl = vm_phys_free_queues[flind][pind]; 126 sbuf_printf(&sbuf, " | %6.6d", fl[oind].lcnt); 127 } 128 sbuf_printf(&sbuf, "\n"); 129 } 130 } 131 sbuf_finish(&sbuf); 132 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 133 sbuf_delete(&sbuf); 134 free(cbuf, M_TEMP); 135 return (error); 136 } 137 138 /* 139 * Outputs the set of physical memory segments. 140 */ 141 static int 142 sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS) 143 { 144 struct sbuf sbuf; 145 struct vm_phys_seg *seg; 146 char *cbuf; 147 const int cbufsize = VM_PHYSSEG_MAX*(VM_NFREEORDER + 1)*81; 148 int error, segind; 149 150 cbuf = malloc(cbufsize, M_TEMP, M_WAITOK | M_ZERO); 151 sbuf_new(&sbuf, cbuf, cbufsize, SBUF_FIXEDLEN); 152 for (segind = 0; segind < vm_phys_nsegs; segind++) { 153 sbuf_printf(&sbuf, "\nSEGMENT %d:\n\n", segind); 154 seg = &vm_phys_segs[segind]; 155 sbuf_printf(&sbuf, "start: %#jx\n", 156 (uintmax_t)seg->start); 157 sbuf_printf(&sbuf, "end: %#jx\n", 158 (uintmax_t)seg->end); 159 sbuf_printf(&sbuf, "free list: %p\n", seg->free_queues); 160 } 161 sbuf_finish(&sbuf); 162 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 163 sbuf_delete(&sbuf); 164 free(cbuf, M_TEMP); 165 return (error); 166 } 167 168 /* 169 * Create a physical memory segment. 170 */ 171 static void 172 vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind) 173 { 174 struct vm_phys_seg *seg; 175 #ifdef VM_PHYSSEG_SPARSE 176 long pages; 177 int segind; 178 179 pages = 0; 180 for (segind = 0; segind < vm_phys_nsegs; segind++) { 181 seg = &vm_phys_segs[segind]; 182 pages += atop(seg->end - seg->start); 183 } 184 #endif 185 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, 186 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); 187 seg = &vm_phys_segs[vm_phys_nsegs++]; 188 seg->start = start; 189 seg->end = end; 190 #ifdef VM_PHYSSEG_SPARSE 191 seg->first_page = &vm_page_array[pages]; 192 #else 193 seg->first_page = PHYS_TO_VM_PAGE(start); 194 #endif 195 seg->free_queues = &vm_phys_free_queues[flind]; 196 } 197 198 /* 199 * Initialize the physical memory allocator. 200 */ 201 void 202 vm_phys_init(void) 203 { 204 struct vm_freelist *fl; 205 int flind, i, oind, pind; 206 207 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 208 #ifdef VM_FREELIST_ISADMA 209 if (phys_avail[i] < 16777216) { 210 if (phys_avail[i + 1] > 16777216) { 211 vm_phys_create_seg(phys_avail[i], 16777216, 212 VM_FREELIST_ISADMA); 213 vm_phys_create_seg(16777216, phys_avail[i + 1], 214 VM_FREELIST_DEFAULT); 215 } else { 216 vm_phys_create_seg(phys_avail[i], 217 phys_avail[i + 1], VM_FREELIST_ISADMA); 218 } 219 if (VM_FREELIST_ISADMA >= vm_nfreelists) 220 vm_nfreelists = VM_FREELIST_ISADMA + 1; 221 } else 222 #endif 223 #ifdef VM_FREELIST_HIGHMEM 224 if (phys_avail[i + 1] > VM_HIGHMEM_ADDRESS) { 225 if (phys_avail[i] < VM_HIGHMEM_ADDRESS) { 226 vm_phys_create_seg(phys_avail[i], 227 VM_HIGHMEM_ADDRESS, VM_FREELIST_DEFAULT); 228 vm_phys_create_seg(VM_HIGHMEM_ADDRESS, 229 phys_avail[i + 1], VM_FREELIST_HIGHMEM); 230 } else { 231 vm_phys_create_seg(phys_avail[i], 232 phys_avail[i + 1], VM_FREELIST_HIGHMEM); 233 } 234 if (VM_FREELIST_HIGHMEM >= vm_nfreelists) 235 vm_nfreelists = VM_FREELIST_HIGHMEM + 1; 236 } else 237 #endif 238 vm_phys_create_seg(phys_avail[i], phys_avail[i + 1], 239 VM_FREELIST_DEFAULT); 240 } 241 for (flind = 0; flind < vm_nfreelists; flind++) { 242 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 243 fl = vm_phys_free_queues[flind][pind]; 244 for (oind = 0; oind < VM_NFREEORDER; oind++) 245 TAILQ_INIT(&fl[oind].pl); 246 } 247 } 248 } 249 250 /* 251 * Split a contiguous, power of two-sized set of physical pages. 252 */ 253 static __inline void 254 vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order) 255 { 256 vm_page_t m_buddy; 257 258 while (oind > order) { 259 oind--; 260 m_buddy = &m[1 << oind]; 261 KASSERT(m_buddy->order == VM_NFREEORDER, 262 ("vm_phys_split_pages: page %p has unexpected order %d", 263 m_buddy, m_buddy->order)); 264 m_buddy->order = oind; 265 TAILQ_INSERT_HEAD(&fl[oind].pl, m_buddy, pageq); 266 fl[oind].lcnt++; 267 } 268 } 269 270 /* 271 * Initialize a physical page and add it to the free lists. 272 */ 273 void 274 vm_phys_add_page(vm_paddr_t pa) 275 { 276 vm_page_t m; 277 278 cnt.v_page_count++; 279 m = vm_phys_paddr_to_vm_page(pa); 280 m->phys_addr = pa; 281 m->segind = vm_phys_paddr_to_segind(pa); 282 m->flags = PG_FREE; 283 KASSERT(m->order == VM_NFREEORDER, 284 ("vm_phys_add_page: page %p has unexpected order %d", 285 m, m->order)); 286 m->pool = VM_FREEPOOL_DEFAULT; 287 pmap_page_init(m); 288 mtx_lock(&vm_page_queue_free_mtx); 289 cnt.v_free_count++; 290 vm_phys_free_pages(m, 0); 291 mtx_unlock(&vm_page_queue_free_mtx); 292 } 293 294 /* 295 * Allocate a contiguous, power of two-sized set of physical pages 296 * from the free lists. 297 * 298 * The free page queues must be locked. 299 */ 300 vm_page_t 301 vm_phys_alloc_pages(int pool, int order) 302 { 303 struct vm_freelist *fl; 304 struct vm_freelist *alt; 305 int flind, oind, pind; 306 vm_page_t m; 307 308 KASSERT(pool < VM_NFREEPOOL, 309 ("vm_phys_alloc_pages: pool %d is out of range", pool)); 310 KASSERT(order < VM_NFREEORDER, 311 ("vm_phys_alloc_pages: order %d is out of range", order)); 312 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 313 for (flind = 0; flind < vm_nfreelists; flind++) { 314 fl = vm_phys_free_queues[flind][pool]; 315 for (oind = order; oind < VM_NFREEORDER; oind++) { 316 m = TAILQ_FIRST(&fl[oind].pl); 317 if (m != NULL) { 318 TAILQ_REMOVE(&fl[oind].pl, m, pageq); 319 fl[oind].lcnt--; 320 m->order = VM_NFREEORDER; 321 vm_phys_split_pages(m, oind, fl, order); 322 return (m); 323 } 324 } 325 326 /* 327 * The given pool was empty. Find the largest 328 * contiguous, power-of-two-sized set of pages in any 329 * pool. Transfer these pages to the given pool, and 330 * use them to satisfy the allocation. 331 */ 332 for (oind = VM_NFREEORDER - 1; oind >= order; oind--) { 333 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 334 alt = vm_phys_free_queues[flind][pind]; 335 m = TAILQ_FIRST(&alt[oind].pl); 336 if (m != NULL) { 337 TAILQ_REMOVE(&alt[oind].pl, m, pageq); 338 alt[oind].lcnt--; 339 m->order = VM_NFREEORDER; 340 vm_phys_set_pool(pool, m, oind); 341 vm_phys_split_pages(m, oind, fl, order); 342 return (m); 343 } 344 } 345 } 346 } 347 return (NULL); 348 } 349 350 /* 351 * Allocate physical memory from phys_avail[]. 352 */ 353 vm_paddr_t 354 vm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment) 355 { 356 vm_paddr_t pa; 357 int i; 358 359 size = round_page(size); 360 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 361 if (phys_avail[i + 1] - phys_avail[i] < size) 362 continue; 363 pa = phys_avail[i]; 364 phys_avail[i] += size; 365 return (pa); 366 } 367 panic("vm_phys_bootstrap_alloc"); 368 } 369 370 /* 371 * Find the vm_page corresponding to the given physical address. 372 */ 373 vm_page_t 374 vm_phys_paddr_to_vm_page(vm_paddr_t pa) 375 { 376 struct vm_phys_seg *seg; 377 int segind; 378 379 for (segind = 0; segind < vm_phys_nsegs; segind++) { 380 seg = &vm_phys_segs[segind]; 381 if (pa >= seg->start && pa < seg->end) 382 return (&seg->first_page[atop(pa - seg->start)]); 383 } 384 panic("vm_phys_paddr_to_vm_page: paddr %#jx is not in any segment", 385 (uintmax_t)pa); 386 } 387 388 /* 389 * Find the segment containing the given physical address. 390 */ 391 static int 392 vm_phys_paddr_to_segind(vm_paddr_t pa) 393 { 394 struct vm_phys_seg *seg; 395 int segind; 396 397 for (segind = 0; segind < vm_phys_nsegs; segind++) { 398 seg = &vm_phys_segs[segind]; 399 if (pa >= seg->start && pa < seg->end) 400 return (segind); 401 } 402 panic("vm_phys_paddr_to_segind: paddr %#jx is not in any segment" , 403 (uintmax_t)pa); 404 } 405 406 /* 407 * Free a contiguous, power of two-sized set of physical pages. 408 * 409 * The free page queues must be locked. 410 */ 411 void 412 vm_phys_free_pages(vm_page_t m, int order) 413 { 414 struct vm_freelist *fl; 415 struct vm_phys_seg *seg; 416 vm_paddr_t pa, pa_buddy; 417 vm_page_t m_buddy; 418 419 KASSERT(m->order == VM_NFREEORDER, 420 ("vm_phys_free_pages: page %p has unexpected order %d", 421 m, m->order)); 422 KASSERT(m->pool < VM_NFREEPOOL, 423 ("vm_phys_free_pages: page %p has unexpected pool %d", 424 m, m->pool)); 425 KASSERT(order < VM_NFREEORDER, 426 ("vm_phys_free_pages: order %d is out of range", order)); 427 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 428 pa = VM_PAGE_TO_PHYS(m); 429 seg = &vm_phys_segs[m->segind]; 430 while (order < VM_NFREEORDER - 1) { 431 pa_buddy = pa ^ (1 << (PAGE_SHIFT + order)); 432 if (pa_buddy < seg->start || 433 pa_buddy >= seg->end) 434 break; 435 m_buddy = &seg->first_page[atop(pa_buddy - seg->start)]; 436 if (m_buddy->order != order) 437 break; 438 fl = (*seg->free_queues)[m_buddy->pool]; 439 TAILQ_REMOVE(&fl[m_buddy->order].pl, m_buddy, pageq); 440 fl[m_buddy->order].lcnt--; 441 m_buddy->order = VM_NFREEORDER; 442 if (m_buddy->pool != m->pool) 443 vm_phys_set_pool(m->pool, m_buddy, order); 444 order++; 445 pa &= ~((1 << (PAGE_SHIFT + order)) - 1); 446 m = &seg->first_page[atop(pa - seg->start)]; 447 } 448 m->order = order; 449 fl = (*seg->free_queues)[m->pool]; 450 TAILQ_INSERT_TAIL(&fl[order].pl, m, pageq); 451 fl[order].lcnt++; 452 } 453 454 /* 455 * Set the pool for a contiguous, power of two-sized set of physical pages. 456 */ 457 void 458 vm_phys_set_pool(int pool, vm_page_t m, int order) 459 { 460 vm_page_t m_tmp; 461 462 for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++) 463 m_tmp->pool = pool; 464 } 465 466 /* 467 * Remove the given physical page "m" from the free lists. 468 * 469 * The free page queues must be locked. 470 */ 471 void 472 vm_phys_unfree_page(vm_page_t m) 473 { 474 struct vm_freelist *fl; 475 struct vm_phys_seg *seg; 476 vm_paddr_t pa, pa_half; 477 vm_page_t m_set, m_tmp; 478 int order; 479 480 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 481 482 /* 483 * First, find the contiguous, power of two-sized set of free 484 * physical pages containing the given physical page "m" and 485 * assign it to "m_set". 486 */ 487 seg = &vm_phys_segs[m->segind]; 488 for (m_set = m, order = 0; m_set->order == VM_NFREEORDER && 489 order < VM_NFREEORDER; ) { 490 order++; 491 pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order)); 492 KASSERT(pa >= seg->start && pa < seg->end, 493 ("vm_phys_unfree_page: paddr %#jx is not within segment %p", 494 (uintmax_t)pa, seg)); 495 m_set = &seg->first_page[atop(pa - seg->start)]; 496 } 497 KASSERT(m_set->order >= order, ("vm_phys_unfree_page: page %p's order" 498 " (%d) is less than expected (%d)", m_set, m_set->order, order)); 499 KASSERT(m_set->order < VM_NFREEORDER, 500 ("vm_phys_unfree_page: page %p has unexpected order %d", 501 m_set, m_set->order)); 502 KASSERT(order < VM_NFREEORDER, 503 ("vm_phys_unfree_page: order %d is out of range", order)); 504 505 /* 506 * Next, remove "m_set" from the free lists. Finally, extract 507 * "m" from "m_set" using an iterative algorithm: While "m_set" 508 * is larger than a page, shrink "m_set" by returning the half 509 * of "m_set" that does not contain "m" to the free lists. 510 */ 511 fl = (*seg->free_queues)[m_set->pool]; 512 order = m_set->order; 513 TAILQ_REMOVE(&fl[order].pl, m_set, pageq); 514 fl[order].lcnt--; 515 m_set->order = VM_NFREEORDER; 516 while (order > 0) { 517 order--; 518 pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order)); 519 if (m->phys_addr < pa_half) 520 m_tmp = &seg->first_page[atop(pa_half - seg->start)]; 521 else { 522 m_tmp = m_set; 523 m_set = &seg->first_page[atop(pa_half - seg->start)]; 524 } 525 m_tmp->order = order; 526 TAILQ_INSERT_HEAD(&fl[order].pl, m_tmp, pageq); 527 fl[order].lcnt++; 528 } 529 KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); 530 } 531 532 /* 533 * Try to zero one physical page. Used by an idle priority thread. 534 */ 535 boolean_t 536 vm_phys_zero_pages_idle(void) 537 { 538 static struct vm_freelist *fl = vm_phys_free_queues[0][0]; 539 static int flind, oind, pind; 540 vm_page_t m, m_tmp; 541 542 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 543 for (;;) { 544 TAILQ_FOREACH_REVERSE(m, &fl[oind].pl, pglist, pageq) { 545 for (m_tmp = m; m_tmp < &m[1 << oind]; m_tmp++) { 546 if ((m_tmp->flags & (PG_CACHED | PG_ZERO)) == 0) { 547 vm_phys_unfree_page(m_tmp); 548 cnt.v_free_count--; 549 mtx_unlock(&vm_page_queue_free_mtx); 550 pmap_zero_page_idle(m_tmp); 551 m_tmp->flags |= PG_ZERO; 552 mtx_lock(&vm_page_queue_free_mtx); 553 cnt.v_free_count++; 554 vm_phys_free_pages(m_tmp, 0); 555 vm_page_zero_count++; 556 cnt_prezero++; 557 return (TRUE); 558 } 559 } 560 } 561 oind++; 562 if (oind == VM_NFREEORDER) { 563 oind = 0; 564 pind++; 565 if (pind == VM_NFREEPOOL) { 566 pind = 0; 567 flind++; 568 if (flind == vm_nfreelists) 569 flind = 0; 570 } 571 fl = vm_phys_free_queues[flind][pind]; 572 } 573 } 574 } 575 576 /* 577 * Allocate a contiguous set of physical pages of the given size 578 * "npages" from the free lists. All of the physical pages must be at 579 * or above the given physical address "low" and below the given 580 * physical address "high". The given value "alignment" determines the 581 * alignment of the first physical page in the set. If the given value 582 * "boundary" is non-zero, then the set of physical pages cannot cross 583 * any physical address boundary that is a multiple of that value. Both 584 * "alignment" and "boundary" must be a power of two. 585 */ 586 vm_page_t 587 vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high, 588 unsigned long alignment, unsigned long boundary) 589 { 590 struct vm_freelist *fl; 591 struct vm_phys_seg *seg; 592 vm_object_t m_object; 593 vm_paddr_t pa, pa_last, size; 594 vm_page_t m, m_ret; 595 int flind, i, oind, order, pind; 596 597 size = npages << PAGE_SHIFT; 598 KASSERT(size != 0, 599 ("vm_phys_alloc_contig: size must not be 0")); 600 KASSERT((alignment & (alignment - 1)) == 0, 601 ("vm_phys_alloc_contig: alignment must be a power of 2")); 602 KASSERT((boundary & (boundary - 1)) == 0, 603 ("vm_phys_alloc_contig: boundary must be a power of 2")); 604 /* Compute the queue that is the best fit for npages. */ 605 for (order = 0; (1 << order) < npages; order++); 606 mtx_lock(&vm_page_queue_free_mtx); 607 for (flind = 0; flind < vm_nfreelists; flind++) { 608 for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) { 609 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 610 fl = vm_phys_free_queues[flind][pind]; 611 TAILQ_FOREACH(m_ret, &fl[oind].pl, pageq) { 612 /* 613 * A free list may contain physical pages 614 * from one or more segments. 615 */ 616 seg = &vm_phys_segs[m_ret->segind]; 617 if (seg->start > high || 618 low >= seg->end) 619 continue; 620 621 /* 622 * Is the size of this allocation request 623 * larger than the largest block size? 624 */ 625 if (order >= VM_NFREEORDER) { 626 /* 627 * Determine if a sufficient number 628 * of subsequent blocks to satisfy 629 * the allocation request are free. 630 */ 631 pa = VM_PAGE_TO_PHYS(m_ret); 632 pa_last = pa + size; 633 for (;;) { 634 pa += 1 << (PAGE_SHIFT + VM_NFREEORDER - 1); 635 if (pa >= pa_last) 636 break; 637 if (pa < seg->start || 638 pa >= seg->end) 639 break; 640 m = &seg->first_page[atop(pa - seg->start)]; 641 if (m->order != VM_NFREEORDER - 1) 642 break; 643 } 644 /* If not, continue to the next block. */ 645 if (pa < pa_last) 646 continue; 647 } 648 649 /* 650 * Determine if the blocks are within the given range, 651 * satisfy the given alignment, and do not cross the 652 * given boundary. 653 */ 654 pa = VM_PAGE_TO_PHYS(m_ret); 655 if (pa >= low && 656 pa + size <= high && 657 (pa & (alignment - 1)) == 0 && 658 ((pa ^ (pa + size - 1)) & ~(boundary - 1)) == 0) 659 goto done; 660 } 661 } 662 } 663 } 664 mtx_unlock(&vm_page_queue_free_mtx); 665 return (NULL); 666 done: 667 for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { 668 fl = (*seg->free_queues)[m->pool]; 669 TAILQ_REMOVE(&fl[m->order].pl, m, pageq); 670 fl[m->order].lcnt--; 671 m->order = VM_NFREEORDER; 672 } 673 if (m_ret->pool != VM_FREEPOOL_DEFAULT) 674 vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind); 675 fl = (*seg->free_queues)[m_ret->pool]; 676 vm_phys_split_pages(m_ret, oind, fl, order); 677 for (i = 0; i < npages; i++) { 678 m = &m_ret[i]; 679 KASSERT(m->queue == PQ_NONE, 680 ("vm_phys_alloc_contig: page %p has unexpected queue %d", 681 m, m->queue)); 682 m_object = m->object; 683 if ((m->flags & PG_CACHED) != 0) 684 vm_page_cache_remove(m); 685 else { 686 KASSERT(VM_PAGE_IS_FREE(m), 687 ("vm_phys_alloc_contig: page %p is not free", m)); 688 cnt.v_free_count--; 689 } 690 m->valid = VM_PAGE_BITS_ALL; 691 if (m->flags & PG_ZERO) 692 vm_page_zero_count--; 693 /* Don't clear the PG_ZERO flag; we'll need it later. */ 694 m->flags = PG_UNMANAGED | (m->flags & PG_ZERO); 695 m->oflags = 0; 696 KASSERT(m->dirty == 0, 697 ("vm_phys_alloc_contig: page %p was dirty", m)); 698 m->wire_count = 0; 699 m->busy = 0; 700 if (m_object != NULL && 701 m_object->type == OBJT_VNODE && 702 m_object->cache == NULL) { 703 mtx_unlock(&vm_page_queue_free_mtx); 704 vdrop(m_object->handle); 705 mtx_lock(&vm_page_queue_free_mtx); 706 } 707 } 708 for (; i < roundup2(npages, 1 << imin(oind, order)); i++) { 709 m = &m_ret[i]; 710 KASSERT(m->order == VM_NFREEORDER, 711 ("vm_phys_alloc_contig: page %p has unexpected order %d", 712 m, m->order)); 713 vm_phys_free_pages(m, 0); 714 } 715 mtx_unlock(&vm_page_queue_free_mtx); 716 return (m_ret); 717 } 718 719 #ifdef DDB 720 /* 721 * Show the number of physical pages in each of the free lists. 722 */ 723 DB_SHOW_COMMAND(freepages, db_show_freepages) 724 { 725 struct vm_freelist *fl; 726 int flind, oind, pind; 727 728 for (flind = 0; flind < vm_nfreelists; flind++) { 729 db_printf("FREE LIST %d:\n" 730 "\n ORDER (SIZE) | NUMBER" 731 "\n ", flind); 732 for (pind = 0; pind < VM_NFREEPOOL; pind++) 733 db_printf(" | POOL %d", pind); 734 db_printf("\n-- "); 735 for (pind = 0; pind < VM_NFREEPOOL; pind++) 736 db_printf("-- -- "); 737 db_printf("--\n"); 738 for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 739 db_printf(" %2.2d (%6.6dK)", oind, 740 1 << (PAGE_SHIFT - 10 + oind)); 741 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 742 fl = vm_phys_free_queues[flind][pind]; 743 db_printf(" | %6.6d", fl[oind].lcnt); 744 } 745 db_printf("\n"); 746 } 747 db_printf("\n"); 748 } 749 } 750 #endif
Cache object: c992260b027e969e79c1d0e33edd4270
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]