Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/pmap.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * Copyright (c) 2005 Alan L. Cox <alc@cs.rice.edu> 9 * All rights reserved. 10 * 11 * This code is derived from software contributed to Berkeley by 12 * the Systems Programming Group of the University of Utah Computer 13 * Science Department and William Jolitz of UUNET Technologies Inc. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. All advertising materials mentioning features or use of this software 24 * must display the following acknowledgement: 25 * This product includes software developed by the University of 26 * California, Berkeley and its contributors. 27 * 4. Neither the name of the University nor the names of its contributors 28 * may be used to endorse or promote products derived from this software 29 * without specific prior written permission. 30 * 31 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 32 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 33 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 34 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 35 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 36 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 37 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 38 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 39 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 40 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 41 * SUCH DAMAGE. 42 * 43 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 44 */ 45 /*- 46 * Copyright (c) 2003 Networks Associates Technology, Inc. 47 * All rights reserved. 48 * 49 * This software was developed for the FreeBSD Project by Jake Burkholder, 50 * Safeport Network Services, and Network Associates Laboratories, the 51 * Security Research Division of Network Associates, Inc. under 52 * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA 53 * CHATS research program. 54 * 55 * Redistribution and use in source and binary forms, with or without 56 * modification, are permitted provided that the following conditions 57 * are met: 58 * 1. Redistributions of source code must retain the above copyright 59 * notice, this list of conditions and the following disclaimer. 60 * 2. Redistributions in binary form must reproduce the above copyright 61 * notice, this list of conditions and the following disclaimer in the 62 * documentation and/or other materials provided with the distribution. 63 * 64 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 65 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 66 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 67 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 68 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 69 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 70 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 71 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 72 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 73 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 74 * SUCH DAMAGE. 75 */ 76 77 #include <sys/cdefs.h> 78 __FBSDID("$FreeBSD: releng/6.3/sys/i386/i386/pmap.c 173886 2007-11-24 19:45:58Z cvs2svn $"); 79 80 /* 81 * Manages physical address maps. 82 * 83 * In addition to hardware address maps, this 84 * module is called upon to provide software-use-only 85 * maps which may or may not be stored in the same 86 * form as hardware maps. These pseudo-maps are 87 * used to store intermediate results from copy 88 * operations to and from address spaces. 89 * 90 * Since the information managed by this module is 91 * also stored by the logical address mapping module, 92 * this module may throw away valid virtual-to-physical 93 * mappings at almost any time. However, invalidations 94 * of virtual-to-physical mappings must be done as 95 * requested. 96 * 97 * In order to cope with hardware architectures which 98 * make virtual-to-physical map invalidates expensive, 99 * this module may delay invalidate or reduced protection 100 * operations until such time as they are actually 101 * necessary. This module is given full information as 102 * to which processors are currently using which maps, 103 * and to when physical maps must be made correct. 104 */ 105 106 #include "opt_cpu.h" 107 #include "opt_pmap.h" 108 #include "opt_msgbuf.h" 109 #include "opt_xbox.h" 110 111 #include <sys/param.h> 112 #include <sys/systm.h> 113 #include <sys/kernel.h> 114 #include <sys/lock.h> 115 #include <sys/malloc.h> 116 #include <sys/mman.h> 117 #include <sys/msgbuf.h> 118 #include <sys/mutex.h> 119 #include <sys/proc.h> 120 #include <sys/sx.h> 121 #include <sys/vmmeter.h> 122 #include <sys/sched.h> 123 #include <sys/sysctl.h> 124 #ifdef SMP 125 #include <sys/smp.h> 126 #endif 127 128 #ifdef XBOX 129 #include <machine/xbox.h> 130 #endif 131 132 #include <vm/vm.h> 133 #include <vm/vm_param.h> 134 #include <vm/vm_kern.h> 135 #include <vm/vm_page.h> 136 #include <vm/vm_map.h> 137 #include <vm/vm_object.h> 138 #include <vm/vm_extern.h> 139 #include <vm/vm_pageout.h> 140 #include <vm/vm_pager.h> 141 #include <vm/uma.h> 142 143 #include <machine/cpu.h> 144 #include <machine/cputypes.h> 145 #include <machine/md_var.h> 146 #include <machine/pcb.h> 147 #include <machine/specialreg.h> 148 #ifdef SMP 149 #include <machine/smp.h> 150 #endif 151 152 #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU) 153 #define CPU_ENABLE_SSE 154 #endif 155 156 #ifndef PMAP_SHPGPERPROC 157 #define PMAP_SHPGPERPROC 200 158 #endif 159 160 #if defined(DIAGNOSTIC) 161 #define PMAP_DIAGNOSTIC 162 #endif 163 164 #if !defined(PMAP_DIAGNOSTIC) 165 #define PMAP_INLINE __inline 166 #else 167 #define PMAP_INLINE 168 #endif 169 170 /* 171 * Get PDEs and PTEs for user/kernel address space 172 */ 173 #define pmap_pde(m, v) (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT])) 174 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT]) 175 176 #define pmap_pde_v(pte) ((*(int *)pte & PG_V) != 0) 177 #define pmap_pte_w(pte) ((*(int *)pte & PG_W) != 0) 178 #define pmap_pte_m(pte) ((*(int *)pte & PG_M) != 0) 179 #define pmap_pte_u(pte) ((*(int *)pte & PG_A) != 0) 180 #define pmap_pte_v(pte) ((*(int *)pte & PG_V) != 0) 181 182 #define pmap_pte_set_w(pte, v) ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \ 183 atomic_clear_int((u_int *)(pte), PG_W)) 184 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v))) 185 186 struct pmap kernel_pmap_store; 187 LIST_HEAD(pmaplist, pmap); 188 static struct pmaplist allpmaps; 189 static struct mtx allpmaps_lock; 190 191 vm_paddr_t avail_end; /* PA of last available physical page */ 192 vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 193 vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 194 int pgeflag = 0; /* PG_G or-in */ 195 int pseflag = 0; /* PG_PS or-in */ 196 197 static int nkpt; 198 vm_offset_t kernel_vm_end; 199 extern u_int32_t KERNend; 200 201 #ifdef PAE 202 static uma_zone_t pdptzone; 203 #endif 204 205 /* 206 * Data for the pv entry allocation mechanism 207 */ 208 static uma_zone_t pvzone; 209 static struct vm_object pvzone_obj; 210 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; 211 int pmap_pagedaemon_waken; 212 213 /* 214 * All those kernel PT submaps that BSD is so fond of 215 */ 216 struct sysmaps { 217 struct mtx lock; 218 pt_entry_t *CMAP1; 219 pt_entry_t *CMAP2; 220 caddr_t CADDR1; 221 caddr_t CADDR2; 222 }; 223 static struct sysmaps sysmaps_pcpu[MAXCPU]; 224 pt_entry_t *CMAP1 = 0; 225 static pt_entry_t *CMAP3; 226 caddr_t CADDR1 = 0, ptvmmap = 0; 227 static caddr_t CADDR3; 228 struct msgbuf *msgbufp = 0; 229 230 /* 231 * Crashdump maps. 232 */ 233 static caddr_t crashdumpmap; 234 235 #ifdef SMP 236 extern pt_entry_t *SMPpt; 237 #endif 238 static pt_entry_t *PMAP1 = 0, *PMAP2; 239 static pt_entry_t *PADDR1 = 0, *PADDR2; 240 #ifdef SMP 241 static int PMAP1cpu; 242 static int PMAP1changedcpu; 243 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 244 &PMAP1changedcpu, 0, 245 "Number of times pmap_pte_quick changed CPU with same PMAP1"); 246 #endif 247 static int PMAP1changed; 248 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 249 &PMAP1changed, 0, 250 "Number of times pmap_pte_quick changed PMAP1"); 251 static int PMAP1unchanged; 252 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 253 &PMAP1unchanged, 0, 254 "Number of times pmap_pte_quick didn't change PMAP1"); 255 static struct mtx PMAP2mutex; 256 257 static PMAP_INLINE void free_pv_entry(pv_entry_t pv); 258 static pv_entry_t get_pv_entry(void); 259 static void pmap_clear_ptes(vm_page_t m, int bit); 260 261 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, 262 vm_page_t m, vm_prot_t prot, vm_page_t mpte); 263 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, 264 vm_page_t *free); 265 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va); 266 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, 267 vm_offset_t va); 268 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m); 269 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, 270 vm_page_t m); 271 272 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); 273 274 static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); 275 static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free); 276 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va); 277 static void pmap_pte_release(pt_entry_t *pte); 278 static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t *); 279 static vm_offset_t pmap_kmem_choose(vm_offset_t addr); 280 #ifdef PAE 281 static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait); 282 #endif 283 284 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t)); 285 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t)); 286 287 /* 288 * Move the kernel virtual free pointer to the next 289 * 4MB. This is used to help improve performance 290 * by using a large (4MB) page for much of the kernel 291 * (.text, .data, .bss) 292 */ 293 static vm_offset_t 294 pmap_kmem_choose(vm_offset_t addr) 295 { 296 vm_offset_t newaddr = addr; 297 298 #ifndef DISABLE_PSE 299 if (cpu_feature & CPUID_PSE) 300 newaddr = (addr + PDRMASK) & ~PDRMASK; 301 #endif 302 return newaddr; 303 } 304 305 /* 306 * Bootstrap the system enough to run with virtual memory. 307 * 308 * On the i386 this is called after mapping has already been enabled 309 * and just syncs the pmap module with what has already been done. 310 * [We can't call it easily with mapping off since the kernel is not 311 * mapped with PA == VA, hence we would have to relocate every address 312 * from the linked base (virtual) address "KERNBASE" to the actual 313 * (physical) address starting relative to 0] 314 */ 315 void 316 pmap_bootstrap(firstaddr, loadaddr) 317 vm_paddr_t firstaddr; 318 vm_paddr_t loadaddr; 319 { 320 vm_offset_t va; 321 pt_entry_t *pte, *unused; 322 struct sysmaps *sysmaps; 323 int i; 324 325 /* 326 * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too 327 * large. It should instead be correctly calculated in locore.s and 328 * not based on 'first' (which is a physical address, not a virtual 329 * address, for the start of unused physical memory). The kernel 330 * page tables are NOT double mapped and thus should not be included 331 * in this calculation. 332 */ 333 virtual_avail = (vm_offset_t) KERNBASE + firstaddr; 334 virtual_avail = pmap_kmem_choose(virtual_avail); 335 336 virtual_end = VM_MAX_KERNEL_ADDRESS; 337 338 /* 339 * Initialize the kernel pmap (which is statically allocated). 340 */ 341 PMAP_LOCK_INIT(kernel_pmap); 342 kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD); 343 #ifdef PAE 344 kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT); 345 #endif 346 kernel_pmap->pm_active = -1; /* don't allow deactivation */ 347 TAILQ_INIT(&kernel_pmap->pm_pvlist); 348 LIST_INIT(&allpmaps); 349 mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN); 350 mtx_lock_spin(&allpmaps_lock); 351 LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list); 352 mtx_unlock_spin(&allpmaps_lock); 353 nkpt = NKPT; 354 355 /* 356 * Reserve some special page table entries/VA space for temporary 357 * mapping of pages. 358 */ 359 #define SYSMAP(c, p, v, n) \ 360 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); 361 362 va = virtual_avail; 363 pte = vtopte(va); 364 365 /* 366 * CMAP1/CMAP2 are used for zeroing and copying pages. 367 * CMAP3 is used for the idle process page zeroing. 368 */ 369 for (i = 0; i < MAXCPU; i++) { 370 sysmaps = &sysmaps_pcpu[i]; 371 mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); 372 SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1) 373 SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1) 374 } 375 SYSMAP(caddr_t, CMAP1, CADDR1, 1) 376 SYSMAP(caddr_t, CMAP3, CADDR3, 1) 377 *CMAP3 = 0; 378 379 /* 380 * Crashdump maps. 381 */ 382 SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS) 383 384 /* 385 * ptvmmap is used for reading arbitrary physical pages via /dev/mem. 386 */ 387 SYSMAP(caddr_t, unused, ptvmmap, 1) 388 389 /* 390 * msgbufp is used to map the system message buffer. 391 */ 392 SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(MSGBUF_SIZE))) 393 394 /* 395 * ptemap is used for pmap_pte_quick 396 */ 397 SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1); 398 SYSMAP(pt_entry_t *, PMAP2, PADDR2, 1); 399 400 mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF); 401 402 virtual_avail = va; 403 404 *CMAP1 = 0; 405 406 #ifdef XBOX 407 /* FIXME: This is gross, but needed for the XBOX. Since we are in such 408 * an early stadium, we cannot yet neatly map video memory ... :-( 409 * Better fixes are very welcome! 410 */ 411 if (!arch_i386_is_xbox) 412 #endif 413 for (i = 0; i < NKPT; i++) 414 PTD[i] = 0; 415 416 /* Initialize the PAT MSR if present. */ 417 pmap_init_pat(); 418 419 /* Turn on PG_G on kernel page(s) */ 420 pmap_set_pg(); 421 } 422 423 /* 424 * Setup the PAT MSR. 425 */ 426 void 427 pmap_init_pat(void) 428 { 429 uint64_t pat_msr; 430 431 /* Bail if this CPU doesn't implement PAT. */ 432 if (!(cpu_feature & CPUID_PAT)) 433 return; 434 435 #ifdef PAT_WORKS 436 /* 437 * Leave the indices 0-3 at the default of WB, WT, UC, and UC-. 438 * Program 4 and 5 as WP and WC. 439 * Leave 6 and 7 as UC and UC-. 440 */ 441 pat_msr = rdmsr(MSR_PAT); 442 pat_msr &= ~(PAT_MASK(4) | PAT_MASK(5)); 443 pat_msr |= PAT_VALUE(4, PAT_WRITE_PROTECTED) | 444 PAT_VALUE(5, PAT_WRITE_COMBINING); 445 #else 446 /* 447 * Due to some Intel errata, we can only safely use the lower 4 448 * PAT entries. Thus, just replace PAT Index 2 with WC instead 449 * of UC-. 450 * 451 * Intel Pentium III Processor Specification Update 452 * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B 453 * or Mode C Paging) 454 * 455 * Intel Pentium IV Processor Specification Update 456 * Errata N46 (PAT Index MSB May Be Calculated Incorrectly) 457 */ 458 pat_msr = rdmsr(MSR_PAT); 459 pat_msr &= ~PAT_MASK(2); 460 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING); 461 #endif 462 wrmsr(MSR_PAT, pat_msr); 463 } 464 465 /* 466 * Set PG_G on kernel pages. Only the BSP calls this when SMP is turned on. 467 */ 468 void 469 pmap_set_pg(void) 470 { 471 pd_entry_t pdir; 472 pt_entry_t *pte; 473 vm_offset_t va, endva; 474 int i; 475 476 if (pgeflag == 0) 477 return; 478 479 i = KERNLOAD/NBPDR; 480 endva = KERNBASE + KERNend; 481 482 if (pseflag) { 483 va = KERNBASE + KERNLOAD; 484 while (va < endva) { 485 pdir = kernel_pmap->pm_pdir[KPTDI+i]; 486 pdir |= pgeflag; 487 kernel_pmap->pm_pdir[KPTDI+i] = PTD[KPTDI+i] = pdir; 488 invltlb(); /* Play it safe, invltlb() every time */ 489 i++; 490 va += NBPDR; 491 } 492 } else { 493 va = (vm_offset_t)btext; 494 while (va < endva) { 495 pte = vtopte(va); 496 if (*pte) 497 *pte |= pgeflag; 498 invltlb(); /* Play it safe, invltlb() every time */ 499 va += PAGE_SIZE; 500 } 501 } 502 } 503 504 /* 505 * Initialize a vm_page's machine-dependent fields. 506 */ 507 void 508 pmap_page_init(vm_page_t m) 509 { 510 511 TAILQ_INIT(&m->md.pv_list); 512 m->md.pv_list_count = 0; 513 } 514 515 #ifdef PAE 516 517 static MALLOC_DEFINE(M_PMAPPDPT, "pmap", "pmap pdpt"); 518 519 static void * 520 pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 521 { 522 *flags = UMA_SLAB_PRIV; 523 return (contigmalloc(PAGE_SIZE, M_PMAPPDPT, 0, 0x0ULL, 0xffffffffULL, 524 1, 0)); 525 } 526 #endif 527 528 /* 529 * Initialize the pmap module. 530 * Called by vm_init, to initialize any structures that the pmap 531 * system needs to map virtual memory. 532 */ 533 void 534 pmap_init(void) 535 { 536 int shpgperproc = PMAP_SHPGPERPROC; 537 538 /* 539 * Initialize the address space (zone) for the pv entries. Set a 540 * high water mark so that the system can recover from excessive 541 * numbers of pv entries. 542 */ 543 pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 544 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 545 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 546 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count; 547 TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max); 548 pv_entry_high_water = 9 * (pv_entry_max / 10); 549 uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 550 551 #ifdef PAE 552 pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL, 553 NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1, 554 UMA_ZONE_VM | UMA_ZONE_NOFREE); 555 uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf); 556 #endif 557 } 558 559 void 560 pmap_init2() 561 { 562 } 563 564 565 /*************************************************** 566 * Low level helper routines..... 567 ***************************************************/ 568 569 /* 570 * Determine the appropriate bits to set in a PTE or PDE for a specified 571 * caching mode. 572 */ 573 static int 574 pmap_cache_bits(int mode, boolean_t is_pde) 575 { 576 int pat_flag, pat_index, cache_bits; 577 578 /* The PAT bit is different for PTE's and PDE's. */ 579 pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT; 580 581 /* If we don't support PAT, map extended modes to older ones. */ 582 if (!(cpu_feature & CPUID_PAT)) { 583 switch (mode) { 584 case PAT_UNCACHEABLE: 585 case PAT_WRITE_THROUGH: 586 case PAT_WRITE_BACK: 587 break; 588 case PAT_UNCACHED: 589 case PAT_WRITE_COMBINING: 590 case PAT_WRITE_PROTECTED: 591 mode = PAT_UNCACHEABLE; 592 break; 593 } 594 } 595 596 /* Map the caching mode to a PAT index. */ 597 switch (mode) { 598 #ifdef PAT_WORKS 599 case PAT_UNCACHEABLE: 600 pat_index = 3; 601 break; 602 case PAT_WRITE_THROUGH: 603 pat_index = 1; 604 break; 605 case PAT_WRITE_BACK: 606 pat_index = 0; 607 break; 608 case PAT_UNCACHED: 609 pat_index = 2; 610 break; 611 case PAT_WRITE_COMBINING: 612 pat_index = 5; 613 break; 614 case PAT_WRITE_PROTECTED: 615 pat_index = 4; 616 break; 617 #else 618 case PAT_UNCACHED: 619 case PAT_UNCACHEABLE: 620 case PAT_WRITE_PROTECTED: 621 pat_index = 3; 622 break; 623 case PAT_WRITE_THROUGH: 624 pat_index = 1; 625 break; 626 case PAT_WRITE_BACK: 627 pat_index = 0; 628 break; 629 case PAT_WRITE_COMBINING: 630 pat_index = 2; 631 break; 632 #endif 633 default: 634 panic("Unknown caching mode %d\n", mode); 635 } 636 637 /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */ 638 cache_bits = 0; 639 if (pat_index & 0x4) 640 cache_bits |= pat_flag; 641 if (pat_index & 0x2) 642 cache_bits |= PG_NC_PCD; 643 if (pat_index & 0x1) 644 cache_bits |= PG_NC_PWT; 645 return (cache_bits); 646 } 647 #ifdef SMP 648 /* 649 * For SMP, these functions have to use the IPI mechanism for coherence. 650 * 651 * N.B.: Before calling any of the following TLB invalidation functions, 652 * the calling processor must ensure that all stores updating a non- 653 * kernel page table are globally performed. Otherwise, another 654 * processor could cache an old, pre-update entry without being 655 * invalidated. This can happen one of two ways: (1) The pmap becomes 656 * active on another processor after its pm_active field is checked by 657 * one of the following functions but before a store updating the page 658 * table is globally performed. (2) The pmap becomes active on another 659 * processor before its pm_active field is checked but due to 660 * speculative loads one of the following functions stills reads the 661 * pmap as inactive on the other processor. 662 * 663 * The kernel page table is exempt because its pm_active field is 664 * immutable. The kernel page table is always active on every 665 * processor. 666 */ 667 void 668 pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 669 { 670 u_int cpumask; 671 u_int other_cpus; 672 673 sched_pin(); 674 if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { 675 invlpg(va); 676 smp_invlpg(va); 677 } else { 678 cpumask = PCPU_GET(cpumask); 679 other_cpus = PCPU_GET(other_cpus); 680 if (pmap->pm_active & cpumask) 681 invlpg(va); 682 if (pmap->pm_active & other_cpus) 683 smp_masked_invlpg(pmap->pm_active & other_cpus, va); 684 } 685 sched_unpin(); 686 } 687 688 void 689 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 690 { 691 u_int cpumask; 692 u_int other_cpus; 693 vm_offset_t addr; 694 695 sched_pin(); 696 if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { 697 for (addr = sva; addr < eva; addr += PAGE_SIZE) 698 invlpg(addr); 699 smp_invlpg_range(sva, eva); 700 } else { 701 cpumask = PCPU_GET(cpumask); 702 other_cpus = PCPU_GET(other_cpus); 703 if (pmap->pm_active & cpumask) 704 for (addr = sva; addr < eva; addr += PAGE_SIZE) 705 invlpg(addr); 706 if (pmap->pm_active & other_cpus) 707 smp_masked_invlpg_range(pmap->pm_active & other_cpus, 708 sva, eva); 709 } 710 sched_unpin(); 711 } 712 713 void 714 pmap_invalidate_all(pmap_t pmap) 715 { 716 u_int cpumask; 717 u_int other_cpus; 718 719 sched_pin(); 720 if (pmap == kernel_pmap || pmap->pm_active == all_cpus) { 721 invltlb(); 722 smp_invltlb(); 723 } else { 724 cpumask = PCPU_GET(cpumask); 725 other_cpus = PCPU_GET(other_cpus); 726 if (pmap->pm_active & cpumask) 727 invltlb(); 728 if (pmap->pm_active & other_cpus) 729 smp_masked_invltlb(pmap->pm_active & other_cpus); 730 } 731 sched_unpin(); 732 } 733 734 void 735 pmap_invalidate_cache(void) 736 { 737 738 sched_pin(); 739 wbinvd(); 740 smp_cache_flush(); 741 sched_unpin(); 742 } 743 #else /* !SMP */ 744 /* 745 * Normal, non-SMP, 486+ invalidation functions. 746 * We inline these within pmap.c for speed. 747 */ 748 PMAP_INLINE void 749 pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 750 { 751 752 if (pmap == kernel_pmap || pmap->pm_active) 753 invlpg(va); 754 } 755 756 PMAP_INLINE void 757 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 758 { 759 vm_offset_t addr; 760 761 if (pmap == kernel_pmap || pmap->pm_active) 762 for (addr = sva; addr < eva; addr += PAGE_SIZE) 763 invlpg(addr); 764 } 765 766 PMAP_INLINE void 767 pmap_invalidate_all(pmap_t pmap) 768 { 769 770 if (pmap == kernel_pmap || pmap->pm_active) 771 invltlb(); 772 } 773 774 PMAP_INLINE void 775 pmap_invalidate_cache(void) 776 { 777 778 wbinvd(); 779 } 780 #endif /* !SMP */ 781 782 /* 783 * Are we current address space or kernel? N.B. We return FALSE when 784 * a pmap's page table is in use because a kernel thread is borrowing 785 * it. The borrowed page table can change spontaneously, making any 786 * dependence on its continued use subject to a race condition. 787 */ 788 static __inline int 789 pmap_is_current(pmap_t pmap) 790 { 791 792 return (pmap == kernel_pmap || 793 (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) && 794 (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME))); 795 } 796 797 /* 798 * If the given pmap is not the current or kernel pmap, the returned pte must 799 * be released by passing it to pmap_pte_release(). 800 */ 801 pt_entry_t * 802 pmap_pte(pmap_t pmap, vm_offset_t va) 803 { 804 pd_entry_t newpf; 805 pd_entry_t *pde; 806 807 pde = pmap_pde(pmap, va); 808 if (*pde & PG_PS) 809 return (pde); 810 if (*pde != 0) { 811 /* are we current address space or kernel? */ 812 if (pmap_is_current(pmap)) 813 return (vtopte(va)); 814 mtx_lock(&PMAP2mutex); 815 newpf = *pde & PG_FRAME; 816 if ((*PMAP2 & PG_FRAME) != newpf) { 817 *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M; 818 pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2); 819 } 820 return (PADDR2 + (i386_btop(va) & (NPTEPG - 1))); 821 } 822 return (0); 823 } 824 825 /* 826 * Releases a pte that was obtained from pmap_pte(). Be prepared for the pte 827 * being NULL. 828 */ 829 static __inline void 830 pmap_pte_release(pt_entry_t *pte) 831 { 832 833 if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2) 834 mtx_unlock(&PMAP2mutex); 835 } 836 837 static __inline void 838 invlcaddr(void *caddr) 839 { 840 841 invlpg((u_int)caddr); 842 } 843 844 /* 845 * Super fast pmap_pte routine best used when scanning 846 * the pv lists. This eliminates many coarse-grained 847 * invltlb calls. Note that many of the pv list 848 * scans are across different pmaps. It is very wasteful 849 * to do an entire invltlb for checking a single mapping. 850 * 851 * If the given pmap is not the current pmap, vm_page_queue_mtx 852 * must be held and curthread pinned to a CPU. 853 */ 854 static pt_entry_t * 855 pmap_pte_quick(pmap_t pmap, vm_offset_t va) 856 { 857 pd_entry_t newpf; 858 pd_entry_t *pde; 859 860 pde = pmap_pde(pmap, va); 861 if (*pde & PG_PS) 862 return (pde); 863 if (*pde != 0) { 864 /* are we current address space or kernel? */ 865 if (pmap_is_current(pmap)) 866 return (vtopte(va)); 867 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 868 KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); 869 newpf = *pde & PG_FRAME; 870 if ((*PMAP1 & PG_FRAME) != newpf) { 871 *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M; 872 #ifdef SMP 873 PMAP1cpu = PCPU_GET(cpuid); 874 #endif 875 invlcaddr(PADDR1); 876 PMAP1changed++; 877 } else 878 #ifdef SMP 879 if (PMAP1cpu != PCPU_GET(cpuid)) { 880 PMAP1cpu = PCPU_GET(cpuid); 881 invlcaddr(PADDR1); 882 PMAP1changedcpu++; 883 } else 884 #endif 885 PMAP1unchanged++; 886 return (PADDR1 + (i386_btop(va) & (NPTEPG - 1))); 887 } 888 return (0); 889 } 890 891 /* 892 * Routine: pmap_extract 893 * Function: 894 * Extract the physical page address associated 895 * with the given map/virtual_address pair. 896 */ 897 vm_paddr_t 898 pmap_extract(pmap_t pmap, vm_offset_t va) 899 { 900 vm_paddr_t rtval; 901 pt_entry_t *pte; 902 pd_entry_t pde; 903 904 rtval = 0; 905 PMAP_LOCK(pmap); 906 pde = pmap->pm_pdir[va >> PDRSHIFT]; 907 if (pde != 0) { 908 if ((pde & PG_PS) != 0) { 909 rtval = (pde & ~PDRMASK) | (va & PDRMASK); 910 PMAP_UNLOCK(pmap); 911 return rtval; 912 } 913 pte = pmap_pte(pmap, va); 914 rtval = (*pte & PG_FRAME) | (va & PAGE_MASK); 915 pmap_pte_release(pte); 916 } 917 PMAP_UNLOCK(pmap); 918 return (rtval); 919 } 920 921 /* 922 * Routine: pmap_extract_and_hold 923 * Function: 924 * Atomically extract and hold the physical page 925 * with the given pmap and virtual address pair 926 * if that mapping permits the given protection. 927 */ 928 vm_page_t 929 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 930 { 931 pd_entry_t pde; 932 pt_entry_t pte; 933 vm_page_t m; 934 935 m = NULL; 936 vm_page_lock_queues(); 937 PMAP_LOCK(pmap); 938 pde = *pmap_pde(pmap, va); 939 if (pde != 0) { 940 if (pde & PG_PS) { 941 if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) { 942 m = PHYS_TO_VM_PAGE((pde & ~PDRMASK) | 943 (va & PDRMASK)); 944 vm_page_hold(m); 945 } 946 } else { 947 sched_pin(); 948 pte = *pmap_pte_quick(pmap, va); 949 if (pte != 0 && 950 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) { 951 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); 952 vm_page_hold(m); 953 } 954 sched_unpin(); 955 } 956 } 957 vm_page_unlock_queues(); 958 PMAP_UNLOCK(pmap); 959 return (m); 960 } 961 962 /*************************************************** 963 * Low level mapping routines..... 964 ***************************************************/ 965 966 /* 967 * Add a wired page to the kva. 968 * Note: not SMP coherent. 969 */ 970 PMAP_INLINE void 971 pmap_kenter(vm_offset_t va, vm_paddr_t pa) 972 { 973 pt_entry_t *pte; 974 975 pte = vtopte(va); 976 pte_store(pte, pa | PG_RW | PG_V | pgeflag); 977 } 978 979 PMAP_INLINE void 980 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode) 981 { 982 pt_entry_t *pte; 983 984 pte = vtopte(va); 985 pte_store(pte, pa | PG_RW | PG_V | pgeflag | pmap_cache_bits(mode, 0)); 986 } 987 988 /* 989 * Remove a page from the kernel pagetables. 990 * Note: not SMP coherent. 991 */ 992 PMAP_INLINE void 993 pmap_kremove(vm_offset_t va) 994 { 995 pt_entry_t *pte; 996 997 pte = vtopte(va); 998 pte_clear(pte); 999 } 1000 1001 /* 1002 * Used to map a range of physical addresses into kernel 1003 * virtual address space. 1004 * 1005 * The value passed in '*virt' is a suggested virtual address for 1006 * the mapping. Architectures which can support a direct-mapped 1007 * physical to virtual region can return the appropriate address 1008 * within that region, leaving '*virt' unchanged. Other 1009 * architectures should map the pages starting at '*virt' and 1010 * update '*virt' with the first usable address after the mapped 1011 * region. 1012 */ 1013 vm_offset_t 1014 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 1015 { 1016 vm_offset_t va, sva; 1017 1018 va = sva = *virt; 1019 while (start < end) { 1020 pmap_kenter(va, start); 1021 va += PAGE_SIZE; 1022 start += PAGE_SIZE; 1023 } 1024 pmap_invalidate_range(kernel_pmap, sva, va); 1025 *virt = va; 1026 return (sva); 1027 } 1028 1029 1030 /* 1031 * Add a list of wired pages to the kva 1032 * this routine is only used for temporary 1033 * kernel mappings that do not need to have 1034 * page modification or references recorded. 1035 * Note that old mappings are simply written 1036 * over. The page *must* be wired. 1037 * Note: SMP coherent. Uses a ranged shootdown IPI. 1038 */ 1039 void 1040 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) 1041 { 1042 pt_entry_t *endpte, oldpte, *pte; 1043 1044 oldpte = 0; 1045 pte = vtopte(sva); 1046 endpte = pte + count; 1047 while (pte < endpte) { 1048 oldpte |= *pte; 1049 pte_store(pte, VM_PAGE_TO_PHYS(*ma) | pgeflag | PG_RW | PG_V); 1050 pte++; 1051 ma++; 1052 } 1053 if ((oldpte & PG_V) != 0) 1054 pmap_invalidate_range(kernel_pmap, sva, sva + count * 1055 PAGE_SIZE); 1056 } 1057 1058 /* 1059 * This routine tears out page mappings from the 1060 * kernel -- it is meant only for temporary mappings. 1061 * Note: SMP coherent. Uses a ranged shootdown IPI. 1062 */ 1063 void 1064 pmap_qremove(vm_offset_t sva, int count) 1065 { 1066 vm_offset_t va; 1067 1068 va = sva; 1069 while (count-- > 0) { 1070 pmap_kremove(va); 1071 va += PAGE_SIZE; 1072 } 1073 pmap_invalidate_range(kernel_pmap, sva, va); 1074 } 1075 1076 /*************************************************** 1077 * Page table page management routines..... 1078 ***************************************************/ 1079 static PMAP_INLINE void 1080 pmap_free_zero_pages(vm_page_t free) 1081 { 1082 vm_page_t m; 1083 1084 while (free != NULL) { 1085 m = free; 1086 free = m->right; 1087 vm_page_free_zero(m); 1088 } 1089 } 1090 1091 /* 1092 * This routine unholds page table pages, and if the hold count 1093 * drops to zero, then it decrements the wire count. 1094 */ 1095 static PMAP_INLINE int 1096 pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free) 1097 { 1098 1099 --m->wire_count; 1100 if (m->wire_count == 0) 1101 return _pmap_unwire_pte_hold(pmap, m, free); 1102 else 1103 return 0; 1104 } 1105 1106 static int 1107 _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free) 1108 { 1109 vm_offset_t pteva; 1110 1111 /* 1112 * unmap the page table page 1113 */ 1114 pmap->pm_pdir[m->pindex] = 0; 1115 --pmap->pm_stats.resident_count; 1116 1117 /* 1118 * This is a release store so that the ordinary store unmapping 1119 * the page table page is globally performed before TLB shoot- 1120 * down is begun. 1121 */ 1122 atomic_subtract_rel_int(&cnt.v_wire_count, 1); 1123 1124 /* 1125 * Do an invltlb to make the invalidated mapping 1126 * take effect immediately. 1127 */ 1128 pteva = VM_MAXUSER_ADDRESS + i386_ptob(m->pindex); 1129 pmap_invalidate_page(pmap, pteva); 1130 1131 /* 1132 * Put page on a list so that it is released after 1133 * *ALL* TLB shootdown is done 1134 */ 1135 m->right = *free; 1136 *free = m; 1137 1138 return 1; 1139 } 1140 1141 /* 1142 * After removing a page table entry, this routine is used to 1143 * conditionally free the page, and manage the hold/wire counts. 1144 */ 1145 static int 1146 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t *free) 1147 { 1148 pd_entry_t ptepde; 1149 vm_page_t mpte; 1150 1151 if (va >= VM_MAXUSER_ADDRESS) 1152 return 0; 1153 ptepde = *pmap_pde(pmap, va); 1154 mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); 1155 return pmap_unwire_pte_hold(pmap, mpte, free); 1156 } 1157 1158 void 1159 pmap_pinit0(pmap) 1160 struct pmap *pmap; 1161 { 1162 1163 PMAP_LOCK_INIT(pmap); 1164 pmap->pm_pdir = (pd_entry_t *)(KERNBASE + (vm_offset_t)IdlePTD); 1165 #ifdef PAE 1166 pmap->pm_pdpt = (pdpt_entry_t *)(KERNBASE + (vm_offset_t)IdlePDPT); 1167 #endif 1168 pmap->pm_active = 0; 1169 PCPU_SET(curpmap, pmap); 1170 TAILQ_INIT(&pmap->pm_pvlist); 1171 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1172 mtx_lock_spin(&allpmaps_lock); 1173 LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); 1174 mtx_unlock_spin(&allpmaps_lock); 1175 } 1176 1177 /* 1178 * Initialize a preallocated and zeroed pmap structure, 1179 * such as one in a vmspace structure. 1180 */ 1181 void 1182 pmap_pinit(pmap) 1183 register struct pmap *pmap; 1184 { 1185 vm_page_t m, ptdpg[NPGPTD]; 1186 vm_paddr_t pa; 1187 static int color; 1188 int i; 1189 1190 PMAP_LOCK_INIT(pmap); 1191 1192 /* 1193 * No need to allocate page table space yet but we do need a valid 1194 * page directory table. 1195 */ 1196 if (pmap->pm_pdir == NULL) { 1197 pmap->pm_pdir = (pd_entry_t *)kmem_alloc_nofault(kernel_map, 1198 NBPTD); 1199 #ifdef PAE 1200 pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO); 1201 KASSERT(((vm_offset_t)pmap->pm_pdpt & 1202 ((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0, 1203 ("pmap_pinit: pdpt misaligned")); 1204 KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30), 1205 ("pmap_pinit: pdpt above 4g")); 1206 #endif 1207 } 1208 1209 /* 1210 * allocate the page directory page(s) 1211 */ 1212 for (i = 0; i < NPGPTD;) { 1213 m = vm_page_alloc(NULL, color++, 1214 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 1215 VM_ALLOC_ZERO); 1216 if (m == NULL) 1217 VM_WAIT; 1218 else { 1219 ptdpg[i++] = m; 1220 } 1221 } 1222 1223 pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD); 1224 1225 for (i = 0; i < NPGPTD; i++) { 1226 if ((ptdpg[i]->flags & PG_ZERO) == 0) 1227 bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE); 1228 } 1229 1230 mtx_lock_spin(&allpmaps_lock); 1231 LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); 1232 mtx_unlock_spin(&allpmaps_lock); 1233 /* Wire in kernel global address entries. */ 1234 /* XXX copies current process, does not fill in MPPTDI */ 1235 bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t)); 1236 #ifdef SMP 1237 pmap->pm_pdir[MPPTDI] = PTD[MPPTDI]; 1238 #endif 1239 1240 /* install self-referential address mapping entry(s) */ 1241 for (i = 0; i < NPGPTD; i++) { 1242 pa = VM_PAGE_TO_PHYS(ptdpg[i]); 1243 pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M; 1244 #ifdef PAE 1245 pmap->pm_pdpt[i] = pa | PG_V; 1246 #endif 1247 } 1248 1249 pmap->pm_active = 0; 1250 TAILQ_INIT(&pmap->pm_pvlist); 1251 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 1252 } 1253 1254 /* 1255 * this routine is called if the page table page is not 1256 * mapped correctly. 1257 */ 1258 static vm_page_t 1259 _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) 1260 { 1261 vm_paddr_t ptepa; 1262 vm_page_t m; 1263 1264 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1265 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1266 ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1267 1268 /* 1269 * Allocate a page table page. 1270 */ 1271 if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | 1272 VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 1273 if (flags & M_WAITOK) { 1274 PMAP_UNLOCK(pmap); 1275 vm_page_unlock_queues(); 1276 VM_WAIT; 1277 vm_page_lock_queues(); 1278 PMAP_LOCK(pmap); 1279 } 1280 1281 /* 1282 * Indicate the need to retry. While waiting, the page table 1283 * page may have been allocated. 1284 */ 1285 return (NULL); 1286 } 1287 if ((m->flags & PG_ZERO) == 0) 1288 pmap_zero_page(m); 1289 1290 /* 1291 * Map the pagetable page into the process address space, if 1292 * it isn't already there. 1293 */ 1294 1295 pmap->pm_stats.resident_count++; 1296 1297 ptepa = VM_PAGE_TO_PHYS(m); 1298 pmap->pm_pdir[ptepindex] = 1299 (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M); 1300 1301 return m; 1302 } 1303 1304 static vm_page_t 1305 pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) 1306 { 1307 unsigned ptepindex; 1308 pd_entry_t ptepa; 1309 vm_page_t m; 1310 1311 KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || 1312 (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, 1313 ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); 1314 1315 /* 1316 * Calculate pagetable page index 1317 */ 1318 ptepindex = va >> PDRSHIFT; 1319 retry: 1320 /* 1321 * Get the page directory entry 1322 */ 1323 ptepa = pmap->pm_pdir[ptepindex]; 1324 1325 /* 1326 * This supports switching from a 4MB page to a 1327 * normal 4K page. 1328 */ 1329 if (ptepa & PG_PS) { 1330 pmap->pm_pdir[ptepindex] = 0; 1331 ptepa = 0; 1332 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; 1333 pmap_invalidate_all(kernel_pmap); 1334 } 1335 1336 /* 1337 * If the page table page is mapped, we just increment the 1338 * hold count, and activate it. 1339 */ 1340 if (ptepa) { 1341 m = PHYS_TO_VM_PAGE(ptepa); 1342 m->wire_count++; 1343 } else { 1344 /* 1345 * Here if the pte page isn't mapped, or if it has 1346 * been deallocated. 1347 */ 1348 m = _pmap_allocpte(pmap, ptepindex, flags); 1349 if (m == NULL && (flags & M_WAITOK)) 1350 goto retry; 1351 } 1352 return (m); 1353 } 1354 1355 1356 /*************************************************** 1357 * Pmap allocation/deallocation routines. 1358 ***************************************************/ 1359 1360 #ifdef SMP 1361 /* 1362 * Deal with a SMP shootdown of other users of the pmap that we are 1363 * trying to dispose of. This can be a bit hairy. 1364 */ 1365 static u_int *lazymask; 1366 static u_int lazyptd; 1367 static volatile u_int lazywait; 1368 1369 void pmap_lazyfix_action(void); 1370 1371 void 1372 pmap_lazyfix_action(void) 1373 { 1374 u_int mymask = PCPU_GET(cpumask); 1375 1376 if (rcr3() == lazyptd) 1377 load_cr3(PCPU_GET(curpcb)->pcb_cr3); 1378 atomic_clear_int(lazymask, mymask); 1379 atomic_store_rel_int(&lazywait, 1); 1380 } 1381 1382 static void 1383 pmap_lazyfix_self(u_int mymask) 1384 { 1385 1386 if (rcr3() == lazyptd) 1387 load_cr3(PCPU_GET(curpcb)->pcb_cr3); 1388 atomic_clear_int(lazymask, mymask); 1389 } 1390 1391 1392 static void 1393 pmap_lazyfix(pmap_t pmap) 1394 { 1395 u_int mymask; 1396 u_int mask; 1397 register u_int spins; 1398 1399 while ((mask = pmap->pm_active) != 0) { 1400 spins = 50000000; 1401 mask = mask & -mask; /* Find least significant set bit */ 1402 mtx_lock_spin(&smp_ipi_mtx); 1403 #ifdef PAE 1404 lazyptd = vtophys(pmap->pm_pdpt); 1405 #else 1406 lazyptd = vtophys(pmap->pm_pdir); 1407 #endif 1408 mymask = PCPU_GET(cpumask); 1409 if (mask == mymask) { 1410 lazymask = &pmap->pm_active; 1411 pmap_lazyfix_self(mymask); 1412 } else { 1413 atomic_store_rel_int((u_int *)&lazymask, 1414 (u_int)&pmap->pm_active); 1415 atomic_store_rel_int(&lazywait, 0); 1416 ipi_selected(mask, IPI_LAZYPMAP); 1417 while (lazywait == 0) { 1418 ia32_pause(); 1419 if (--spins == 0) 1420 break; 1421 } 1422 } 1423 mtx_unlock_spin(&smp_ipi_mtx); 1424 if (spins == 0) 1425 printf("pmap_lazyfix: spun for 50000000\n"); 1426 } 1427 } 1428 1429 #else /* SMP */ 1430 1431 /* 1432 * Cleaning up on uniprocessor is easy. For various reasons, we're 1433 * unlikely to have to even execute this code, including the fact 1434 * that the cleanup is deferred until the parent does a wait(2), which 1435 * means that another userland process has run. 1436 */ 1437 static void 1438 pmap_lazyfix(pmap_t pmap) 1439 { 1440 u_int cr3; 1441 1442 cr3 = vtophys(pmap->pm_pdir); 1443 if (cr3 == rcr3()) { 1444 load_cr3(PCPU_GET(curpcb)->pcb_cr3); 1445 pmap->pm_active &= ~(PCPU_GET(cpumask)); 1446 } 1447 } 1448 #endif /* SMP */ 1449 1450 /* 1451 * Release any resources held by the given physical map. 1452 * Called when a pmap initialized by pmap_pinit is being released. 1453 * Should only be called if the map contains no valid mappings. 1454 */ 1455 void 1456 pmap_release(pmap_t pmap) 1457 { 1458 vm_page_t m, ptdpg[NPGPTD]; 1459 int i; 1460 1461 KASSERT(pmap->pm_stats.resident_count == 0, 1462 ("pmap_release: pmap resident count %ld != 0", 1463 pmap->pm_stats.resident_count)); 1464 1465 pmap_lazyfix(pmap); 1466 mtx_lock_spin(&allpmaps_lock); 1467 LIST_REMOVE(pmap, pm_list); 1468 mtx_unlock_spin(&allpmaps_lock); 1469 1470 for (i = 0; i < NPGPTD; i++) 1471 ptdpg[i] = PHYS_TO_VM_PAGE(pmap->pm_pdir[PTDPTDI + i]); 1472 1473 bzero(pmap->pm_pdir + PTDPTDI, (nkpt + NPGPTD) * 1474 sizeof(*pmap->pm_pdir)); 1475 #ifdef SMP 1476 pmap->pm_pdir[MPPTDI] = 0; 1477 #endif 1478 1479 pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD); 1480 1481 vm_page_lock_queues(); 1482 for (i = 0; i < NPGPTD; i++) { 1483 m = ptdpg[i]; 1484 #ifdef PAE 1485 KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME), 1486 ("pmap_release: got wrong ptd page")); 1487 #endif 1488 m->wire_count--; 1489 atomic_subtract_int(&cnt.v_wire_count, 1); 1490 vm_page_free_zero(m); 1491 } 1492 vm_page_unlock_queues(); 1493 PMAP_LOCK_DESTROY(pmap); 1494 } 1495 1496 static int 1497 kvm_size(SYSCTL_HANDLER_ARGS) 1498 { 1499 unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE; 1500 1501 return sysctl_handle_long(oidp, &ksize, 0, req); 1502 } 1503 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 1504 0, 0, kvm_size, "IU", "Size of KVM"); 1505 1506 static int 1507 kvm_free(SYSCTL_HANDLER_ARGS) 1508 { 1509 unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; 1510 1511 return sysctl_handle_long(oidp, &kfree, 0, req); 1512 } 1513 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 1514 0, 0, kvm_free, "IU", "Amount of KVM free"); 1515 1516 /* 1517 * grow the number of kernel page table entries, if needed 1518 */ 1519 void 1520 pmap_growkernel(vm_offset_t addr) 1521 { 1522 struct pmap *pmap; 1523 vm_paddr_t ptppaddr; 1524 vm_page_t nkpg; 1525 pd_entry_t newpdir; 1526 pt_entry_t *pde; 1527 1528 mtx_assert(&kernel_map->system_mtx, MA_OWNED); 1529 if (kernel_vm_end == 0) { 1530 kernel_vm_end = KERNBASE; 1531 nkpt = 0; 1532 while (pdir_pde(PTD, kernel_vm_end)) { 1533 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 1534 nkpt++; 1535 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1536 kernel_vm_end = kernel_map->max_offset; 1537 break; 1538 } 1539 } 1540 } 1541 addr = roundup2(addr, PAGE_SIZE * NPTEPG); 1542 if (addr - 1 >= kernel_map->max_offset) 1543 addr = kernel_map->max_offset; 1544 while (kernel_vm_end < addr) { 1545 if (pdir_pde(PTD, kernel_vm_end)) { 1546 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 1547 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1548 kernel_vm_end = kernel_map->max_offset; 1549 break; 1550 } 1551 continue; 1552 } 1553 1554 /* 1555 * This index is bogus, but out of the way 1556 */ 1557 nkpg = vm_page_alloc(NULL, nkpt, 1558 VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED); 1559 if (!nkpg) 1560 panic("pmap_growkernel: no memory to grow kernel"); 1561 1562 nkpt++; 1563 1564 pmap_zero_page(nkpg); 1565 ptppaddr = VM_PAGE_TO_PHYS(nkpg); 1566 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M); 1567 pdir_pde(PTD, kernel_vm_end) = newpdir; 1568 1569 mtx_lock_spin(&allpmaps_lock); 1570 LIST_FOREACH(pmap, &allpmaps, pm_list) { 1571 pde = pmap_pde(pmap, kernel_vm_end); 1572 pde_store(pde, newpdir); 1573 } 1574 mtx_unlock_spin(&allpmaps_lock); 1575 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1); 1576 if (kernel_vm_end - 1 >= kernel_map->max_offset) { 1577 kernel_vm_end = kernel_map->max_offset; 1578 break; 1579 } 1580 } 1581 } 1582 1583 1584 /*************************************************** 1585 * page management routines. 1586 ***************************************************/ 1587 1588 /* 1589 * free the pv_entry back to the free list 1590 */ 1591 static PMAP_INLINE void 1592 free_pv_entry(pv_entry_t pv) 1593 { 1594 pv_entry_count--; 1595 uma_zfree(pvzone, pv); 1596 } 1597 1598 /* 1599 * get a new pv_entry, allocating a block from the system 1600 * when needed. 1601 * the memory allocation is performed bypassing the malloc code 1602 * because of the possibility of allocations at interrupt time. 1603 */ 1604 static pv_entry_t 1605 get_pv_entry(void) 1606 { 1607 pv_entry_count++; 1608 if ((pv_entry_count > pv_entry_high_water) && 1609 (pmap_pagedaemon_waken == 0)) { 1610 pmap_pagedaemon_waken = 1; 1611 wakeup (&vm_pages_needed); 1612 } 1613 return uma_zalloc(pvzone, M_NOWAIT); 1614 } 1615 1616 1617 static void 1618 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va) 1619 { 1620 pv_entry_t pv; 1621 1622 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1623 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1624 if (m->md.pv_list_count < pmap->pm_stats.resident_count) { 1625 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 1626 if (pmap == pv->pv_pmap && va == pv->pv_va) 1627 break; 1628 } 1629 } else { 1630 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) { 1631 if (va == pv->pv_va) 1632 break; 1633 } 1634 } 1635 KASSERT(pv != NULL, ("pmap_remove_entry: pv not found")); 1636 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1637 m->md.pv_list_count--; 1638 if (TAILQ_EMPTY(&m->md.pv_list)) 1639 vm_page_flag_clear(m, PG_WRITEABLE); 1640 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 1641 free_pv_entry(pv); 1642 } 1643 1644 /* 1645 * Create a pv entry for page at pa for 1646 * (pmap, va). 1647 */ 1648 static void 1649 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) 1650 { 1651 pv_entry_t pv; 1652 1653 pv = get_pv_entry(); 1654 if (pv == NULL) 1655 panic("no pv entries: increase vm.pmap.shpgperproc"); 1656 pv->pv_va = va; 1657 pv->pv_pmap = pmap; 1658 1659 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1660 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1661 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1662 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1663 m->md.pv_list_count++; 1664 } 1665 1666 /* 1667 * Conditionally create a pv entry. 1668 */ 1669 static boolean_t 1670 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) 1671 { 1672 pv_entry_t pv; 1673 1674 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1675 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1676 if (pv_entry_count < pv_entry_high_water && 1677 (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) { 1678 pv_entry_count++; 1679 pv->pv_va = va; 1680 pv->pv_pmap = pmap; 1681 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist); 1682 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 1683 m->md.pv_list_count++; 1684 return (TRUE); 1685 } else 1686 return (FALSE); 1687 } 1688 1689 /* 1690 * pmap_remove_pte: do the things to unmap a page in a process 1691 */ 1692 static int 1693 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, vm_page_t *free) 1694 { 1695 pt_entry_t oldpte; 1696 vm_page_t m; 1697 1698 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1699 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1700 oldpte = pte_load_clear(ptq); 1701 if (oldpte & PG_W) 1702 pmap->pm_stats.wired_count -= 1; 1703 /* 1704 * Machines that don't support invlpg, also don't support 1705 * PG_G. 1706 */ 1707 if (oldpte & PG_G) 1708 pmap_invalidate_page(kernel_pmap, va); 1709 pmap->pm_stats.resident_count -= 1; 1710 if (oldpte & PG_MANAGED) { 1711 m = PHYS_TO_VM_PAGE(oldpte); 1712 if (oldpte & PG_M) { 1713 KASSERT((oldpte & PG_RW), 1714 ("pmap_remove_pte: modified page not writable: va: %#x, pte: %#jx", 1715 va, (uintmax_t)oldpte)); 1716 vm_page_dirty(m); 1717 } 1718 if (oldpte & PG_A) 1719 vm_page_flag_set(m, PG_REFERENCED); 1720 pmap_remove_entry(pmap, m, va); 1721 } 1722 return (pmap_unuse_pt(pmap, va, free)); 1723 } 1724 1725 /* 1726 * Remove a single page from a process address space 1727 */ 1728 static void 1729 pmap_remove_page(pmap_t pmap, vm_offset_t va) 1730 { 1731 pt_entry_t *pte; 1732 vm_page_t free = NULL; 1733 1734 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1735 KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); 1736 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 1737 if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0) 1738 return; 1739 pmap_remove_pte(pmap, pte, va, &free); 1740 pmap_invalidate_page(pmap, va); 1741 pmap_free_zero_pages(free); 1742 } 1743 1744 /* 1745 * Remove the given range of addresses from the specified map. 1746 * 1747 * It is assumed that the start and end are properly 1748 * rounded to the page size. 1749 */ 1750 void 1751 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 1752 { 1753 vm_offset_t pdnxt; 1754 pd_entry_t ptpaddr; 1755 pt_entry_t *pte; 1756 vm_page_t free = NULL; 1757 int anyvalid; 1758 1759 /* 1760 * Perform an unsynchronized read. This is, however, safe. 1761 */ 1762 if (pmap->pm_stats.resident_count == 0) 1763 return; 1764 1765 anyvalid = 0; 1766 1767 vm_page_lock_queues(); 1768 sched_pin(); 1769 PMAP_LOCK(pmap); 1770 1771 /* 1772 * special handling of removing one page. a very 1773 * common operation and easy to short circuit some 1774 * code. 1775 */ 1776 if ((sva + PAGE_SIZE == eva) && 1777 ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) { 1778 pmap_remove_page(pmap, sva); 1779 goto out; 1780 } 1781 1782 for (; sva < eva; sva = pdnxt) { 1783 unsigned pdirindex; 1784 1785 /* 1786 * Calculate index for next page table. 1787 */ 1788 pdnxt = (sva + NBPDR) & ~PDRMASK; 1789 if (pmap->pm_stats.resident_count == 0) 1790 break; 1791 1792 pdirindex = sva >> PDRSHIFT; 1793 ptpaddr = pmap->pm_pdir[pdirindex]; 1794 1795 /* 1796 * Weed out invalid mappings. Note: we assume that the page 1797 * directory table is always allocated, and in kernel virtual. 1798 */ 1799 if (ptpaddr == 0) 1800 continue; 1801 1802 /* 1803 * Check for large page. 1804 */ 1805 if ((ptpaddr & PG_PS) != 0) { 1806 pmap->pm_pdir[pdirindex] = 0; 1807 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; 1808 anyvalid = 1; 1809 continue; 1810 } 1811 1812 /* 1813 * Limit our scan to either the end of the va represented 1814 * by the current page table page, or to the end of the 1815 * range being removed. 1816 */ 1817 if (pdnxt > eva) 1818 pdnxt = eva; 1819 1820 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++, 1821 sva += PAGE_SIZE) { 1822 if (*pte == 0) 1823 continue; 1824 1825 /* 1826 * The TLB entry for a PG_G mapping is invalidated 1827 * by pmap_remove_pte(). 1828 */ 1829 if ((*pte & PG_G) == 0) 1830 anyvalid = 1; 1831 if (pmap_remove_pte(pmap, pte, sva, &free)) 1832 break; 1833 } 1834 } 1835 out: 1836 sched_unpin(); 1837 if (anyvalid) { 1838 pmap_invalidate_all(pmap); 1839 pmap_free_zero_pages(free); 1840 } 1841 vm_page_unlock_queues(); 1842 PMAP_UNLOCK(pmap); 1843 } 1844 1845 /* 1846 * Routine: pmap_remove_all 1847 * Function: 1848 * Removes this physical page from 1849 * all physical maps in which it resides. 1850 * Reflects back modify bits to the pager. 1851 * 1852 * Notes: 1853 * Original versions of this routine were very 1854 * inefficient because they iteratively called 1855 * pmap_remove (slow...) 1856 */ 1857 1858 void 1859 pmap_remove_all(vm_page_t m) 1860 { 1861 register pv_entry_t pv; 1862 pt_entry_t *pte, tpte; 1863 vm_page_t free; 1864 1865 #if defined(PMAP_DIAGNOSTIC) 1866 /* 1867 * XXX This makes pmap_remove_all() illegal for non-managed pages! 1868 */ 1869 if (m->flags & PG_FICTITIOUS) { 1870 panic("pmap_remove_all: illegal for unmanaged page, va: 0x%x", 1871 VM_PAGE_TO_PHYS(m)); 1872 } 1873 #endif 1874 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1875 sched_pin(); 1876 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 1877 PMAP_LOCK(pv->pv_pmap); 1878 pv->pv_pmap->pm_stats.resident_count--; 1879 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 1880 tpte = pte_load_clear(pte); 1881 if (tpte & PG_W) 1882 pv->pv_pmap->pm_stats.wired_count--; 1883 if (tpte & PG_A) 1884 vm_page_flag_set(m, PG_REFERENCED); 1885 1886 /* 1887 * Update the vm_page_t clean and reference bits. 1888 */ 1889 if (tpte & PG_M) { 1890 KASSERT((tpte & PG_RW), 1891 ("pmap_remove_all: modified page not writable: va: %#x, pte: %#jx", 1892 pv->pv_va, (uintmax_t)tpte)); 1893 vm_page_dirty(m); 1894 } 1895 free = NULL; 1896 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, &free); 1897 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 1898 pmap_free_zero_pages(free); 1899 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 1900 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 1901 m->md.pv_list_count--; 1902 PMAP_UNLOCK(pv->pv_pmap); 1903 free_pv_entry(pv); 1904 } 1905 vm_page_flag_clear(m, PG_WRITEABLE); 1906 sched_unpin(); 1907 } 1908 1909 /* 1910 * Set the physical protection on the 1911 * specified range of this map as requested. 1912 */ 1913 void 1914 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1915 { 1916 vm_offset_t pdnxt; 1917 pd_entry_t ptpaddr; 1918 pt_entry_t *pte; 1919 int anychanged; 1920 1921 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1922 pmap_remove(pmap, sva, eva); 1923 return; 1924 } 1925 1926 if (prot & VM_PROT_WRITE) 1927 return; 1928 1929 anychanged = 0; 1930 1931 vm_page_lock_queues(); 1932 sched_pin(); 1933 PMAP_LOCK(pmap); 1934 for (; sva < eva; sva = pdnxt) { 1935 unsigned obits, pbits, pdirindex; 1936 1937 pdnxt = (sva + NBPDR) & ~PDRMASK; 1938 1939 pdirindex = sva >> PDRSHIFT; 1940 ptpaddr = pmap->pm_pdir[pdirindex]; 1941 1942 /* 1943 * Weed out invalid mappings. Note: we assume that the page 1944 * directory table is always allocated, and in kernel virtual. 1945 */ 1946 if (ptpaddr == 0) 1947 continue; 1948 1949 /* 1950 * Check for large page. 1951 */ 1952 if ((ptpaddr & PG_PS) != 0) { 1953 pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW); 1954 anychanged = 1; 1955 continue; 1956 } 1957 1958 if (pdnxt > eva) 1959 pdnxt = eva; 1960 1961 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++, 1962 sva += PAGE_SIZE) { 1963 vm_page_t m; 1964 1965 retry: 1966 /* 1967 * Regardless of whether a pte is 32 or 64 bits in 1968 * size, PG_RW, PG_A, and PG_M are among the least 1969 * significant 32 bits. 1970 */ 1971 obits = pbits = *(u_int *)pte; 1972 if (pbits & PG_MANAGED) { 1973 m = NULL; 1974 if (pbits & PG_A) { 1975 m = PHYS_TO_VM_PAGE(*pte); 1976 vm_page_flag_set(m, PG_REFERENCED); 1977 pbits &= ~PG_A; 1978 } 1979 if ((pbits & PG_M) != 0) { 1980 if (m == NULL) 1981 m = PHYS_TO_VM_PAGE(*pte); 1982 vm_page_dirty(m); 1983 } 1984 } 1985 1986 pbits &= ~(PG_RW | PG_M); 1987 1988 if (pbits != obits) { 1989 if (!atomic_cmpset_int((u_int *)pte, obits, 1990 pbits)) 1991 goto retry; 1992 if (obits & PG_G) 1993 pmap_invalidate_page(pmap, sva); 1994 else 1995 anychanged = 1; 1996 } 1997 } 1998 } 1999 sched_unpin(); 2000 if (anychanged) 2001 pmap_invalidate_all(pmap); 2002 vm_page_unlock_queues(); 2003 PMAP_UNLOCK(pmap); 2004 } 2005 2006 /* 2007 * Insert the given physical page (p) at 2008 * the specified virtual address (v) in the 2009 * target physical map with the protection requested. 2010 * 2011 * If specified, the page will be wired down, meaning 2012 * that the related pte can not be reclaimed. 2013 * 2014 * NB: This is the only routine which MAY NOT lazy-evaluate 2015 * or lose information. That is, this routine must actually 2016 * insert this page into the given map NOW. 2017 */ 2018 void 2019 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 2020 boolean_t wired) 2021 { 2022 vm_paddr_t pa; 2023 pd_entry_t *pde; 2024 register pt_entry_t *pte; 2025 vm_paddr_t opa; 2026 pt_entry_t origpte, newpte; 2027 vm_page_t mpte, om; 2028 boolean_t invlva; 2029 2030 va &= PG_FRAME; 2031 #ifdef PMAP_DIAGNOSTIC 2032 if (va > VM_MAX_KERNEL_ADDRESS) 2033 panic("pmap_enter: toobig"); 2034 if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS)) 2035 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)", va); 2036 #endif 2037 2038 mpte = NULL; 2039 2040 vm_page_lock_queues(); 2041 PMAP_LOCK(pmap); 2042 sched_pin(); 2043 2044 /* 2045 * In the case that a page table page is not 2046 * resident, we are creating it here. 2047 */ 2048 if (va < VM_MAXUSER_ADDRESS) { 2049 mpte = pmap_allocpte(pmap, va, M_WAITOK); 2050 } 2051 #if 0 && defined(PMAP_DIAGNOSTIC) 2052 else { 2053 pd_entry_t *pdeaddr = pmap_pde(pmap, va); 2054 origpte = *pdeaddr; 2055 if ((origpte & PG_V) == 0) { 2056 panic("pmap_enter: invalid kernel page table page, pdir=%p, pde=%p, va=%p\n", 2057 pmap->pm_pdir[PTDPTDI], origpte, va); 2058 } 2059 } 2060 #endif 2061 2062 pde = pmap_pde(pmap, va); 2063 if ((*pde & PG_PS) != 0) 2064 panic("pmap_enter: attempted pmap_enter on 4MB page"); 2065 pte = pmap_pte_quick(pmap, va); 2066 2067 /* 2068 * Page Directory table entry not valid, we need a new PT page 2069 */ 2070 if (pte == NULL) { 2071 panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x\n", 2072 (uintmax_t)pmap->pm_pdir[PTDPTDI], va); 2073 } 2074 2075 pa = VM_PAGE_TO_PHYS(m); 2076 om = NULL; 2077 origpte = *pte; 2078 opa = origpte & PG_FRAME; 2079 2080 /* 2081 * Mapping has not changed, must be protection or wiring change. 2082 */ 2083 if (origpte && (opa == pa)) { 2084 /* 2085 * Wiring change, just update stats. We don't worry about 2086 * wiring PT pages as they remain resident as long as there 2087 * are valid mappings in them. Hence, if a user page is wired, 2088 * the PT page will be also. 2089 */ 2090 if (wired && ((origpte & PG_W) == 0)) 2091 pmap->pm_stats.wired_count++; 2092 else if (!wired && (origpte & PG_W)) 2093 pmap->pm_stats.wired_count--; 2094 2095 /* 2096 * Remove extra pte reference 2097 */ 2098 if (mpte) 2099 mpte->wire_count--; 2100 2101 /* 2102 * We might be turning off write access to the page, 2103 * so we go ahead and sense modify status. 2104 */ 2105 if (origpte & PG_MANAGED) { 2106 om = m; 2107 pa |= PG_MANAGED; 2108 } 2109 goto validate; 2110 } 2111 /* 2112 * Mapping has changed, invalidate old range and fall through to 2113 * handle validating new mapping. 2114 */ 2115 if (opa) { 2116 if (origpte & PG_W) 2117 pmap->pm_stats.wired_count--; 2118 if (origpte & PG_MANAGED) { 2119 om = PHYS_TO_VM_PAGE(opa); 2120 pmap_remove_entry(pmap, om, va); 2121 } 2122 if (mpte != NULL) { 2123 mpte->wire_count--; 2124 KASSERT(mpte->wire_count > 0, 2125 ("pmap_enter: missing reference to page table page," 2126 " va: 0x%x", va)); 2127 } 2128 } else 2129 pmap->pm_stats.resident_count++; 2130 2131 /* 2132 * Enter on the PV list if part of our managed memory. 2133 */ 2134 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) { 2135 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 2136 ("pmap_enter: managed mapping within the clean submap")); 2137 pmap_insert_entry(pmap, va, m); 2138 pa |= PG_MANAGED; 2139 } 2140 2141 /* 2142 * Increment counters 2143 */ 2144 if (wired) 2145 pmap->pm_stats.wired_count++; 2146 2147 validate: 2148 /* 2149 * Now validate mapping with desired protection/wiring. 2150 */ 2151 newpte = (pt_entry_t)(pa | PG_V); 2152 if ((prot & VM_PROT_WRITE) != 0) 2153 newpte |= PG_RW; 2154 if (wired) 2155 newpte |= PG_W; 2156 if (va < VM_MAXUSER_ADDRESS) 2157 newpte |= PG_U; 2158 if (pmap == kernel_pmap) 2159 newpte |= pgeflag; 2160 2161 /* 2162 * if the mapping or permission bits are different, we need 2163 * to update the pte. 2164 */ 2165 if ((origpte & ~(PG_M|PG_A)) != newpte) { 2166 if (origpte & PG_V) { 2167 invlva = FALSE; 2168 origpte = pte_load_store(pte, newpte | PG_A); 2169 if (origpte & PG_A) { 2170 if (origpte & PG_MANAGED) 2171 vm_page_flag_set(om, PG_REFERENCED); 2172 if (opa != VM_PAGE_TO_PHYS(m)) 2173 invlva = TRUE; 2174 } 2175 if (origpte & PG_M) { 2176 KASSERT((origpte & PG_RW), 2177 ("pmap_enter: modified page not writable: va: %#x, pte: %#jx", 2178 va, (uintmax_t)origpte)); 2179 if ((origpte & PG_MANAGED) != 0) 2180 vm_page_dirty(om); 2181 if ((prot & VM_PROT_WRITE) == 0) 2182 invlva = TRUE; 2183 } 2184 if (invlva) 2185 pmap_invalidate_page(pmap, va); 2186 } else 2187 pte_store(pte, newpte | PG_A); 2188 } 2189 sched_unpin(); 2190 vm_page_unlock_queues(); 2191 PMAP_UNLOCK(pmap); 2192 } 2193 2194 /* 2195 * Maps a sequence of resident pages belonging to the same object. 2196 * The sequence begins with the given page m_start. This page is 2197 * mapped at the given virtual address start. Each subsequent page is 2198 * mapped at a virtual address that is offset from start by the same 2199 * amount as the page is offset from m_start within the object. The 2200 * last page in the sequence is the page with the largest offset from 2201 * m_start that can be mapped at a virtual address less than the given 2202 * virtual address end. Not every virtual page between start and end 2203 * is mapped; only those for which a resident page exists with the 2204 * corresponding offset from m_start are mapped. 2205 */ 2206 void 2207 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 2208 vm_page_t m_start, vm_prot_t prot) 2209 { 2210 vm_page_t m, mpte; 2211 vm_pindex_t diff, psize; 2212 2213 VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); 2214 psize = atop(end - start); 2215 mpte = NULL; 2216 m = m_start; 2217 PMAP_LOCK(pmap); 2218 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 2219 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m, 2220 prot, mpte); 2221 m = TAILQ_NEXT(m, listq); 2222 } 2223 PMAP_UNLOCK(pmap); 2224 } 2225 2226 /* 2227 * this code makes some *MAJOR* assumptions: 2228 * 1. Current pmap & pmap exists. 2229 * 2. Not wired. 2230 * 3. Read access. 2231 * 4. No page table pages. 2232 * but is *MUCH* faster than pmap_enter... 2233 */ 2234 2235 void 2236 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 2237 { 2238 2239 PMAP_LOCK(pmap); 2240 (void) pmap_enter_quick_locked(pmap, va, m, prot, NULL); 2241 PMAP_UNLOCK(pmap); 2242 } 2243 2244 static vm_page_t 2245 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, 2246 vm_prot_t prot, vm_page_t mpte) 2247 { 2248 pt_entry_t *pte; 2249 vm_paddr_t pa; 2250 vm_page_t free; 2251 2252 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || 2253 (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, 2254 ("pmap_enter_quick_locked: managed mapping within the clean submap")); 2255 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2256 PMAP_LOCK_ASSERT(pmap, MA_OWNED); 2257 2258 /* 2259 * In the case that a page table page is not 2260 * resident, we are creating it here. 2261 */ 2262 if (va < VM_MAXUSER_ADDRESS) { 2263 unsigned ptepindex; 2264 pd_entry_t ptepa; 2265 2266 /* 2267 * Calculate pagetable page index 2268 */ 2269 ptepindex = va >> PDRSHIFT; 2270 if (mpte && (mpte->pindex == ptepindex)) { 2271 mpte->wire_count++; 2272 } else { 2273 /* 2274 * Get the page directory entry 2275 */ 2276 ptepa = pmap->pm_pdir[ptepindex]; 2277 2278 /* 2279 * If the page table page is mapped, we just increment 2280 * the hold count, and activate it. 2281 */ 2282 if (ptepa) { 2283 if (ptepa & PG_PS) 2284 panic("pmap_enter_quick: unexpected mapping into 4MB page"); 2285 mpte = PHYS_TO_VM_PAGE(ptepa); 2286 mpte->wire_count++; 2287 } else { 2288 mpte = _pmap_allocpte(pmap, ptepindex, 2289 M_NOWAIT); 2290 if (mpte == NULL) 2291 return (mpte); 2292 } 2293 } 2294 } else { 2295 mpte = NULL; 2296 } 2297 2298 /* 2299 * This call to vtopte makes the assumption that we are 2300 * entering the page into the current pmap. In order to support 2301 * quick entry into any pmap, one would likely use pmap_pte_quick. 2302 * But that isn't as quick as vtopte. 2303 */ 2304 pte = vtopte(va); 2305 if (*pte) { 2306 if (mpte != NULL) { 2307 mpte->wire_count--; 2308 mpte = NULL; 2309 } 2310 return (mpte); 2311 } 2312 2313 /* 2314 * Enter on the PV list if part of our managed memory. 2315 */ 2316 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 && 2317 !pmap_try_insert_pv_entry(pmap, va, m)) { 2318 if (mpte != NULL) { 2319 free = NULL; 2320 if (pmap_unwire_pte_hold(pmap, mpte, &free)) { 2321 pmap_invalidate_page(pmap, va); 2322 pmap_free_zero_pages(free); 2323 } 2324 2325 mpte = NULL; 2326 } 2327 return (mpte); 2328 } 2329 2330 /* 2331 * Increment counters 2332 */ 2333 pmap->pm_stats.resident_count++; 2334 2335 pa = VM_PAGE_TO_PHYS(m); 2336 2337 /* 2338 * Now validate mapping with RO protection 2339 */ 2340 if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) 2341 pte_store(pte, pa | PG_V | PG_U); 2342 else 2343 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED); 2344 return mpte; 2345 } 2346 2347 /* 2348 * Make a temporary mapping for a physical address. This is only intended 2349 * to be used for panic dumps. 2350 */ 2351 void * 2352 pmap_kenter_temporary(vm_paddr_t pa, int i) 2353 { 2354 vm_offset_t va; 2355 2356 va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); 2357 pmap_kenter(va, pa); 2358 invlpg(va); 2359 return ((void *)crashdumpmap); 2360 } 2361 2362 /* 2363 * This code maps large physical mmap regions into the 2364 * processor address space. Note that some shortcuts 2365 * are taken, but the code works. 2366 */ 2367 void 2368 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, 2369 vm_object_t object, vm_pindex_t pindex, 2370 vm_size_t size) 2371 { 2372 vm_page_t p; 2373 2374 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 2375 KASSERT(object->type == OBJT_DEVICE, 2376 ("pmap_object_init_pt: non-device object")); 2377 if (pseflag && 2378 ((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0)) { 2379 int i; 2380 vm_page_t m[1]; 2381 unsigned int ptepindex; 2382 int npdes; 2383 pd_entry_t ptepa; 2384 2385 PMAP_LOCK(pmap); 2386 if (pmap->pm_pdir[ptepindex = (addr >> PDRSHIFT)]) 2387 goto out; 2388 PMAP_UNLOCK(pmap); 2389 retry: 2390 p = vm_page_lookup(object, pindex); 2391 if (p != NULL) { 2392 vm_page_lock_queues(); 2393 if (vm_page_sleep_if_busy(p, FALSE, "init4p")) 2394 goto retry; 2395 } else { 2396 p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL); 2397 if (p == NULL) 2398 return; 2399 m[0] = p; 2400 2401 if (vm_pager_get_pages(object, m, 1, 0) != VM_PAGER_OK) { 2402 vm_page_lock_queues(); 2403 vm_page_free(p); 2404 vm_page_unlock_queues(); 2405 return; 2406 } 2407 2408 p = vm_page_lookup(object, pindex); 2409 vm_page_lock_queues(); 2410 vm_page_wakeup(p); 2411 } 2412 vm_page_unlock_queues(); 2413 2414 ptepa = VM_PAGE_TO_PHYS(p); 2415 if (ptepa & (NBPDR - 1)) 2416 return; 2417 2418 p->valid = VM_PAGE_BITS_ALL; 2419 2420 PMAP_LOCK(pmap); 2421 pmap->pm_stats.resident_count += size >> PAGE_SHIFT; 2422 npdes = size >> PDRSHIFT; 2423 for(i = 0; i < npdes; i++) { 2424 pde_store(&pmap->pm_pdir[ptepindex], 2425 ptepa | PG_U | PG_RW | PG_V | PG_PS); 2426 ptepa += NBPDR; 2427 ptepindex += 1; 2428 } 2429 pmap_invalidate_all(pmap); 2430 out: 2431 PMAP_UNLOCK(pmap); 2432 } 2433 } 2434 2435 /* 2436 * Routine: pmap_change_wiring 2437 * Function: Change the wiring attribute for a map/virtual-address 2438 * pair. 2439 * In/out conditions: 2440 * The mapping must already exist in the pmap. 2441 */ 2442 void 2443 pmap_change_wiring(pmap, va, wired) 2444 register pmap_t pmap; 2445 vm_offset_t va; 2446 boolean_t wired; 2447 { 2448 register pt_entry_t *pte; 2449 2450 PMAP_LOCK(pmap); 2451 pte = pmap_pte(pmap, va); 2452 2453 if (wired && !pmap_pte_w(pte)) 2454 pmap->pm_stats.wired_count++; 2455 else if (!wired && pmap_pte_w(pte)) 2456 pmap->pm_stats.wired_count--; 2457 2458 /* 2459 * Wiring is not a hardware characteristic so there is no need to 2460 * invalidate TLB. 2461 */ 2462 pmap_pte_set_w(pte, wired); 2463 pmap_pte_release(pte); 2464 PMAP_UNLOCK(pmap); 2465 } 2466 2467 2468 2469 /* 2470 * Copy the range specified by src_addr/len 2471 * from the source map to the range dst_addr/len 2472 * in the destination map. 2473 * 2474 * This routine is only advisory and need not do anything. 2475 */ 2476 2477 void 2478 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, 2479 vm_offset_t src_addr) 2480 { 2481 vm_page_t free; 2482 vm_offset_t addr; 2483 vm_offset_t end_addr = src_addr + len; 2484 vm_offset_t pdnxt; 2485 2486 if (dst_addr != src_addr) 2487 return; 2488 2489 if (!pmap_is_current(src_pmap)) 2490 return; 2491 2492 vm_page_lock_queues(); 2493 if (dst_pmap < src_pmap) { 2494 PMAP_LOCK(dst_pmap); 2495 PMAP_LOCK(src_pmap); 2496 } else { 2497 PMAP_LOCK(src_pmap); 2498 PMAP_LOCK(dst_pmap); 2499 } 2500 sched_pin(); 2501 for (addr = src_addr; addr < end_addr; addr = pdnxt) { 2502 pt_entry_t *src_pte, *dst_pte; 2503 vm_page_t dstmpte, srcmpte; 2504 pd_entry_t srcptepaddr; 2505 unsigned ptepindex; 2506 2507 if (addr >= UPT_MIN_ADDRESS) 2508 panic("pmap_copy: invalid to pmap_copy page tables"); 2509 2510 pdnxt = (addr + NBPDR) & ~PDRMASK; 2511 ptepindex = addr >> PDRSHIFT; 2512 2513 srcptepaddr = src_pmap->pm_pdir[ptepindex]; 2514 if (srcptepaddr == 0) 2515 continue; 2516 2517 if (srcptepaddr & PG_PS) { 2518 if (dst_pmap->pm_pdir[ptepindex] == 0) { 2519 dst_pmap->pm_pdir[ptepindex] = srcptepaddr & 2520 ~PG_W; 2521 dst_pmap->pm_stats.resident_count += 2522 NBPDR / PAGE_SIZE; 2523 } 2524 continue; 2525 } 2526 2527 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); 2528 if (srcmpte->wire_count == 0) 2529 panic("pmap_copy: source page table page is unused"); 2530 2531 if (pdnxt > end_addr) 2532 pdnxt = end_addr; 2533 2534 src_pte = vtopte(addr); 2535 while (addr < pdnxt) { 2536 pt_entry_t ptetemp; 2537 ptetemp = *src_pte; 2538 /* 2539 * we only virtual copy managed pages 2540 */ 2541 if ((ptetemp & PG_MANAGED) != 0) { 2542 dstmpte = pmap_allocpte(dst_pmap, addr, 2543 M_NOWAIT); 2544 if (dstmpte == NULL) 2545 break; 2546 dst_pte = pmap_pte_quick(dst_pmap, addr); 2547 if (*dst_pte == 0 && 2548 pmap_try_insert_pv_entry(dst_pmap, addr, 2549 PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) { 2550 /* 2551 * Clear the wired, modified, and 2552 * accessed (referenced) bits 2553 * during the copy. 2554 */ 2555 *dst_pte = ptetemp & ~(PG_W | PG_M | 2556 PG_A); 2557 dst_pmap->pm_stats.resident_count++; 2558 } else { 2559 free = NULL; 2560 if (pmap_unwire_pte_hold( dst_pmap, 2561 dstmpte, &free)) { 2562 pmap_invalidate_page(dst_pmap, 2563 addr); 2564 pmap_free_zero_pages(free); 2565 } 2566 } 2567 if (dstmpte->wire_count >= srcmpte->wire_count) 2568 break; 2569 } 2570 addr += PAGE_SIZE; 2571 src_pte++; 2572 } 2573 } 2574 sched_unpin(); 2575 vm_page_unlock_queues(); 2576 PMAP_UNLOCK(src_pmap); 2577 PMAP_UNLOCK(dst_pmap); 2578 } 2579 2580 static __inline void 2581 pagezero(void *page) 2582 { 2583 #if defined(I686_CPU) 2584 if (cpu_class == CPUCLASS_686) { 2585 #if defined(CPU_ENABLE_SSE) 2586 if (cpu_feature & CPUID_SSE2) 2587 sse2_pagezero(page); 2588 else 2589 #endif 2590 i686_pagezero(page); 2591 } else 2592 #endif 2593 bzero(page, PAGE_SIZE); 2594 } 2595 2596 /* 2597 * pmap_zero_page zeros the specified hardware page by mapping 2598 * the page into KVM and using bzero to clear its contents. 2599 */ 2600 void 2601 pmap_zero_page(vm_page_t m) 2602 { 2603 struct sysmaps *sysmaps; 2604 2605 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; 2606 mtx_lock(&sysmaps->lock); 2607 if (*sysmaps->CMAP2) 2608 panic("pmap_zero_page: CMAP2 busy"); 2609 sched_pin(); 2610 *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M; 2611 invlcaddr(sysmaps->CADDR2); 2612 pagezero(sysmaps->CADDR2); 2613 *sysmaps->CMAP2 = 0; 2614 sched_unpin(); 2615 mtx_unlock(&sysmaps->lock); 2616 } 2617 2618 /* 2619 * pmap_zero_page_area zeros the specified hardware page by mapping 2620 * the page into KVM and using bzero to clear its contents. 2621 * 2622 * off and size may not cover an area beyond a single hardware page. 2623 */ 2624 void 2625 pmap_zero_page_area(vm_page_t m, int off, int size) 2626 { 2627 struct sysmaps *sysmaps; 2628 2629 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; 2630 mtx_lock(&sysmaps->lock); 2631 if (*sysmaps->CMAP2) 2632 panic("pmap_zero_page: CMAP2 busy"); 2633 sched_pin(); 2634 *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M; 2635 invlcaddr(sysmaps->CADDR2); 2636 if (off == 0 && size == PAGE_SIZE) 2637 pagezero(sysmaps->CADDR2); 2638 else 2639 bzero((char *)sysmaps->CADDR2 + off, size); 2640 *sysmaps->CMAP2 = 0; 2641 sched_unpin(); 2642 mtx_unlock(&sysmaps->lock); 2643 } 2644 2645 /* 2646 * pmap_zero_page_idle zeros the specified hardware page by mapping 2647 * the page into KVM and using bzero to clear its contents. This 2648 * is intended to be called from the vm_pagezero process only and 2649 * outside of Giant. 2650 */ 2651 void 2652 pmap_zero_page_idle(vm_page_t m) 2653 { 2654 2655 if (*CMAP3) 2656 panic("pmap_zero_page: CMAP3 busy"); 2657 sched_pin(); 2658 *CMAP3 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M; 2659 invlcaddr(CADDR3); 2660 pagezero(CADDR3); 2661 *CMAP3 = 0; 2662 sched_unpin(); 2663 } 2664 2665 /* 2666 * pmap_copy_page copies the specified (machine independent) 2667 * page by mapping the page into virtual memory and using 2668 * bcopy to copy the page, one machine dependent page at a 2669 * time. 2670 */ 2671 void 2672 pmap_copy_page(vm_page_t src, vm_page_t dst) 2673 { 2674 struct sysmaps *sysmaps; 2675 2676 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; 2677 mtx_lock(&sysmaps->lock); 2678 if (*sysmaps->CMAP1) 2679 panic("pmap_copy_page: CMAP1 busy"); 2680 if (*sysmaps->CMAP2) 2681 panic("pmap_copy_page: CMAP2 busy"); 2682 sched_pin(); 2683 invlpg((u_int)sysmaps->CADDR1); 2684 invlpg((u_int)sysmaps->CADDR2); 2685 *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A; 2686 *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M; 2687 bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE); 2688 *sysmaps->CMAP1 = 0; 2689 *sysmaps->CMAP2 = 0; 2690 sched_unpin(); 2691 mtx_unlock(&sysmaps->lock); 2692 } 2693 2694 /* 2695 * Returns true if the pmap's pv is one of the first 2696 * 16 pvs linked to from this page. This count may 2697 * be changed upwards or downwards in the future; it 2698 * is only necessary that true be returned for a small 2699 * subset of pmaps for proper page aging. 2700 */ 2701 boolean_t 2702 pmap_page_exists_quick(pmap, m) 2703 pmap_t pmap; 2704 vm_page_t m; 2705 { 2706 pv_entry_t pv; 2707 int loops = 0; 2708 2709 if (m->flags & PG_FICTITIOUS) 2710 return FALSE; 2711 2712 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2713 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2714 if (pv->pv_pmap == pmap) { 2715 return TRUE; 2716 } 2717 loops++; 2718 if (loops >= 16) 2719 break; 2720 } 2721 return (FALSE); 2722 } 2723 2724 #define PMAP_REMOVE_PAGES_CURPROC_ONLY 2725 /* 2726 * Remove all pages from specified address space 2727 * this aids process exit speeds. Also, this code 2728 * is special cased for current process only, but 2729 * can have the more generic (and slightly slower) 2730 * mode enabled. This is much faster than pmap_remove 2731 * in the case of running down an entire address space. 2732 */ 2733 void 2734 pmap_remove_pages(pmap, sva, eva) 2735 pmap_t pmap; 2736 vm_offset_t sva, eva; 2737 { 2738 pt_entry_t *pte, tpte; 2739 vm_page_t m, free = NULL; 2740 pv_entry_t pv, npv; 2741 2742 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY 2743 if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) { 2744 printf("warning: pmap_remove_pages called with non-current pmap\n"); 2745 return; 2746 } 2747 #endif 2748 vm_page_lock_queues(); 2749 PMAP_LOCK(pmap); 2750 sched_pin(); 2751 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 2752 2753 if (pv->pv_va >= eva || pv->pv_va < sva) { 2754 npv = TAILQ_NEXT(pv, pv_plist); 2755 continue; 2756 } 2757 2758 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY 2759 pte = vtopte(pv->pv_va); 2760 #else 2761 pte = pmap_pte_quick(pmap, pv->pv_va); 2762 #endif 2763 tpte = *pte; 2764 2765 if (tpte == 0) { 2766 printf("TPTE at %p IS ZERO @ VA %08x\n", 2767 pte, pv->pv_va); 2768 panic("bad pte"); 2769 } 2770 2771 /* 2772 * We cannot remove wired pages from a process' mapping at this time 2773 */ 2774 if (tpte & PG_W) { 2775 npv = TAILQ_NEXT(pv, pv_plist); 2776 continue; 2777 } 2778 2779 m = PHYS_TO_VM_PAGE(tpte); 2780 KASSERT(m->phys_addr == (tpte & PG_FRAME), 2781 ("vm_page_t %p phys_addr mismatch %016jx %016jx", 2782 m, (uintmax_t)m->phys_addr, (uintmax_t)tpte)); 2783 2784 KASSERT(m < &vm_page_array[vm_page_array_size], 2785 ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte)); 2786 2787 pmap->pm_stats.resident_count--; 2788 2789 pte_clear(pte); 2790 2791 /* 2792 * Update the vm_page_t clean and reference bits. 2793 */ 2794 if (tpte & PG_M) { 2795 vm_page_dirty(m); 2796 } 2797 2798 npv = TAILQ_NEXT(pv, pv_plist); 2799 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); 2800 2801 m->md.pv_list_count--; 2802 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2803 if (TAILQ_EMPTY(&m->md.pv_list)) 2804 vm_page_flag_clear(m, PG_WRITEABLE); 2805 2806 pmap_unuse_pt(pmap, pv->pv_va, &free); 2807 free_pv_entry(pv); 2808 } 2809 sched_unpin(); 2810 pmap_invalidate_all(pmap); 2811 pmap_free_zero_pages(free); 2812 vm_page_unlock_queues(); 2813 PMAP_UNLOCK(pmap); 2814 } 2815 2816 /* 2817 * pmap_is_modified: 2818 * 2819 * Return whether or not the specified physical page was modified 2820 * in any physical maps. 2821 */ 2822 boolean_t 2823 pmap_is_modified(vm_page_t m) 2824 { 2825 pv_entry_t pv; 2826 pt_entry_t *pte; 2827 boolean_t rv; 2828 2829 rv = FALSE; 2830 if (m->flags & PG_FICTITIOUS) 2831 return (rv); 2832 2833 sched_pin(); 2834 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2835 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2836 PMAP_LOCK(pv->pv_pmap); 2837 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 2838 rv = (*pte & PG_M) != 0; 2839 PMAP_UNLOCK(pv->pv_pmap); 2840 if (rv) 2841 break; 2842 } 2843 sched_unpin(); 2844 return (rv); 2845 } 2846 2847 /* 2848 * pmap_is_prefaultable: 2849 * 2850 * Return whether or not the specified virtual address is elgible 2851 * for prefault. 2852 */ 2853 boolean_t 2854 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2855 { 2856 pt_entry_t *pte; 2857 boolean_t rv; 2858 2859 rv = FALSE; 2860 PMAP_LOCK(pmap); 2861 if (*pmap_pde(pmap, addr)) { 2862 pte = vtopte(addr); 2863 rv = *pte == 0; 2864 } 2865 PMAP_UNLOCK(pmap); 2866 return (rv); 2867 } 2868 2869 /* 2870 * Clear the given bit in each of the given page's ptes. The bit is 2871 * expressed as a 32-bit mask. Consequently, if the pte is 64 bits in 2872 * size, only a bit within the least significant 32 can be cleared. 2873 */ 2874 static __inline void 2875 pmap_clear_ptes(vm_page_t m, int bit) 2876 { 2877 register pv_entry_t pv; 2878 pt_entry_t pbits, *pte; 2879 2880 if ((m->flags & PG_FICTITIOUS) || 2881 (bit == PG_RW && (m->flags & PG_WRITEABLE) == 0)) 2882 return; 2883 2884 sched_pin(); 2885 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2886 /* 2887 * Loop over all current mappings setting/clearing as appropos If 2888 * setting RO do we need to clear the VAC? 2889 */ 2890 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 2891 PMAP_LOCK(pv->pv_pmap); 2892 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 2893 retry: 2894 pbits = *pte; 2895 if (pbits & bit) { 2896 if (bit == PG_RW) { 2897 /* 2898 * Regardless of whether a pte is 32 or 64 bits 2899 * in size, PG_RW and PG_M are among the least 2900 * significant 32 bits. 2901 */ 2902 if (!atomic_cmpset_int((u_int *)pte, pbits, 2903 pbits & ~(PG_RW | PG_M))) 2904 goto retry; 2905 if (pbits & PG_M) { 2906 vm_page_dirty(m); 2907 } 2908 } else { 2909 atomic_clear_int((u_int *)pte, bit); 2910 } 2911 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2912 } 2913 PMAP_UNLOCK(pv->pv_pmap); 2914 } 2915 if (bit == PG_RW) 2916 vm_page_flag_clear(m, PG_WRITEABLE); 2917 sched_unpin(); 2918 } 2919 2920 /* 2921 * pmap_page_protect: 2922 * 2923 * Lower the permission for all mappings to a given page. 2924 */ 2925 void 2926 pmap_page_protect(vm_page_t m, vm_prot_t prot) 2927 { 2928 if ((prot & VM_PROT_WRITE) == 0) { 2929 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 2930 pmap_clear_ptes(m, PG_RW); 2931 } else { 2932 pmap_remove_all(m); 2933 } 2934 } 2935 } 2936 2937 /* 2938 * pmap_ts_referenced: 2939 * 2940 * Return a count of reference bits for a page, clearing those bits. 2941 * It is not necessary for every reference bit to be cleared, but it 2942 * is necessary that 0 only be returned when there are truly no 2943 * reference bits set. 2944 * 2945 * XXX: The exact number of bits to check and clear is a matter that 2946 * should be tested and standardized at some point in the future for 2947 * optimal aging of shared pages. 2948 */ 2949 int 2950 pmap_ts_referenced(vm_page_t m) 2951 { 2952 register pv_entry_t pv, pvf, pvn; 2953 pt_entry_t *pte; 2954 pt_entry_t v; 2955 int rtval = 0; 2956 2957 if (m->flags & PG_FICTITIOUS) 2958 return (rtval); 2959 2960 sched_pin(); 2961 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 2962 if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 2963 2964 pvf = pv; 2965 2966 do { 2967 pvn = TAILQ_NEXT(pv, pv_list); 2968 2969 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2970 2971 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list); 2972 2973 PMAP_LOCK(pv->pv_pmap); 2974 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); 2975 2976 if (pte && ((v = pte_load(pte)) & PG_A) != 0) { 2977 atomic_clear_int((u_int *)pte, PG_A); 2978 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2979 2980 rtval++; 2981 if (rtval > 4) { 2982 PMAP_UNLOCK(pv->pv_pmap); 2983 break; 2984 } 2985 } 2986 PMAP_UNLOCK(pv->pv_pmap); 2987 } while ((pv = pvn) != NULL && pv != pvf); 2988 } 2989 sched_unpin(); 2990 2991 return (rtval); 2992 } 2993 2994 /* 2995 * Clear the modify bits on the specified physical page. 2996 */ 2997 void 2998 pmap_clear_modify(vm_page_t m) 2999 { 3000 pmap_clear_ptes(m, PG_M); 3001 } 3002 3003 /* 3004 * pmap_clear_reference: 3005 * 3006 * Clear the reference bit on the specified physical page. 3007 */ 3008 void 3009 pmap_clear_reference(vm_page_t m) 3010 { 3011 pmap_clear_ptes(m, PG_A); 3012 } 3013 3014 /* 3015 * Miscellaneous support routines follow 3016 */ 3017 3018 /* 3019 * Map a set of physical memory pages into the kernel virtual 3020 * address space. Return a pointer to where it is mapped. This 3021 * routine is intended to be used for mapping device memory, 3022 * NOT real memory. 3023 */ 3024 void * 3025 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) 3026 { 3027 vm_offset_t va, tmpva, offset; 3028 3029 offset = pa & PAGE_MASK; 3030 size = roundup(offset + size, PAGE_SIZE); 3031 pa = pa & PG_FRAME; 3032 3033 if (pa < KERNLOAD && pa + size <= KERNLOAD) 3034 va = KERNBASE + pa; 3035 else 3036 va = kmem_alloc_nofault(kernel_map, size); 3037 if (!va) 3038 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 3039 3040 for (tmpva = va; size > 0; ) { 3041 pmap_kenter_attr(tmpva, pa, mode); 3042 size -= PAGE_SIZE; 3043 tmpva += PAGE_SIZE; 3044 pa += PAGE_SIZE; 3045 } 3046 pmap_invalidate_range(kernel_pmap, va, tmpva); 3047 pmap_invalidate_cache(); 3048 return ((void *)(va + offset)); 3049 } 3050 3051 void * 3052 pmap_mapdev(vm_paddr_t pa, vm_size_t size) 3053 { 3054 3055 return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE)); 3056 } 3057 3058 void * 3059 pmap_mapbios(vm_paddr_t pa, vm_size_t size) 3060 { 3061 3062 return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK)); 3063 } 3064 3065 void 3066 pmap_unmapdev(va, size) 3067 vm_offset_t va; 3068 vm_size_t size; 3069 { 3070 vm_offset_t base, offset, tmpva; 3071 3072 if (va >= KERNBASE && va + size <= KERNBASE + KERNLOAD) 3073 return; 3074 base = va & PG_FRAME; 3075 offset = va & PAGE_MASK; 3076 size = roundup(offset + size, PAGE_SIZE); 3077 for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE) 3078 pmap_kremove(tmpva); 3079 pmap_invalidate_range(kernel_pmap, va, tmpva); 3080 kmem_free(kernel_map, base, size); 3081 } 3082 3083 int 3084 pmap_change_attr(va, size, mode) 3085 vm_offset_t va; 3086 vm_size_t size; 3087 int mode; 3088 { 3089 vm_offset_t base, offset, tmpva; 3090 pt_entry_t *pte; 3091 u_int opte, npte; 3092 pd_entry_t *pde; 3093 3094 base = va & PG_FRAME; 3095 offset = va & PAGE_MASK; 3096 size = roundup(offset + size, PAGE_SIZE); 3097 3098 /* Only supported on kernel virtual addresses. */ 3099 if (base <= VM_MAXUSER_ADDRESS) 3100 return (EINVAL); 3101 3102 /* 4MB pages and pages that aren't mapped aren't supported. */ 3103 for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE) { 3104 pde = pmap_pde(kernel_pmap, tmpva); 3105 if (*pde & PG_PS) 3106 return (EINVAL); 3107 if (*pde == 0) 3108 return (EINVAL); 3109 pte = vtopte(va); 3110 if (*pte == 0) 3111 return (EINVAL); 3112 } 3113 3114 /* 3115 * Ok, all the pages exist and are 4k, so run through them updating 3116 * their cache mode. 3117 */ 3118 for (tmpva = base; size > 0; ) { 3119 pte = vtopte(tmpva); 3120 3121 /* 3122 * The cache mode bits are all in the low 32-bits of the 3123 * PTE, so we can just spin on updating the low 32-bits. 3124 */ 3125 do { 3126 opte = *(u_int *)pte; 3127 npte = opte & ~(PG_PTE_PAT | PG_NC_PCD | PG_NC_PWT); 3128 npte |= pmap_cache_bits(mode, 0); 3129 } while (npte != opte && 3130 !atomic_cmpset_int((u_int *)pte, opte, npte)); 3131 tmpva += PAGE_SIZE; 3132 size -= PAGE_SIZE; 3133 } 3134 3135 /* 3136 * Flush CPU caches to make sure any data isn't cached that shouldn't 3137 * be, etc. 3138 */ 3139 pmap_invalidate_range(kernel_pmap, base, tmpva); 3140 pmap_invalidate_cache(); 3141 return (0); 3142 } 3143 3144 /* 3145 * perform the pmap work for mincore 3146 */ 3147 int 3148 pmap_mincore(pmap, addr) 3149 pmap_t pmap; 3150 vm_offset_t addr; 3151 { 3152 pt_entry_t *ptep, pte; 3153 vm_page_t m; 3154 int val = 0; 3155 3156 PMAP_LOCK(pmap); 3157 ptep = pmap_pte(pmap, addr); 3158 pte = (ptep != NULL) ? *ptep : 0; 3159 pmap_pte_release(ptep); 3160 PMAP_UNLOCK(pmap); 3161 3162 if (pte != 0) { 3163 vm_paddr_t pa; 3164 3165 val = MINCORE_INCORE; 3166 if ((pte & PG_MANAGED) == 0) 3167 return val; 3168 3169 pa = pte & PG_FRAME; 3170 3171 m = PHYS_TO_VM_PAGE(pa); 3172 3173 /* 3174 * Modified by us 3175 */ 3176 if (pte & PG_M) 3177 val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER; 3178 else { 3179 /* 3180 * Modified by someone else 3181 */ 3182 vm_page_lock_queues(); 3183 if (m->dirty || pmap_is_modified(m)) 3184 val |= MINCORE_MODIFIED_OTHER; 3185 vm_page_unlock_queues(); 3186 } 3187 /* 3188 * Referenced by us 3189 */ 3190 if (pte & PG_A) 3191 val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER; 3192 else { 3193 /* 3194 * Referenced by someone else 3195 */ 3196 vm_page_lock_queues(); 3197 if ((m->flags & PG_REFERENCED) || 3198 pmap_ts_referenced(m)) { 3199 val |= MINCORE_REFERENCED_OTHER; 3200 vm_page_flag_set(m, PG_REFERENCED); 3201 } 3202 vm_page_unlock_queues(); 3203 } 3204 } 3205 return val; 3206 } 3207 3208 void 3209 pmap_activate(struct thread *td) 3210 { 3211 pmap_t pmap, oldpmap; 3212 u_int32_t cr3; 3213 3214 critical_enter(); 3215 pmap = vmspace_pmap(td->td_proc->p_vmspace); 3216 oldpmap = PCPU_GET(curpmap); 3217 #if defined(SMP) 3218 atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask)); 3219 atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask)); 3220 #else 3221 oldpmap->pm_active &= ~1; 3222 pmap->pm_active |= 1; 3223 #endif 3224 #ifdef PAE 3225 cr3 = vtophys(pmap->pm_pdpt); 3226 #else 3227 cr3 = vtophys(pmap->pm_pdir); 3228 #endif 3229 /* 3230 * pmap_activate is for the current thread on the current cpu 3231 */ 3232 td->td_pcb->pcb_cr3 = cr3; 3233 load_cr3(cr3); 3234 PCPU_SET(curpmap, pmap); 3235 critical_exit(); 3236 } 3237 3238 vm_offset_t 3239 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) 3240 { 3241 3242 if ((obj == NULL) || (size < NBPDR) || (obj->type != OBJT_DEVICE)) { 3243 return addr; 3244 } 3245 3246 addr = (addr + PDRMASK) & ~PDRMASK; 3247 return addr; 3248 } 3249 3250 3251 #if defined(PMAP_DEBUG) 3252 pmap_pid_dump(int pid) 3253 { 3254 pmap_t pmap; 3255 struct proc *p; 3256 int npte = 0; 3257 int index; 3258 3259 sx_slock(&allproc_lock); 3260 FOREACH_PROC_IN_SYSTEM(p) { 3261 if (p->p_pid != pid) 3262 continue; 3263 3264 if (p->p_vmspace) { 3265 int i,j; 3266 index = 0; 3267 pmap = vmspace_pmap(p->p_vmspace); 3268 for (i = 0; i < NPDEPTD; i++) { 3269 pd_entry_t *pde; 3270 pt_entry_t *pte; 3271 vm_offset_t base = i << PDRSHIFT; 3272 3273 pde = &pmap->pm_pdir[i]; 3274 if (pde && pmap_pde_v(pde)) { 3275 for (j = 0; j < NPTEPG; j++) { 3276 vm_offset_t va = base + (j << PAGE_SHIFT); 3277 if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) { 3278 if (index) { 3279 index = 0; 3280 printf("\n"); 3281 } 3282 sx_sunlock(&allproc_lock); 3283 return npte; 3284 } 3285 pte = pmap_pte(pmap, va); 3286 if (pte && pmap_pte_v(pte)) { 3287 pt_entry_t pa; 3288 vm_page_t m; 3289 pa = *pte; 3290 m = PHYS_TO_VM_PAGE(pa); 3291 printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x", 3292 va, pa, m->hold_count, m->wire_count, m->flags); 3293 npte++; 3294 index++; 3295 if (index >= 2) { 3296 index = 0; 3297 printf("\n"); 3298 } else { 3299 printf(" "); 3300 } 3301 } 3302 } 3303 } 3304 } 3305 } 3306 } 3307 sx_sunlock(&allproc_lock); 3308 return npte; 3309 } 3310 #endif 3311 3312 #if defined(DEBUG) 3313 3314 static void pads(pmap_t pm); 3315 void pmap_pvdump(vm_offset_t pa); 3316 3317 /* print address space of pmap*/ 3318 static void 3319 pads(pm) 3320 pmap_t pm; 3321 { 3322 int i, j; 3323 vm_paddr_t va; 3324 pt_entry_t *ptep; 3325 3326 if (pm == kernel_pmap) 3327 return; 3328 for (i = 0; i < NPDEPTD; i++) 3329 if (pm->pm_pdir[i]) 3330 for (j = 0; j < NPTEPG; j++) { 3331 va = (i << PDRSHIFT) + (j << PAGE_SHIFT); 3332 if (pm == kernel_pmap && va < KERNBASE) 3333 continue; 3334 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) 3335 continue; 3336 ptep = pmap_pte(pm, va); 3337 if (pmap_pte_v(ptep)) 3338 printf("%x:%x ", va, *ptep); 3339 }; 3340 3341 } 3342 3343 void 3344 pmap_pvdump(pa) 3345 vm_paddr_t pa; 3346 { 3347 pv_entry_t pv; 3348 vm_page_t m; 3349 3350 printf("pa %x", pa); 3351 m = PHYS_TO_VM_PAGE(pa); 3352 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { 3353 printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va); 3354 pads(pv->pv_pmap); 3355 } 3356 printf(" "); 3357 } 3358 #endif
Cache object: 13c32304debbabe85e0b1fc4ed12927d
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]