The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/pmap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-4-Clause
    3  *
    4  * Copyright (c) 1991 Regents of the University of California.
    5  * All rights reserved.
    6  * Copyright (c) 1994 John S. Dyson
    7  * All rights reserved.
    8  * Copyright (c) 1994 David Greenman
    9  * All rights reserved.
   10  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
   11  * All rights reserved.
   12  *
   13  * This code is derived from software contributed to Berkeley by
   14  * the Systems Programming Group of the University of Utah Computer
   15  * Science Department and William Jolitz of UUNET Technologies Inc.
   16  *
   17  * Redistribution and use in source and binary forms, with or without
   18  * modification, are permitted provided that the following conditions
   19  * are met:
   20  * 1. Redistributions of source code must retain the above copyright
   21  *    notice, this list of conditions and the following disclaimer.
   22  * 2. Redistributions in binary form must reproduce the above copyright
   23  *    notice, this list of conditions and the following disclaimer in the
   24  *    documentation and/or other materials provided with the distribution.
   25  * 3. All advertising materials mentioning features or use of this software
   26  *    must display the following acknowledgement:
   27  *      This product includes software developed by the University of
   28  *      California, Berkeley and its contributors.
   29  * 4. Neither the name of the University nor the names of its contributors
   30  *    may be used to endorse or promote products derived from this software
   31  *    without specific prior written permission.
   32  *
   33  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   34  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   35  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   36  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   37  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   38  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   39  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   40  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   41  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   42  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   43  * SUCH DAMAGE.
   44  *
   45  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   46  */
   47 /*-
   48  * Copyright (c) 2003 Networks Associates Technology, Inc.
   49  * All rights reserved.
   50  * Copyright (c) 2018 The FreeBSD Foundation
   51  * All rights reserved.
   52  *
   53  * This software was developed for the FreeBSD Project by Jake Burkholder,
   54  * Safeport Network Services, and Network Associates Laboratories, the
   55  * Security Research Division of Network Associates, Inc. under
   56  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   57  * CHATS research program.
   58  *
   59  * Portions of this software were developed by
   60  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
   61  * the FreeBSD Foundation.
   62  *
   63  * Redistribution and use in source and binary forms, with or without
   64  * modification, are permitted provided that the following conditions
   65  * are met:
   66  * 1. Redistributions of source code must retain the above copyright
   67  *    notice, this list of conditions and the following disclaimer.
   68  * 2. Redistributions in binary form must reproduce the above copyright
   69  *    notice, this list of conditions and the following disclaimer in the
   70  *    documentation and/or other materials provided with the distribution.
   71  *
   72  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   73  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   74  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   75  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   76  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   77  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   78  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   79  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   80  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   81  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   82  * SUCH DAMAGE.
   83  */
   84 
   85 #include <sys/cdefs.h>
   86 __FBSDID("$FreeBSD$");
   87 
   88 /*
   89  *      Manages physical address maps.
   90  *
   91  *      Since the information managed by this module is
   92  *      also stored by the logical address mapping module,
   93  *      this module may throw away valid virtual-to-physical
   94  *      mappings at almost any time.  However, invalidations
   95  *      of virtual-to-physical mappings must be done as
   96  *      requested.
   97  *
   98  *      In order to cope with hardware architectures which
   99  *      make virtual-to-physical map invalidates expensive,
  100  *      this module may delay invalidate or reduced protection
  101  *      operations until such time as they are actually
  102  *      necessary.  This module is given full information as
  103  *      to which processors are currently using which maps,
  104  *      and to when physical maps must be made correct.
  105  */
  106 
  107 #include "opt_apic.h"
  108 #include "opt_cpu.h"
  109 #include "opt_pmap.h"
  110 #include "opt_smp.h"
  111 #include "opt_vm.h"
  112 
  113 #include <sys/param.h>
  114 #include <sys/systm.h>
  115 #include <sys/kernel.h>
  116 #include <sys/ktr.h>
  117 #include <sys/lock.h>
  118 #include <sys/malloc.h>
  119 #include <sys/mman.h>
  120 #include <sys/msgbuf.h>
  121 #include <sys/mutex.h>
  122 #include <sys/proc.h>
  123 #include <sys/rwlock.h>
  124 #include <sys/sbuf.h>
  125 #include <sys/sf_buf.h>
  126 #include <sys/sx.h>
  127 #include <sys/vmmeter.h>
  128 #include <sys/sched.h>
  129 #include <sys/sysctl.h>
  130 #include <sys/smp.h>
  131 #include <sys/vmem.h>
  132 
  133 #include <vm/vm.h>
  134 #include <vm/vm_param.h>
  135 #include <vm/vm_kern.h>
  136 #include <vm/vm_page.h>
  137 #include <vm/vm_map.h>
  138 #include <vm/vm_object.h>
  139 #include <vm/vm_extern.h>
  140 #include <vm/vm_pageout.h>
  141 #include <vm/vm_pager.h>
  142 #include <vm/vm_phys.h>
  143 #include <vm/vm_radix.h>
  144 #include <vm/vm_reserv.h>
  145 #include <vm/uma.h>
  146 
  147 #ifdef DEV_APIC
  148 #include <sys/bus.h>
  149 #include <machine/intr_machdep.h>
  150 #include <x86/apicvar.h>
  151 #endif
  152 #include <x86/ifunc.h>
  153 #include <machine/bootinfo.h>
  154 #include <machine/cpu.h>
  155 #include <machine/cputypes.h>
  156 #include <machine/md_var.h>
  157 #include <machine/pcb.h>
  158 #include <machine/specialreg.h>
  159 #ifdef SMP
  160 #include <machine/smp.h>
  161 #endif
  162 #include <machine/pmap_base.h>
  163 
  164 #if !defined(DIAGNOSTIC)
  165 #ifdef __GNUC_GNU_INLINE__
  166 #define PMAP_INLINE     __attribute__((__gnu_inline__)) inline
  167 #else
  168 #define PMAP_INLINE     extern inline
  169 #endif
  170 #else
  171 #define PMAP_INLINE
  172 #endif
  173 
  174 #ifdef PV_STATS
  175 #define PV_STAT(x)      do { x ; } while (0)
  176 #else
  177 #define PV_STAT(x)      do { } while (0)
  178 #endif
  179 
  180 #define pa_index(pa)    ((pa) >> PDRSHIFT)
  181 #define pa_to_pvh(pa)   (&pv_table[pa_index(pa)])
  182 
  183 /*
  184  * PTmap is recursive pagemap at top of virtual address space.
  185  * Within PTmap, the page directory can be found (third indirection).
  186  */
  187 #define PTmap   ((pt_entry_t *)(PTDPTDI << PDRSHIFT))
  188 #define PTD     ((pd_entry_t *)((PTDPTDI << PDRSHIFT) + (PTDPTDI * PAGE_SIZE)))
  189 #define PTDpde  ((pd_entry_t *)((PTDPTDI << PDRSHIFT) + (PTDPTDI * PAGE_SIZE) + \
  190     (PTDPTDI * PDESIZE)))
  191 
  192 /*
  193  * Translate a virtual address to the kernel virtual address of its page table
  194  * entry (PTE).  This can be used recursively.  If the address of a PTE as
  195  * previously returned by this macro is itself given as the argument, then the
  196  * address of the page directory entry (PDE) that maps the PTE will be
  197  * returned.
  198  *
  199  * This macro may be used before pmap_bootstrap() is called.
  200  */
  201 #define vtopte(va)      (PTmap + i386_btop(va))
  202 
  203 /*
  204  * Get PDEs and PTEs for user/kernel address space
  205  */
  206 #define pmap_pde(m, v)  (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
  207 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
  208 
  209 #define pmap_pde_v(pte)         ((*(int *)pte & PG_V) != 0)
  210 #define pmap_pte_w(pte)         ((*(int *)pte & PG_W) != 0)
  211 #define pmap_pte_m(pte)         ((*(int *)pte & PG_M) != 0)
  212 #define pmap_pte_u(pte)         ((*(int *)pte & PG_A) != 0)
  213 #define pmap_pte_v(pte)         ((*(int *)pte & PG_V) != 0)
  214 
  215 #define pmap_pte_set_w(pte, v)  ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \
  216     atomic_clear_int((u_int *)(pte), PG_W))
  217 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
  218 
  219 static int pgeflag = 0;         /* PG_G or-in */
  220 static int pseflag = 0;         /* PG_PS or-in */
  221 
  222 static int nkpt = NKPT;
  223 
  224 #ifdef PMAP_PAE_COMP
  225 pt_entry_t pg_nx;
  226 static uma_zone_t pdptzone;
  227 #else
  228 #define pg_nx   0
  229 #endif
  230 
  231 _Static_assert(VM_MAXUSER_ADDRESS == VADDR(TRPTDI, 0), "VM_MAXUSER_ADDRESS");
  232 _Static_assert(VM_MAX_KERNEL_ADDRESS <= VADDR(PTDPTDI, 0),
  233     "VM_MAX_KERNEL_ADDRESS");
  234 _Static_assert(PMAP_MAP_LOW == VADDR(LOWPTDI, 0), "PMAP_MAP_LOW");
  235 _Static_assert(KERNLOAD == (KERNPTDI << PDRSHIFT), "KERNLOAD");
  236 
  237 extern int pat_works;
  238 extern int pg_ps_enabled;
  239 
  240 extern int elf32_nxstack;
  241 
  242 #define PAT_INDEX_SIZE  8
  243 static int pat_index[PAT_INDEX_SIZE];   /* cache mode to PAT index conversion */
  244 
  245 /*
  246  * pmap_mapdev support pre initialization (i.e. console)
  247  */
  248 #define PMAP_PREINIT_MAPPING_COUNT      8
  249 static struct pmap_preinit_mapping {
  250         vm_paddr_t      pa;
  251         vm_offset_t     va;
  252         vm_size_t       sz;
  253         int             mode;
  254 } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT];
  255 static int pmap_initialized;
  256 
  257 static struct rwlock_padalign pvh_global_lock;
  258 
  259 /*
  260  * Data for the pv entry allocation mechanism
  261  */
  262 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
  263 extern int pv_entry_max, pv_entry_count;
  264 static int pv_entry_high_water = 0;
  265 static struct md_page *pv_table;
  266 extern int shpgperproc;
  267 
  268 static struct pv_chunk *pv_chunkbase;   /* KVA block for pv_chunks */
  269 static int pv_maxchunks;                /* How many chunks we have KVA for */
  270 static vm_offset_t pv_vafree;           /* freelist stored in the PTE */
  271 
  272 /*
  273  * All those kernel PT submaps that BSD is so fond of
  274  */
  275 static pt_entry_t *CMAP3;
  276 static pd_entry_t *KPTD;
  277 static caddr_t CADDR3;
  278 
  279 /*
  280  * Crashdump maps.
  281  */
  282 static caddr_t crashdumpmap;
  283 
  284 static pt_entry_t *PMAP1 = NULL, *PMAP2, *PMAP3;
  285 static pt_entry_t *PADDR1 = NULL, *PADDR2, *PADDR3;
  286 #ifdef SMP
  287 static int PMAP1cpu, PMAP3cpu;
  288 extern int PMAP1changedcpu;
  289 #endif
  290 extern int PMAP1changed;
  291 extern int PMAP1unchanged;
  292 static struct mtx PMAP2mutex;
  293 
  294 /*
  295  * Internal flags for pmap_enter()'s helper functions.
  296  */
  297 #define PMAP_ENTER_NORECLAIM    0x1000000       /* Don't reclaim PV entries. */
  298 #define PMAP_ENTER_NOREPLACE    0x2000000       /* Don't replace mappings. */
  299 
  300 static void     free_pv_chunk(struct pv_chunk *pc);
  301 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  302 static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
  303 static void     pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  304 static bool     pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,
  305                     u_int flags);
  306 #if VM_NRESERVLEVEL > 0
  307 static void     pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  308 #endif
  309 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  310 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  311                     vm_offset_t va);
  312 static int      pmap_pvh_wired_mappings(struct md_page *pvh, int count);
  313 
  314 static void     pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);
  315 static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  316 static bool     pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
  317                     vm_prot_t prot);
  318 static int      pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
  319                     u_int flags, vm_page_t m);
  320 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  321     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  322 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted);
  323 static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
  324                     pd_entry_t pde);
  325 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
  326 static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
  327 static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
  328 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
  329 static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
  330 static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
  331 #if VM_NRESERVLEVEL > 0
  332 static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  333 #endif
  334 static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
  335     vm_prot_t prot);
  336 static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
  337 static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
  338     struct spglist *free);
  339 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
  340     struct spglist *free);
  341 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
  342 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free);
  343 static bool     pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
  344                     struct spglist *free);
  345 static void pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va);
  346 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  347 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  348     vm_page_t m);
  349 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  350     pd_entry_t newpde);
  351 static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
  352 
  353 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);
  354 
  355 static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags);
  356 static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free);
  357 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
  358 static void pmap_pte_release(pt_entry_t *pte);
  359 static int pmap_unuse_pt(pmap_t, vm_offset_t, struct spglist *);
  360 #ifdef PMAP_PAE_COMP
  361 static void *pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain,
  362     uint8_t *flags, int wait);
  363 #endif
  364 static void pmap_init_trm(void);
  365 static void pmap_invalidate_all_int(pmap_t pmap);
  366 
  367 static __inline void pagezero(void *page);
  368 
  369 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  370 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  371 
  372 extern char _end[];
  373 extern u_long physfree; /* phys addr of next free page */
  374 extern u_long vm86phystk;/* PA of vm86/bios stack */
  375 extern u_long vm86paddr;/* address of vm86 region */
  376 extern int vm86pa;      /* phys addr of vm86 region */
  377 extern u_long KERNend;  /* phys addr end of kernel (just after bss) */
  378 #ifdef PMAP_PAE_COMP
  379 pd_entry_t *IdlePTD_pae;        /* phys addr of kernel PTD */
  380 pdpt_entry_t *IdlePDPT; /* phys addr of kernel PDPT */
  381 pt_entry_t *KPTmap_pae; /* address of kernel page tables */
  382 #define IdlePTD IdlePTD_pae
  383 #define KPTmap  KPTmap_pae
  384 #else
  385 pd_entry_t *IdlePTD_nopae;
  386 pt_entry_t *KPTmap_nopae;
  387 #define IdlePTD IdlePTD_nopae
  388 #define KPTmap  KPTmap_nopae
  389 #endif
  390 extern u_long KPTphys;  /* phys addr of kernel page tables */
  391 extern u_long tramp_idleptd;
  392 
  393 static u_long
  394 allocpages(u_int cnt, u_long *physfree)
  395 {
  396         u_long res;
  397 
  398         res = *physfree;
  399         *physfree += PAGE_SIZE * cnt;
  400         bzero((void *)res, PAGE_SIZE * cnt);
  401         return (res);
  402 }
  403 
  404 static void
  405 pmap_cold_map(u_long pa, u_long va, u_long cnt)
  406 {
  407         pt_entry_t *pt;
  408 
  409         for (pt = (pt_entry_t *)KPTphys + atop(va); cnt > 0;
  410             cnt--, pt++, va += PAGE_SIZE, pa += PAGE_SIZE)
  411                 *pt = pa | PG_V | PG_RW | PG_A | PG_M;
  412 }
  413 
  414 static void
  415 pmap_cold_mapident(u_long pa, u_long cnt)
  416 {
  417 
  418         pmap_cold_map(pa, pa, cnt);
  419 }
  420 
  421 _Static_assert(LOWPTDI * 2 * NBPDR == KERNBASE,
  422     "Broken double-map of zero PTD");
  423 
  424 static void
  425 __CONCAT(PMTYPE, remap_lower)(bool enable)
  426 {
  427         int i;
  428 
  429         for (i = 0; i < LOWPTDI; i++)
  430                 IdlePTD[i] = enable ? IdlePTD[LOWPTDI + i] : 0;
  431         load_cr3(rcr3());               /* invalidate TLB */
  432 }
  433 
  434 /*
  435  * Called from locore.s before paging is enabled.  Sets up the first
  436  * kernel page table.  Since kernel is mapped with PA == VA, this code
  437  * does not require relocations.
  438  */
  439 void
  440 __CONCAT(PMTYPE, cold)(void)
  441 {
  442         pt_entry_t *pt;
  443         u_long a;
  444         u_int cr3, ncr4;
  445 
  446         physfree = (u_long)&_end;
  447         if (bootinfo.bi_esymtab != 0)
  448                 physfree = bootinfo.bi_esymtab;
  449         if (bootinfo.bi_kernend != 0)
  450                 physfree = bootinfo.bi_kernend;
  451         physfree = roundup2(physfree, NBPDR);
  452         KERNend = physfree;
  453 
  454         /* Allocate Kernel Page Tables */
  455         KPTphys = allocpages(NKPT, &physfree);
  456         KPTmap = (pt_entry_t *)KPTphys;
  457 
  458         /* Allocate Page Table Directory */
  459 #ifdef PMAP_PAE_COMP
  460         /* XXX only need 32 bytes (easier for now) */
  461         IdlePDPT = (pdpt_entry_t *)allocpages(1, &physfree);
  462 #endif
  463         IdlePTD = (pd_entry_t *)allocpages(NPGPTD, &physfree);
  464 
  465         /*
  466          * Allocate KSTACK.  Leave a guard page between IdlePTD and
  467          * proc0kstack, to control stack overflow for thread0 and
  468          * prevent corruption of the page table.  We leak the guard
  469          * physical memory due to 1:1 mappings.
  470          */
  471         allocpages(1, &physfree);
  472         proc0kstack = allocpages(TD0_KSTACK_PAGES, &physfree);
  473 
  474         /* vm86/bios stack */
  475         vm86phystk = allocpages(1, &physfree);
  476 
  477         /* pgtable + ext + IOPAGES */
  478         vm86paddr = vm86pa = allocpages(3, &physfree);
  479 
  480         /* Install page tables into PTD.  Page table page 1 is wasted. */
  481         for (a = 0; a < NKPT; a++)
  482                 IdlePTD[a] = (KPTphys + ptoa(a)) | PG_V | PG_RW | PG_A | PG_M;
  483 
  484 #ifdef PMAP_PAE_COMP
  485         /* PAE install PTD pointers into PDPT */
  486         for (a = 0; a < NPGPTD; a++)
  487                 IdlePDPT[a] = ((u_int)IdlePTD + ptoa(a)) | PG_V;
  488 #endif
  489 
  490         /*
  491          * Install recursive mapping for kernel page tables into
  492          * itself.
  493          */
  494         for (a = 0; a < NPGPTD; a++)
  495                 IdlePTD[PTDPTDI + a] = ((u_int)IdlePTD + ptoa(a)) | PG_V |
  496                     PG_RW;
  497 
  498         /*
  499          * Initialize page table pages mapping physical address zero
  500          * through the (physical) end of the kernel.  Many of these
  501          * pages must be reserved, and we reserve them all and map
  502          * them linearly for convenience.  We do this even if we've
  503          * enabled PSE above; we'll just switch the corresponding
  504          * kernel PDEs before we turn on paging.
  505          *
  506          * This and all other page table entries allow read and write
  507          * access for various reasons.  Kernel mappings never have any
  508          * access restrictions.
  509          */
  510         pmap_cold_mapident(0, atop(NBPDR) * LOWPTDI);
  511         pmap_cold_map(0, NBPDR * LOWPTDI, atop(NBPDR) * LOWPTDI);
  512         pmap_cold_mapident(KERNBASE, atop(KERNend - KERNBASE));
  513 
  514         /* Map page table directory */
  515 #ifdef PMAP_PAE_COMP
  516         pmap_cold_mapident((u_long)IdlePDPT, 1);
  517 #endif
  518         pmap_cold_mapident((u_long)IdlePTD, NPGPTD);
  519 
  520         /* Map early KPTmap.  It is really pmap_cold_mapident. */
  521         pmap_cold_map(KPTphys, (u_long)KPTmap, NKPT);
  522 
  523         /* Map proc0kstack */
  524         pmap_cold_mapident(proc0kstack, TD0_KSTACK_PAGES);
  525         /* ISA hole already mapped */
  526 
  527         pmap_cold_mapident(vm86phystk, 1);
  528         pmap_cold_mapident(vm86pa, 3);
  529 
  530         /* Map page 0 into the vm86 page table */
  531         *(pt_entry_t *)vm86pa = 0 | PG_RW | PG_U | PG_A | PG_M | PG_V;
  532 
  533         /* ...likewise for the ISA hole for vm86 */
  534         for (pt = (pt_entry_t *)vm86pa + atop(ISA_HOLE_START), a = 0;
  535             a < atop(ISA_HOLE_LENGTH); a++, pt++)
  536                 *pt = (ISA_HOLE_START + ptoa(a)) | PG_RW | PG_U | PG_A |
  537                     PG_M | PG_V;
  538 
  539         /* Enable PSE, PGE, VME, and PAE if configured. */
  540         ncr4 = 0;
  541         if ((cpu_feature & CPUID_PSE) != 0) {
  542                 ncr4 |= CR4_PSE;
  543                 pseflag = PG_PS;
  544                 /*
  545                  * Superpage mapping of the kernel text.  Existing 4k
  546                  * page table pages are wasted.
  547                  */
  548                 for (a = KERNBASE; a < KERNend; a += NBPDR)
  549                         IdlePTD[a >> PDRSHIFT] = a | PG_PS | PG_A | PG_M |
  550                             PG_RW | PG_V;
  551         }
  552         if ((cpu_feature & CPUID_PGE) != 0) {
  553                 ncr4 |= CR4_PGE;
  554                 pgeflag = PG_G;
  555         }
  556         ncr4 |= (cpu_feature & CPUID_VME) != 0 ? CR4_VME : 0;
  557 #ifdef PMAP_PAE_COMP
  558         ncr4 |= CR4_PAE;
  559 #endif
  560         if (ncr4 != 0)
  561                 load_cr4(rcr4() | ncr4);
  562 
  563         /* Now enable paging */
  564 #ifdef PMAP_PAE_COMP
  565         cr3 = (u_int)IdlePDPT;
  566         if ((cpu_feature & CPUID_PAT) == 0)
  567                 wbinvd();
  568 #else
  569         cr3 = (u_int)IdlePTD;
  570 #endif
  571         tramp_idleptd = cr3;
  572         load_cr3(cr3);
  573         load_cr0(rcr0() | CR0_PG);
  574 
  575         /*
  576          * Now running relocated at KERNBASE where the system is
  577          * linked to run.
  578          */
  579 
  580         /*
  581          * Remove the lowest part of the double mapping of low memory
  582          * to get some null pointer checks.
  583          */
  584         __CONCAT(PMTYPE, remap_lower)(false);
  585 
  586         kernel_vm_end = /* 0 + */ NKPT * NBPDR;
  587 #ifdef PMAP_PAE_COMP
  588         i386_pmap_VM_NFREEORDER = VM_NFREEORDER_PAE;
  589         i386_pmap_VM_LEVEL_0_ORDER = VM_LEVEL_0_ORDER_PAE;
  590         i386_pmap_PDRSHIFT = PDRSHIFT_PAE;
  591 #else
  592         i386_pmap_VM_NFREEORDER = VM_NFREEORDER_NOPAE;
  593         i386_pmap_VM_LEVEL_0_ORDER = VM_LEVEL_0_ORDER_NOPAE;
  594         i386_pmap_PDRSHIFT = PDRSHIFT_NOPAE;
  595 #endif
  596 }
  597 
  598 static void
  599 __CONCAT(PMTYPE, set_nx)(void)
  600 {
  601 
  602 #ifdef PMAP_PAE_COMP
  603         if ((amd_feature & AMDID_NX) == 0)
  604                 return;
  605         pg_nx = PG_NX;
  606         elf32_nxstack = 1;
  607         /* EFER.EFER_NXE is set in initializecpu(). */
  608 #endif
  609 }
  610 
  611 /*
  612  *      Bootstrap the system enough to run with virtual memory.
  613  *
  614  *      On the i386 this is called after pmap_cold() created initial
  615  *      kernel page table and enabled paging, and just syncs the pmap
  616  *      module with what has already been done.
  617  */
  618 static void
  619 __CONCAT(PMTYPE, bootstrap)(vm_paddr_t firstaddr)
  620 {
  621         vm_offset_t va;
  622         pt_entry_t *pte, *unused;
  623         struct pcpu *pc;
  624         u_long res;
  625         int i;
  626 
  627         res = atop(firstaddr - (vm_paddr_t)KERNLOAD);
  628 
  629         /*
  630          * Add a physical memory segment (vm_phys_seg) corresponding to the
  631          * preallocated kernel page table pages so that vm_page structures
  632          * representing these pages will be created.  The vm_page structures
  633          * are required for promotion of the corresponding kernel virtual
  634          * addresses to superpage mappings.
  635          */
  636         vm_phys_early_add_seg(KPTphys, KPTphys + ptoa(nkpt));
  637 
  638         /*
  639          * Initialize the first available kernel virtual address.
  640          * However, using "firstaddr" may waste a few pages of the
  641          * kernel virtual address space, because pmap_cold() may not
  642          * have mapped every physical page that it allocated.
  643          * Preferably, pmap_cold() would provide a first unused
  644          * virtual address in addition to "firstaddr".
  645          */
  646         virtual_avail = (vm_offset_t)firstaddr;
  647         virtual_end = VM_MAX_KERNEL_ADDRESS;
  648 
  649         /*
  650          * Initialize the kernel pmap (which is statically allocated).
  651          * Count bootstrap data as being resident in case any of this data is
  652          * later unmapped (using pmap_remove()) and freed.
  653          */
  654         PMAP_LOCK_INIT(kernel_pmap);
  655         kernel_pmap->pm_pdir = IdlePTD;
  656 #ifdef PMAP_PAE_COMP
  657         kernel_pmap->pm_pdpt = IdlePDPT;
  658 #endif
  659         CPU_FILL(&kernel_pmap->pm_active);      /* don't allow deactivation */
  660         kernel_pmap->pm_stats.resident_count = res;
  661         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  662 
  663         /*
  664          * Initialize the global pv list lock.
  665          */
  666         rw_init(&pvh_global_lock, "pmap pv global");
  667 
  668         /*
  669          * Reserve some special page table entries/VA space for temporary
  670          * mapping of pages.
  671          */
  672 #define SYSMAP(c, p, v, n)      \
  673         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  674 
  675         va = virtual_avail;
  676         pte = vtopte(va);
  677 
  678         /*
  679          * Initialize temporary map objects on the current CPU for use
  680          * during early boot.
  681          * CMAP1/CMAP2 are used for zeroing and copying pages.
  682          * CMAP3 is used for the boot-time memory test.
  683          */
  684         pc = get_pcpu();
  685         mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
  686         SYSMAP(caddr_t, pc->pc_cmap_pte1, pc->pc_cmap_addr1, 1)
  687         SYSMAP(caddr_t, pc->pc_cmap_pte2, pc->pc_cmap_addr2, 1)
  688         SYSMAP(vm_offset_t, pte, pc->pc_qmap_addr, 1)
  689 
  690         SYSMAP(caddr_t, CMAP3, CADDR3, 1);
  691 
  692         /*
  693          * Crashdump maps.
  694          */
  695         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  696 
  697         /*
  698          * ptvmmap is used for reading arbitrary physical pages via /dev/mem.
  699          */
  700         SYSMAP(caddr_t, unused, ptvmmap, 1)
  701 
  702         /*
  703          * msgbufp is used to map the system message buffer.
  704          */
  705         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(msgbufsize)))
  706 
  707         /*
  708          * KPTmap is used by pmap_kextract().
  709          *
  710          * KPTmap is first initialized by pmap_cold().  However, that initial
  711          * KPTmap can only support NKPT page table pages.  Here, a larger
  712          * KPTmap is created that can support KVA_PAGES page table pages.
  713          */
  714         SYSMAP(pt_entry_t *, KPTD, KPTmap, KVA_PAGES)
  715 
  716         for (i = 0; i < NKPT; i++)
  717                 KPTD[i] = (KPTphys + ptoa(i)) | PG_RW | PG_V;
  718 
  719         /*
  720          * PADDR1 and PADDR2 are used by pmap_pte_quick() and pmap_pte(),
  721          * respectively.
  722          */
  723         SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1)
  724         SYSMAP(pt_entry_t *, PMAP2, PADDR2, 1)
  725         SYSMAP(pt_entry_t *, PMAP3, PADDR3, 1)
  726 
  727         mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
  728 
  729         virtual_avail = va;
  730 
  731         /*
  732          * Initialize the PAT MSR if present.
  733          * pmap_init_pat() clears and sets CR4_PGE, which, as a
  734          * side-effect, invalidates stale PG_G TLB entries that might
  735          * have been created in our pre-boot environment.  We assume
  736          * that PAT support implies PGE and in reverse, PGE presence
  737          * comes with PAT.  Both features were added for Pentium Pro.
  738          */
  739         pmap_init_pat();
  740 }
  741 
  742 static void
  743 pmap_init_reserved_pages(void)
  744 {
  745         struct pcpu *pc;
  746         vm_offset_t pages;
  747         int i;
  748 
  749 #ifdef PMAP_PAE_COMP
  750         if (!pae_mode)
  751                 return;
  752 #else
  753         if (pae_mode)
  754                 return;
  755 #endif
  756         CPU_FOREACH(i) {
  757                 pc = pcpu_find(i);
  758                 mtx_init(&pc->pc_copyout_mlock, "cpmlk", NULL, MTX_DEF |
  759                     MTX_NEW);
  760                 pc->pc_copyout_maddr = kva_alloc(ptoa(2));
  761                 if (pc->pc_copyout_maddr == 0)
  762                         panic("unable to allocate non-sleepable copyout KVA");
  763                 sx_init(&pc->pc_copyout_slock, "cpslk");
  764                 pc->pc_copyout_saddr = kva_alloc(ptoa(2));
  765                 if (pc->pc_copyout_saddr == 0)
  766                         panic("unable to allocate sleepable copyout KVA");
  767                 pc->pc_pmap_eh_va = kva_alloc(ptoa(1));
  768                 if (pc->pc_pmap_eh_va == 0)
  769                         panic("unable to allocate pmap_extract_and_hold KVA");
  770                 pc->pc_pmap_eh_ptep = (char *)vtopte(pc->pc_pmap_eh_va);
  771 
  772                 /*
  773                  * Skip if the mappings have already been initialized,
  774                  * i.e. this is the BSP.
  775                  */
  776                 if (pc->pc_cmap_addr1 != 0)
  777                         continue;
  778 
  779                 mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
  780                 pages = kva_alloc(PAGE_SIZE * 3);
  781                 if (pages == 0)
  782                         panic("unable to allocate CMAP KVA");
  783                 pc->pc_cmap_pte1 = vtopte(pages);
  784                 pc->pc_cmap_pte2 = vtopte(pages + PAGE_SIZE);
  785                 pc->pc_cmap_addr1 = (caddr_t)pages;
  786                 pc->pc_cmap_addr2 = (caddr_t)(pages + PAGE_SIZE);
  787                 pc->pc_qmap_addr = pages + ptoa(2);
  788         }
  789 }
  790 
  791 SYSINIT(rpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_reserved_pages, NULL);
  792 
  793 /*
  794  * Setup the PAT MSR.
  795  */
  796 static void
  797 __CONCAT(PMTYPE, init_pat)(void)
  798 {
  799         int pat_table[PAT_INDEX_SIZE];
  800         uint64_t pat_msr;
  801         u_long cr0, cr4;
  802         int i;
  803 
  804         /* Set default PAT index table. */
  805         for (i = 0; i < PAT_INDEX_SIZE; i++)
  806                 pat_table[i] = -1;
  807         pat_table[PAT_WRITE_BACK] = 0;
  808         pat_table[PAT_WRITE_THROUGH] = 1;
  809         pat_table[PAT_UNCACHEABLE] = 3;
  810         pat_table[PAT_WRITE_COMBINING] = 3;
  811         pat_table[PAT_WRITE_PROTECTED] = 3;
  812         pat_table[PAT_UNCACHED] = 3;
  813 
  814         /*
  815          * Bail if this CPU doesn't implement PAT.
  816          * We assume that PAT support implies PGE.
  817          */
  818         if ((cpu_feature & CPUID_PAT) == 0) {
  819                 for (i = 0; i < PAT_INDEX_SIZE; i++)
  820                         pat_index[i] = pat_table[i];
  821                 pat_works = 0;
  822                 return;
  823         }
  824 
  825         /*
  826          * Due to some Intel errata, we can only safely use the lower 4
  827          * PAT entries.
  828          *
  829          *   Intel Pentium III Processor Specification Update
  830          * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
  831          * or Mode C Paging)
  832          *
  833          *   Intel Pentium IV  Processor Specification Update
  834          * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
  835          */
  836         if (cpu_vendor_id == CPU_VENDOR_INTEL &&
  837             !(CPUID_TO_FAMILY(cpu_id) == 6 && CPUID_TO_MODEL(cpu_id) >= 0xe))
  838                 pat_works = 0;
  839 
  840         /* Initialize default PAT entries. */
  841         pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |
  842             PAT_VALUE(1, PAT_WRITE_THROUGH) |
  843             PAT_VALUE(2, PAT_UNCACHED) |
  844             PAT_VALUE(3, PAT_UNCACHEABLE) |
  845             PAT_VALUE(4, PAT_WRITE_BACK) |
  846             PAT_VALUE(5, PAT_WRITE_THROUGH) |
  847             PAT_VALUE(6, PAT_UNCACHED) |
  848             PAT_VALUE(7, PAT_UNCACHEABLE);
  849 
  850         if (pat_works) {
  851                 /*
  852                  * Leave the indices 0-3 at the default of WB, WT, UC-, and UC.
  853                  * Program 5 and 6 as WP and WC.
  854                  * Leave 4 and 7 as WB and UC.
  855                  */
  856                 pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6));
  857                 pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) |
  858                     PAT_VALUE(6, PAT_WRITE_COMBINING);
  859                 pat_table[PAT_UNCACHED] = 2;
  860                 pat_table[PAT_WRITE_PROTECTED] = 5;
  861                 pat_table[PAT_WRITE_COMBINING] = 6;
  862         } else {
  863                 /*
  864                  * Just replace PAT Index 2 with WC instead of UC-.
  865                  */
  866                 pat_msr &= ~PAT_MASK(2);
  867                 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  868                 pat_table[PAT_WRITE_COMBINING] = 2;
  869         }
  870 
  871         /* Disable PGE. */
  872         cr4 = rcr4();
  873         load_cr4(cr4 & ~CR4_PGE);
  874 
  875         /* Disable caches (CD = 1, NW = 0). */
  876         cr0 = rcr0();
  877         load_cr0((cr0 & ~CR0_NW) | CR0_CD);
  878 
  879         /* Flushes caches and TLBs. */
  880         wbinvd();
  881         invltlb();
  882 
  883         /* Update PAT and index table. */
  884         wrmsr(MSR_PAT, pat_msr);
  885         for (i = 0; i < PAT_INDEX_SIZE; i++)
  886                 pat_index[i] = pat_table[i];
  887 
  888         /* Flush caches and TLBs again. */
  889         wbinvd();
  890         invltlb();
  891 
  892         /* Restore caches and PGE. */
  893         load_cr0(cr0);
  894         load_cr4(cr4);
  895 }
  896 
  897 #ifdef PMAP_PAE_COMP
  898 static void *
  899 pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
  900     int wait)
  901 {
  902 
  903         /* Inform UMA that this allocator uses kernel_map/object. */
  904         *flags = UMA_SLAB_KERNEL;
  905         return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
  906             bytes, wait, 0x0ULL, 0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
  907 }
  908 #endif
  909 
  910 /*
  911  * Abuse the pte nodes for unmapped kva to thread a kva freelist through.
  912  * Requirements:
  913  *  - Must deal with pages in order to ensure that none of the PG_* bits
  914  *    are ever set, PG_V in particular.
  915  *  - Assumes we can write to ptes without pte_store() atomic ops, even
  916  *    on PAE systems.  This should be ok.
  917  *  - Assumes nothing will ever test these addresses for 0 to indicate
  918  *    no mapping instead of correctly checking PG_V.
  919  *  - Assumes a vm_offset_t will fit in a pte (true for i386).
  920  * Because PG_V is never set, there can be no mappings to invalidate.
  921  */
  922 static vm_offset_t
  923 pmap_ptelist_alloc(vm_offset_t *head)
  924 {
  925         pt_entry_t *pte;
  926         vm_offset_t va;
  927 
  928         va = *head;
  929         if (va == 0)
  930                 panic("pmap_ptelist_alloc: exhausted ptelist KVA");
  931         pte = vtopte(va);
  932         *head = *pte;
  933         if (*head & PG_V)
  934                 panic("pmap_ptelist_alloc: va with PG_V set!");
  935         *pte = 0;
  936         return (va);
  937 }
  938 
  939 static void
  940 pmap_ptelist_free(vm_offset_t *head, vm_offset_t va)
  941 {
  942         pt_entry_t *pte;
  943 
  944         if (va & PG_V)
  945                 panic("pmap_ptelist_free: freeing va with PG_V set!");
  946         pte = vtopte(va);
  947         *pte = *head;           /* virtual! PG_V is 0 though */
  948         *head = va;
  949 }
  950 
  951 static void
  952 pmap_ptelist_init(vm_offset_t *head, void *base, int npages)
  953 {
  954         int i;
  955         vm_offset_t va;
  956 
  957         *head = 0;
  958         for (i = npages - 1; i >= 0; i--) {
  959                 va = (vm_offset_t)base + i * PAGE_SIZE;
  960                 pmap_ptelist_free(head, va);
  961         }
  962 }
  963 
  964 /*
  965  *      Initialize the pmap module.
  966  *      Called by vm_init, to initialize any structures that the pmap
  967  *      system needs to map virtual memory.
  968  */
  969 static void
  970 __CONCAT(PMTYPE, init)(void)
  971 {
  972         struct pmap_preinit_mapping *ppim;
  973         vm_page_t mpte;
  974         vm_size_t s;
  975         int i, pv_npg;
  976 
  977         /*
  978          * Initialize the vm page array entries for the kernel pmap's
  979          * page table pages.
  980          */ 
  981         PMAP_LOCK(kernel_pmap);
  982         for (i = 0; i < NKPT; i++) {
  983                 mpte = PHYS_TO_VM_PAGE(KPTphys + ptoa(i));
  984                 KASSERT(mpte >= vm_page_array &&
  985                     mpte < &vm_page_array[vm_page_array_size],
  986                     ("pmap_init: page table page is out of range"));
  987                 mpte->pindex = i + KPTDI;
  988                 mpte->phys_addr = KPTphys + ptoa(i);
  989                 mpte->ref_count = 1;
  990 
  991                 /*
  992                  * Collect the page table pages that were replaced by a 2/4MB
  993                  * page.  They are filled with equivalent 4KB page mappings.
  994                  */
  995                 if (pseflag != 0 &&
  996                     KERNBASE <= i << PDRSHIFT && i << PDRSHIFT < KERNend &&
  997                     pmap_insert_pt_page(kernel_pmap, mpte, true))
  998                         panic("pmap_init: pmap_insert_pt_page failed");
  999         }
 1000         PMAP_UNLOCK(kernel_pmap);
 1001         vm_wire_add(NKPT);
 1002 
 1003         /*
 1004          * Initialize the address space (zone) for the pv entries.  Set a
 1005          * high water mark so that the system can recover from excessive
 1006          * numbers of pv entries.
 1007          */
 1008         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
 1009         pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count;
 1010         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
 1011         pv_entry_max = roundup(pv_entry_max, _NPCPV);
 1012         pv_entry_high_water = 9 * (pv_entry_max / 10);
 1013 
 1014         /*
 1015          * If the kernel is running on a virtual machine, then it must assume
 1016          * that MCA is enabled by the hypervisor.  Moreover, the kernel must
 1017          * be prepared for the hypervisor changing the vendor and family that
 1018          * are reported by CPUID.  Consequently, the workaround for AMD Family
 1019          * 10h Erratum 383 is enabled if the processor's feature set does not
 1020          * include at least one feature that is only supported by older Intel
 1021          * or newer AMD processors.
 1022          */
 1023         if (vm_guest != VM_GUEST_NO && (cpu_feature & CPUID_SS) == 0 &&
 1024             (cpu_feature2 & (CPUID2_SSSE3 | CPUID2_SSE41 | CPUID2_AESNI |
 1025             CPUID2_AVX | CPUID2_XSAVE)) == 0 && (amd_feature2 & (AMDID2_XOP |
 1026             AMDID2_FMA4)) == 0)
 1027                 workaround_erratum383 = 1;
 1028 
 1029         /*
 1030          * Are large page mappings supported and enabled?
 1031          */
 1032         TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
 1033         if (pseflag == 0)
 1034                 pg_ps_enabled = 0;
 1035         else if (pg_ps_enabled) {
 1036                 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
 1037                     ("pmap_init: can't assign to pagesizes[1]"));
 1038                 pagesizes[1] = NBPDR;
 1039         }
 1040 
 1041         /*
 1042          * Calculate the size of the pv head table for superpages.
 1043          * Handle the possibility that "vm_phys_segs[...].end" is zero.
 1044          */
 1045         pv_npg = trunc_4mpage(vm_phys_segs[vm_phys_nsegs - 1].end -
 1046             PAGE_SIZE) / NBPDR + 1;
 1047 
 1048         /*
 1049          * Allocate memory for the pv head table for superpages.
 1050          */
 1051         s = (vm_size_t)(pv_npg * sizeof(struct md_page));
 1052         s = round_page(s);
 1053         pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO);
 1054         for (i = 0; i < pv_npg; i++)
 1055                 TAILQ_INIT(&pv_table[i].pv_list);
 1056 
 1057         pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc);
 1058         pv_chunkbase = (struct pv_chunk *)kva_alloc(PAGE_SIZE * pv_maxchunks);
 1059         if (pv_chunkbase == NULL)
 1060                 panic("pmap_init: not enough kvm for pv chunks");
 1061         pmap_ptelist_init(&pv_vafree, pv_chunkbase, pv_maxchunks);
 1062 #ifdef PMAP_PAE_COMP
 1063         pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL,
 1064             NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1,
 1065             UMA_ZONE_CONTIG | UMA_ZONE_VM | UMA_ZONE_NOFREE);
 1066         uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf);
 1067 #endif
 1068 
 1069         pmap_initialized = 1;
 1070         pmap_init_trm();
 1071 
 1072         if (!bootverbose)
 1073                 return;
 1074         for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 1075                 ppim = pmap_preinit_mapping + i;
 1076                 if (ppim->va == 0)
 1077                         continue;
 1078                 printf("PPIM %u: PA=%#jx, VA=%#x, size=%#x, mode=%#x\n", i,
 1079                     (uintmax_t)ppim->pa, ppim->va, ppim->sz, ppim->mode);
 1080         }
 1081 
 1082 }
 1083 
 1084 extern u_long pmap_pde_demotions;
 1085 extern u_long pmap_pde_mappings;
 1086 extern u_long pmap_pde_p_failures;
 1087 extern u_long pmap_pde_promotions;
 1088 
 1089 /***************************************************
 1090  * Low level helper routines.....
 1091  ***************************************************/
 1092 
 1093 static boolean_t
 1094 __CONCAT(PMTYPE, is_valid_memattr)(pmap_t pmap __unused, vm_memattr_t mode)
 1095 {
 1096 
 1097         return (mode >= 0 && mode < PAT_INDEX_SIZE &&
 1098             pat_index[(int)mode] >= 0);
 1099 }
 1100 
 1101 /*
 1102  * Determine the appropriate bits to set in a PTE or PDE for a specified
 1103  * caching mode.
 1104  */
 1105 static int
 1106 __CONCAT(PMTYPE, cache_bits)(pmap_t pmap, int mode, boolean_t is_pde)
 1107 {
 1108         int cache_bits, pat_flag, pat_idx;
 1109 
 1110         if (!pmap_is_valid_memattr(pmap, mode))
 1111                 panic("Unknown caching mode %d\n", mode);
 1112 
 1113         /* The PAT bit is different for PTE's and PDE's. */
 1114         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
 1115 
 1116         /* Map the caching mode to a PAT index. */
 1117         pat_idx = pat_index[mode];
 1118 
 1119         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
 1120         cache_bits = 0;
 1121         if (pat_idx & 0x4)
 1122                 cache_bits |= pat_flag;
 1123         if (pat_idx & 0x2)
 1124                 cache_bits |= PG_NC_PCD;
 1125         if (pat_idx & 0x1)
 1126                 cache_bits |= PG_NC_PWT;
 1127         return (cache_bits);
 1128 }
 1129 
 1130 static int
 1131 pmap_pat_index(pmap_t pmap, pt_entry_t pte, bool is_pde)
 1132 {
 1133         int pat_flag, pat_idx;
 1134 
 1135         if ((cpu_feature & CPUID_PAT) == 0)
 1136                 return (0);
 1137 
 1138         pat_idx = 0;
 1139         /* The PAT bit is different for PTE's and PDE's. */
 1140         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
 1141 
 1142         if ((pte & pat_flag) != 0)
 1143                 pat_idx |= 0x4;
 1144         if ((pte & PG_NC_PCD) != 0)
 1145                 pat_idx |= 0x2;
 1146         if ((pte & PG_NC_PWT) != 0)
 1147                 pat_idx |= 0x1;
 1148 
 1149         /* See pmap_init_pat(). */
 1150         if (pat_works) {
 1151                 if (pat_idx == 4)
 1152                         pat_idx = 0;
 1153                 if (pat_idx == 7)
 1154                         pat_idx = 3;
 1155         } else {
 1156                 /* XXXKIB */
 1157         }
 1158 
 1159         return (pat_idx);
 1160 }
 1161 
 1162 static bool
 1163 __CONCAT(PMTYPE, ps_enabled)(pmap_t pmap __unused)
 1164 {
 1165 
 1166         return (pg_ps_enabled);
 1167 }
 1168 
 1169 /*
 1170  * The caller is responsible for maintaining TLB consistency.
 1171  */
 1172 static void
 1173 pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde)
 1174 {
 1175         pd_entry_t *pde;
 1176 
 1177         pde = pmap_pde(kernel_pmap, va);
 1178         pde_store(pde, newpde);
 1179 }
 1180 
 1181 /*
 1182  * After changing the page size for the specified virtual address in the page
 1183  * table, flush the corresponding entries from the processor's TLB.  Only the
 1184  * calling processor's TLB is affected.
 1185  *
 1186  * The calling thread must be pinned to a processor.
 1187  */
 1188 static void
 1189 pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
 1190 {
 1191 
 1192         if ((newpde & PG_PS) == 0)
 1193                 /* Demotion: flush a specific 2MB page mapping. */
 1194                 invlpg(va);
 1195         else /* if ((newpde & PG_G) == 0) */
 1196                 /*
 1197                  * Promotion: flush every 4KB page mapping from the TLB
 1198                  * because there are too many to flush individually.
 1199                  */
 1200                 invltlb();
 1201 }
 1202 
 1203 #ifdef SMP
 1204 
 1205 static void
 1206 pmap_curcpu_cb_dummy(pmap_t pmap __unused, vm_offset_t addr1 __unused,
 1207     vm_offset_t addr2 __unused)
 1208 {
 1209 }
 1210 
 1211 /*
 1212  * For SMP, these functions have to use the IPI mechanism for coherence.
 1213  *
 1214  * N.B.: Before calling any of the following TLB invalidation functions,
 1215  * the calling processor must ensure that all stores updating a non-
 1216  * kernel page table are globally performed.  Otherwise, another
 1217  * processor could cache an old, pre-update entry without being
 1218  * invalidated.  This can happen one of two ways: (1) The pmap becomes
 1219  * active on another processor after its pm_active field is checked by
 1220  * one of the following functions but before a store updating the page
 1221  * table is globally performed. (2) The pmap becomes active on another
 1222  * processor before its pm_active field is checked but due to
 1223  * speculative loads one of the following functions stills reads the
 1224  * pmap as inactive on the other processor.
 1225  * 
 1226  * The kernel page table is exempt because its pm_active field is
 1227  * immutable.  The kernel page table is always active on every
 1228  * processor.
 1229  */
 1230 static void
 1231 pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)
 1232 {
 1233         cpuset_t *mask, other_cpus;
 1234         u_int cpuid;
 1235 
 1236         sched_pin();
 1237         if (pmap == kernel_pmap) {
 1238                 invlpg(va);
 1239                 mask = &all_cpus;
 1240         } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1241                 mask = &all_cpus;
 1242         } else {
 1243                 cpuid = PCPU_GET(cpuid);
 1244                 other_cpus = all_cpus;
 1245                 CPU_CLR(cpuid, &other_cpus);
 1246                 CPU_AND(&other_cpus, &pmap->pm_active);
 1247                 mask = &other_cpus;
 1248         }
 1249         smp_masked_invlpg(*mask, va, pmap, pmap_curcpu_cb_dummy);
 1250         sched_unpin();
 1251 }
 1252 
 1253 /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */
 1254 #define PMAP_INVLPG_THRESHOLD   (4 * 1024 * PAGE_SIZE)
 1255 
 1256 static void
 1257 pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1258 {
 1259         cpuset_t *mask, other_cpus;
 1260         vm_offset_t addr;
 1261         u_int cpuid;
 1262 
 1263         if (eva - sva >= PMAP_INVLPG_THRESHOLD) {
 1264                 pmap_invalidate_all_int(pmap);
 1265                 return;
 1266         }
 1267 
 1268         sched_pin();
 1269         if (pmap == kernel_pmap) {
 1270                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1271                         invlpg(addr);
 1272                 mask = &all_cpus;
 1273         } else  if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1274                 mask = &all_cpus;
 1275         } else {
 1276                 cpuid = PCPU_GET(cpuid);
 1277                 other_cpus = all_cpus;
 1278                 CPU_CLR(cpuid, &other_cpus);
 1279                 CPU_AND(&other_cpus, &pmap->pm_active);
 1280                 mask = &other_cpus;
 1281         }
 1282         smp_masked_invlpg_range(*mask, sva, eva, pmap, pmap_curcpu_cb_dummy);
 1283         sched_unpin();
 1284 }
 1285 
 1286 static void
 1287 pmap_invalidate_all_int(pmap_t pmap)
 1288 {
 1289         cpuset_t *mask, other_cpus;
 1290         u_int cpuid;
 1291 
 1292         sched_pin();
 1293         if (pmap == kernel_pmap) {
 1294                 invltlb();
 1295                 mask = &all_cpus;
 1296         } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1297                 mask = &all_cpus;
 1298         } else {
 1299                 cpuid = PCPU_GET(cpuid);
 1300                 other_cpus = all_cpus;
 1301                 CPU_CLR(cpuid, &other_cpus);
 1302                 CPU_AND(&other_cpus, &pmap->pm_active);
 1303                 mask = &other_cpus;
 1304         }
 1305         smp_masked_invltlb(*mask, pmap, pmap_curcpu_cb_dummy);
 1306         sched_unpin();
 1307 }
 1308 
 1309 static void
 1310 pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused,
 1311     vm_offset_t addr1 __unused, vm_offset_t addr2 __unused)
 1312 {
 1313         wbinvd();
 1314 }
 1315 
 1316 static void
 1317 __CONCAT(PMTYPE, invalidate_cache)(void)
 1318 {
 1319         smp_cache_flush(pmap_invalidate_cache_curcpu_cb);
 1320 }
 1321 
 1322 struct pde_action {
 1323         cpuset_t invalidate;    /* processors that invalidate their TLB */
 1324         vm_offset_t va;
 1325         pd_entry_t *pde;
 1326         pd_entry_t newpde;
 1327         u_int store;            /* processor that updates the PDE */
 1328 };
 1329 
 1330 static void
 1331 pmap_update_pde_kernel(void *arg)
 1332 {
 1333         struct pde_action *act = arg;
 1334         pd_entry_t *pde;
 1335 
 1336         if (act->store == PCPU_GET(cpuid)) {
 1337                 pde = pmap_pde(kernel_pmap, act->va);
 1338                 pde_store(pde, act->newpde);
 1339         }
 1340 }
 1341 
 1342 static void
 1343 pmap_update_pde_user(void *arg)
 1344 {
 1345         struct pde_action *act = arg;
 1346 
 1347         if (act->store == PCPU_GET(cpuid))
 1348                 pde_store(act->pde, act->newpde);
 1349 }
 1350 
 1351 static void
 1352 pmap_update_pde_teardown(void *arg)
 1353 {
 1354         struct pde_action *act = arg;
 1355 
 1356         if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
 1357                 pmap_update_pde_invalidate(act->va, act->newpde);
 1358 }
 1359 
 1360 /*
 1361  * Change the page size for the specified virtual address in a way that
 1362  * prevents any possibility of the TLB ever having two entries that map the
 1363  * same virtual address using different page sizes.  This is the recommended
 1364  * workaround for Erratum 383 on AMD Family 10h processors.  It prevents a
 1365  * machine check exception for a TLB state that is improperly diagnosed as a
 1366  * hardware error.
 1367  */
 1368 static void
 1369 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1370 {
 1371         struct pde_action act;
 1372         cpuset_t active, other_cpus;
 1373         u_int cpuid;
 1374 
 1375         sched_pin();
 1376         cpuid = PCPU_GET(cpuid);
 1377         other_cpus = all_cpus;
 1378         CPU_CLR(cpuid, &other_cpus);
 1379         if (pmap == kernel_pmap)
 1380                 active = all_cpus;
 1381         else
 1382                 active = pmap->pm_active;
 1383         if (CPU_OVERLAP(&active, &other_cpus)) {
 1384                 act.store = cpuid;
 1385                 act.invalidate = active;
 1386                 act.va = va;
 1387                 act.pde = pde;
 1388                 act.newpde = newpde;
 1389                 CPU_SET(cpuid, &active);
 1390                 smp_rendezvous_cpus(active,
 1391                     smp_no_rendezvous_barrier, pmap == kernel_pmap ?
 1392                     pmap_update_pde_kernel : pmap_update_pde_user,
 1393                     pmap_update_pde_teardown, &act);
 1394         } else {
 1395                 if (pmap == kernel_pmap)
 1396                         pmap_kenter_pde(va, newpde);
 1397                 else
 1398                         pde_store(pde, newpde);
 1399                 if (CPU_ISSET(cpuid, &active))
 1400                         pmap_update_pde_invalidate(va, newpde);
 1401         }
 1402         sched_unpin();
 1403 }
 1404 #else /* !SMP */
 1405 /*
 1406  * Normal, non-SMP, 486+ invalidation functions.
 1407  * We inline these within pmap.c for speed.
 1408  */
 1409 static void
 1410 pmap_invalidate_page_int(pmap_t pmap, vm_offset_t va)
 1411 {
 1412 
 1413         if (pmap == kernel_pmap)
 1414                 invlpg(va);
 1415 }
 1416 
 1417 static void
 1418 pmap_invalidate_range_int(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1419 {
 1420         vm_offset_t addr;
 1421 
 1422         if (pmap == kernel_pmap)
 1423                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1424                         invlpg(addr);
 1425 }
 1426 
 1427 static void
 1428 pmap_invalidate_all_int(pmap_t pmap)
 1429 {
 1430 
 1431         if (pmap == kernel_pmap)
 1432                 invltlb();
 1433 }
 1434 
 1435 static void
 1436 __CONCAT(PMTYPE, invalidate_cache)(void)
 1437 {
 1438 
 1439         wbinvd();
 1440 }
 1441 
 1442 static void
 1443 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1444 {
 1445 
 1446         if (pmap == kernel_pmap)
 1447                 pmap_kenter_pde(va, newpde);
 1448         else
 1449                 pde_store(pde, newpde);
 1450         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1451                 pmap_update_pde_invalidate(va, newpde);
 1452 }
 1453 #endif /* !SMP */
 1454 
 1455 static void
 1456 __CONCAT(PMTYPE, invalidate_page)(pmap_t pmap, vm_offset_t va)
 1457 {
 1458 
 1459         pmap_invalidate_page_int(pmap, va);
 1460 }
 1461 
 1462 static void
 1463 __CONCAT(PMTYPE, invalidate_range)(pmap_t pmap, vm_offset_t sva,
 1464     vm_offset_t eva)
 1465 {
 1466 
 1467         pmap_invalidate_range_int(pmap, sva, eva);
 1468 }
 1469 
 1470 static void
 1471 __CONCAT(PMTYPE, invalidate_all)(pmap_t pmap)
 1472 {
 1473 
 1474         pmap_invalidate_all_int(pmap);
 1475 }
 1476 
 1477 static void
 1478 pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
 1479 {
 1480 
 1481         /*
 1482          * When the PDE has PG_PROMOTED set, the 2- or 4MB page mapping was
 1483          * created by a promotion that did not invalidate the 512 or 1024 4KB
 1484          * page mappings that might exist in the TLB.  Consequently, at this
 1485          * point, the TLB may hold both 4KB and 2- or 4MB page mappings for
 1486          * the address range [va, va + NBPDR).  Therefore, the entire range
 1487          * must be invalidated here.  In contrast, when PG_PROMOTED is clear,
 1488          * the TLB will not hold any 4KB page mappings for the address range
 1489          * [va, va + NBPDR), and so a single INVLPG suffices to invalidate the
 1490          * 2- or 4MB page mapping from the TLB.
 1491          */
 1492         if ((pde & PG_PROMOTED) != 0)
 1493                 pmap_invalidate_range_int(pmap, va, va + NBPDR - 1);
 1494         else
 1495                 pmap_invalidate_page_int(pmap, va);
 1496 }
 1497 
 1498 /*
 1499  * Are we current address space or kernel?
 1500  */
 1501 static __inline int
 1502 pmap_is_current(pmap_t pmap)
 1503 {
 1504 
 1505         return (pmap == kernel_pmap);
 1506 }
 1507 
 1508 /*
 1509  * If the given pmap is not the current or kernel pmap, the returned pte must
 1510  * be released by passing it to pmap_pte_release().
 1511  */
 1512 static pt_entry_t *
 1513 __CONCAT(PMTYPE, pte)(pmap_t pmap, vm_offset_t va)
 1514 {
 1515         pd_entry_t newpf;
 1516         pd_entry_t *pde;
 1517 
 1518         pde = pmap_pde(pmap, va);
 1519         if (*pde & PG_PS)
 1520                 return (pde);
 1521         if (*pde != 0) {
 1522                 /* are we current address space or kernel? */
 1523                 if (pmap_is_current(pmap))
 1524                         return (vtopte(va));
 1525                 mtx_lock(&PMAP2mutex);
 1526                 newpf = *pde & PG_FRAME;
 1527                 if ((*PMAP2 & PG_FRAME) != newpf) {
 1528                         *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1529                         pmap_invalidate_page_int(kernel_pmap,
 1530                             (vm_offset_t)PADDR2);
 1531                 }
 1532                 return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
 1533         }
 1534         return (NULL);
 1535 }
 1536 
 1537 /*
 1538  * Releases a pte that was obtained from pmap_pte().  Be prepared for the pte
 1539  * being NULL.
 1540  */
 1541 static __inline void
 1542 pmap_pte_release(pt_entry_t *pte)
 1543 {
 1544 
 1545         if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2)
 1546                 mtx_unlock(&PMAP2mutex);
 1547 }
 1548 
 1549 /*
 1550  * NB:  The sequence of updating a page table followed by accesses to the
 1551  * corresponding pages is subject to the situation described in the "AMD64
 1552  * Architecture Programmer's Manual Volume 2: System Programming" rev. 3.23,
 1553  * "7.3.1 Special Coherency Considerations".  Therefore, issuing the INVLPG
 1554  * right after modifying the PTE bits is crucial.
 1555  */
 1556 static __inline void
 1557 invlcaddr(void *caddr)
 1558 {
 1559 
 1560         invlpg((u_int)caddr);
 1561 }
 1562 
 1563 /*
 1564  * Super fast pmap_pte routine best used when scanning
 1565  * the pv lists.  This eliminates many coarse-grained
 1566  * invltlb calls.  Note that many of the pv list
 1567  * scans are across different pmaps.  It is very wasteful
 1568  * to do an entire invltlb for checking a single mapping.
 1569  *
 1570  * If the given pmap is not the current pmap, pvh_global_lock
 1571  * must be held and curthread pinned to a CPU.
 1572  */
 1573 static pt_entry_t *
 1574 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
 1575 {
 1576         pd_entry_t newpf;
 1577         pd_entry_t *pde;
 1578 
 1579         pde = pmap_pde(pmap, va);
 1580         if (*pde & PG_PS)
 1581                 return (pde);
 1582         if (*pde != 0) {
 1583                 /* are we current address space or kernel? */
 1584                 if (pmap_is_current(pmap))
 1585                         return (vtopte(va));
 1586                 rw_assert(&pvh_global_lock, RA_WLOCKED);
 1587                 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 1588                 newpf = *pde & PG_FRAME;
 1589                 if ((*PMAP1 & PG_FRAME) != newpf) {
 1590                         *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1591 #ifdef SMP
 1592                         PMAP1cpu = PCPU_GET(cpuid);
 1593 #endif
 1594                         invlcaddr(PADDR1);
 1595                         PMAP1changed++;
 1596                 } else
 1597 #ifdef SMP
 1598                 if (PMAP1cpu != PCPU_GET(cpuid)) {
 1599                         PMAP1cpu = PCPU_GET(cpuid);
 1600                         invlcaddr(PADDR1);
 1601                         PMAP1changedcpu++;
 1602                 } else
 1603 #endif
 1604                         PMAP1unchanged++;
 1605                 return (PADDR1 + (i386_btop(va) & (NPTEPG - 1)));
 1606         }
 1607         return (0);
 1608 }
 1609 
 1610 static pt_entry_t *
 1611 pmap_pte_quick3(pmap_t pmap, vm_offset_t va)
 1612 {
 1613         pd_entry_t newpf;
 1614         pd_entry_t *pde;
 1615 
 1616         pde = pmap_pde(pmap, va);
 1617         if (*pde & PG_PS)
 1618                 return (pde);
 1619         if (*pde != 0) {
 1620                 rw_assert(&pvh_global_lock, RA_WLOCKED);
 1621                 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 1622                 newpf = *pde & PG_FRAME;
 1623                 if ((*PMAP3 & PG_FRAME) != newpf) {
 1624                         *PMAP3 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1625 #ifdef SMP
 1626                         PMAP3cpu = PCPU_GET(cpuid);
 1627 #endif
 1628                         invlcaddr(PADDR3);
 1629                         PMAP1changed++;
 1630                 } else
 1631 #ifdef SMP
 1632                 if (PMAP3cpu != PCPU_GET(cpuid)) {
 1633                         PMAP3cpu = PCPU_GET(cpuid);
 1634                         invlcaddr(PADDR3);
 1635                         PMAP1changedcpu++;
 1636                 } else
 1637 #endif
 1638                         PMAP1unchanged++;
 1639                 return (PADDR3 + (i386_btop(va) & (NPTEPG - 1)));
 1640         }
 1641         return (0);
 1642 }
 1643 
 1644 static pt_entry_t
 1645 pmap_pte_ufast(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
 1646 {
 1647         pt_entry_t *eh_ptep, pte, *ptep;
 1648 
 1649         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1650         pde &= PG_FRAME;
 1651         critical_enter();
 1652         eh_ptep = (pt_entry_t *)PCPU_GET(pmap_eh_ptep);
 1653         if ((*eh_ptep & PG_FRAME) != pde) {
 1654                 *eh_ptep = pde | PG_RW | PG_V | PG_A | PG_M;
 1655                 invlcaddr((void *)PCPU_GET(pmap_eh_va));
 1656         }
 1657         ptep = (pt_entry_t *)PCPU_GET(pmap_eh_va) + (i386_btop(va) &
 1658             (NPTEPG - 1));
 1659         pte = *ptep;
 1660         critical_exit();
 1661         return (pte);
 1662 }
 1663 
 1664 /*
 1665  * Extract from the kernel page table the physical address that is mapped by
 1666  * the given virtual address "va".
 1667  *
 1668  * This function may be used before pmap_bootstrap() is called.
 1669  */
 1670 static vm_paddr_t
 1671 __CONCAT(PMTYPE, kextract)(vm_offset_t va)
 1672 {
 1673         vm_paddr_t pa;
 1674 
 1675         if ((pa = pte_load(&PTD[va >> PDRSHIFT])) & PG_PS) {
 1676                 pa = (pa & PG_PS_FRAME) | (va & PDRMASK);
 1677         } else {
 1678                 /*
 1679                  * Beware of a concurrent promotion that changes the PDE at
 1680                  * this point!  For example, vtopte() must not be used to
 1681                  * access the PTE because it would use the new PDE.  It is,
 1682                  * however, safe to use the old PDE because the page table
 1683                  * page is preserved by the promotion.
 1684                  */
 1685                 pa = KPTmap[i386_btop(va)];
 1686                 pa = (pa & PG_FRAME) | (va & PAGE_MASK);
 1687         }
 1688         return (pa);
 1689 }
 1690 
 1691 /*
 1692  *      Routine:        pmap_extract
 1693  *      Function:
 1694  *              Extract the physical page address associated
 1695  *              with the given map/virtual_address pair.
 1696  */
 1697 static vm_paddr_t
 1698 __CONCAT(PMTYPE, extract)(pmap_t pmap, vm_offset_t va)
 1699 {
 1700         vm_paddr_t rtval;
 1701         pt_entry_t pte;
 1702         pd_entry_t pde;
 1703 
 1704         rtval = 0;
 1705         PMAP_LOCK(pmap);
 1706         pde = pmap->pm_pdir[va >> PDRSHIFT];
 1707         if (pde != 0) {
 1708                 if ((pde & PG_PS) != 0)
 1709                         rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
 1710                 else {
 1711                         pte = pmap_pte_ufast(pmap, va, pde);
 1712                         rtval = (pte & PG_FRAME) | (va & PAGE_MASK);
 1713                 }
 1714         }
 1715         PMAP_UNLOCK(pmap);
 1716         return (rtval);
 1717 }
 1718 
 1719 /*
 1720  *      Routine:        pmap_extract_and_hold
 1721  *      Function:
 1722  *              Atomically extract and hold the physical page
 1723  *              with the given pmap and virtual address pair
 1724  *              if that mapping permits the given protection.
 1725  */
 1726 static vm_page_t
 1727 __CONCAT(PMTYPE, extract_and_hold)(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 1728 {
 1729         pd_entry_t pde;
 1730         pt_entry_t pte;
 1731         vm_page_t m;
 1732 
 1733         m = NULL;
 1734         PMAP_LOCK(pmap);
 1735         pde = *pmap_pde(pmap, va);
 1736         if (pde != 0) {
 1737                 if (pde & PG_PS) {
 1738                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0)
 1739                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
 1740                                     (va & PDRMASK));
 1741                 } else {
 1742                         pte = pmap_pte_ufast(pmap, va, pde);
 1743                         if (pte != 0 &&
 1744                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0))
 1745                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
 1746                 }
 1747                 if (m != NULL && !vm_page_wire_mapped(m))
 1748                         m = NULL;
 1749         }
 1750         PMAP_UNLOCK(pmap);
 1751         return (m);
 1752 }
 1753 
 1754 /***************************************************
 1755  * Low level mapping routines.....
 1756  ***************************************************/
 1757 
 1758 /*
 1759  * Add a wired page to the kva.
 1760  * Note: not SMP coherent.
 1761  *
 1762  * This function may be used before pmap_bootstrap() is called.
 1763  */
 1764 static void
 1765 __CONCAT(PMTYPE, kenter)(vm_offset_t va, vm_paddr_t pa)
 1766 {
 1767         pt_entry_t *pte;
 1768 
 1769         pte = vtopte(va);
 1770         pte_store(pte, pa | PG_RW | PG_V);
 1771 }
 1772 
 1773 static __inline void
 1774 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
 1775 {
 1776         pt_entry_t *pte;
 1777 
 1778         pte = vtopte(va);
 1779         pte_store(pte, pa | PG_RW | PG_V | pmap_cache_bits(kernel_pmap,
 1780             mode, 0));
 1781 }
 1782 
 1783 /*
 1784  * Remove a page from the kernel pagetables.
 1785  * Note: not SMP coherent.
 1786  *
 1787  * This function may be used before pmap_bootstrap() is called.
 1788  */
 1789 static void
 1790 __CONCAT(PMTYPE, kremove)(vm_offset_t va)
 1791 {
 1792         pt_entry_t *pte;
 1793 
 1794         pte = vtopte(va);
 1795         pte_clear(pte);
 1796 }
 1797 
 1798 /*
 1799  *      Used to map a range of physical addresses into kernel
 1800  *      virtual address space.
 1801  *
 1802  *      The value passed in '*virt' is a suggested virtual address for
 1803  *      the mapping. Architectures which can support a direct-mapped
 1804  *      physical to virtual region can return the appropriate address
 1805  *      within that region, leaving '*virt' unchanged. Other
 1806  *      architectures should map the pages starting at '*virt' and
 1807  *      update '*virt' with the first usable address after the mapped
 1808  *      region.
 1809  */
 1810 static vm_offset_t
 1811 __CONCAT(PMTYPE, map)(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end,
 1812     int prot)
 1813 {
 1814         vm_offset_t va, sva;
 1815         vm_paddr_t superpage_offset;
 1816         pd_entry_t newpde;
 1817 
 1818         va = *virt;
 1819         /*
 1820          * Does the physical address range's size and alignment permit at
 1821          * least one superpage mapping to be created?
 1822          */ 
 1823         superpage_offset = start & PDRMASK;
 1824         if ((end - start) - ((NBPDR - superpage_offset) & PDRMASK) >= NBPDR) {
 1825                 /*
 1826                  * Increase the starting virtual address so that its alignment
 1827                  * does not preclude the use of superpage mappings.
 1828                  */
 1829                 if ((va & PDRMASK) < superpage_offset)
 1830                         va = (va & ~PDRMASK) + superpage_offset;
 1831                 else if ((va & PDRMASK) > superpage_offset)
 1832                         va = ((va + PDRMASK) & ~PDRMASK) + superpage_offset;
 1833         }
 1834         sva = va;
 1835         while (start < end) {
 1836                 if ((start & PDRMASK) == 0 && end - start >= NBPDR &&
 1837                     pseflag != 0) {
 1838                         KASSERT((va & PDRMASK) == 0,
 1839                             ("pmap_map: misaligned va %#x", va));
 1840                         newpde = start | PG_PS | PG_RW | PG_V;
 1841                         pmap_kenter_pde(va, newpde);
 1842                         va += NBPDR;
 1843                         start += NBPDR;
 1844                 } else {
 1845                         pmap_kenter(va, start);
 1846                         va += PAGE_SIZE;
 1847                         start += PAGE_SIZE;
 1848                 }
 1849         }
 1850         pmap_invalidate_range_int(kernel_pmap, sva, va);
 1851         *virt = va;
 1852         return (sva);
 1853 }
 1854 
 1855 /*
 1856  * Add a list of wired pages to the kva
 1857  * this routine is only used for temporary
 1858  * kernel mappings that do not need to have
 1859  * page modification or references recorded.
 1860  * Note that old mappings are simply written
 1861  * over.  The page *must* be wired.
 1862  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1863  */
 1864 static void
 1865 __CONCAT(PMTYPE, qenter)(vm_offset_t sva, vm_page_t *ma, int count)
 1866 {
 1867         pt_entry_t *endpte, oldpte, pa, *pte;
 1868         vm_page_t m;
 1869 
 1870         oldpte = 0;
 1871         pte = vtopte(sva);
 1872         endpte = pte + count;
 1873         while (pte < endpte) {
 1874                 m = *ma++;
 1875                 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(kernel_pmap,
 1876                     m->md.pat_mode, 0);
 1877                 if ((*pte & (PG_FRAME | PG_PTE_CACHE)) != pa) {
 1878                         oldpte |= *pte;
 1879                         pte_store(pte, pa | pg_nx | PG_RW | PG_V);
 1880                 }
 1881                 pte++;
 1882         }
 1883         if (__predict_false((oldpte & PG_V) != 0))
 1884                 pmap_invalidate_range_int(kernel_pmap, sva, sva + count *
 1885                     PAGE_SIZE);
 1886 }
 1887 
 1888 /*
 1889  * This routine tears out page mappings from the
 1890  * kernel -- it is meant only for temporary mappings.
 1891  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1892  */
 1893 static void
 1894 __CONCAT(PMTYPE, qremove)(vm_offset_t sva, int count)
 1895 {
 1896         vm_offset_t va;
 1897 
 1898         va = sva;
 1899         while (count-- > 0) {
 1900                 pmap_kremove(va);
 1901                 va += PAGE_SIZE;
 1902         }
 1903         pmap_invalidate_range_int(kernel_pmap, sva, va);
 1904 }
 1905 
 1906 /***************************************************
 1907  * Page table page management routines.....
 1908  ***************************************************/
 1909 /*
 1910  * Schedule the specified unused page table page to be freed.  Specifically,
 1911  * add the page to the specified list of pages that will be released to the
 1912  * physical memory manager after the TLB has been updated.
 1913  */
 1914 static __inline void
 1915 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
 1916     boolean_t set_PG_ZERO)
 1917 {
 1918 
 1919         if (set_PG_ZERO)
 1920                 m->flags |= PG_ZERO;
 1921         else
 1922                 m->flags &= ~PG_ZERO;
 1923         SLIST_INSERT_HEAD(free, m, plinks.s.ss);
 1924 }
 1925 
 1926 /*
 1927  * Inserts the specified page table page into the specified pmap's collection
 1928  * of idle page table pages.  Each of a pmap's page table pages is responsible
 1929  * for mapping a distinct range of virtual addresses.  The pmap's collection is
 1930  * ordered by this virtual address range.
 1931  *
 1932  * If "promoted" is false, then the page table page "mpte" must be zero filled.
 1933  */
 1934 static __inline int
 1935 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted)
 1936 {
 1937 
 1938         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1939         mpte->valid = promoted ? VM_PAGE_BITS_ALL : 0;
 1940         return (vm_radix_insert(&pmap->pm_root, mpte));
 1941 }
 1942 
 1943 /*
 1944  * Removes the page table page mapping the specified virtual address from the
 1945  * specified pmap's collection of idle page table pages, and returns it.
 1946  * Otherwise, returns NULL if there is no page table page corresponding to the
 1947  * specified virtual address.
 1948  */
 1949 static __inline vm_page_t
 1950 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
 1951 {
 1952 
 1953         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1954         return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT));
 1955 }
 1956 
 1957 /*
 1958  * Decrements a page table page's reference count, which is used to record the
 1959  * number of valid page table entries within the page.  If the reference count
 1960  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
 1961  * page table page was unmapped and FALSE otherwise.
 1962  */
 1963 static inline boolean_t
 1964 pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
 1965 {
 1966 
 1967         --m->ref_count;
 1968         if (m->ref_count == 0) {
 1969                 _pmap_unwire_ptp(pmap, m, free);
 1970                 return (TRUE);
 1971         } else
 1972                 return (FALSE);
 1973 }
 1974 
 1975 static void
 1976 _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
 1977 {
 1978 
 1979         /*
 1980          * unmap the page table page
 1981          */
 1982         pmap->pm_pdir[m->pindex] = 0;
 1983         --pmap->pm_stats.resident_count;
 1984 
 1985         /*
 1986          * There is not need to invalidate the recursive mapping since
 1987          * we never instantiate such mapping for the usermode pmaps,
 1988          * and never remove page table pages from the kernel pmap.
 1989          * Put page on a list so that it is released since all TLB
 1990          * shootdown is done.
 1991          */
 1992         MPASS(pmap != kernel_pmap);
 1993         pmap_add_delayed_free_list(m, free, TRUE);
 1994 }
 1995 
 1996 /*
 1997  * After removing a page table entry, this routine is used to
 1998  * conditionally free the page, and manage the reference count.
 1999  */
 2000 static int
 2001 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free)
 2002 {
 2003         pd_entry_t ptepde;
 2004         vm_page_t mpte;
 2005 
 2006         if (pmap == kernel_pmap)
 2007                 return (0);
 2008         ptepde = *pmap_pde(pmap, va);
 2009         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 2010         return (pmap_unwire_ptp(pmap, mpte, free));
 2011 }
 2012 
 2013 /*
 2014  * Release a page table page reference after a failed attempt to create a
 2015  * mapping.
 2016  */
 2017 static void
 2018 pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte)
 2019 {
 2020         struct spglist free;
 2021 
 2022         SLIST_INIT(&free);
 2023         if (pmap_unwire_ptp(pmap, mpte, &free)) {
 2024                 /*
 2025                  * Although "va" was never mapped, paging-structure caches
 2026                  * could nonetheless have entries that refer to the freed
 2027                  * page table pages.  Invalidate those entries.
 2028                  */
 2029                 pmap_invalidate_page_int(pmap, va);
 2030                 vm_page_free_pages_toq(&free, true);
 2031         }
 2032 }
 2033 
 2034 /*
 2035  * Initialize the pmap for the swapper process.
 2036  */
 2037 static void
 2038 __CONCAT(PMTYPE, pinit0)(pmap_t pmap)
 2039 {
 2040 
 2041         PMAP_LOCK_INIT(pmap);
 2042         pmap->pm_pdir = IdlePTD;
 2043 #ifdef PMAP_PAE_COMP
 2044         pmap->pm_pdpt = IdlePDPT;
 2045 #endif
 2046         pmap->pm_root.rt_root = 0;
 2047         CPU_ZERO(&pmap->pm_active);
 2048         TAILQ_INIT(&pmap->pm_pvchunk);
 2049         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 2050         pmap_activate_boot(pmap);
 2051 }
 2052 
 2053 /*
 2054  * Initialize a preallocated and zeroed pmap structure,
 2055  * such as one in a vmspace structure.
 2056  */
 2057 static int
 2058 __CONCAT(PMTYPE, pinit)(pmap_t pmap)
 2059 {
 2060         vm_page_t m;
 2061         int i;
 2062 
 2063         /*
 2064          * No need to allocate page table space yet but we do need a valid
 2065          * page directory table.
 2066          */
 2067         if (pmap->pm_pdir == NULL) {
 2068                 pmap->pm_pdir = (pd_entry_t *)kva_alloc(NBPTD);
 2069                 if (pmap->pm_pdir == NULL)
 2070                         return (0);
 2071 #ifdef PMAP_PAE_COMP
 2072                 pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
 2073                 KASSERT(((vm_offset_t)pmap->pm_pdpt &
 2074                     ((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0,
 2075                     ("pmap_pinit: pdpt misaligned"));
 2076                 KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
 2077                     ("pmap_pinit: pdpt above 4g"));
 2078 #endif
 2079                 pmap->pm_root.rt_root = 0;
 2080         }
 2081         KASSERT(vm_radix_is_empty(&pmap->pm_root),
 2082             ("pmap_pinit: pmap has reserved page table page(s)"));
 2083 
 2084         /*
 2085          * allocate the page directory page(s)
 2086          */
 2087         for (i = 0; i < NPGPTD; i++) {
 2088                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 2089                     VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK);
 2090                 pmap->pm_ptdpg[i] = m;
 2091 #ifdef PMAP_PAE_COMP
 2092                 pmap->pm_pdpt[i] = VM_PAGE_TO_PHYS(m) | PG_V;
 2093 #endif
 2094         }
 2095 
 2096         pmap_qenter((vm_offset_t)pmap->pm_pdir, pmap->pm_ptdpg, NPGPTD);
 2097 #ifdef PMAP_PAE_COMP
 2098         if ((cpu_feature & CPUID_PAT) == 0) {
 2099                 pmap_invalidate_cache_range(
 2100                     trunc_page((vm_offset_t)pmap->pm_pdpt),
 2101                     round_page((vm_offset_t)pmap->pm_pdpt +
 2102                     NPGPTD * sizeof(pdpt_entry_t)));
 2103         }
 2104 #endif
 2105 
 2106         for (i = 0; i < NPGPTD; i++)
 2107                 if ((pmap->pm_ptdpg[i]->flags & PG_ZERO) == 0)
 2108                         pagezero(pmap->pm_pdir + (i * NPDEPG));
 2109 
 2110         /* Install the trampoline mapping. */
 2111         pmap->pm_pdir[TRPTDI] = PTD[TRPTDI];
 2112 
 2113         CPU_ZERO(&pmap->pm_active);
 2114         TAILQ_INIT(&pmap->pm_pvchunk);
 2115         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 2116 
 2117         return (1);
 2118 }
 2119 
 2120 /*
 2121  * this routine is called if the page table page is not
 2122  * mapped correctly.
 2123  */
 2124 static vm_page_t
 2125 _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags)
 2126 {
 2127         vm_paddr_t ptepa;
 2128         vm_page_t m;
 2129 
 2130         /*
 2131          * Allocate a page table page.
 2132          */
 2133         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 2134             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 2135                 if ((flags & PMAP_ENTER_NOSLEEP) == 0) {
 2136                         PMAP_UNLOCK(pmap);
 2137                         rw_wunlock(&pvh_global_lock);
 2138                         vm_wait(NULL);
 2139                         rw_wlock(&pvh_global_lock);
 2140                         PMAP_LOCK(pmap);
 2141                 }
 2142 
 2143                 /*
 2144                  * Indicate the need to retry.  While waiting, the page table
 2145                  * page may have been allocated.
 2146                  */
 2147                 return (NULL);
 2148         }
 2149         if ((m->flags & PG_ZERO) == 0)
 2150                 pmap_zero_page(m);
 2151 
 2152         /*
 2153          * Map the pagetable page into the process address space, if
 2154          * it isn't already there.
 2155          */
 2156 
 2157         pmap->pm_stats.resident_count++;
 2158 
 2159         ptepa = VM_PAGE_TO_PHYS(m);
 2160         pmap->pm_pdir[ptepindex] =
 2161                 (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
 2162 
 2163         return (m);
 2164 }
 2165 
 2166 static vm_page_t
 2167 pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)
 2168 {
 2169         u_int ptepindex;
 2170         pd_entry_t ptepa;
 2171         vm_page_t m;
 2172 
 2173         /*
 2174          * Calculate pagetable page index
 2175          */
 2176         ptepindex = va >> PDRSHIFT;
 2177 retry:
 2178         /*
 2179          * Get the page directory entry
 2180          */
 2181         ptepa = pmap->pm_pdir[ptepindex];
 2182 
 2183         /*
 2184          * This supports switching from a 4MB page to a
 2185          * normal 4K page.
 2186          */
 2187         if (ptepa & PG_PS) {
 2188                 (void)pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va);
 2189                 ptepa = pmap->pm_pdir[ptepindex];
 2190         }
 2191 
 2192         /*
 2193          * If the page table page is mapped, we just increment the
 2194          * hold count, and activate it.
 2195          */
 2196         if (ptepa) {
 2197                 m = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 2198                 m->ref_count++;
 2199         } else {
 2200                 /*
 2201                  * Here if the pte page isn't mapped, or if it has
 2202                  * been deallocated. 
 2203                  */
 2204                 m = _pmap_allocpte(pmap, ptepindex, flags);
 2205                 if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0)
 2206                         goto retry;
 2207         }
 2208         return (m);
 2209 }
 2210 
 2211 /***************************************************
 2212 * Pmap allocation/deallocation routines.
 2213  ***************************************************/
 2214 
 2215 /*
 2216  * Release any resources held by the given physical map.
 2217  * Called when a pmap initialized by pmap_pinit is being released.
 2218  * Should only be called if the map contains no valid mappings.
 2219  */
 2220 static void
 2221 __CONCAT(PMTYPE, release)(pmap_t pmap)
 2222 {
 2223         vm_page_t m;
 2224         int i;
 2225 
 2226         KASSERT(pmap->pm_stats.resident_count == 0,
 2227             ("pmap_release: pmap resident count %ld != 0",
 2228             pmap->pm_stats.resident_count));
 2229         KASSERT(vm_radix_is_empty(&pmap->pm_root),
 2230             ("pmap_release: pmap has reserved page table page(s)"));
 2231         KASSERT(CPU_EMPTY(&pmap->pm_active),
 2232             ("releasing active pmap %p", pmap));
 2233 
 2234         pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
 2235 
 2236         for (i = 0; i < NPGPTD; i++) {
 2237                 m = pmap->pm_ptdpg[i];
 2238 #ifdef PMAP_PAE_COMP
 2239                 KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
 2240                     ("pmap_release: got wrong ptd page"));
 2241 #endif
 2242                 vm_page_unwire_noq(m);
 2243                 vm_page_free(m);
 2244         }
 2245 }
 2246 
 2247 /*
 2248  * grow the number of kernel page table entries, if needed
 2249  */
 2250 static void
 2251 __CONCAT(PMTYPE, growkernel)(vm_offset_t addr)
 2252 {
 2253         vm_paddr_t ptppaddr;
 2254         vm_page_t nkpg;
 2255         pd_entry_t newpdir;
 2256 
 2257         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 2258         addr = roundup2(addr, NBPDR);
 2259         if (addr - 1 >= vm_map_max(kernel_map))
 2260                 addr = vm_map_max(kernel_map);
 2261         while (kernel_vm_end < addr) {
 2262                 if (pdir_pde(PTD, kernel_vm_end)) {
 2263                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2264                         if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
 2265                                 kernel_vm_end = vm_map_max(kernel_map);
 2266                                 break;
 2267                         }
 2268                         continue;
 2269                 }
 2270 
 2271                 nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDRSHIFT,
 2272                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 2273                     VM_ALLOC_ZERO);
 2274                 if (nkpg == NULL)
 2275                         panic("pmap_growkernel: no memory to grow kernel");
 2276 
 2277                 nkpt++;
 2278 
 2279                 if ((nkpg->flags & PG_ZERO) == 0)
 2280                         pmap_zero_page(nkpg);
 2281                 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
 2282                 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
 2283                 pdir_pde(KPTD, kernel_vm_end) = newpdir;
 2284 
 2285                 pmap_kenter_pde(kernel_vm_end, newpdir);
 2286                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2287                 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
 2288                         kernel_vm_end = vm_map_max(kernel_map);
 2289                         break;
 2290                 }
 2291         }
 2292 }
 2293 
 2294 /***************************************************
 2295  * page management routines.
 2296  ***************************************************/
 2297 
 2298 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 2299 CTASSERT(_NPCM == 11);
 2300 CTASSERT(_NPCPV == 336);
 2301 
 2302 static __inline struct pv_chunk *
 2303 pv_to_chunk(pv_entry_t pv)
 2304 {
 2305 
 2306         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
 2307 }
 2308 
 2309 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 2310 
 2311 #define PC_FREE0_9      0xfffffffful    /* Free values for index 0 through 9 */
 2312 #define PC_FREE10       0x0000fffful    /* Free values for index 10 */
 2313 
 2314 static const uint32_t pc_freemask[_NPCM] = {
 2315         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2316         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2317         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2318         PC_FREE0_9, PC_FREE10
 2319 };
 2320 
 2321 #ifdef PV_STATS
 2322 extern int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 2323 extern long pv_entry_frees, pv_entry_allocs;
 2324 extern int pv_entry_spare;
 2325 #endif
 2326 
 2327 /*
 2328  * We are in a serious low memory condition.  Resort to
 2329  * drastic measures to free some pages so we can allocate
 2330  * another pv entry chunk.
 2331  */
 2332 static vm_page_t
 2333 pmap_pv_reclaim(pmap_t locked_pmap)
 2334 {
 2335         struct pch newtail;
 2336         struct pv_chunk *pc;
 2337         struct md_page *pvh;
 2338         pd_entry_t *pde;
 2339         pmap_t pmap;
 2340         pt_entry_t *pte, tpte;
 2341         pv_entry_t pv;
 2342         vm_offset_t va;
 2343         vm_page_t m, m_pc;
 2344         struct spglist free;
 2345         uint32_t inuse;
 2346         int bit, field, freed;
 2347 
 2348         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
 2349         pmap = NULL;
 2350         m_pc = NULL;
 2351         SLIST_INIT(&free);
 2352         TAILQ_INIT(&newtail);
 2353         while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 ||
 2354             SLIST_EMPTY(&free))) {
 2355                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2356                 if (pmap != pc->pc_pmap) {
 2357                         if (pmap != NULL) {
 2358                                 pmap_invalidate_all_int(pmap);
 2359                                 if (pmap != locked_pmap)
 2360                                         PMAP_UNLOCK(pmap);
 2361                         }
 2362                         pmap = pc->pc_pmap;
 2363                         /* Avoid deadlock and lock recursion. */
 2364                         if (pmap > locked_pmap)
 2365                                 PMAP_LOCK(pmap);
 2366                         else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
 2367                                 pmap = NULL;
 2368                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2369                                 continue;
 2370                         }
 2371                 }
 2372 
 2373                 /*
 2374                  * Destroy every non-wired, 4 KB page mapping in the chunk.
 2375                  */
 2376                 freed = 0;
 2377                 for (field = 0; field < _NPCM; field++) {
 2378                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
 2379                             inuse != 0; inuse &= ~(1UL << bit)) {
 2380                                 bit = bsfl(inuse);
 2381                                 pv = &pc->pc_pventry[field * 32 + bit];
 2382                                 va = pv->pv_va;
 2383                                 pde = pmap_pde(pmap, va);
 2384                                 if ((*pde & PG_PS) != 0)
 2385                                         continue;
 2386                                 pte = __CONCAT(PMTYPE, pte)(pmap, va);
 2387                                 tpte = *pte;
 2388                                 if ((tpte & PG_W) == 0)
 2389                                         tpte = pte_load_clear(pte);
 2390                                 pmap_pte_release(pte);
 2391                                 if ((tpte & PG_W) != 0)
 2392                                         continue;
 2393                                 KASSERT(tpte != 0,
 2394                                     ("pmap_pv_reclaim: pmap %p va %x zero pte",
 2395                                     pmap, va));
 2396                                 if ((tpte & PG_G) != 0)
 2397                                         pmap_invalidate_page_int(pmap, va);
 2398                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2399                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2400                                         vm_page_dirty(m);
 2401                                 if ((tpte & PG_A) != 0)
 2402                                         vm_page_aflag_set(m, PGA_REFERENCED);
 2403                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 2404                                 if (TAILQ_EMPTY(&m->md.pv_list) &&
 2405                                     (m->flags & PG_FICTITIOUS) == 0) {
 2406                                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2407                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 2408                                                 vm_page_aflag_clear(m,
 2409                                                     PGA_WRITEABLE);
 2410                                         }
 2411                                 }
 2412                                 pc->pc_map[field] |= 1UL << bit;
 2413                                 pmap_unuse_pt(pmap, va, &free);
 2414                                 freed++;
 2415                         }
 2416                 }
 2417                 if (freed == 0) {
 2418                         TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2419                         continue;
 2420                 }
 2421                 /* Every freed mapping is for a 4 KB page. */
 2422                 pmap->pm_stats.resident_count -= freed;
 2423                 PV_STAT(pv_entry_frees += freed);
 2424                 PV_STAT(pv_entry_spare += freed);
 2425                 pv_entry_count -= freed;
 2426                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2427                 for (field = 0; field < _NPCM; field++)
 2428                         if (pc->pc_map[field] != pc_freemask[field]) {
 2429                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2430                                     pc_list);
 2431                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2432 
 2433                                 /*
 2434                                  * One freed pv entry in locked_pmap is
 2435                                  * sufficient.
 2436                                  */
 2437                                 if (pmap == locked_pmap)
 2438                                         goto out;
 2439                                 break;
 2440                         }
 2441                 if (field == _NPCM) {
 2442                         PV_STAT(pv_entry_spare -= _NPCPV);
 2443                         PV_STAT(pc_chunk_count--);
 2444                         PV_STAT(pc_chunk_frees++);
 2445                         /* Entire chunk is free; return it. */
 2446                         m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2447                         pmap_qremove((vm_offset_t)pc, 1);
 2448                         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2449                         break;
 2450                 }
 2451         }
 2452 out:
 2453         TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
 2454         if (pmap != NULL) {
 2455                 pmap_invalidate_all_int(pmap);
 2456                 if (pmap != locked_pmap)
 2457                         PMAP_UNLOCK(pmap);
 2458         }
 2459         if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) {
 2460                 m_pc = SLIST_FIRST(&free);
 2461                 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
 2462                 /* Recycle a freed page table page. */
 2463                 m_pc->ref_count = 1;
 2464         }
 2465         vm_page_free_pages_toq(&free, true);
 2466         return (m_pc);
 2467 }
 2468 
 2469 /*
 2470  * free the pv_entry back to the free list
 2471  */
 2472 static void
 2473 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 2474 {
 2475         struct pv_chunk *pc;
 2476         int idx, field, bit;
 2477 
 2478         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2479         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2480         PV_STAT(pv_entry_frees++);
 2481         PV_STAT(pv_entry_spare++);
 2482         pv_entry_count--;
 2483         pc = pv_to_chunk(pv);
 2484         idx = pv - &pc->pc_pventry[0];
 2485         field = idx / 32;
 2486         bit = idx % 32;
 2487         pc->pc_map[field] |= 1ul << bit;
 2488         for (idx = 0; idx < _NPCM; idx++)
 2489                 if (pc->pc_map[idx] != pc_freemask[idx]) {
 2490                         /*
 2491                          * 98% of the time, pc is already at the head of the
 2492                          * list.  If it isn't already, move it to the head.
 2493                          */
 2494                         if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
 2495                             pc)) {
 2496                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2497                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2498                                     pc_list);
 2499                         }
 2500                         return;
 2501                 }
 2502         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2503         free_pv_chunk(pc);
 2504 }
 2505 
 2506 static void
 2507 free_pv_chunk(struct pv_chunk *pc)
 2508 {
 2509         vm_page_t m;
 2510 
 2511         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2512         PV_STAT(pv_entry_spare -= _NPCPV);
 2513         PV_STAT(pc_chunk_count--);
 2514         PV_STAT(pc_chunk_frees++);
 2515         /* entire chunk is free, return it */
 2516         m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2517         pmap_qremove((vm_offset_t)pc, 1);
 2518         vm_page_unwire_noq(m);
 2519         vm_page_free(m);
 2520         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2521 }
 2522 
 2523 /*
 2524  * get a new pv_entry, allocating a block from the system
 2525  * when needed.
 2526  */
 2527 static pv_entry_t
 2528 get_pv_entry(pmap_t pmap, boolean_t try)
 2529 {
 2530         static const struct timeval printinterval = { 60, 0 };
 2531         static struct timeval lastprint;
 2532         int bit, field;
 2533         pv_entry_t pv;
 2534         struct pv_chunk *pc;
 2535         vm_page_t m;
 2536 
 2537         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2538         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2539         PV_STAT(pv_entry_allocs++);
 2540         pv_entry_count++;
 2541         if (pv_entry_count > pv_entry_high_water)
 2542                 if (ratecheck(&lastprint, &printinterval))
 2543                         printf("Approaching the limit on PV entries, consider "
 2544                             "increasing either the vm.pmap.shpgperproc or the "
 2545                             "vm.pmap.pv_entries tunable.\n");
 2546 retry:
 2547         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 2548         if (pc != NULL) {
 2549                 for (field = 0; field < _NPCM; field++) {
 2550                         if (pc->pc_map[field]) {
 2551                                 bit = bsfl(pc->pc_map[field]);
 2552                                 break;
 2553                         }
 2554                 }
 2555                 if (field < _NPCM) {
 2556                         pv = &pc->pc_pventry[field * 32 + bit];
 2557                         pc->pc_map[field] &= ~(1ul << bit);
 2558                         /* If this was the last item, move it to tail */
 2559                         for (field = 0; field < _NPCM; field++)
 2560                                 if (pc->pc_map[field] != 0) {
 2561                                         PV_STAT(pv_entry_spare--);
 2562                                         return (pv);    /* not full, return */
 2563                                 }
 2564                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2565                         TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 2566                         PV_STAT(pv_entry_spare--);
 2567                         return (pv);
 2568                 }
 2569         }
 2570         /*
 2571          * Access to the ptelist "pv_vafree" is synchronized by the pvh
 2572          * global lock.  If "pv_vafree" is currently non-empty, it will
 2573          * remain non-empty until pmap_ptelist_alloc() completes.
 2574          */
 2575         if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 2576             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 2577                 if (try) {
 2578                         pv_entry_count--;
 2579                         PV_STAT(pc_chunk_tryfail++);
 2580                         return (NULL);
 2581                 }
 2582                 m = pmap_pv_reclaim(pmap);
 2583                 if (m == NULL)
 2584                         goto retry;
 2585         }
 2586         PV_STAT(pc_chunk_count++);
 2587         PV_STAT(pc_chunk_allocs++);
 2588         pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree);
 2589         pmap_qenter((vm_offset_t)pc, &m, 1);
 2590         pc->pc_pmap = pmap;
 2591         pc->pc_map[0] = pc_freemask[0] & ~1ul;  /* preallocated bit 0 */
 2592         for (field = 1; field < _NPCM; field++)
 2593                 pc->pc_map[field] = pc_freemask[field];
 2594         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 2595         pv = &pc->pc_pventry[0];
 2596         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2597         PV_STAT(pv_entry_spare += _NPCPV - 1);
 2598         return (pv);
 2599 }
 2600 
 2601 static __inline pv_entry_t
 2602 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2603 {
 2604         pv_entry_t pv;
 2605 
 2606         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2607         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 2608                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 2609                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 2610                         break;
 2611                 }
 2612         }
 2613         return (pv);
 2614 }
 2615 
 2616 static void
 2617 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2618 {
 2619         struct md_page *pvh;
 2620         pv_entry_t pv;
 2621         vm_offset_t va_last;
 2622         vm_page_t m;
 2623 
 2624         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2625         KASSERT((pa & PDRMASK) == 0,
 2626             ("pmap_pv_demote_pde: pa is not 4mpage aligned"));
 2627 
 2628         /*
 2629          * Transfer the 4mpage's pv entry for this mapping to the first
 2630          * page's pv list.
 2631          */
 2632         pvh = pa_to_pvh(pa);
 2633         va = trunc_4mpage(va);
 2634         pv = pmap_pvh_remove(pvh, pmap, va);
 2635         KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
 2636         m = PHYS_TO_VM_PAGE(pa);
 2637         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2638         /* Instantiate the remaining NPTEPG - 1 pv entries. */
 2639         va_last = va + NBPDR - PAGE_SIZE;
 2640         do {
 2641                 m++;
 2642                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 2643                     ("pmap_pv_demote_pde: page %p is not managed", m));
 2644                 va += PAGE_SIZE;
 2645                 pmap_insert_entry(pmap, va, m);
 2646         } while (va < va_last);
 2647 }
 2648 
 2649 #if VM_NRESERVLEVEL > 0
 2650 static void
 2651 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2652 {
 2653         struct md_page *pvh;
 2654         pv_entry_t pv;
 2655         vm_offset_t va_last;
 2656         vm_page_t m;
 2657 
 2658         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2659         KASSERT((pa & PDRMASK) == 0,
 2660             ("pmap_pv_promote_pde: pa is not 4mpage aligned"));
 2661 
 2662         /*
 2663          * Transfer the first page's pv entry for this mapping to the
 2664          * 4mpage's pv list.  Aside from avoiding the cost of a call
 2665          * to get_pv_entry(), a transfer avoids the possibility that
 2666          * get_pv_entry() calls pmap_collect() and that pmap_collect()
 2667          * removes one of the mappings that is being promoted.
 2668          */
 2669         m = PHYS_TO_VM_PAGE(pa);
 2670         va = trunc_4mpage(va);
 2671         pv = pmap_pvh_remove(&m->md, pmap, va);
 2672         KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
 2673         pvh = pa_to_pvh(pa);
 2674         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 2675         /* Free the remaining NPTEPG - 1 pv entries. */
 2676         va_last = va + NBPDR - PAGE_SIZE;
 2677         do {
 2678                 m++;
 2679                 va += PAGE_SIZE;
 2680                 pmap_pvh_free(&m->md, pmap, va);
 2681         } while (va < va_last);
 2682 }
 2683 #endif /* VM_NRESERVLEVEL > 0 */
 2684 
 2685 static void
 2686 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2687 {
 2688         pv_entry_t pv;
 2689 
 2690         pv = pmap_pvh_remove(pvh, pmap, va);
 2691         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 2692         free_pv_entry(pmap, pv);
 2693 }
 2694 
 2695 static void
 2696 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 2697 {
 2698         struct md_page *pvh;
 2699 
 2700         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2701         pmap_pvh_free(&m->md, pmap, va);
 2702         if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) {
 2703                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2704                 if (TAILQ_EMPTY(&pvh->pv_list))
 2705                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 2706         }
 2707 }
 2708 
 2709 /*
 2710  * Create a pv entry for page at pa for
 2711  * (pmap, va).
 2712  */
 2713 static void
 2714 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2715 {
 2716         pv_entry_t pv;
 2717 
 2718         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2719         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2720         pv = get_pv_entry(pmap, FALSE);
 2721         pv->pv_va = va;
 2722         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2723 }
 2724 
 2725 /*
 2726  * Conditionally create a pv entry.
 2727  */
 2728 static boolean_t
 2729 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2730 {
 2731         pv_entry_t pv;
 2732 
 2733         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2734         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2735         if (pv_entry_count < pv_entry_high_water && 
 2736             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 2737                 pv->pv_va = va;
 2738                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2739                 return (TRUE);
 2740         } else
 2741                 return (FALSE);
 2742 }
 2743 
 2744 /*
 2745  * Create the pv entries for each of the pages within a superpage.
 2746  */
 2747 static bool
 2748 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags)
 2749 {
 2750         struct md_page *pvh;
 2751         pv_entry_t pv;
 2752         bool noreclaim;
 2753 
 2754         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2755         noreclaim = (flags & PMAP_ENTER_NORECLAIM) != 0;
 2756         if ((noreclaim && pv_entry_count >= pv_entry_high_water) ||
 2757             (pv = get_pv_entry(pmap, noreclaim)) == NULL)
 2758                 return (false);
 2759         pv->pv_va = va;
 2760         pvh = pa_to_pvh(pde & PG_PS_FRAME);
 2761         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 2762         return (true);
 2763 }
 2764 
 2765 /*
 2766  * Fills a page table page with mappings to consecutive physical pages.
 2767  */
 2768 static void
 2769 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
 2770 {
 2771         pt_entry_t *pte;
 2772 
 2773         for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
 2774                 *pte = newpte;  
 2775                 newpte += PAGE_SIZE;
 2776         }
 2777 }
 2778 
 2779 /*
 2780  * Tries to demote a 2- or 4MB page mapping.  If demotion fails, the
 2781  * 2- or 4MB page mapping is invalidated.
 2782  */
 2783 static boolean_t
 2784 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2785 {
 2786         pd_entry_t newpde, oldpde;
 2787         pt_entry_t *firstpte, newpte;
 2788         vm_paddr_t mptepa;
 2789         vm_page_t mpte;
 2790         struct spglist free;
 2791         vm_offset_t sva;
 2792 
 2793         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2794         oldpde = *pde;
 2795         KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
 2796             ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
 2797         if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
 2798             NULL) {
 2799                 KASSERT((oldpde & PG_W) == 0,
 2800                     ("pmap_demote_pde: page table page for a wired mapping"
 2801                     " is missing"));
 2802 
 2803                 /*
 2804                  * Invalidate the 2- or 4MB page mapping and return
 2805                  * "failure" if the mapping was never accessed or the
 2806                  * allocation of the new page table page fails.
 2807                  */
 2808                 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
 2809                     va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL |
 2810                     VM_ALLOC_WIRED)) == NULL) {
 2811                         SLIST_INIT(&free);
 2812                         sva = trunc_4mpage(va);
 2813                         pmap_remove_pde(pmap, pde, sva, &free);
 2814                         if ((oldpde & PG_G) == 0)
 2815                                 pmap_invalidate_pde_page(pmap, sva, oldpde);
 2816                         vm_page_free_pages_toq(&free, true);
 2817                         CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
 2818                             " in pmap %p", va, pmap);
 2819                         return (FALSE);
 2820                 }
 2821                 if (pmap != kernel_pmap) {
 2822                         mpte->ref_count = NPTEPG;
 2823                         pmap->pm_stats.resident_count++;
 2824                 }
 2825         }
 2826         mptepa = VM_PAGE_TO_PHYS(mpte);
 2827 
 2828         /*
 2829          * If the page mapping is in the kernel's address space, then the
 2830          * KPTmap can provide access to the page table page.  Otherwise,
 2831          * temporarily map the page table page (mpte) into the kernel's
 2832          * address space at either PADDR1 or PADDR2. 
 2833          */
 2834         if (pmap == kernel_pmap)
 2835                 firstpte = &KPTmap[i386_btop(trunc_4mpage(va))];
 2836         else if (curthread->td_pinned > 0 && rw_wowned(&pvh_global_lock)) {
 2837                 if ((*PMAP1 & PG_FRAME) != mptepa) {
 2838                         *PMAP1 = mptepa | PG_RW | PG_V | PG_A | PG_M;
 2839 #ifdef SMP
 2840                         PMAP1cpu = PCPU_GET(cpuid);
 2841 #endif
 2842                         invlcaddr(PADDR1);
 2843                         PMAP1changed++;
 2844                 } else
 2845 #ifdef SMP
 2846                 if (PMAP1cpu != PCPU_GET(cpuid)) {
 2847                         PMAP1cpu = PCPU_GET(cpuid);
 2848                         invlcaddr(PADDR1);
 2849                         PMAP1changedcpu++;
 2850                 } else
 2851 #endif
 2852                         PMAP1unchanged++;
 2853                 firstpte = PADDR1;
 2854         } else {
 2855                 mtx_lock(&PMAP2mutex);
 2856                 if ((*PMAP2 & PG_FRAME) != mptepa) {
 2857                         *PMAP2 = mptepa | PG_RW | PG_V | PG_A | PG_M;
 2858                         pmap_invalidate_page_int(kernel_pmap,
 2859                             (vm_offset_t)PADDR2);
 2860                 }
 2861                 firstpte = PADDR2;
 2862         }
 2863         newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
 2864         KASSERT((oldpde & PG_A) != 0,
 2865             ("pmap_demote_pde: oldpde is missing PG_A"));
 2866         KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
 2867             ("pmap_demote_pde: oldpde is missing PG_M"));
 2868         newpte = oldpde & ~PG_PS;
 2869         if ((newpte & PG_PDE_PAT) != 0)
 2870                 newpte ^= PG_PDE_PAT | PG_PTE_PAT;
 2871 
 2872         /*
 2873          * If the page table page is not leftover from an earlier promotion,
 2874          * initialize it.
 2875          */
 2876         if (mpte->valid == 0)
 2877                 pmap_fill_ptp(firstpte, newpte);
 2878 
 2879         KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
 2880             ("pmap_demote_pde: firstpte and newpte map different physical"
 2881             " addresses"));
 2882 
 2883         /*
 2884          * If the mapping has changed attributes, update the page table
 2885          * entries.
 2886          */ 
 2887         if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
 2888                 pmap_fill_ptp(firstpte, newpte);
 2889 
 2890         /*
 2891          * Demote the mapping.  This pmap is locked.  The old PDE has
 2892          * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
 2893          * set.  Thus, there is no danger of a race with another
 2894          * processor changing the setting of PG_A and/or PG_M between
 2895          * the read above and the store below. 
 2896          */
 2897         if (workaround_erratum383)
 2898                 pmap_update_pde(pmap, va, pde, newpde);
 2899         else if (pmap == kernel_pmap)
 2900                 pmap_kenter_pde(va, newpde);
 2901         else
 2902                 pde_store(pde, newpde); 
 2903         if (firstpte == PADDR2)
 2904                 mtx_unlock(&PMAP2mutex);
 2905 
 2906         /*
 2907          * Invalidate the recursive mapping of the page table page.
 2908          */
 2909         pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));
 2910 
 2911         /*
 2912          * Demote the pv entry.  This depends on the earlier demotion
 2913          * of the mapping.  Specifically, the (re)creation of a per-
 2914          * page pv entry might trigger the execution of pmap_collect(),
 2915          * which might reclaim a newly (re)created per-page pv entry
 2916          * and destroy the associated mapping.  In order to destroy
 2917          * the mapping, the PDE must have already changed from mapping
 2918          * the 2mpage to referencing the page table page.
 2919          */
 2920         if ((oldpde & PG_MANAGED) != 0)
 2921                 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);
 2922 
 2923         pmap_pde_demotions++;
 2924         CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#x"
 2925             " in pmap %p", va, pmap);
 2926         return (TRUE);
 2927 }
 2928 
 2929 /*
 2930  * Removes a 2- or 4MB page mapping from the kernel pmap.
 2931  */
 2932 static void
 2933 pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2934 {
 2935         pd_entry_t newpde;
 2936         vm_paddr_t mptepa;
 2937         vm_page_t mpte;
 2938 
 2939         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2940         mpte = pmap_remove_pt_page(pmap, va);
 2941         if (mpte == NULL)
 2942                 panic("pmap_remove_kernel_pde: Missing pt page.");
 2943 
 2944         mptepa = VM_PAGE_TO_PHYS(mpte);
 2945         newpde = mptepa | PG_M | PG_A | PG_RW | PG_V;
 2946 
 2947         /*
 2948          * If this page table page was unmapped by a promotion, then it
 2949          * contains valid mappings.  Zero it to invalidate those mappings.
 2950          */
 2951         if (mpte->valid != 0)
 2952                 pagezero((void *)&KPTmap[i386_btop(trunc_4mpage(va))]);
 2953 
 2954         /*
 2955          * Remove the mapping.
 2956          */
 2957         if (workaround_erratum383)
 2958                 pmap_update_pde(pmap, va, pde, newpde);
 2959         else 
 2960                 pmap_kenter_pde(va, newpde);
 2961 
 2962         /*
 2963          * Invalidate the recursive mapping of the page table page.
 2964          */
 2965         pmap_invalidate_page_int(pmap, (vm_offset_t)vtopte(va));
 2966 }
 2967 
 2968 /*
 2969  * pmap_remove_pde: do the things to unmap a superpage in a process
 2970  */
 2971 static void
 2972 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 2973     struct spglist *free)
 2974 {
 2975         struct md_page *pvh;
 2976         pd_entry_t oldpde;
 2977         vm_offset_t eva, va;
 2978         vm_page_t m, mpte;
 2979 
 2980         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2981         KASSERT((sva & PDRMASK) == 0,
 2982             ("pmap_remove_pde: sva is not 4mpage aligned"));
 2983         oldpde = pte_load_clear(pdq);
 2984         if (oldpde & PG_W)
 2985                 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
 2986 
 2987         /*
 2988          * Machines that don't support invlpg, also don't support
 2989          * PG_G.
 2990          */
 2991         if ((oldpde & PG_G) != 0)
 2992                 pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
 2993 
 2994         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 2995         if (oldpde & PG_MANAGED) {
 2996                 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
 2997                 pmap_pvh_free(pvh, pmap, sva);
 2998                 eva = sva + NBPDR;
 2999                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 3000                     va < eva; va += PAGE_SIZE, m++) {
 3001                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3002                                 vm_page_dirty(m);
 3003                         if (oldpde & PG_A)
 3004                                 vm_page_aflag_set(m, PGA_REFERENCED);
 3005                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 3006                             TAILQ_EMPTY(&pvh->pv_list))
 3007                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 3008                 }
 3009         }
 3010         if (pmap == kernel_pmap) {
 3011                 pmap_remove_kernel_pde(pmap, pdq, sva);
 3012         } else {
 3013                 mpte = pmap_remove_pt_page(pmap, sva);
 3014                 if (mpte != NULL) {
 3015                         KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
 3016                             ("pmap_remove_pde: pte page not promoted"));
 3017                         pmap->pm_stats.resident_count--;
 3018                         KASSERT(mpte->ref_count == NPTEPG,
 3019                             ("pmap_remove_pde: pte page ref count error"));
 3020                         mpte->ref_count = 0;
 3021                         pmap_add_delayed_free_list(mpte, free, FALSE);
 3022                 }
 3023         }
 3024 }
 3025 
 3026 /*
 3027  * pmap_remove_pte: do the things to unmap a page in a process
 3028  */
 3029 static int
 3030 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
 3031     struct spglist *free)
 3032 {
 3033         pt_entry_t oldpte;
 3034         vm_page_t m;
 3035 
 3036         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3037         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3038         oldpte = pte_load_clear(ptq);
 3039         KASSERT(oldpte != 0,
 3040             ("pmap_remove_pte: pmap %p va %x zero pte", pmap, va));
 3041         if (oldpte & PG_W)
 3042                 pmap->pm_stats.wired_count -= 1;
 3043         /*
 3044          * Machines that don't support invlpg, also don't support
 3045          * PG_G.
 3046          */
 3047         if (oldpte & PG_G)
 3048                 pmap_invalidate_page_int(kernel_pmap, va);
 3049         pmap->pm_stats.resident_count -= 1;
 3050         if (oldpte & PG_MANAGED) {
 3051                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 3052                 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3053                         vm_page_dirty(m);
 3054                 if (oldpte & PG_A)
 3055                         vm_page_aflag_set(m, PGA_REFERENCED);
 3056                 pmap_remove_entry(pmap, m, va);
 3057         }
 3058         return (pmap_unuse_pt(pmap, va, free));
 3059 }
 3060 
 3061 /*
 3062  * Remove a single page from a process address space
 3063  */
 3064 static void
 3065 pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)
 3066 {
 3067         pt_entry_t *pte;
 3068 
 3069         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3070         KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 3071         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3072         if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
 3073                 return;
 3074         pmap_remove_pte(pmap, pte, va, free);
 3075         pmap_invalidate_page_int(pmap, va);
 3076 }
 3077 
 3078 /*
 3079  * Removes the specified range of addresses from the page table page.
 3080  */
 3081 static bool
 3082 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
 3083     struct spglist *free)
 3084 {
 3085         pt_entry_t *pte;
 3086         bool anyvalid;
 3087 
 3088         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3089         KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 3090         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3091         anyvalid = false;
 3092         for (pte = pmap_pte_quick(pmap, sva); sva != eva; pte++,
 3093             sva += PAGE_SIZE) {
 3094                 if (*pte == 0)
 3095                         continue;
 3096 
 3097                 /*
 3098                  * The TLB entry for a PG_G mapping is invalidated by
 3099                  * pmap_remove_pte().
 3100                  */
 3101                 if ((*pte & PG_G) == 0)
 3102                         anyvalid = true;
 3103 
 3104                 if (pmap_remove_pte(pmap, pte, sva, free))
 3105                         break;
 3106         }
 3107         return (anyvalid);
 3108 }
 3109 
 3110 /*
 3111  *      Remove the given range of addresses from the specified map.
 3112  *
 3113  *      It is assumed that the start and end are properly
 3114  *      rounded to the page size.
 3115  */
 3116 static void
 3117 __CONCAT(PMTYPE, remove)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 3118 {
 3119         vm_offset_t pdnxt;
 3120         pd_entry_t ptpaddr;
 3121         struct spglist free;
 3122         int anyvalid;
 3123 
 3124         /*
 3125          * Perform an unsynchronized read.  This is, however, safe.
 3126          */
 3127         if (pmap->pm_stats.resident_count == 0)
 3128                 return;
 3129 
 3130         anyvalid = 0;
 3131         SLIST_INIT(&free);
 3132 
 3133         rw_wlock(&pvh_global_lock);
 3134         sched_pin();
 3135         PMAP_LOCK(pmap);
 3136 
 3137         /*
 3138          * special handling of removing one page.  a very
 3139          * common operation and easy to short circuit some
 3140          * code.
 3141          */
 3142         if ((sva + PAGE_SIZE == eva) && 
 3143             ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
 3144                 pmap_remove_page(pmap, sva, &free);
 3145                 goto out;
 3146         }
 3147 
 3148         for (; sva < eva; sva = pdnxt) {
 3149                 u_int pdirindex;
 3150 
 3151                 /*
 3152                  * Calculate index for next page table.
 3153                  */
 3154                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 3155                 if (pdnxt < sva)
 3156                         pdnxt = eva;
 3157                 if (pmap->pm_stats.resident_count == 0)
 3158                         break;
 3159 
 3160                 pdirindex = sva >> PDRSHIFT;
 3161                 ptpaddr = pmap->pm_pdir[pdirindex];
 3162 
 3163                 /*
 3164                  * Weed out invalid mappings. Note: we assume that the page
 3165                  * directory table is always allocated, and in kernel virtual.
 3166                  */
 3167                 if (ptpaddr == 0)
 3168                         continue;
 3169 
 3170                 /*
 3171                  * Check for large page.
 3172                  */
 3173                 if ((ptpaddr & PG_PS) != 0) {
 3174                         /*
 3175                          * Are we removing the entire large page?  If not,
 3176                          * demote the mapping and fall through.
 3177                          */
 3178                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 3179                                 /*
 3180                                  * The TLB entry for a PG_G mapping is
 3181                                  * invalidated by pmap_remove_pde().
 3182                                  */
 3183                                 if ((ptpaddr & PG_G) == 0)
 3184                                         anyvalid = 1;
 3185                                 pmap_remove_pde(pmap,
 3186                                     &pmap->pm_pdir[pdirindex], sva, &free);
 3187                                 continue;
 3188                         } else if (!pmap_demote_pde(pmap,
 3189                             &pmap->pm_pdir[pdirindex], sva)) {
 3190                                 /* The large page mapping was destroyed. */
 3191                                 continue;
 3192                         }
 3193                 }
 3194 
 3195                 /*
 3196                  * Limit our scan to either the end of the va represented
 3197                  * by the current page table page, or to the end of the
 3198                  * range being removed.
 3199                  */
 3200                 if (pdnxt > eva)
 3201                         pdnxt = eva;
 3202 
 3203                 if (pmap_remove_ptes(pmap, sva, pdnxt, &free))
 3204                         anyvalid = 1;
 3205         }
 3206 out:
 3207         sched_unpin();
 3208         if (anyvalid)
 3209                 pmap_invalidate_all_int(pmap);
 3210         rw_wunlock(&pvh_global_lock);
 3211         PMAP_UNLOCK(pmap);
 3212         vm_page_free_pages_toq(&free, true);
 3213 }
 3214 
 3215 /*
 3216  *      Routine:        pmap_remove_all
 3217  *      Function:
 3218  *              Removes this physical page from
 3219  *              all physical maps in which it resides.
 3220  *              Reflects back modify bits to the pager.
 3221  *
 3222  *      Notes:
 3223  *              Original versions of this routine were very
 3224  *              inefficient because they iteratively called
 3225  *              pmap_remove (slow...)
 3226  */
 3227 
 3228 static void
 3229 __CONCAT(PMTYPE, remove_all)(vm_page_t m)
 3230 {
 3231         struct md_page *pvh;
 3232         pv_entry_t pv;
 3233         pmap_t pmap;
 3234         pt_entry_t *pte, tpte;
 3235         pd_entry_t *pde;
 3236         vm_offset_t va;
 3237         struct spglist free;
 3238 
 3239         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3240             ("pmap_remove_all: page %p is not managed", m));
 3241         SLIST_INIT(&free);
 3242         rw_wlock(&pvh_global_lock);
 3243         sched_pin();
 3244         if ((m->flags & PG_FICTITIOUS) != 0)
 3245                 goto small_mappings;
 3246         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 3247         while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
 3248                 va = pv->pv_va;
 3249                 pmap = PV_PMAP(pv);
 3250                 PMAP_LOCK(pmap);
 3251                 pde = pmap_pde(pmap, va);
 3252                 (void)pmap_demote_pde(pmap, pde, va);
 3253                 PMAP_UNLOCK(pmap);
 3254         }
 3255 small_mappings:
 3256         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 3257                 pmap = PV_PMAP(pv);
 3258                 PMAP_LOCK(pmap);
 3259                 pmap->pm_stats.resident_count--;
 3260                 pde = pmap_pde(pmap, pv->pv_va);
 3261                 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
 3262                     " a 4mpage in page %p's pv list", m));
 3263                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3264                 tpte = pte_load_clear(pte);
 3265                 KASSERT(tpte != 0, ("pmap_remove_all: pmap %p va %x zero pte",
 3266                     pmap, pv->pv_va));
 3267                 if (tpte & PG_W)
 3268                         pmap->pm_stats.wired_count--;
 3269                 if (tpte & PG_A)
 3270                         vm_page_aflag_set(m, PGA_REFERENCED);
 3271 
 3272                 /*
 3273                  * Update the vm_page_t clean and reference bits.
 3274                  */
 3275                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3276                         vm_page_dirty(m);
 3277                 pmap_unuse_pt(pmap, pv->pv_va, &free);
 3278                 pmap_invalidate_page_int(pmap, pv->pv_va);
 3279                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 3280                 free_pv_entry(pmap, pv);
 3281                 PMAP_UNLOCK(pmap);
 3282         }
 3283         vm_page_aflag_clear(m, PGA_WRITEABLE);
 3284         sched_unpin();
 3285         rw_wunlock(&pvh_global_lock);
 3286         vm_page_free_pages_toq(&free, true);
 3287 }
 3288 
 3289 /*
 3290  * pmap_protect_pde: do the things to protect a 4mpage in a process
 3291  */
 3292 static boolean_t
 3293 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
 3294 {
 3295         pd_entry_t newpde, oldpde;
 3296         vm_page_t m, mt;
 3297         boolean_t anychanged;
 3298 
 3299         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3300         KASSERT((sva & PDRMASK) == 0,
 3301             ("pmap_protect_pde: sva is not 4mpage aligned"));
 3302         anychanged = FALSE;
 3303 retry:
 3304         oldpde = newpde = *pde;
 3305         if ((prot & VM_PROT_WRITE) == 0) {
 3306                 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
 3307                     (PG_MANAGED | PG_M | PG_RW)) {
 3308                         m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 3309                         for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 3310                                 vm_page_dirty(mt);
 3311                 }
 3312                 newpde &= ~(PG_RW | PG_M);
 3313         }
 3314 #ifdef PMAP_PAE_COMP
 3315         if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
 3316                 newpde |= pg_nx;
 3317 #endif
 3318         if (newpde != oldpde) {
 3319                 /*
 3320                  * As an optimization to future operations on this PDE, clear
 3321                  * PG_PROMOTED.  The impending invalidation will remove any
 3322                  * lingering 4KB page mappings from the TLB.
 3323                  */
 3324                 if (!pde_cmpset(pde, oldpde, newpde & ~PG_PROMOTED))
 3325                         goto retry;
 3326                 if ((oldpde & PG_G) != 0)
 3327                         pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
 3328                 else
 3329                         anychanged = TRUE;
 3330         }
 3331         return (anychanged);
 3332 }
 3333 
 3334 /*
 3335  *      Set the physical protection on the
 3336  *      specified range of this map as requested.
 3337  */
 3338 static void
 3339 __CONCAT(PMTYPE, protect)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
 3340     vm_prot_t prot)
 3341 {
 3342         vm_offset_t pdnxt;
 3343         pd_entry_t ptpaddr;
 3344         pt_entry_t *pte;
 3345         boolean_t anychanged, pv_lists_locked;
 3346 
 3347         KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
 3348         if (prot == VM_PROT_NONE) {
 3349                 pmap_remove(pmap, sva, eva);
 3350                 return;
 3351         }
 3352 
 3353 #ifdef PMAP_PAE_COMP
 3354         if ((prot & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
 3355             (VM_PROT_WRITE | VM_PROT_EXECUTE))
 3356                 return;
 3357 #else
 3358         if (prot & VM_PROT_WRITE)
 3359                 return;
 3360 #endif
 3361 
 3362         if (pmap_is_current(pmap))
 3363                 pv_lists_locked = FALSE;
 3364         else {
 3365                 pv_lists_locked = TRUE;
 3366 resume:
 3367                 rw_wlock(&pvh_global_lock);
 3368                 sched_pin();
 3369         }
 3370         anychanged = FALSE;
 3371 
 3372         PMAP_LOCK(pmap);
 3373         for (; sva < eva; sva = pdnxt) {
 3374                 pt_entry_t obits, pbits;
 3375                 u_int pdirindex;
 3376 
 3377                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 3378                 if (pdnxt < sva)
 3379                         pdnxt = eva;
 3380 
 3381                 pdirindex = sva >> PDRSHIFT;
 3382                 ptpaddr = pmap->pm_pdir[pdirindex];
 3383 
 3384                 /*
 3385                  * Weed out invalid mappings. Note: we assume that the page
 3386                  * directory table is always allocated, and in kernel virtual.
 3387                  */
 3388                 if (ptpaddr == 0)
 3389                         continue;
 3390 
 3391                 /*
 3392                  * Check for large page.
 3393                  */
 3394                 if ((ptpaddr & PG_PS) != 0) {
 3395                         /*
 3396                          * Are we protecting the entire large page?  If not,
 3397                          * demote the mapping and fall through.
 3398                          */
 3399                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 3400                                 /*
 3401                                  * The TLB entry for a PG_G mapping is
 3402                                  * invalidated by pmap_protect_pde().
 3403                                  */
 3404                                 if (pmap_protect_pde(pmap,
 3405                                     &pmap->pm_pdir[pdirindex], sva, prot))
 3406                                         anychanged = TRUE;
 3407                                 continue;
 3408                         } else {
 3409                                 if (!pv_lists_locked) {
 3410                                         pv_lists_locked = TRUE;
 3411                                         if (!rw_try_wlock(&pvh_global_lock)) {
 3412                                                 if (anychanged)
 3413                                                         pmap_invalidate_all_int(
 3414                                                             pmap);
 3415                                                 PMAP_UNLOCK(pmap);
 3416                                                 goto resume;
 3417                                         }
 3418                                         sched_pin();
 3419                                 }
 3420                                 if (!pmap_demote_pde(pmap,
 3421                                     &pmap->pm_pdir[pdirindex], sva)) {
 3422                                         /*
 3423                                          * The large page mapping was
 3424                                          * destroyed.
 3425                                          */
 3426                                         continue;
 3427                                 }
 3428                         }
 3429                 }
 3430 
 3431                 if (pdnxt > eva)
 3432                         pdnxt = eva;
 3433 
 3434                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 3435                     sva += PAGE_SIZE) {
 3436                         vm_page_t m;
 3437 
 3438 retry:
 3439                         /*
 3440                          * Regardless of whether a pte is 32 or 64 bits in
 3441                          * size, PG_RW, PG_A, and PG_M are among the least
 3442                          * significant 32 bits.
 3443                          */
 3444                         obits = pbits = *pte;
 3445                         if ((pbits & PG_V) == 0)
 3446                                 continue;
 3447 
 3448                         if ((prot & VM_PROT_WRITE) == 0) {
 3449                                 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
 3450                                     (PG_MANAGED | PG_M | PG_RW)) {
 3451                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 3452                                         vm_page_dirty(m);
 3453                                 }
 3454                                 pbits &= ~(PG_RW | PG_M);
 3455                         }
 3456 #ifdef PMAP_PAE_COMP
 3457                         if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
 3458                                 pbits |= pg_nx;
 3459 #endif
 3460 
 3461                         if (pbits != obits) {
 3462 #ifdef PMAP_PAE_COMP
 3463                                 if (!atomic_cmpset_64(pte, obits, pbits))
 3464                                         goto retry;
 3465 #else
 3466                                 if (!atomic_cmpset_int((u_int *)pte, obits,
 3467                                     pbits))
 3468                                         goto retry;
 3469 #endif
 3470                                 if (obits & PG_G)
 3471                                         pmap_invalidate_page_int(pmap, sva);
 3472                                 else
 3473                                         anychanged = TRUE;
 3474                         }
 3475                 }
 3476         }
 3477         if (anychanged)
 3478                 pmap_invalidate_all_int(pmap);
 3479         if (pv_lists_locked) {
 3480                 sched_unpin();
 3481                 rw_wunlock(&pvh_global_lock);
 3482         }
 3483         PMAP_UNLOCK(pmap);
 3484 }
 3485 
 3486 #if VM_NRESERVLEVEL > 0
 3487 /*
 3488  * Tries to promote the 512 or 1024, contiguous 4KB page mappings that are
 3489  * within a single page table page (PTP) to a single 2- or 4MB page mapping.
 3490  * For promotion to occur, two conditions must be met: (1) the 4KB page
 3491  * mappings must map aligned, contiguous physical memory and (2) the 4KB page
 3492  * mappings must have identical characteristics.
 3493  *
 3494  * Managed (PG_MANAGED) mappings within the kernel address space are not
 3495  * promoted.  The reason is that kernel PDEs are replicated in each pmap but
 3496  * pmap_clear_ptes() and pmap_ts_referenced() only read the PDE from the kernel
 3497  * pmap.
 3498  */
 3499 static void
 3500 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 3501 {
 3502         pd_entry_t newpde;
 3503         pt_entry_t *firstpte, oldpte, pa, *pte;
 3504         vm_offset_t oldpteva;
 3505         vm_page_t mpte;
 3506 
 3507         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3508 
 3509         /*
 3510          * Examine the first PTE in the specified PTP.  Abort if this PTE is
 3511          * either invalid, unused, or does not map the first 4KB physical page
 3512          * within a 2- or 4MB page.
 3513          */
 3514         firstpte = pmap_pte_quick(pmap, trunc_4mpage(va));
 3515 setpde:
 3516         newpde = *firstpte;
 3517         if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
 3518                 pmap_pde_p_failures++;
 3519                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3520                     " in pmap %p", va, pmap);
 3521                 return;
 3522         }
 3523         if ((*firstpte & PG_MANAGED) != 0 && pmap == kernel_pmap) {
 3524                 pmap_pde_p_failures++;
 3525                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3526                     " in pmap %p", va, pmap);
 3527                 return;
 3528         }
 3529         if ((newpde & (PG_M | PG_RW)) == PG_RW) {
 3530                 /*
 3531                  * When PG_M is already clear, PG_RW can be cleared without
 3532                  * a TLB invalidation.
 3533                  */
 3534                 if (!atomic_cmpset_int((u_int *)firstpte, newpde, newpde &
 3535                     ~PG_RW))  
 3536                         goto setpde;
 3537                 newpde &= ~PG_RW;
 3538         }
 3539 
 3540         /* 
 3541          * Examine each of the other PTEs in the specified PTP.  Abort if this
 3542          * PTE maps an unexpected 4KB physical page or does not have identical
 3543          * characteristics to the first PTE.
 3544          */
 3545         pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + NBPDR - PAGE_SIZE;
 3546         for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
 3547 setpte:
 3548                 oldpte = *pte;
 3549                 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
 3550                         pmap_pde_p_failures++;
 3551                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3552                             " in pmap %p", va, pmap);
 3553                         return;
 3554                 }
 3555                 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
 3556                         /*
 3557                          * When PG_M is already clear, PG_RW can be cleared
 3558                          * without a TLB invalidation.
 3559                          */
 3560                         if (!atomic_cmpset_int((u_int *)pte, oldpte,
 3561                             oldpte & ~PG_RW))
 3562                                 goto setpte;
 3563                         oldpte &= ~PG_RW;
 3564                         oldpteva = (oldpte & PG_FRAME & PDRMASK) |
 3565                             (va & ~PDRMASK);
 3566                         CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#x"
 3567                             " in pmap %p", oldpteva, pmap);
 3568                 }
 3569                 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
 3570                         pmap_pde_p_failures++;
 3571                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3572                             " in pmap %p", va, pmap);
 3573                         return;
 3574                 }
 3575                 pa -= PAGE_SIZE;
 3576         }
 3577 
 3578         /*
 3579          * Save the page table page in its current state until the PDE
 3580          * mapping the superpage is demoted by pmap_demote_pde() or
 3581          * destroyed by pmap_remove_pde(). 
 3582          */
 3583         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 3584         KASSERT(mpte >= vm_page_array &&
 3585             mpte < &vm_page_array[vm_page_array_size],
 3586             ("pmap_promote_pde: page table page is out of range"));
 3587         KASSERT(mpte->pindex == va >> PDRSHIFT,
 3588             ("pmap_promote_pde: page table page's pindex is wrong"));
 3589         if (pmap_insert_pt_page(pmap, mpte, true)) {
 3590                 pmap_pde_p_failures++;
 3591                 CTR2(KTR_PMAP,
 3592                     "pmap_promote_pde: failure for va %#x in pmap %p", va,
 3593                     pmap);
 3594                 return;
 3595         }
 3596 
 3597         /*
 3598          * Promote the pv entries.
 3599          */
 3600         if ((newpde & PG_MANAGED) != 0)
 3601                 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
 3602 
 3603         /*
 3604          * Propagate the PAT index to its proper position.
 3605          */
 3606         if ((newpde & PG_PTE_PAT) != 0)
 3607                 newpde ^= PG_PDE_PAT | PG_PTE_PAT;
 3608 
 3609         /*
 3610          * Map the superpage.
 3611          */
 3612         if (workaround_erratum383)
 3613                 pmap_update_pde(pmap, va, pde, PG_PS | newpde);
 3614         else if (pmap == kernel_pmap)
 3615                 pmap_kenter_pde(va, PG_PROMOTED | PG_PS | newpde);
 3616         else
 3617                 pde_store(pde, PG_PROMOTED | PG_PS | newpde);
 3618 
 3619         pmap_pde_promotions++;
 3620         CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x"
 3621             " in pmap %p", va, pmap);
 3622 }
 3623 #endif /* VM_NRESERVLEVEL > 0 */
 3624 
 3625 /*
 3626  *      Insert the given physical page (p) at
 3627  *      the specified virtual address (v) in the
 3628  *      target physical map with the protection requested.
 3629  *
 3630  *      If specified, the page will be wired down, meaning
 3631  *      that the related pte can not be reclaimed.
 3632  *
 3633  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 3634  *      or lose information.  That is, this routine must actually
 3635  *      insert this page into the given map NOW.
 3636  */
 3637 static int
 3638 __CONCAT(PMTYPE, enter)(pmap_t pmap, vm_offset_t va, vm_page_t m,
 3639     vm_prot_t prot, u_int flags, int8_t psind)
 3640 {
 3641         pd_entry_t *pde;
 3642         pt_entry_t *pte;
 3643         pt_entry_t newpte, origpte;
 3644         pv_entry_t pv;
 3645         vm_paddr_t opa, pa;
 3646         vm_page_t mpte, om;
 3647         int rv;
 3648 
 3649         va = trunc_page(va);
 3650         KASSERT((pmap == kernel_pmap && va < VM_MAX_KERNEL_ADDRESS) ||
 3651             (pmap != kernel_pmap && va < VM_MAXUSER_ADDRESS),
 3652             ("pmap_enter: toobig k%d %#x", pmap == kernel_pmap, va));
 3653         KASSERT(va < PMAP_TRM_MIN_ADDRESS,
 3654             ("pmap_enter: invalid to pmap_enter into trampoline (va: 0x%x)",
 3655             va));
 3656         KASSERT(pmap != kernel_pmap || (m->oflags & VPO_UNMANAGED) != 0 ||
 3657             va < kmi.clean_sva || va >= kmi.clean_eva,
 3658             ("pmap_enter: managed mapping within the clean submap"));
 3659         if ((m->oflags & VPO_UNMANAGED) == 0)
 3660                 VM_PAGE_OBJECT_BUSY_ASSERT(m);
 3661         KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
 3662             ("pmap_enter: flags %u has reserved bits set", flags));
 3663         pa = VM_PAGE_TO_PHYS(m);
 3664         newpte = (pt_entry_t)(pa | PG_A | PG_V);
 3665         if ((flags & VM_PROT_WRITE) != 0)
 3666                 newpte |= PG_M;
 3667         if ((prot & VM_PROT_WRITE) != 0)
 3668                 newpte |= PG_RW;
 3669         KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
 3670             ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
 3671 #ifdef PMAP_PAE_COMP
 3672         if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
 3673                 newpte |= pg_nx;
 3674 #endif
 3675         if ((flags & PMAP_ENTER_WIRED) != 0)
 3676                 newpte |= PG_W;
 3677         if (pmap != kernel_pmap)
 3678                 newpte |= PG_U;
 3679         newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);
 3680         if ((m->oflags & VPO_UNMANAGED) == 0)
 3681                 newpte |= PG_MANAGED;
 3682 
 3683         rw_wlock(&pvh_global_lock);
 3684         PMAP_LOCK(pmap);
 3685         sched_pin();
 3686         if (psind == 1) {
 3687                 /* Assert the required virtual and physical alignment. */ 
 3688                 KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned"));
 3689                 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
 3690                 rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m);
 3691                 goto out;
 3692         }
 3693 
 3694         pde = pmap_pde(pmap, va);
 3695         if (pmap != kernel_pmap) {
 3696                 /*
 3697                  * va is for UVA.
 3698                  * In the case that a page table page is not resident,
 3699                  * we are creating it here.  pmap_allocpte() handles
 3700                  * demotion.
 3701                  */
 3702                 mpte = pmap_allocpte(pmap, va, flags);
 3703                 if (mpte == NULL) {
 3704                         KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0,
 3705                             ("pmap_allocpte failed with sleep allowed"));
 3706                         rv = KERN_RESOURCE_SHORTAGE;
 3707                         goto out;
 3708                 }
 3709         } else {
 3710                 /*
 3711                  * va is for KVA, so pmap_demote_pde() will never fail
 3712                  * to install a page table page.  PG_V is also
 3713                  * asserted by pmap_demote_pde().
 3714                  */
 3715                 mpte = NULL;
 3716                 KASSERT(pde != NULL && (*pde & PG_V) != 0,
 3717                     ("KVA %#x invalid pde pdir %#jx", va,
 3718                     (uintmax_t)pmap->pm_pdir[PTDPTDI]));
 3719                 if ((*pde & PG_PS) != 0)
 3720                         pmap_demote_pde(pmap, pde, va);
 3721         }
 3722         pte = pmap_pte_quick(pmap, va);
 3723 
 3724         /*
 3725          * Page Directory table entry is not valid, which should not
 3726          * happen.  We should have either allocated the page table
 3727          * page or demoted the existing mapping above.
 3728          */
 3729         if (pte == NULL) {
 3730                 panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x",
 3731                     (uintmax_t)pmap->pm_pdir[PTDPTDI], va);
 3732         }
 3733 
 3734         origpte = *pte;
 3735         pv = NULL;
 3736 
 3737         /*
 3738          * Is the specified virtual address already mapped?
 3739          */
 3740         if ((origpte & PG_V) != 0) {
 3741                 /*
 3742                  * Wiring change, just update stats. We don't worry about
 3743                  * wiring PT pages as they remain resident as long as there
 3744                  * are valid mappings in them. Hence, if a user page is wired,
 3745                  * the PT page will be also.
 3746                  */
 3747                 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
 3748                         pmap->pm_stats.wired_count++;
 3749                 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
 3750                         pmap->pm_stats.wired_count--;
 3751 
 3752                 /*
 3753                  * Remove the extra PT page reference.
 3754                  */
 3755                 if (mpte != NULL) {
 3756                         mpte->ref_count--;
 3757                         KASSERT(mpte->ref_count > 0,
 3758                             ("pmap_enter: missing reference to page table page,"
 3759                              " va: 0x%x", va));
 3760                 }
 3761 
 3762                 /*
 3763                  * Has the physical page changed?
 3764                  */
 3765                 opa = origpte & PG_FRAME;
 3766                 if (opa == pa) {
 3767                         /*
 3768                          * No, might be a protection or wiring change.
 3769                          */
 3770                         if ((origpte & PG_MANAGED) != 0 &&
 3771                             (newpte & PG_RW) != 0)
 3772                                 vm_page_aflag_set(m, PGA_WRITEABLE);
 3773                         if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
 3774                                 goto unchanged;
 3775                         goto validate;
 3776                 }
 3777 
 3778                 /*
 3779                  * The physical page has changed.  Temporarily invalidate
 3780                  * the mapping.  This ensures that all threads sharing the
 3781                  * pmap keep a consistent view of the mapping, which is
 3782                  * necessary for the correct handling of COW faults.  It
 3783                  * also permits reuse of the old mapping's PV entry,
 3784                  * avoiding an allocation.
 3785                  *
 3786                  * For consistency, handle unmanaged mappings the same way.
 3787                  */
 3788                 origpte = pte_load_clear(pte);
 3789                 KASSERT((origpte & PG_FRAME) == opa,
 3790                     ("pmap_enter: unexpected pa update for %#x", va));
 3791                 if ((origpte & PG_MANAGED) != 0) {
 3792                         om = PHYS_TO_VM_PAGE(opa);
 3793 
 3794                         /*
 3795                          * The pmap lock is sufficient to synchronize with
 3796                          * concurrent calls to pmap_page_test_mappings() and
 3797                          * pmap_ts_referenced().
 3798                          */
 3799                         if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3800                                 vm_page_dirty(om);
 3801                         if ((origpte & PG_A) != 0) {
 3802                                 pmap_invalidate_page_int(pmap, va);
 3803                                 vm_page_aflag_set(om, PGA_REFERENCED);
 3804                         }
 3805                         pv = pmap_pvh_remove(&om->md, pmap, va);
 3806                         KASSERT(pv != NULL,
 3807                             ("pmap_enter: no PV entry for %#x", va));
 3808                         if ((newpte & PG_MANAGED) == 0)
 3809                                 free_pv_entry(pmap, pv);
 3810                         if ((om->a.flags & PGA_WRITEABLE) != 0 &&
 3811                             TAILQ_EMPTY(&om->md.pv_list) &&
 3812                             ((om->flags & PG_FICTITIOUS) != 0 ||
 3813                             TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
 3814                                 vm_page_aflag_clear(om, PGA_WRITEABLE);
 3815                 } else {
 3816                         /*
 3817                          * Since this mapping is unmanaged, assume that PG_A
 3818                          * is set.
 3819                          */
 3820                         pmap_invalidate_page_int(pmap, va);
 3821                 }
 3822                 origpte = 0;
 3823         } else {
 3824                 /*
 3825                  * Increment the counters.
 3826                  */
 3827                 if ((newpte & PG_W) != 0)
 3828                         pmap->pm_stats.wired_count++;
 3829                 pmap->pm_stats.resident_count++;
 3830         }
 3831 
 3832         /*
 3833          * Enter on the PV list if part of our managed memory.
 3834          */
 3835         if ((newpte & PG_MANAGED) != 0) {
 3836                 if (pv == NULL) {
 3837                         pv = get_pv_entry(pmap, FALSE);
 3838                         pv->pv_va = va;
 3839                 }
 3840                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 3841                 if ((newpte & PG_RW) != 0)
 3842                         vm_page_aflag_set(m, PGA_WRITEABLE);
 3843         }
 3844 
 3845         /*
 3846          * Update the PTE.
 3847          */
 3848         if ((origpte & PG_V) != 0) {
 3849 validate:
 3850                 origpte = pte_load_store(pte, newpte);
 3851                 KASSERT((origpte & PG_FRAME) == pa,
 3852                     ("pmap_enter: unexpected pa update for %#x", va));
 3853                 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
 3854                     (PG_M | PG_RW)) {
 3855                         if ((origpte & PG_MANAGED) != 0)
 3856                                 vm_page_dirty(m);
 3857 
 3858                         /*
 3859                          * Although the PTE may still have PG_RW set, TLB
 3860                          * invalidation may nonetheless be required because
 3861                          * the PTE no longer has PG_M set.
 3862                          */
 3863                 }
 3864 #ifdef PMAP_PAE_COMP
 3865                 else if ((origpte & PG_NX) != 0 || (newpte & PG_NX) == 0) {
 3866                         /*
 3867                          * This PTE change does not require TLB invalidation.
 3868                          */
 3869                         goto unchanged;
 3870                 }
 3871 #endif
 3872                 if ((origpte & PG_A) != 0)
 3873                         pmap_invalidate_page_int(pmap, va);
 3874         } else
 3875                 pte_store_zero(pte, newpte);
 3876 
 3877 unchanged:
 3878 
 3879 #if VM_NRESERVLEVEL > 0
 3880         /*
 3881          * If both the page table page and the reservation are fully
 3882          * populated, then attempt promotion.
 3883          */
 3884         if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
 3885             pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 &&
 3886             vm_reserv_level_iffullpop(m) == 0)
 3887                 pmap_promote_pde(pmap, pde, va);
 3888 #endif
 3889 
 3890         rv = KERN_SUCCESS;
 3891 out:
 3892         sched_unpin();
 3893         rw_wunlock(&pvh_global_lock);
 3894         PMAP_UNLOCK(pmap);
 3895         return (rv);
 3896 }
 3897 
 3898 /*
 3899  * Tries to create a read- and/or execute-only 2 or 4 MB page mapping.  Returns
 3900  * true if successful.  Returns false if (1) a mapping already exists at the
 3901  * specified virtual address or (2) a PV entry cannot be allocated without
 3902  * reclaiming another PV entry.
 3903  */
 3904 static bool
 3905 pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 3906 {
 3907         pd_entry_t newpde;
 3908 
 3909         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3910         newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 1) |
 3911             PG_PS | PG_V;
 3912         if ((m->oflags & VPO_UNMANAGED) == 0)
 3913                 newpde |= PG_MANAGED;
 3914 #ifdef PMAP_PAE_COMP
 3915         if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
 3916                 newpde |= pg_nx;
 3917 #endif
 3918         if (pmap != kernel_pmap)
 3919                 newpde |= PG_U;
 3920         return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
 3921             PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL) ==
 3922             KERN_SUCCESS);
 3923 }
 3924 
 3925 /*
 3926  * Returns true if every page table entry in the page table page that maps
 3927  * the specified kernel virtual address is zero.
 3928  */
 3929 static bool
 3930 pmap_every_pte_zero(vm_offset_t va)
 3931 {
 3932         pt_entry_t *pt_end, *pte;
 3933 
 3934         KASSERT((va & PDRMASK) == 0, ("va is misaligned"));
 3935         pte = vtopte(va);
 3936         for (pt_end = pte + NPTEPG; pte < pt_end; pte++) {
 3937                 if (*pte != 0)
 3938                         return (false);
 3939         }
 3940         return (true);
 3941 }
 3942 
 3943 /*
 3944  * Tries to create the specified 2 or 4 MB page mapping.  Returns KERN_SUCCESS
 3945  * if the mapping was created, and either KERN_FAILURE or
 3946  * KERN_RESOURCE_SHORTAGE otherwise.  Returns KERN_FAILURE if
 3947  * PMAP_ENTER_NOREPLACE was specified and a mapping already exists at the
 3948  * specified virtual address.  Returns KERN_RESOURCE_SHORTAGE if
 3949  * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
 3950  *
 3951  * The parameter "m" is only used when creating a managed, writeable mapping.
 3952  */
 3953 static int
 3954 pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,
 3955     vm_page_t m)
 3956 {
 3957         struct spglist free;
 3958         pd_entry_t oldpde, *pde;
 3959         vm_page_t mt;
 3960 
 3961         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3962         KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
 3963             ("pmap_enter_pde: newpde is missing PG_M"));
 3964         KASSERT(pmap == kernel_pmap || (newpde & PG_W) == 0,
 3965             ("pmap_enter_pde: cannot create wired user mapping"));
 3966         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3967         pde = pmap_pde(pmap, va);
 3968         oldpde = *pde;
 3969         if ((oldpde & PG_V) != 0) {
 3970                 if ((flags & PMAP_ENTER_NOREPLACE) != 0 && (pmap !=
 3971                     kernel_pmap || (oldpde & PG_PS) != 0 ||
 3972                     !pmap_every_pte_zero(va))) {
 3973                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3974                             " in pmap %p", va, pmap);
 3975                         return (KERN_FAILURE);
 3976                 }
 3977                 /* Break the existing mapping(s). */
 3978                 SLIST_INIT(&free);
 3979                 if ((oldpde & PG_PS) != 0) {
 3980                         /*
 3981                          * If the PDE resulted from a promotion, then a
 3982                          * reserved PT page could be freed.
 3983                          */
 3984                         (void)pmap_remove_pde(pmap, pde, va, &free);
 3985                         if ((oldpde & PG_G) == 0)
 3986                                 pmap_invalidate_pde_page(pmap, va, oldpde);
 3987                 } else {
 3988                         if (pmap_remove_ptes(pmap, va, va + NBPDR, &free))
 3989                                pmap_invalidate_all_int(pmap);
 3990                 }
 3991                 if (pmap != kernel_pmap) {
 3992                         vm_page_free_pages_toq(&free, true);
 3993                         KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
 3994                             pde));
 3995                 } else {
 3996                         KASSERT(SLIST_EMPTY(&free),
 3997                             ("pmap_enter_pde: freed kernel page table page"));
 3998 
 3999                         /*
 4000                          * Both pmap_remove_pde() and pmap_remove_ptes() will
 4001                          * leave the kernel page table page zero filled.
 4002                          */
 4003                         mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 4004                         if (pmap_insert_pt_page(pmap, mt, false))
 4005                                 panic("pmap_enter_pde: trie insert failed");
 4006                 }
 4007         }
 4008         if ((newpde & PG_MANAGED) != 0) {
 4009                 /*
 4010                  * Abort this mapping if its PV entry could not be created.
 4011                  */
 4012                 if (!pmap_pv_insert_pde(pmap, va, newpde, flags)) {
 4013                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 4014                             " in pmap %p", va, pmap);
 4015                         return (KERN_RESOURCE_SHORTAGE);
 4016                 }
 4017                 if ((newpde & PG_RW) != 0) {
 4018                         for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4019                                 vm_page_aflag_set(mt, PGA_WRITEABLE);
 4020                 }
 4021         }
 4022 
 4023         /*
 4024          * Increment counters.
 4025          */
 4026         if ((newpde & PG_W) != 0)
 4027                 pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;
 4028         pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
 4029 
 4030         /*
 4031          * Map the superpage.  (This is not a promoted mapping; there will not
 4032          * be any lingering 4KB page mappings in the TLB.)
 4033          */
 4034         pde_store(pde, newpde);
 4035 
 4036         pmap_pde_mappings++;
 4037         CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx in pmap %p",
 4038             va, pmap);
 4039         return (KERN_SUCCESS);
 4040 }
 4041 
 4042 /*
 4043  * Maps a sequence of resident pages belonging to the same object.
 4044  * The sequence begins with the given page m_start.  This page is
 4045  * mapped at the given virtual address start.  Each subsequent page is
 4046  * mapped at a virtual address that is offset from start by the same
 4047  * amount as the page is offset from m_start within the object.  The
 4048  * last page in the sequence is the page with the largest offset from
 4049  * m_start that can be mapped at a virtual address less than the given
 4050  * virtual address end.  Not every virtual page between start and end
 4051  * is mapped; only those for which a resident page exists with the
 4052  * corresponding offset from m_start are mapped.
 4053  */
 4054 static void
 4055 __CONCAT(PMTYPE, enter_object)(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 4056     vm_page_t m_start, vm_prot_t prot)
 4057 {
 4058         vm_offset_t va;
 4059         vm_page_t m, mpte;
 4060         vm_pindex_t diff, psize;
 4061 
 4062         VM_OBJECT_ASSERT_LOCKED(m_start->object);
 4063 
 4064         psize = atop(end - start);
 4065         mpte = NULL;
 4066         m = m_start;
 4067         rw_wlock(&pvh_global_lock);
 4068         PMAP_LOCK(pmap);
 4069         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 4070                 va = start + ptoa(diff);
 4071                 if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
 4072                     m->psind == 1 && pg_ps_enabled &&
 4073                     pmap_enter_4mpage(pmap, va, m, prot))
 4074                         m = &m[NBPDR / PAGE_SIZE - 1];
 4075                 else
 4076                         mpte = pmap_enter_quick_locked(pmap, va, m, prot,
 4077                             mpte);
 4078                 m = TAILQ_NEXT(m, listq);
 4079         }
 4080         rw_wunlock(&pvh_global_lock);
 4081         PMAP_UNLOCK(pmap);
 4082 }
 4083 
 4084 /*
 4085  * this code makes some *MAJOR* assumptions:
 4086  * 1. Current pmap & pmap exists.
 4087  * 2. Not wired.
 4088  * 3. Read access.
 4089  * 4. No page table pages.
 4090  * but is *MUCH* faster than pmap_enter...
 4091  */
 4092 
 4093 static void
 4094 __CONCAT(PMTYPE, enter_quick)(pmap_t pmap, vm_offset_t va, vm_page_t m,
 4095     vm_prot_t prot)
 4096 {
 4097 
 4098         rw_wlock(&pvh_global_lock);
 4099         PMAP_LOCK(pmap);
 4100         (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);
 4101         rw_wunlock(&pvh_global_lock);
 4102         PMAP_UNLOCK(pmap);
 4103 }
 4104 
 4105 static vm_page_t
 4106 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 4107     vm_prot_t prot, vm_page_t mpte)
 4108 {
 4109         pt_entry_t newpte, *pte;
 4110 
 4111         KASSERT(pmap != kernel_pmap || va < kmi.clean_sva ||
 4112             va >= kmi.clean_eva || (m->oflags & VPO_UNMANAGED) != 0,
 4113             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 4114         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4115         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4116 
 4117         /*
 4118          * In the case that a page table page is not
 4119          * resident, we are creating it here.
 4120          */
 4121         if (pmap != kernel_pmap) {
 4122                 u_int ptepindex;
 4123                 pd_entry_t ptepa;
 4124 
 4125                 /*
 4126                  * Calculate pagetable page index
 4127                  */
 4128                 ptepindex = va >> PDRSHIFT;
 4129                 if (mpte && (mpte->pindex == ptepindex)) {
 4130                         mpte->ref_count++;
 4131                 } else {
 4132                         /*
 4133                          * Get the page directory entry
 4134                          */
 4135                         ptepa = pmap->pm_pdir[ptepindex];
 4136 
 4137                         /*
 4138                          * If the page table page is mapped, we just increment
 4139                          * the hold count, and activate it.
 4140                          */
 4141                         if (ptepa) {
 4142                                 if (ptepa & PG_PS)
 4143                                         return (NULL);
 4144                                 mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 4145                                 mpte->ref_count++;
 4146                         } else {
 4147                                 mpte = _pmap_allocpte(pmap, ptepindex,
 4148                                     PMAP_ENTER_NOSLEEP);
 4149                                 if (mpte == NULL)
 4150                                         return (mpte);
 4151                         }
 4152                 }
 4153         } else {
 4154                 mpte = NULL;
 4155         }
 4156 
 4157         sched_pin();
 4158         pte = pmap_pte_quick(pmap, va);
 4159         if (*pte) {
 4160                 if (mpte != NULL)
 4161                         mpte->ref_count--;
 4162                 sched_unpin();
 4163                 return (NULL);
 4164         }
 4165 
 4166         /*
 4167          * Enter on the PV list if part of our managed memory.
 4168          */
 4169         if ((m->oflags & VPO_UNMANAGED) == 0 &&
 4170             !pmap_try_insert_pv_entry(pmap, va, m)) {
 4171                 if (mpte != NULL)
 4172                         pmap_abort_ptp(pmap, va, mpte);
 4173                 sched_unpin();
 4174                 return (NULL);
 4175         }
 4176 
 4177         /*
 4178          * Increment counters
 4179          */
 4180         pmap->pm_stats.resident_count++;
 4181 
 4182         newpte = VM_PAGE_TO_PHYS(m) | PG_V |
 4183             pmap_cache_bits(pmap, m->md.pat_mode, 0);
 4184         if ((m->oflags & VPO_UNMANAGED) == 0)
 4185                 newpte |= PG_MANAGED;
 4186 #ifdef PMAP_PAE_COMP
 4187         if ((prot & VM_PROT_EXECUTE) == 0 && !i386_read_exec)
 4188                 newpte |= pg_nx;
 4189 #endif
 4190         if (pmap != kernel_pmap)
 4191                 newpte |= PG_U;
 4192         pte_store_zero(pte, newpte);
 4193         sched_unpin();
 4194         return (mpte);
 4195 }
 4196 
 4197 /*
 4198  * Make a temporary mapping for a physical address.  This is only intended
 4199  * to be used for panic dumps.
 4200  */
 4201 static void *
 4202 __CONCAT(PMTYPE, kenter_temporary)(vm_paddr_t pa, int i)
 4203 {
 4204         vm_offset_t va;
 4205 
 4206         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 4207         pmap_kenter(va, pa);
 4208         invlpg(va);
 4209         return ((void *)crashdumpmap);
 4210 }
 4211 
 4212 /*
 4213  * This code maps large physical mmap regions into the
 4214  * processor address space.  Note that some shortcuts
 4215  * are taken, but the code works.
 4216  */
 4217 static void
 4218 __CONCAT(PMTYPE, object_init_pt)(pmap_t pmap, vm_offset_t addr,
 4219     vm_object_t object, vm_pindex_t pindex, vm_size_t size)
 4220 {
 4221         pd_entry_t *pde;
 4222         vm_paddr_t pa, ptepa;
 4223         vm_page_t p;
 4224         int pat_mode;
 4225 
 4226         VM_OBJECT_ASSERT_WLOCKED(object);
 4227         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 4228             ("pmap_object_init_pt: non-device object"));
 4229         if (pg_ps_enabled &&
 4230             (addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
 4231                 if (!vm_object_populate(object, pindex, pindex + atop(size)))
 4232                         return;
 4233                 p = vm_page_lookup(object, pindex);
 4234                 KASSERT(p->valid == VM_PAGE_BITS_ALL,
 4235                     ("pmap_object_init_pt: invalid page %p", p));
 4236                 pat_mode = p->md.pat_mode;
 4237 
 4238                 /*
 4239                  * Abort the mapping if the first page is not physically
 4240                  * aligned to a 2/4MB page boundary.
 4241                  */
 4242                 ptepa = VM_PAGE_TO_PHYS(p);
 4243                 if (ptepa & (NBPDR - 1))
 4244                         return;
 4245 
 4246                 /*
 4247                  * Skip the first page.  Abort the mapping if the rest of
 4248                  * the pages are not physically contiguous or have differing
 4249                  * memory attributes.
 4250                  */
 4251                 p = TAILQ_NEXT(p, listq);
 4252                 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
 4253                     pa += PAGE_SIZE) {
 4254                         KASSERT(p->valid == VM_PAGE_BITS_ALL,
 4255                             ("pmap_object_init_pt: invalid page %p", p));
 4256                         if (pa != VM_PAGE_TO_PHYS(p) ||
 4257                             pat_mode != p->md.pat_mode)
 4258                                 return;
 4259                         p = TAILQ_NEXT(p, listq);
 4260                 }
 4261 
 4262                 /*
 4263                  * Map using 2/4MB pages.  Since "ptepa" is 2/4M aligned and
 4264                  * "size" is a multiple of 2/4M, adding the PAT setting to
 4265                  * "pa" will not affect the termination of this loop.
 4266                  */
 4267                 PMAP_LOCK(pmap);
 4268                 for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, 1);
 4269                     pa < ptepa + size; pa += NBPDR) {
 4270                         pde = pmap_pde(pmap, addr);
 4271                         if (*pde == 0) {
 4272                                 pde_store(pde, pa | PG_PS | PG_M | PG_A |
 4273                                     PG_U | PG_RW | PG_V);
 4274                                 pmap->pm_stats.resident_count += NBPDR /
 4275                                     PAGE_SIZE;
 4276                                 pmap_pde_mappings++;
 4277                         }
 4278                         /* Else continue on if the PDE is already valid. */
 4279                         addr += NBPDR;
 4280                 }
 4281                 PMAP_UNLOCK(pmap);
 4282         }
 4283 }
 4284 
 4285 /*
 4286  *      Clear the wired attribute from the mappings for the specified range of
 4287  *      addresses in the given pmap.  Every valid mapping within that range
 4288  *      must have the wired attribute set.  In contrast, invalid mappings
 4289  *      cannot have the wired attribute set, so they are ignored.
 4290  *
 4291  *      The wired attribute of the page table entry is not a hardware feature,
 4292  *      so there is no need to invalidate any TLB entries.
 4293  */
 4294 static void
 4295 __CONCAT(PMTYPE, unwire)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 4296 {
 4297         vm_offset_t pdnxt;
 4298         pd_entry_t *pde;
 4299         pt_entry_t *pte;
 4300         boolean_t pv_lists_locked;
 4301 
 4302         if (pmap_is_current(pmap))
 4303                 pv_lists_locked = FALSE;
 4304         else {
 4305                 pv_lists_locked = TRUE;
 4306 resume:
 4307                 rw_wlock(&pvh_global_lock);
 4308                 sched_pin();
 4309         }
 4310         PMAP_LOCK(pmap);
 4311         for (; sva < eva; sva = pdnxt) {
 4312                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 4313                 if (pdnxt < sva)
 4314                         pdnxt = eva;
 4315                 pde = pmap_pde(pmap, sva);
 4316                 if ((*pde & PG_V) == 0)
 4317                         continue;
 4318                 if ((*pde & PG_PS) != 0) {
 4319                         if ((*pde & PG_W) == 0)
 4320                                 panic("pmap_unwire: pde %#jx is missing PG_W",
 4321                                     (uintmax_t)*pde);
 4322 
 4323                         /*
 4324                          * Are we unwiring the entire large page?  If not,
 4325                          * demote the mapping and fall through.
 4326                          */
 4327                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 4328                                 /*
 4329                                  * Regardless of whether a pde (or pte) is 32
 4330                                  * or 64 bits in size, PG_W is among the least
 4331                                  * significant 32 bits.
 4332                                  */
 4333                                 atomic_clear_int((u_int *)pde, PG_W);
 4334                                 pmap->pm_stats.wired_count -= NBPDR /
 4335                                     PAGE_SIZE;
 4336                                 continue;
 4337                         } else {
 4338                                 if (!pv_lists_locked) {
 4339                                         pv_lists_locked = TRUE;
 4340                                         if (!rw_try_wlock(&pvh_global_lock)) {
 4341                                                 PMAP_UNLOCK(pmap);
 4342                                                 /* Repeat sva. */
 4343                                                 goto resume;
 4344                                         }
 4345                                         sched_pin();
 4346                                 }
 4347                                 if (!pmap_demote_pde(pmap, pde, sva))
 4348                                         panic("pmap_unwire: demotion failed");
 4349                         }
 4350                 }
 4351                 if (pdnxt > eva)
 4352                         pdnxt = eva;
 4353                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 4354                     sva += PAGE_SIZE) {
 4355                         if ((*pte & PG_V) == 0)
 4356                                 continue;
 4357                         if ((*pte & PG_W) == 0)
 4358                                 panic("pmap_unwire: pte %#jx is missing PG_W",
 4359                                     (uintmax_t)*pte);
 4360 
 4361                         /*
 4362                          * PG_W must be cleared atomically.  Although the pmap
 4363                          * lock synchronizes access to PG_W, another processor
 4364                          * could be setting PG_M and/or PG_A concurrently.
 4365                          *
 4366                          * PG_W is among the least significant 32 bits.
 4367                          */
 4368                         atomic_clear_int((u_int *)pte, PG_W);
 4369                         pmap->pm_stats.wired_count--;
 4370                 }
 4371         }
 4372         if (pv_lists_locked) {
 4373                 sched_unpin();
 4374                 rw_wunlock(&pvh_global_lock);
 4375         }
 4376         PMAP_UNLOCK(pmap);
 4377 }
 4378 
 4379 /*
 4380  *      Copy the range specified by src_addr/len
 4381  *      from the source map to the range dst_addr/len
 4382  *      in the destination map.
 4383  *
 4384  *      This routine is only advisory and need not do anything.  Since
 4385  *      current pmap is always the kernel pmap when executing in
 4386  *      kernel, and we do not copy from the kernel pmap to a user
 4387  *      pmap, this optimization is not usable in 4/4G full split i386
 4388  *      world.
 4389  */
 4390 
 4391 static void
 4392 __CONCAT(PMTYPE, copy)(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
 4393     vm_size_t len, vm_offset_t src_addr)
 4394 {
 4395         pt_entry_t *src_pte, *dst_pte, ptetemp;
 4396         pd_entry_t srcptepaddr;
 4397         vm_page_t dstmpte, srcmpte;
 4398         vm_offset_t addr, end_addr, pdnxt;
 4399         u_int ptepindex;
 4400 
 4401         if (dst_addr != src_addr)
 4402                 return;
 4403 
 4404         end_addr = src_addr + len;
 4405 
 4406         rw_wlock(&pvh_global_lock);
 4407         if (dst_pmap < src_pmap) {
 4408                 PMAP_LOCK(dst_pmap);
 4409                 PMAP_LOCK(src_pmap);
 4410         } else {
 4411                 PMAP_LOCK(src_pmap);
 4412                 PMAP_LOCK(dst_pmap);
 4413         }
 4414         sched_pin();
 4415         for (addr = src_addr; addr < end_addr; addr = pdnxt) {
 4416                 KASSERT(addr < PMAP_TRM_MIN_ADDRESS,
 4417                     ("pmap_copy: invalid to pmap_copy the trampoline"));
 4418 
 4419                 pdnxt = (addr + NBPDR) & ~PDRMASK;
 4420                 if (pdnxt < addr)
 4421                         pdnxt = end_addr;
 4422                 ptepindex = addr >> PDRSHIFT;
 4423 
 4424                 srcptepaddr = src_pmap->pm_pdir[ptepindex];
 4425                 if (srcptepaddr == 0)
 4426                         continue;
 4427 
 4428                 if (srcptepaddr & PG_PS) {
 4429                         if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr)
 4430                                 continue;
 4431                         if (dst_pmap->pm_pdir[ptepindex] == 0 &&
 4432                             ((srcptepaddr & PG_MANAGED) == 0 ||
 4433                             pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr,
 4434                             PMAP_ENTER_NORECLAIM))) {
 4435                                 dst_pmap->pm_pdir[ptepindex] = srcptepaddr &
 4436                                     ~PG_W;
 4437                                 dst_pmap->pm_stats.resident_count +=
 4438                                     NBPDR / PAGE_SIZE;
 4439                                 pmap_pde_mappings++;
 4440                         }
 4441                         continue;
 4442                 }
 4443 
 4444                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 4445                 KASSERT(srcmpte->ref_count > 0,
 4446                     ("pmap_copy: source page table page is unused"));
 4447 
 4448                 if (pdnxt > end_addr)
 4449                         pdnxt = end_addr;
 4450 
 4451                 src_pte = pmap_pte_quick3(src_pmap, addr);
 4452                 while (addr < pdnxt) {
 4453                         ptetemp = *src_pte;
 4454                         /*
 4455                          * we only virtual copy managed pages
 4456                          */
 4457                         if ((ptetemp & PG_MANAGED) != 0) {
 4458                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 4459                                     PMAP_ENTER_NOSLEEP);
 4460                                 if (dstmpte == NULL)
 4461                                         goto out;
 4462                                 dst_pte = pmap_pte_quick(dst_pmap, addr);
 4463                                 if (*dst_pte == 0 &&
 4464                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 4465                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
 4466                                         /*
 4467                                          * Clear the wired, modified, and
 4468                                          * accessed (referenced) bits
 4469                                          * during the copy.
 4470                                          */
 4471                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 4472                                             PG_A);
 4473                                         dst_pmap->pm_stats.resident_count++;
 4474                                 } else {
 4475                                         pmap_abort_ptp(dst_pmap, addr, dstmpte);
 4476                                         goto out;
 4477                                 }
 4478                                 if (dstmpte->ref_count >= srcmpte->ref_count)
 4479                                         break;
 4480                         }
 4481                         addr += PAGE_SIZE;
 4482                         src_pte++;
 4483                 }
 4484         }
 4485 out:
 4486         sched_unpin();
 4487         rw_wunlock(&pvh_global_lock);
 4488         PMAP_UNLOCK(src_pmap);
 4489         PMAP_UNLOCK(dst_pmap);
 4490 }
 4491 
 4492 /*
 4493  * Zero 1 page of virtual memory mapped from a hardware page by the caller.
 4494  */
 4495 static __inline void
 4496 pagezero(void *page)
 4497 {
 4498 #if defined(I686_CPU)
 4499         if (cpu_class == CPUCLASS_686) {
 4500                 if (cpu_feature & CPUID_SSE2)
 4501                         sse2_pagezero(page);
 4502                 else
 4503                         i686_pagezero(page);
 4504         } else
 4505 #endif
 4506                 bzero(page, PAGE_SIZE);
 4507 }
 4508 
 4509 /*
 4510  * Zero the specified hardware page.
 4511  */
 4512 static void
 4513 __CONCAT(PMTYPE, zero_page)(vm_page_t m)
 4514 {
 4515         pt_entry_t *cmap_pte2;
 4516         struct pcpu *pc;
 4517 
 4518         sched_pin();
 4519         pc = get_pcpu();
 4520         cmap_pte2 = pc->pc_cmap_pte2;
 4521         mtx_lock(&pc->pc_cmap_lock);
 4522         if (*cmap_pte2)
 4523                 panic("pmap_zero_page: CMAP2 busy");
 4524         *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 4525             pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0);
 4526         invlcaddr(pc->pc_cmap_addr2);
 4527         pagezero(pc->pc_cmap_addr2);
 4528         *cmap_pte2 = 0;
 4529 
 4530         /*
 4531          * Unpin the thread before releasing the lock.  Otherwise the thread
 4532          * could be rescheduled while still bound to the current CPU, only
 4533          * to unpin itself immediately upon resuming execution.
 4534          */
 4535         sched_unpin();
 4536         mtx_unlock(&pc->pc_cmap_lock);
 4537 }
 4538 
 4539 /*
 4540  * Zero an an area within a single hardware page.  off and size must not
 4541  * cover an area beyond a single hardware page.
 4542  */
 4543 static void
 4544 __CONCAT(PMTYPE, zero_page_area)(vm_page_t m, int off, int size)
 4545 {
 4546         pt_entry_t *cmap_pte2;
 4547         struct pcpu *pc;
 4548 
 4549         sched_pin();
 4550         pc = get_pcpu();
 4551         cmap_pte2 = pc->pc_cmap_pte2;
 4552         mtx_lock(&pc->pc_cmap_lock);
 4553         if (*cmap_pte2)
 4554                 panic("pmap_zero_page_area: CMAP2 busy");
 4555         *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 4556             pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0);
 4557         invlcaddr(pc->pc_cmap_addr2);
 4558         if (off == 0 && size == PAGE_SIZE) 
 4559                 pagezero(pc->pc_cmap_addr2);
 4560         else
 4561                 bzero(pc->pc_cmap_addr2 + off, size);
 4562         *cmap_pte2 = 0;
 4563         sched_unpin();
 4564         mtx_unlock(&pc->pc_cmap_lock);
 4565 }
 4566 
 4567 /*
 4568  * Copy 1 specified hardware page to another.
 4569  */
 4570 static void
 4571 __CONCAT(PMTYPE, copy_page)(vm_page_t src, vm_page_t dst)
 4572 {
 4573         pt_entry_t *cmap_pte1, *cmap_pte2;
 4574         struct pcpu *pc;
 4575 
 4576         sched_pin();
 4577         pc = get_pcpu();
 4578         cmap_pte1 = pc->pc_cmap_pte1; 
 4579         cmap_pte2 = pc->pc_cmap_pte2;
 4580         mtx_lock(&pc->pc_cmap_lock);
 4581         if (*cmap_pte1)
 4582                 panic("pmap_copy_page: CMAP1 busy");
 4583         if (*cmap_pte2)
 4584                 panic("pmap_copy_page: CMAP2 busy");
 4585         *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A |
 4586             pmap_cache_bits(kernel_pmap, src->md.pat_mode, 0);
 4587         invlcaddr(pc->pc_cmap_addr1);
 4588         *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M |
 4589             pmap_cache_bits(kernel_pmap, dst->md.pat_mode, 0);
 4590         invlcaddr(pc->pc_cmap_addr2);
 4591         bcopy(pc->pc_cmap_addr1, pc->pc_cmap_addr2, PAGE_SIZE);
 4592         *cmap_pte1 = 0;
 4593         *cmap_pte2 = 0;
 4594         sched_unpin();
 4595         mtx_unlock(&pc->pc_cmap_lock);
 4596 }
 4597 
 4598 static void
 4599 __CONCAT(PMTYPE, copy_pages)(vm_page_t ma[], vm_offset_t a_offset,
 4600     vm_page_t mb[], vm_offset_t b_offset, int xfersize)
 4601 {
 4602         vm_page_t a_pg, b_pg;
 4603         char *a_cp, *b_cp;
 4604         vm_offset_t a_pg_offset, b_pg_offset;
 4605         pt_entry_t *cmap_pte1, *cmap_pte2;
 4606         struct pcpu *pc;
 4607         int cnt;
 4608 
 4609         sched_pin();
 4610         pc = get_pcpu();
 4611         cmap_pte1 = pc->pc_cmap_pte1; 
 4612         cmap_pte2 = pc->pc_cmap_pte2;
 4613         mtx_lock(&pc->pc_cmap_lock);
 4614         if (*cmap_pte1 != 0)
 4615                 panic("pmap_copy_pages: CMAP1 busy");
 4616         if (*cmap_pte2 != 0)
 4617                 panic("pmap_copy_pages: CMAP2 busy");
 4618         while (xfersize > 0) {
 4619                 a_pg = ma[a_offset >> PAGE_SHIFT];
 4620                 a_pg_offset = a_offset & PAGE_MASK;
 4621                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
 4622                 b_pg = mb[b_offset >> PAGE_SHIFT];
 4623                 b_pg_offset = b_offset & PAGE_MASK;
 4624                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 4625                 *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A |
 4626                     pmap_cache_bits(kernel_pmap, a_pg->md.pat_mode, 0);
 4627                 invlcaddr(pc->pc_cmap_addr1);
 4628                 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A |
 4629                     PG_M | pmap_cache_bits(kernel_pmap, b_pg->md.pat_mode, 0);
 4630                 invlcaddr(pc->pc_cmap_addr2);
 4631                 a_cp = pc->pc_cmap_addr1 + a_pg_offset;
 4632                 b_cp = pc->pc_cmap_addr2 + b_pg_offset;
 4633                 bcopy(a_cp, b_cp, cnt);
 4634                 a_offset += cnt;
 4635                 b_offset += cnt;
 4636                 xfersize -= cnt;
 4637         }
 4638         *cmap_pte1 = 0;
 4639         *cmap_pte2 = 0;
 4640         sched_unpin();
 4641         mtx_unlock(&pc->pc_cmap_lock);
 4642 }
 4643 
 4644 /*
 4645  * Returns true if the pmap's pv is one of the first
 4646  * 16 pvs linked to from this page.  This count may
 4647  * be changed upwards or downwards in the future; it
 4648  * is only necessary that true be returned for a small
 4649  * subset of pmaps for proper page aging.
 4650  */
 4651 static boolean_t
 4652 __CONCAT(PMTYPE, page_exists_quick)(pmap_t pmap, vm_page_t m)
 4653 {
 4654         struct md_page *pvh;
 4655         pv_entry_t pv;
 4656         int loops = 0;
 4657         boolean_t rv;
 4658 
 4659         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4660             ("pmap_page_exists_quick: page %p is not managed", m));
 4661         rv = FALSE;
 4662         rw_wlock(&pvh_global_lock);
 4663         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 4664                 if (PV_PMAP(pv) == pmap) {
 4665                         rv = TRUE;
 4666                         break;
 4667                 }
 4668                 loops++;
 4669                 if (loops >= 16)
 4670                         break;
 4671         }
 4672         if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
 4673                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4674                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4675                         if (PV_PMAP(pv) == pmap) {
 4676                                 rv = TRUE;
 4677                                 break;
 4678                         }
 4679                         loops++;
 4680                         if (loops >= 16)
 4681                                 break;
 4682                 }
 4683         }
 4684         rw_wunlock(&pvh_global_lock);
 4685         return (rv);
 4686 }
 4687 
 4688 /*
 4689  *      pmap_page_wired_mappings:
 4690  *
 4691  *      Return the number of managed mappings to the given physical page
 4692  *      that are wired.
 4693  */
 4694 static int
 4695 __CONCAT(PMTYPE, page_wired_mappings)(vm_page_t m)
 4696 {
 4697         int count;
 4698 
 4699         count = 0;
 4700         if ((m->oflags & VPO_UNMANAGED) != 0)
 4701                 return (count);
 4702         rw_wlock(&pvh_global_lock);
 4703         count = pmap_pvh_wired_mappings(&m->md, count);
 4704         if ((m->flags & PG_FICTITIOUS) == 0) {
 4705             count = pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)),
 4706                 count);
 4707         }
 4708         rw_wunlock(&pvh_global_lock);
 4709         return (count);
 4710 }
 4711 
 4712 /*
 4713  *      pmap_pvh_wired_mappings:
 4714  *
 4715  *      Return the updated number "count" of managed mappings that are wired.
 4716  */
 4717 static int
 4718 pmap_pvh_wired_mappings(struct md_page *pvh, int count)
 4719 {
 4720         pmap_t pmap;
 4721         pt_entry_t *pte;
 4722         pv_entry_t pv;
 4723 
 4724         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4725         sched_pin();
 4726         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4727                 pmap = PV_PMAP(pv);
 4728                 PMAP_LOCK(pmap);
 4729                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4730                 if ((*pte & PG_W) != 0)
 4731                         count++;
 4732                 PMAP_UNLOCK(pmap);
 4733         }
 4734         sched_unpin();
 4735         return (count);
 4736 }
 4737 
 4738 /*
 4739  * Returns TRUE if the given page is mapped individually or as part of
 4740  * a 4mpage.  Otherwise, returns FALSE.
 4741  */
 4742 static boolean_t
 4743 __CONCAT(PMTYPE, page_is_mapped)(vm_page_t m)
 4744 {
 4745         boolean_t rv;
 4746 
 4747         if ((m->oflags & VPO_UNMANAGED) != 0)
 4748                 return (FALSE);
 4749         rw_wlock(&pvh_global_lock);
 4750         rv = !TAILQ_EMPTY(&m->md.pv_list) ||
 4751             ((m->flags & PG_FICTITIOUS) == 0 &&
 4752             !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
 4753         rw_wunlock(&pvh_global_lock);
 4754         return (rv);
 4755 }
 4756 
 4757 /*
 4758  * Remove all pages from specified address space
 4759  * this aids process exit speeds.  Also, this code
 4760  * is special cased for current process only, but
 4761  * can have the more generic (and slightly slower)
 4762  * mode enabled.  This is much faster than pmap_remove
 4763  * in the case of running down an entire address space.
 4764  */
 4765 static void
 4766 __CONCAT(PMTYPE, remove_pages)(pmap_t pmap)
 4767 {
 4768         pt_entry_t *pte, tpte;
 4769         vm_page_t m, mpte, mt;
 4770         pv_entry_t pv;
 4771         struct md_page *pvh;
 4772         struct pv_chunk *pc, *npc;
 4773         struct spglist free;
 4774         int field, idx;
 4775         int32_t bit;
 4776         uint32_t inuse, bitmask;
 4777         int allfree;
 4778 
 4779         if (pmap != PCPU_GET(curpmap)) {
 4780                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 4781                 return;
 4782         }
 4783         SLIST_INIT(&free);
 4784         rw_wlock(&pvh_global_lock);
 4785         PMAP_LOCK(pmap);
 4786         sched_pin();
 4787         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 4788                 KASSERT(pc->pc_pmap == pmap, ("Wrong pmap %p %p", pmap,
 4789                     pc->pc_pmap));
 4790                 allfree = 1;
 4791                 for (field = 0; field < _NPCM; field++) {
 4792                         inuse = ~pc->pc_map[field] & pc_freemask[field];
 4793                         while (inuse != 0) {
 4794                                 bit = bsfl(inuse);
 4795                                 bitmask = 1UL << bit;
 4796                                 idx = field * 32 + bit;
 4797                                 pv = &pc->pc_pventry[idx];
 4798                                 inuse &= ~bitmask;
 4799 
 4800                                 pte = pmap_pde(pmap, pv->pv_va);
 4801                                 tpte = *pte;
 4802                                 if ((tpte & PG_PS) == 0) {
 4803                                         pte = pmap_pte_quick(pmap, pv->pv_va);
 4804                                         tpte = *pte & ~PG_PTE_PAT;
 4805                                 }
 4806 
 4807                                 if (tpte == 0) {
 4808                                         printf(
 4809                                             "TPTE at %p  IS ZERO @ VA %08x\n",
 4810                                             pte, pv->pv_va);
 4811                                         panic("bad pte");
 4812                                 }
 4813 
 4814 /*
 4815  * We cannot remove wired pages from a process' mapping at this time
 4816  */
 4817                                 if (tpte & PG_W) {
 4818                                         allfree = 0;
 4819                                         continue;
 4820                                 }
 4821 
 4822                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 4823                                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 4824                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 4825                                     m, (uintmax_t)m->phys_addr,
 4826                                     (uintmax_t)tpte));
 4827 
 4828                                 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
 4829                                     m < &vm_page_array[vm_page_array_size],
 4830                                     ("pmap_remove_pages: bad tpte %#jx",
 4831                                     (uintmax_t)tpte));
 4832 
 4833                                 pte_clear(pte);
 4834 
 4835                                 /*
 4836                                  * Update the vm_page_t clean/reference bits.
 4837                                  */
 4838                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4839                                         if ((tpte & PG_PS) != 0) {
 4840                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4841                                                         vm_page_dirty(mt);
 4842                                         } else
 4843                                                 vm_page_dirty(m);
 4844                                 }
 4845 
 4846                                 /* Mark free */
 4847                                 PV_STAT(pv_entry_frees++);
 4848                                 PV_STAT(pv_entry_spare++);
 4849                                 pv_entry_count--;
 4850                                 pc->pc_map[field] |= bitmask;
 4851                                 if ((tpte & PG_PS) != 0) {
 4852                                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 4853                                         pvh = pa_to_pvh(tpte & PG_PS_FRAME);
 4854                                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 4855                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 4856                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4857                                                         if (TAILQ_EMPTY(&mt->md.pv_list))
 4858                                                                 vm_page_aflag_clear(mt, PGA_WRITEABLE);
 4859                                         }
 4860                                         mpte = pmap_remove_pt_page(pmap, pv->pv_va);
 4861                                         if (mpte != NULL) {
 4862                                                 KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
 4863                                                     ("pmap_remove_pages: pte page not promoted"));
 4864                                                 pmap->pm_stats.resident_count--;
 4865                                                 KASSERT(mpte->ref_count == NPTEPG,
 4866                                                     ("pmap_remove_pages: pte page ref count error"));
 4867                                                 mpte->ref_count = 0;
 4868                                                 pmap_add_delayed_free_list(mpte, &free, FALSE);
 4869                                         }
 4870                                 } else {
 4871                                         pmap->pm_stats.resident_count--;
 4872                                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 4873                                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 4874                                             (m->flags & PG_FICTITIOUS) == 0) {
 4875                                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4876                                                 if (TAILQ_EMPTY(&pvh->pv_list))
 4877                                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 4878                                         }
 4879                                         pmap_unuse_pt(pmap, pv->pv_va, &free);
 4880                                 }
 4881                         }
 4882                 }
 4883                 if (allfree) {
 4884                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 4885                         free_pv_chunk(pc);
 4886                 }
 4887         }
 4888         sched_unpin();
 4889         pmap_invalidate_all_int(pmap);
 4890         rw_wunlock(&pvh_global_lock);
 4891         PMAP_UNLOCK(pmap);
 4892         vm_page_free_pages_toq(&free, true);
 4893 }
 4894 
 4895 /*
 4896  *      pmap_is_modified:
 4897  *
 4898  *      Return whether or not the specified physical page was modified
 4899  *      in any physical maps.
 4900  */
 4901 static boolean_t
 4902 __CONCAT(PMTYPE, is_modified)(vm_page_t m)
 4903 {
 4904         boolean_t rv;
 4905 
 4906         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4907             ("pmap_is_modified: page %p is not managed", m));
 4908 
 4909         /*
 4910          * If the page is not busied then this check is racy.
 4911          */
 4912         if (!pmap_page_is_write_mapped(m))
 4913                 return (FALSE);
 4914         rw_wlock(&pvh_global_lock);
 4915         rv = pmap_is_modified_pvh(&m->md) ||
 4916             ((m->flags & PG_FICTITIOUS) == 0 &&
 4917             pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4918         rw_wunlock(&pvh_global_lock);
 4919         return (rv);
 4920 }
 4921 
 4922 /*
 4923  * Returns TRUE if any of the given mappings were used to modify
 4924  * physical memory.  Otherwise, returns FALSE.  Both page and 2mpage
 4925  * mappings are supported.
 4926  */
 4927 static boolean_t
 4928 pmap_is_modified_pvh(struct md_page *pvh)
 4929 {
 4930         pv_entry_t pv;
 4931         pt_entry_t *pte;
 4932         pmap_t pmap;
 4933         boolean_t rv;
 4934 
 4935         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4936         rv = FALSE;
 4937         sched_pin();
 4938         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4939                 pmap = PV_PMAP(pv);
 4940                 PMAP_LOCK(pmap);
 4941                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4942                 rv = (*pte & (PG_M | PG_RW)) == (PG_M | PG_RW);
 4943                 PMAP_UNLOCK(pmap);
 4944                 if (rv)
 4945                         break;
 4946         }
 4947         sched_unpin();
 4948         return (rv);
 4949 }
 4950 
 4951 /*
 4952  *      pmap_is_prefaultable:
 4953  *
 4954  *      Return whether or not the specified virtual address is elgible
 4955  *      for prefault.
 4956  */
 4957 static boolean_t
 4958 __CONCAT(PMTYPE, is_prefaultable)(pmap_t pmap, vm_offset_t addr)
 4959 {
 4960         pd_entry_t pde;
 4961         boolean_t rv;
 4962 
 4963         rv = FALSE;
 4964         PMAP_LOCK(pmap);
 4965         pde = *pmap_pde(pmap, addr);
 4966         if (pde != 0 && (pde & PG_PS) == 0)
 4967                 rv = pmap_pte_ufast(pmap, addr, pde) == 0;
 4968         PMAP_UNLOCK(pmap);
 4969         return (rv);
 4970 }
 4971 
 4972 /*
 4973  *      pmap_is_referenced:
 4974  *
 4975  *      Return whether or not the specified physical page was referenced
 4976  *      in any physical maps.
 4977  */
 4978 static boolean_t
 4979 __CONCAT(PMTYPE, is_referenced)(vm_page_t m)
 4980 {
 4981         boolean_t rv;
 4982 
 4983         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4984             ("pmap_is_referenced: page %p is not managed", m));
 4985         rw_wlock(&pvh_global_lock);
 4986         rv = pmap_is_referenced_pvh(&m->md) ||
 4987             ((m->flags & PG_FICTITIOUS) == 0 &&
 4988             pmap_is_referenced_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4989         rw_wunlock(&pvh_global_lock);
 4990         return (rv);
 4991 }
 4992 
 4993 /*
 4994  * Returns TRUE if any of the given mappings were referenced and FALSE
 4995  * otherwise.  Both page and 4mpage mappings are supported.
 4996  */
 4997 static boolean_t
 4998 pmap_is_referenced_pvh(struct md_page *pvh)
 4999 {
 5000         pv_entry_t pv;
 5001         pt_entry_t *pte;
 5002         pmap_t pmap;
 5003         boolean_t rv;
 5004 
 5005         rw_assert(&pvh_global_lock, RA_WLOCKED);
 5006         rv = FALSE;
 5007         sched_pin();
 5008         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 5009                 pmap = PV_PMAP(pv);
 5010                 PMAP_LOCK(pmap);
 5011                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5012                 rv = (*pte & (PG_A | PG_V)) == (PG_A | PG_V);
 5013                 PMAP_UNLOCK(pmap);
 5014                 if (rv)
 5015                         break;
 5016         }
 5017         sched_unpin();
 5018         return (rv);
 5019 }
 5020 
 5021 /*
 5022  * Clear the write and modified bits in each of the given page's mappings.
 5023  */
 5024 static void
 5025 __CONCAT(PMTYPE, remove_write)(vm_page_t m)
 5026 {
 5027         struct md_page *pvh;
 5028         pv_entry_t next_pv, pv;
 5029         pmap_t pmap;
 5030         pd_entry_t *pde;
 5031         pt_entry_t oldpte, *pte;
 5032         vm_offset_t va;
 5033 
 5034         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5035             ("pmap_remove_write: page %p is not managed", m));
 5036         vm_page_assert_busied(m);
 5037 
 5038         if (!pmap_page_is_write_mapped(m))
 5039                 return;
 5040         rw_wlock(&pvh_global_lock);
 5041         sched_pin();
 5042         if ((m->flags & PG_FICTITIOUS) != 0)
 5043                 goto small_mappings;
 5044         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5045         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
 5046                 va = pv->pv_va;
 5047                 pmap = PV_PMAP(pv);
 5048                 PMAP_LOCK(pmap);
 5049                 pde = pmap_pde(pmap, va);
 5050                 if ((*pde & PG_RW) != 0)
 5051                         (void)pmap_demote_pde(pmap, pde, va);
 5052                 PMAP_UNLOCK(pmap);
 5053         }
 5054 small_mappings:
 5055         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5056                 pmap = PV_PMAP(pv);
 5057                 PMAP_LOCK(pmap);
 5058                 pde = pmap_pde(pmap, pv->pv_va);
 5059                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_write: found"
 5060                     " a 4mpage in page %p's pv list", m));
 5061                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5062 retry:
 5063                 oldpte = *pte;
 5064                 if ((oldpte & PG_RW) != 0) {
 5065                         /*
 5066                          * Regardless of whether a pte is 32 or 64 bits
 5067                          * in size, PG_RW and PG_M are among the least
 5068                          * significant 32 bits.
 5069                          */
 5070                         if (!atomic_cmpset_int((u_int *)pte, oldpte,
 5071                             oldpte & ~(PG_RW | PG_M)))
 5072                                 goto retry;
 5073                         if ((oldpte & PG_M) != 0)
 5074                                 vm_page_dirty(m);
 5075                         pmap_invalidate_page_int(pmap, pv->pv_va);
 5076                 }
 5077                 PMAP_UNLOCK(pmap);
 5078         }
 5079         vm_page_aflag_clear(m, PGA_WRITEABLE);
 5080         sched_unpin();
 5081         rw_wunlock(&pvh_global_lock);
 5082 }
 5083 
 5084 /*
 5085  *      pmap_ts_referenced:
 5086  *
 5087  *      Return a count of reference bits for a page, clearing those bits.
 5088  *      It is not necessary for every reference bit to be cleared, but it
 5089  *      is necessary that 0 only be returned when there are truly no
 5090  *      reference bits set.
 5091  *
 5092  *      As an optimization, update the page's dirty field if a modified bit is
 5093  *      found while counting reference bits.  This opportunistic update can be
 5094  *      performed at low cost and can eliminate the need for some future calls
 5095  *      to pmap_is_modified().  However, since this function stops after
 5096  *      finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
 5097  *      dirty pages.  Those dirty pages will only be detected by a future call
 5098  *      to pmap_is_modified().
 5099  */
 5100 static int
 5101 __CONCAT(PMTYPE, ts_referenced)(vm_page_t m)
 5102 {
 5103         struct md_page *pvh;
 5104         pv_entry_t pv, pvf;
 5105         pmap_t pmap;
 5106         pd_entry_t *pde;
 5107         pt_entry_t *pte;
 5108         vm_paddr_t pa;
 5109         int rtval = 0;
 5110 
 5111         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5112             ("pmap_ts_referenced: page %p is not managed", m));
 5113         pa = VM_PAGE_TO_PHYS(m);
 5114         pvh = pa_to_pvh(pa);
 5115         rw_wlock(&pvh_global_lock);
 5116         sched_pin();
 5117         if ((m->flags & PG_FICTITIOUS) != 0 ||
 5118             (pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
 5119                 goto small_mappings;
 5120         pv = pvf;
 5121         do {
 5122                 pmap = PV_PMAP(pv);
 5123                 PMAP_LOCK(pmap);
 5124                 pde = pmap_pde(pmap, pv->pv_va);
 5125                 if ((*pde & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5126                         /*
 5127                          * Although "*pde" is mapping a 2/4MB page, because
 5128                          * this function is called at a 4KB page granularity,
 5129                          * we only update the 4KB page under test.
 5130                          */
 5131                         vm_page_dirty(m);
 5132                 }
 5133                 if ((*pde & PG_A) != 0) {
 5134                         /*
 5135                          * Since this reference bit is shared by either 1024
 5136                          * or 512 4KB pages, it should not be cleared every
 5137                          * time it is tested.  Apply a simple "hash" function
 5138                          * on the physical page number, the virtual superpage
 5139                          * number, and the pmap address to select one 4KB page
 5140                          * out of the 1024 or 512 on which testing the
 5141                          * reference bit will result in clearing that bit.
 5142                          * This function is designed to avoid the selection of
 5143                          * the same 4KB page for every 2- or 4MB page mapping.
 5144                          *
 5145                          * On demotion, a mapping that hasn't been referenced
 5146                          * is simply destroyed.  To avoid the possibility of a
 5147                          * subsequent page fault on a demoted wired mapping,
 5148                          * always leave its reference bit set.  Moreover,
 5149                          * since the superpage is wired, the current state of
 5150                          * its reference bit won't affect page replacement.
 5151                          */
 5152                         if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PDRSHIFT) ^
 5153                             (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
 5154                             (*pde & PG_W) == 0) {
 5155                                 atomic_clear_int((u_int *)pde, PG_A);
 5156                                 pmap_invalidate_page_int(pmap, pv->pv_va);
 5157                         }
 5158                         rtval++;
 5159                 }
 5160                 PMAP_UNLOCK(pmap);
 5161                 /* Rotate the PV list if it has more than one entry. */
 5162                 if (TAILQ_NEXT(pv, pv_next) != NULL) {
 5163                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 5164                         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 5165                 }
 5166                 if (rtval >= PMAP_TS_REFERENCED_MAX)
 5167                         goto out;
 5168         } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
 5169 small_mappings:
 5170         if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
 5171                 goto out;
 5172         pv = pvf;
 5173         do {
 5174                 pmap = PV_PMAP(pv);
 5175                 PMAP_LOCK(pmap);
 5176                 pde = pmap_pde(pmap, pv->pv_va);
 5177                 KASSERT((*pde & PG_PS) == 0,
 5178                     ("pmap_ts_referenced: found a 4mpage in page %p's pv list",
 5179                     m));
 5180                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5181                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 5182                         vm_page_dirty(m);
 5183                 if ((*pte & PG_A) != 0) {
 5184                         atomic_clear_int((u_int *)pte, PG_A);
 5185                         pmap_invalidate_page_int(pmap, pv->pv_va);
 5186                         rtval++;
 5187                 }
 5188                 PMAP_UNLOCK(pmap);
 5189                 /* Rotate the PV list if it has more than one entry. */
 5190                 if (TAILQ_NEXT(pv, pv_next) != NULL) {
 5191                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 5192                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 5193                 }
 5194         } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && rtval <
 5195             PMAP_TS_REFERENCED_MAX);
 5196 out:
 5197         sched_unpin();
 5198         rw_wunlock(&pvh_global_lock);
 5199         return (rtval);
 5200 }
 5201 
 5202 /*
 5203  *      Apply the given advice to the specified range of addresses within the
 5204  *      given pmap.  Depending on the advice, clear the referenced and/or
 5205  *      modified flags in each mapping and set the mapped page's dirty field.
 5206  */
 5207 static void
 5208 __CONCAT(PMTYPE, advise)(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
 5209     int advice)
 5210 {
 5211         pd_entry_t oldpde, *pde;
 5212         pt_entry_t *pte;
 5213         vm_offset_t va, pdnxt;
 5214         vm_page_t m;
 5215         bool anychanged, pv_lists_locked;
 5216 
 5217         if (advice != MADV_DONTNEED && advice != MADV_FREE)
 5218                 return;
 5219         if (pmap_is_current(pmap))
 5220                 pv_lists_locked = false;
 5221         else {
 5222                 pv_lists_locked = true;
 5223 resume:
 5224                 rw_wlock(&pvh_global_lock);
 5225                 sched_pin();
 5226         }
 5227         anychanged = false;
 5228         PMAP_LOCK(pmap);
 5229         for (; sva < eva; sva = pdnxt) {
 5230                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 5231                 if (pdnxt < sva)
 5232                         pdnxt = eva;
 5233                 pde = pmap_pde(pmap, sva);
 5234                 oldpde = *pde;
 5235                 if ((oldpde & PG_V) == 0)
 5236                         continue;
 5237                 else if ((oldpde & PG_PS) != 0) {
 5238                         if ((oldpde & PG_MANAGED) == 0)
 5239                                 continue;
 5240                         if (!pv_lists_locked) {
 5241                                 pv_lists_locked = true;
 5242                                 if (!rw_try_wlock(&pvh_global_lock)) {
 5243                                         if (anychanged)
 5244                                                 pmap_invalidate_all_int(pmap);
 5245                                         PMAP_UNLOCK(pmap);
 5246                                         goto resume;
 5247                                 }
 5248                                 sched_pin();
 5249                         }
 5250                         if (!pmap_demote_pde(pmap, pde, sva)) {
 5251                                 /*
 5252                                  * The large page mapping was destroyed.
 5253                                  */
 5254                                 continue;
 5255                         }
 5256 
 5257                         /*
 5258                          * Unless the page mappings are wired, remove the
 5259                          * mapping to a single page so that a subsequent
 5260                          * access may repromote.  Choosing the last page
 5261                          * within the address range [sva, min(pdnxt, eva))
 5262                          * generally results in more repromotions.  Since the
 5263                          * underlying page table page is fully populated, this
 5264                          * removal never frees a page table page.
 5265                          */
 5266                         if ((oldpde & PG_W) == 0) {
 5267                                 va = eva;
 5268                                 if (va > pdnxt)
 5269                                         va = pdnxt;
 5270                                 va -= PAGE_SIZE;
 5271                                 KASSERT(va >= sva,
 5272                                     ("pmap_advise: no address gap"));
 5273                                 pte = pmap_pte_quick(pmap, va);
 5274                                 KASSERT((*pte & PG_V) != 0,
 5275                                     ("pmap_advise: invalid PTE"));
 5276                                 pmap_remove_pte(pmap, pte, va, NULL);
 5277                                 anychanged = true;
 5278                         }
 5279                 }
 5280                 if (pdnxt > eva)
 5281                         pdnxt = eva;
 5282                 va = pdnxt;
 5283                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 5284                     sva += PAGE_SIZE) {
 5285                         if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
 5286                                 goto maybe_invlrng;
 5287                         else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5288                                 if (advice == MADV_DONTNEED) {
 5289                                         /*
 5290                                          * Future calls to pmap_is_modified()
 5291                                          * can be avoided by making the page
 5292                                          * dirty now.
 5293                                          */
 5294                                         m = PHYS_TO_VM_PAGE(*pte & PG_FRAME);
 5295                                         vm_page_dirty(m);
 5296                                 }
 5297                                 atomic_clear_int((u_int *)pte, PG_M | PG_A);
 5298                         } else if ((*pte & PG_A) != 0)
 5299                                 atomic_clear_int((u_int *)pte, PG_A);
 5300                         else
 5301                                 goto maybe_invlrng;
 5302                         if ((*pte & PG_G) != 0) {
 5303                                 if (va == pdnxt)
 5304                                         va = sva;
 5305                         } else
 5306                                 anychanged = true;
 5307                         continue;
 5308 maybe_invlrng:
 5309                         if (va != pdnxt) {
 5310                                 pmap_invalidate_range_int(pmap, va, sva);
 5311                                 va = pdnxt;
 5312                         }
 5313                 }
 5314                 if (va != pdnxt)
 5315                         pmap_invalidate_range_int(pmap, va, sva);
 5316         }
 5317         if (anychanged)
 5318                 pmap_invalidate_all_int(pmap);
 5319         if (pv_lists_locked) {
 5320                 sched_unpin();
 5321                 rw_wunlock(&pvh_global_lock);
 5322         }
 5323         PMAP_UNLOCK(pmap);
 5324 }
 5325 
 5326 /*
 5327  *      Clear the modify bits on the specified physical page.
 5328  */
 5329 static void
 5330 __CONCAT(PMTYPE, clear_modify)(vm_page_t m)
 5331 {
 5332         struct md_page *pvh;
 5333         pv_entry_t next_pv, pv;
 5334         pmap_t pmap;
 5335         pd_entry_t oldpde, *pde;
 5336         pt_entry_t *pte;
 5337         vm_offset_t va;
 5338 
 5339         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5340             ("pmap_clear_modify: page %p is not managed", m));
 5341         vm_page_assert_busied(m);
 5342 
 5343         if (!pmap_page_is_write_mapped(m))
 5344                 return;
 5345         rw_wlock(&pvh_global_lock);
 5346         sched_pin();
 5347         if ((m->flags & PG_FICTITIOUS) != 0)
 5348                 goto small_mappings;
 5349         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5350         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
 5351                 va = pv->pv_va;
 5352                 pmap = PV_PMAP(pv);
 5353                 PMAP_LOCK(pmap);
 5354                 pde = pmap_pde(pmap, va);
 5355                 oldpde = *pde;
 5356                 /* If oldpde has PG_RW set, then it also has PG_M set. */
 5357                 if ((oldpde & PG_RW) != 0 &&
 5358                     pmap_demote_pde(pmap, pde, va) &&
 5359                     (oldpde & PG_W) == 0) {
 5360                         /*
 5361                          * Write protect the mapping to a single page so that
 5362                          * a subsequent write access may repromote.
 5363                          */
 5364                         va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME);
 5365                         pte = pmap_pte_quick(pmap, va);
 5366                         /*
 5367                          * Regardless of whether a pte is 32 or 64 bits
 5368                          * in size, PG_RW and PG_M are among the least
 5369                          * significant 32 bits.
 5370                          */
 5371                         atomic_clear_int((u_int *)pte, PG_M | PG_RW);
 5372                         vm_page_dirty(m);
 5373                         pmap_invalidate_page_int(pmap, va);
 5374                 }
 5375                 PMAP_UNLOCK(pmap);
 5376         }
 5377 small_mappings:
 5378         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5379                 pmap = PV_PMAP(pv);
 5380                 PMAP_LOCK(pmap);
 5381                 pde = pmap_pde(pmap, pv->pv_va);
 5382                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
 5383                     " a 4mpage in page %p's pv list", m));
 5384                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5385                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5386                         /*
 5387                          * Regardless of whether a pte is 32 or 64 bits
 5388                          * in size, PG_M is among the least significant
 5389                          * 32 bits. 
 5390                          */
 5391                         atomic_clear_int((u_int *)pte, PG_M);
 5392                         pmap_invalidate_page_int(pmap, pv->pv_va);
 5393                 }
 5394                 PMAP_UNLOCK(pmap);
 5395         }
 5396         sched_unpin();
 5397         rw_wunlock(&pvh_global_lock);
 5398 }
 5399 
 5400 /*
 5401  * Miscellaneous support routines follow
 5402  */
 5403 
 5404 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
 5405 static __inline void
 5406 pmap_pte_attr(pt_entry_t *pte, int cache_bits)
 5407 {
 5408         u_int opte, npte;
 5409 
 5410         /*
 5411          * The cache mode bits are all in the low 32-bits of the
 5412          * PTE, so we can just spin on updating the low 32-bits.
 5413          */
 5414         do {
 5415                 opte = *(u_int *)pte;
 5416                 npte = opte & ~PG_PTE_CACHE;
 5417                 npte |= cache_bits;
 5418         } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
 5419 }
 5420 
 5421 /* Adjust the cache mode for a 2/4MB page mapped via a PDE. */
 5422 static __inline void
 5423 pmap_pde_attr(pd_entry_t *pde, int cache_bits)
 5424 {
 5425         u_int opde, npde;
 5426 
 5427         /*
 5428          * The cache mode bits are all in the low 32-bits of the
 5429          * PDE, so we can just spin on updating the low 32-bits.
 5430          */
 5431         do {
 5432                 opde = *(u_int *)pde;
 5433                 npde = opde & ~PG_PDE_CACHE;
 5434                 npde |= cache_bits;
 5435         } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
 5436 }
 5437 
 5438 /*
 5439  * Map a set of physical memory pages into the kernel virtual
 5440  * address space. Return a pointer to where it is mapped. This
 5441  * routine is intended to be used for mapping device memory,
 5442  * NOT real memory.
 5443  */
 5444 static void *
 5445 __CONCAT(PMTYPE, mapdev_attr)(vm_paddr_t pa, vm_size_t size, int mode,
 5446     int flags)
 5447 {
 5448         struct pmap_preinit_mapping *ppim;
 5449         vm_offset_t va, offset;
 5450         vm_page_t m;
 5451         vm_size_t tmpsize;
 5452         int i;
 5453 
 5454         offset = pa & PAGE_MASK;
 5455         size = round_page(offset + size);
 5456         pa = pa & PG_FRAME;
 5457 
 5458         if (pa < PMAP_MAP_LOW && pa + size <= PMAP_MAP_LOW) {
 5459                 va = pa + PMAP_MAP_LOW;
 5460                 if ((flags & MAPDEV_SETATTR) == 0)
 5461                         return ((void *)(va + offset));
 5462         } else if (!pmap_initialized) {
 5463                 va = 0;
 5464                 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 5465                         ppim = pmap_preinit_mapping + i;
 5466                         if (ppim->va == 0) {
 5467                                 ppim->pa = pa;
 5468                                 ppim->sz = size;
 5469                                 ppim->mode = mode;
 5470                                 ppim->va = virtual_avail;
 5471                                 virtual_avail += size;
 5472                                 va = ppim->va;
 5473                                 break;
 5474                         }
 5475                 }
 5476                 if (va == 0)
 5477                         panic("%s: too many preinit mappings", __func__);
 5478         } else {
 5479                 /*
 5480                  * If we have a preinit mapping, re-use it.
 5481                  */
 5482                 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 5483                         ppim = pmap_preinit_mapping + i;
 5484                         if (ppim->pa == pa && ppim->sz == size &&
 5485                             (ppim->mode == mode ||
 5486                             (flags & MAPDEV_SETATTR) == 0))
 5487                                 return ((void *)(ppim->va + offset));
 5488                 }
 5489                 va = kva_alloc(size);
 5490                 if (va == 0)
 5491                         panic("%s: Couldn't allocate KVA", __func__);
 5492         }
 5493         for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) {
 5494                 if ((flags & MAPDEV_SETATTR) == 0 && pmap_initialized) {
 5495                         m = PHYS_TO_VM_PAGE(pa);
 5496                         if (m != NULL && VM_PAGE_TO_PHYS(m) == pa) {
 5497                                 pmap_kenter_attr(va + tmpsize, pa + tmpsize,
 5498                                     m->md.pat_mode);
 5499                                 continue;
 5500                         }
 5501                 }
 5502                 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
 5503         }
 5504         pmap_invalidate_range_int(kernel_pmap, va, va + tmpsize);
 5505         pmap_invalidate_cache_range(va, va + size);
 5506         return ((void *)(va + offset));
 5507 }
 5508 
 5509 static void
 5510 __CONCAT(PMTYPE, unmapdev)(vm_offset_t va, vm_size_t size)
 5511 {
 5512         struct pmap_preinit_mapping *ppim;
 5513         vm_offset_t offset;
 5514         int i;
 5515 
 5516         if (va >= PMAP_MAP_LOW && va <= KERNBASE && va + size <= KERNBASE)
 5517                 return;
 5518         offset = va & PAGE_MASK;
 5519         size = round_page(offset + size);
 5520         va = trunc_page(va);
 5521         for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 5522                 ppim = pmap_preinit_mapping + i;
 5523                 if (ppim->va == va && ppim->sz == size) {
 5524                         if (pmap_initialized)
 5525                                 return;
 5526                         ppim->pa = 0;
 5527                         ppim->va = 0;
 5528                         ppim->sz = 0;
 5529                         ppim->mode = 0;
 5530                         if (va + size == virtual_avail)
 5531                                 virtual_avail = va;
 5532                         return;
 5533                 }
 5534         }
 5535         if (pmap_initialized) {
 5536                 pmap_qremove(va, atop(size));
 5537                 kva_free(va, size);
 5538         }
 5539 }
 5540 
 5541 /*
 5542  * Sets the memory attribute for the specified page.
 5543  */
 5544 static void
 5545 __CONCAT(PMTYPE, page_set_memattr)(vm_page_t m, vm_memattr_t ma)
 5546 {
 5547 
 5548         m->md.pat_mode = ma;
 5549         if ((m->flags & PG_FICTITIOUS) != 0)
 5550                 return;
 5551 
 5552         /*
 5553          * If "m" is a normal page, flush it from the cache.
 5554          * See pmap_invalidate_cache_range().
 5555          *
 5556          * First, try to find an existing mapping of the page by sf
 5557          * buffer. sf_buf_invalidate_cache() modifies mapping and
 5558          * flushes the cache.
 5559          */    
 5560         if (sf_buf_invalidate_cache(m))
 5561                 return;
 5562 
 5563         /*
 5564          * If page is not mapped by sf buffer, but CPU does not
 5565          * support self snoop, map the page transient and do
 5566          * invalidation. In the worst case, whole cache is flushed by
 5567          * pmap_invalidate_cache_range().
 5568          */
 5569         if ((cpu_feature & CPUID_SS) == 0)
 5570                 pmap_flush_page(m);
 5571 }
 5572 
 5573 static void
 5574 __CONCAT(PMTYPE, flush_page)(vm_page_t m)
 5575 {
 5576         pt_entry_t *cmap_pte2;
 5577         struct pcpu *pc;
 5578         vm_offset_t sva, eva;
 5579         bool useclflushopt;
 5580 
 5581         useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0;
 5582         if (useclflushopt || (cpu_feature & CPUID_CLFSH) != 0) {
 5583                 sched_pin();
 5584                 pc = get_pcpu();
 5585                 cmap_pte2 = pc->pc_cmap_pte2; 
 5586                 mtx_lock(&pc->pc_cmap_lock);
 5587                 if (*cmap_pte2)
 5588                         panic("pmap_flush_page: CMAP2 busy");
 5589                 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
 5590                     PG_A | PG_M | pmap_cache_bits(kernel_pmap, m->md.pat_mode,
 5591                     0);
 5592                 invlcaddr(pc->pc_cmap_addr2);
 5593                 sva = (vm_offset_t)pc->pc_cmap_addr2;
 5594                 eva = sva + PAGE_SIZE;
 5595 
 5596                 /*
 5597                  * Use mfence or sfence despite the ordering implied by
 5598                  * mtx_{un,}lock() because clflush on non-Intel CPUs
 5599                  * and clflushopt are not guaranteed to be ordered by
 5600                  * any other instruction.
 5601                  */
 5602                 if (useclflushopt)
 5603                         sfence();
 5604                 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 5605                         mfence();
 5606                 for (; sva < eva; sva += cpu_clflush_line_size) {
 5607                         if (useclflushopt)
 5608                                 clflushopt(sva);
 5609                         else
 5610                                 clflush(sva);
 5611                 }
 5612                 if (useclflushopt)
 5613                         sfence();
 5614                 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 5615                         mfence();
 5616                 *cmap_pte2 = 0;
 5617                 sched_unpin();
 5618                 mtx_unlock(&pc->pc_cmap_lock);
 5619         } else
 5620                 pmap_invalidate_cache();
 5621 }
 5622 
 5623 /*
 5624  * Changes the specified virtual address range's memory type to that given by
 5625  * the parameter "mode".  The specified virtual address range must be
 5626  * completely contained within either the kernel map.
 5627  *
 5628  * Returns zero if the change completed successfully, and either EINVAL or
 5629  * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
 5630  * of the virtual address range was not mapped, and ENOMEM is returned if
 5631  * there was insufficient memory available to complete the change.
 5632  */
 5633 static int
 5634 __CONCAT(PMTYPE, change_attr)(vm_offset_t va, vm_size_t size, int mode)
 5635 {
 5636         vm_offset_t base, offset, tmpva;
 5637         pd_entry_t *pde;
 5638         pt_entry_t *pte;
 5639         int cache_bits_pte, cache_bits_pde;
 5640         boolean_t changed;
 5641 
 5642         base = trunc_page(va);
 5643         offset = va & PAGE_MASK;
 5644         size = round_page(offset + size);
 5645 
 5646         /*
 5647          * Only supported on kernel virtual addresses above the recursive map.
 5648          */
 5649         if (base < VM_MIN_KERNEL_ADDRESS)
 5650                 return (EINVAL);
 5651 
 5652         cache_bits_pde = pmap_cache_bits(kernel_pmap, mode, 1);
 5653         cache_bits_pte = pmap_cache_bits(kernel_pmap, mode, 0);
 5654         changed = FALSE;
 5655 
 5656         /*
 5657          * Pages that aren't mapped aren't supported.  Also break down
 5658          * 2/4MB pages into 4KB pages if required.
 5659          */
 5660         PMAP_LOCK(kernel_pmap);
 5661         for (tmpva = base; tmpva < base + size; ) {
 5662                 pde = pmap_pde(kernel_pmap, tmpva);
 5663                 if (*pde == 0) {
 5664                         PMAP_UNLOCK(kernel_pmap);
 5665                         return (EINVAL);
 5666                 }
 5667                 if (*pde & PG_PS) {
 5668                         /*
 5669                          * If the current 2/4MB page already has
 5670                          * the required memory type, then we need not
 5671                          * demote this page.  Just increment tmpva to
 5672                          * the next 2/4MB page frame.
 5673                          */
 5674                         if ((*pde & PG_PDE_CACHE) == cache_bits_pde) {
 5675                                 tmpva = trunc_4mpage(tmpva) + NBPDR;
 5676                                 continue;
 5677                         }
 5678 
 5679                         /*
 5680                          * If the current offset aligns with a 2/4MB
 5681                          * page frame and there is at least 2/4MB left
 5682                          * within the range, then we need not break
 5683                          * down this page into 4KB pages.
 5684                          */
 5685                         if ((tmpva & PDRMASK) == 0 &&
 5686                             tmpva + PDRMASK < base + size) {
 5687                                 tmpva += NBPDR;
 5688                                 continue;
 5689                         }
 5690                         if (!pmap_demote_pde(kernel_pmap, pde, tmpva)) {
 5691                                 PMAP_UNLOCK(kernel_pmap);
 5692                                 return (ENOMEM);
 5693                         }
 5694                 }
 5695                 pte = vtopte(tmpva);
 5696                 if (*pte == 0) {
 5697                         PMAP_UNLOCK(kernel_pmap);
 5698                         return (EINVAL);
 5699                 }
 5700                 tmpva += PAGE_SIZE;
 5701         }
 5702         PMAP_UNLOCK(kernel_pmap);
 5703 
 5704         /*
 5705          * Ok, all the pages exist, so run through them updating their
 5706          * cache mode if required.
 5707          */
 5708         for (tmpva = base; tmpva < base + size; ) {
 5709                 pde = pmap_pde(kernel_pmap, tmpva);
 5710                 if (*pde & PG_PS) {
 5711                         if ((*pde & PG_PDE_CACHE) != cache_bits_pde) {
 5712                                 pmap_pde_attr(pde, cache_bits_pde);
 5713                                 changed = TRUE;
 5714                         }
 5715                         tmpva = trunc_4mpage(tmpva) + NBPDR;
 5716                 } else {
 5717                         pte = vtopte(tmpva);
 5718                         if ((*pte & PG_PTE_CACHE) != cache_bits_pte) {
 5719                                 pmap_pte_attr(pte, cache_bits_pte);
 5720                                 changed = TRUE;
 5721                         }
 5722                         tmpva += PAGE_SIZE;
 5723                 }
 5724         }
 5725 
 5726         /*
 5727          * Flush CPU caches to make sure any data isn't cached that
 5728          * shouldn't be, etc.
 5729          */
 5730         if (changed) {
 5731                 pmap_invalidate_range_int(kernel_pmap, base, tmpva);
 5732                 pmap_invalidate_cache_range(base, tmpva);
 5733         }
 5734         return (0);
 5735 }
 5736 
 5737 /*
 5738  * Perform the pmap work for mincore(2).  If the page is not both referenced and
 5739  * modified by this pmap, returns its physical address so that the caller can
 5740  * find other mappings.
 5741  */
 5742 static int
 5743 __CONCAT(PMTYPE, mincore)(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)
 5744 {
 5745         pd_entry_t pde;
 5746         pt_entry_t pte;
 5747         vm_paddr_t pa;
 5748         int val;
 5749 
 5750         PMAP_LOCK(pmap);
 5751         pde = *pmap_pde(pmap, addr);
 5752         if (pde != 0) {
 5753                 if ((pde & PG_PS) != 0) {
 5754                         pte = pde;
 5755                         /* Compute the physical address of the 4KB page. */
 5756                         pa = ((pde & PG_PS_FRAME) | (addr & PDRMASK)) &
 5757                             PG_FRAME;
 5758                         val = MINCORE_PSIND(1);
 5759                 } else {
 5760                         pte = pmap_pte_ufast(pmap, addr, pde);
 5761                         pa = pte & PG_FRAME;
 5762                         val = 0;
 5763                 }
 5764         } else {
 5765                 pte = 0;
 5766                 pa = 0;
 5767                 val = 0;
 5768         }
 5769         if ((pte & PG_V) != 0) {
 5770                 val |= MINCORE_INCORE;
 5771                 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 5772                         val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
 5773                 if ((pte & PG_A) != 0)
 5774                         val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
 5775         }
 5776         if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
 5777             (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
 5778             (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
 5779                 *pap = pa;
 5780         }
 5781         PMAP_UNLOCK(pmap);
 5782         return (val);
 5783 }
 5784 
 5785 static void
 5786 __CONCAT(PMTYPE, activate)(struct thread *td)
 5787 {
 5788         pmap_t  pmap, oldpmap;
 5789         u_int   cpuid;
 5790         u_int32_t  cr3;
 5791 
 5792         critical_enter();
 5793         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 5794         oldpmap = PCPU_GET(curpmap);
 5795         cpuid = PCPU_GET(cpuid);
 5796 #if defined(SMP)
 5797         CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
 5798         CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
 5799 #else
 5800         CPU_CLR(cpuid, &oldpmap->pm_active);
 5801         CPU_SET(cpuid, &pmap->pm_active);
 5802 #endif
 5803 #ifdef PMAP_PAE_COMP
 5804         cr3 = vtophys(pmap->pm_pdpt);
 5805 #else
 5806         cr3 = vtophys(pmap->pm_pdir);
 5807 #endif
 5808         /*
 5809          * pmap_activate is for the current thread on the current cpu
 5810          */
 5811         td->td_pcb->pcb_cr3 = cr3;
 5812         PCPU_SET(curpmap, pmap);
 5813         critical_exit();
 5814 }
 5815 
 5816 static void
 5817 __CONCAT(PMTYPE, activate_boot)(pmap_t pmap)
 5818 {
 5819         u_int cpuid;
 5820 
 5821         cpuid = PCPU_GET(cpuid);
 5822 #if defined(SMP)
 5823         CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
 5824 #else
 5825         CPU_SET(cpuid, &pmap->pm_active);
 5826 #endif
 5827         PCPU_SET(curpmap, pmap);
 5828 }
 5829 
 5830 /*
 5831  *      Increase the starting virtual address of the given mapping if a
 5832  *      different alignment might result in more superpage mappings.
 5833  */
 5834 static void
 5835 __CONCAT(PMTYPE, align_superpage)(vm_object_t object, vm_ooffset_t offset,
 5836     vm_offset_t *addr, vm_size_t size)
 5837 {
 5838         vm_offset_t superpage_offset;
 5839 
 5840         if (size < NBPDR)
 5841                 return;
 5842         if (object != NULL && (object->flags & OBJ_COLORED) != 0)
 5843                 offset += ptoa(object->pg_color);
 5844         superpage_offset = offset & PDRMASK;
 5845         if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR ||
 5846             (*addr & PDRMASK) == superpage_offset)
 5847                 return;
 5848         if ((*addr & PDRMASK) < superpage_offset)
 5849                 *addr = (*addr & ~PDRMASK) + superpage_offset;
 5850         else
 5851                 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
 5852 }
 5853 
 5854 static vm_offset_t
 5855 __CONCAT(PMTYPE, quick_enter_page)(vm_page_t m)
 5856 {
 5857         vm_offset_t qaddr;
 5858         pt_entry_t *pte;
 5859 
 5860         critical_enter();
 5861         qaddr = PCPU_GET(qmap_addr);
 5862         pte = vtopte(qaddr);
 5863 
 5864         KASSERT(*pte == 0,
 5865             ("pmap_quick_enter_page: PTE busy %#jx", (uintmax_t)*pte));
 5866         *pte = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 5867             pmap_cache_bits(kernel_pmap, pmap_page_get_memattr(m), 0);
 5868         invlpg(qaddr);
 5869 
 5870         return (qaddr);
 5871 }
 5872 
 5873 static void
 5874 __CONCAT(PMTYPE, quick_remove_page)(vm_offset_t addr)
 5875 {
 5876         vm_offset_t qaddr;
 5877         pt_entry_t *pte;
 5878 
 5879         qaddr = PCPU_GET(qmap_addr);
 5880         pte = vtopte(qaddr);
 5881 
 5882         KASSERT(*pte != 0, ("pmap_quick_remove_page: PTE not in use"));
 5883         KASSERT(addr == qaddr, ("pmap_quick_remove_page: invalid address"));
 5884 
 5885         *pte = 0;
 5886         critical_exit();
 5887 }
 5888 
 5889 static vmem_t *pmap_trm_arena;
 5890 static vmem_addr_t pmap_trm_arena_last = PMAP_TRM_MIN_ADDRESS;
 5891 static int trm_guard = PAGE_SIZE;
 5892 
 5893 static int
 5894 pmap_trm_import(void *unused __unused, vmem_size_t size, int flags,
 5895     vmem_addr_t *addrp)
 5896 {
 5897         vm_page_t m;
 5898         vmem_addr_t af, addr, prev_addr;
 5899         pt_entry_t *trm_pte;
 5900 
 5901         prev_addr = atomic_load_long(&pmap_trm_arena_last);
 5902         size = round_page(size) + trm_guard;
 5903         for (;;) {
 5904                 if (prev_addr + size < prev_addr || prev_addr + size < size ||
 5905                     prev_addr + size > PMAP_TRM_MAX_ADDRESS)
 5906                         return (ENOMEM);
 5907                 addr = prev_addr + size;
 5908                 if (atomic_fcmpset_int(&pmap_trm_arena_last, &prev_addr, addr))
 5909                         break;
 5910         }
 5911         prev_addr += trm_guard;
 5912         trm_pte = PTmap + atop(prev_addr);
 5913         for (af = prev_addr; af < addr; af += PAGE_SIZE) {
 5914                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_NOBUSY |
 5915                     VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_WAITOK);
 5916                 pte_store(&trm_pte[atop(af - prev_addr)], VM_PAGE_TO_PHYS(m) |
 5917                     PG_M | PG_A | PG_RW | PG_V | pgeflag |
 5918                     pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, FALSE));
 5919         }
 5920         *addrp = prev_addr;
 5921         return (0);
 5922 }
 5923 
 5924 void
 5925 pmap_init_trm(void)
 5926 {
 5927         vm_page_t pd_m;
 5928 
 5929         TUNABLE_INT_FETCH("machdep.trm_guard", &trm_guard);
 5930         if ((trm_guard & PAGE_MASK) != 0)
 5931                 trm_guard = 0;
 5932         pmap_trm_arena = vmem_create("i386trampoline", 0, 0, 1, 0, M_WAITOK);
 5933         vmem_set_import(pmap_trm_arena, pmap_trm_import, NULL, NULL, PAGE_SIZE);
 5934         pd_m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_NOBUSY |
 5935             VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_WAITOK | VM_ALLOC_ZERO);
 5936         if ((pd_m->flags & PG_ZERO) == 0)
 5937                 pmap_zero_page(pd_m);
 5938         PTD[TRPTDI] = VM_PAGE_TO_PHYS(pd_m) | PG_M | PG_A | PG_RW | PG_V |
 5939             pmap_cache_bits(kernel_pmap, VM_MEMATTR_DEFAULT, TRUE);
 5940 }
 5941 
 5942 static void *
 5943 __CONCAT(PMTYPE, trm_alloc)(size_t size, int flags)
 5944 {
 5945         vmem_addr_t res;
 5946         int error;
 5947 
 5948         MPASS((flags & ~(M_WAITOK | M_NOWAIT | M_ZERO)) == 0);
 5949         error = vmem_xalloc(pmap_trm_arena, roundup2(size, 4), sizeof(int),
 5950             0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, flags | M_FIRSTFIT, &res);
 5951         if (error != 0)
 5952                 return (NULL);
 5953         if ((flags & M_ZERO) != 0)
 5954                 bzero((void *)res, size);
 5955         return ((void *)res);
 5956 }
 5957 
 5958 static void
 5959 __CONCAT(PMTYPE, trm_free)(void *addr, size_t size)
 5960 {
 5961 
 5962         vmem_free(pmap_trm_arena, (uintptr_t)addr, roundup2(size, 4));
 5963 }
 5964 
 5965 static void
 5966 __CONCAT(PMTYPE, ksetrw)(vm_offset_t va)
 5967 {
 5968 
 5969         *vtopte(va) |= PG_RW;
 5970 }
 5971 
 5972 static void
 5973 __CONCAT(PMTYPE, remap_lowptdi)(bool enable)
 5974 {
 5975 
 5976         PTD[KPTDI] = enable ? PTD[LOWPTDI] : 0;
 5977         invltlb_glob();
 5978 }
 5979 
 5980 static vm_offset_t
 5981 __CONCAT(PMTYPE, get_map_low)(void)
 5982 {
 5983 
 5984         return (PMAP_MAP_LOW);
 5985 }
 5986 
 5987 static vm_offset_t
 5988 __CONCAT(PMTYPE, get_vm_maxuser_address)(void)
 5989 {
 5990 
 5991         return (VM_MAXUSER_ADDRESS);
 5992 }
 5993 
 5994 static vm_paddr_t
 5995 __CONCAT(PMTYPE, pg_frame)(vm_paddr_t pa)
 5996 {
 5997 
 5998         return (pa & PG_FRAME);
 5999 }
 6000 
 6001 static void
 6002 __CONCAT(PMTYPE, sf_buf_map)(struct sf_buf *sf)
 6003 {
 6004         pt_entry_t opte, *ptep;
 6005 
 6006         /*
 6007          * Update the sf_buf's virtual-to-physical mapping, flushing the
 6008          * virtual address from the TLB.  Since the reference count for
 6009          * the sf_buf's old mapping was zero, that mapping is not
 6010          * currently in use.  Consequently, there is no need to exchange
 6011          * the old and new PTEs atomically, even under PAE.
 6012          */
 6013         ptep = vtopte(sf->kva);
 6014         opte = *ptep;
 6015         *ptep = VM_PAGE_TO_PHYS(sf->m) | PG_RW | PG_V |
 6016             pmap_cache_bits(kernel_pmap, sf->m->md.pat_mode, 0);
 6017 
 6018         /*
 6019          * Avoid unnecessary TLB invalidations: If the sf_buf's old
 6020          * virtual-to-physical mapping was not used, then any processor
 6021          * that has invalidated the sf_buf's virtual address from its TLB
 6022          * since the last used mapping need not invalidate again.
 6023          */
 6024 #ifdef SMP
 6025         if ((opte & (PG_V | PG_A)) ==  (PG_V | PG_A))
 6026                 CPU_ZERO(&sf->cpumask);
 6027 #else
 6028         if ((opte & (PG_V | PG_A)) ==  (PG_V | PG_A))
 6029                 pmap_invalidate_page_int(kernel_pmap, sf->kva);
 6030 #endif
 6031 }
 6032 
 6033 static void
 6034 __CONCAT(PMTYPE, cp_slow0_map)(vm_offset_t kaddr, int plen, vm_page_t *ma)
 6035 {
 6036         pt_entry_t *pte;
 6037         int i;
 6038 
 6039         for (i = 0, pte = vtopte(kaddr); i < plen; i++, pte++) {
 6040                 *pte = PG_V | PG_RW | PG_A | PG_M | VM_PAGE_TO_PHYS(ma[i]) |
 6041                     pmap_cache_bits(kernel_pmap, pmap_page_get_memattr(ma[i]),
 6042                     FALSE);
 6043                 invlpg(kaddr + ptoa(i));
 6044         }
 6045 }
 6046 
 6047 static u_int
 6048 __CONCAT(PMTYPE, get_kcr3)(void)
 6049 {
 6050 
 6051 #ifdef PMAP_PAE_COMP
 6052         return ((u_int)IdlePDPT);
 6053 #else
 6054         return ((u_int)IdlePTD);
 6055 #endif
 6056 }
 6057 
 6058 static u_int
 6059 __CONCAT(PMTYPE, get_cr3)(pmap_t pmap)
 6060 {
 6061 
 6062 #ifdef PMAP_PAE_COMP
 6063         return ((u_int)vtophys(pmap->pm_pdpt));
 6064 #else
 6065         return ((u_int)vtophys(pmap->pm_pdir));
 6066 #endif
 6067 }
 6068 
 6069 static caddr_t
 6070 __CONCAT(PMTYPE, cmap3)(vm_paddr_t pa, u_int pte_bits)
 6071 {
 6072         pt_entry_t *pte;
 6073 
 6074         pte = CMAP3;
 6075         *pte = pa | pte_bits;
 6076         invltlb();
 6077         return (CADDR3);
 6078 }
 6079 
 6080 static void
 6081 __CONCAT(PMTYPE, basemem_setup)(u_int basemem)
 6082 {
 6083         pt_entry_t *pte;
 6084         int i;
 6085 
 6086         /*
 6087          * Map pages between basemem and ISA_HOLE_START, if any, r/w into
 6088          * the vm86 page table so that vm86 can scribble on them using
 6089          * the vm86 map too.  XXX: why 2 ways for this and only 1 way for
 6090          * page 0, at least as initialized here?
 6091          */
 6092         pte = (pt_entry_t *)vm86paddr;
 6093         for (i = basemem / 4; i < 160; i++)
 6094                 pte[i] = (i << PAGE_SHIFT) | PG_V | PG_RW | PG_U;
 6095 }
 6096 
 6097 struct bios16_pmap_handle {
 6098         pt_entry_t      *pte;
 6099         pd_entry_t      *ptd;
 6100         pt_entry_t      orig_ptd;
 6101 };
 6102 
 6103 static void *
 6104 __CONCAT(PMTYPE, bios16_enter)(void)
 6105 {
 6106         struct bios16_pmap_handle *h;
 6107 
 6108         /*
 6109          * no page table, so create one and install it.
 6110          */
 6111         h = malloc(sizeof(struct bios16_pmap_handle), M_TEMP, M_WAITOK);
 6112         h->pte = (pt_entry_t *)malloc(PAGE_SIZE, M_TEMP, M_WAITOK);
 6113         h->ptd = IdlePTD;
 6114         *h->pte = vm86phystk | PG_RW | PG_V;
 6115         h->orig_ptd = *h->ptd;
 6116         *h->ptd = vtophys(h->pte) | PG_RW | PG_V;
 6117         pmap_invalidate_all_int(kernel_pmap);   /* XXX insurance for now */
 6118         return (h);
 6119 }
 6120 
 6121 static void
 6122 __CONCAT(PMTYPE, bios16_leave)(void *arg)
 6123 {
 6124         struct bios16_pmap_handle *h;
 6125 
 6126         h = arg;
 6127         *h->ptd = h->orig_ptd;          /* remove page table */
 6128         /*
 6129          * XXX only needs to be invlpg(0) but that doesn't work on the 386
 6130          */
 6131         pmap_invalidate_all_int(kernel_pmap);
 6132         free(h->pte, M_TEMP);           /* ... and free it */
 6133 }
 6134 
 6135 struct pmap_kernel_map_range {
 6136         vm_offset_t sva;
 6137         pt_entry_t attrs;
 6138         int ptes;
 6139         int pdes;
 6140         int pdpes;
 6141 };
 6142 
 6143 static void
 6144 sysctl_kmaps_dump(struct sbuf *sb, struct pmap_kernel_map_range *range,
 6145     vm_offset_t eva)
 6146 {
 6147         const char *mode;
 6148         int i, pat_idx;
 6149 
 6150         if (eva <= range->sva)
 6151                 return;
 6152 
 6153         pat_idx = pmap_pat_index(kernel_pmap, range->attrs, true);
 6154         for (i = 0; i < PAT_INDEX_SIZE; i++)
 6155                 if (pat_index[i] == pat_idx)
 6156                         break;
 6157 
 6158         switch (i) {
 6159         case PAT_WRITE_BACK:
 6160                 mode = "WB";
 6161                 break;
 6162         case PAT_WRITE_THROUGH:
 6163                 mode = "WT";
 6164                 break;
 6165         case PAT_UNCACHEABLE:
 6166                 mode = "UC";
 6167                 break;
 6168         case PAT_UNCACHED:
 6169                 mode = "U-";
 6170                 break;
 6171         case PAT_WRITE_PROTECTED:
 6172                 mode = "WP";
 6173                 break;
 6174         case PAT_WRITE_COMBINING:
 6175                 mode = "WC";
 6176                 break;
 6177         default:
 6178                 printf("%s: unknown PAT mode %#x for range 0x%08x-0x%08x\n",
 6179                     __func__, pat_idx, range->sva, eva);
 6180                 mode = "??";
 6181                 break;
 6182         }
 6183 
 6184         sbuf_printf(sb, "0x%08x-0x%08x r%c%c%c%c %s %d %d %d\n",
 6185             range->sva, eva,
 6186             (range->attrs & PG_RW) != 0 ? 'w' : '-',
 6187             (range->attrs & pg_nx) != 0 ? '-' : 'x',
 6188             (range->attrs & PG_U) != 0 ? 'u' : 's',
 6189             (range->attrs & PG_G) != 0 ? 'g' : '-',
 6190             mode, range->pdpes, range->pdes, range->ptes);
 6191 
 6192         /* Reset to sentinel value. */
 6193         range->sva = 0xffffffff;
 6194 }
 6195 
 6196 /*
 6197  * Determine whether the attributes specified by a page table entry match those
 6198  * being tracked by the current range.  This is not quite as simple as a direct
 6199  * flag comparison since some PAT modes have multiple representations.
 6200  */
 6201 static bool
 6202 sysctl_kmaps_match(struct pmap_kernel_map_range *range, pt_entry_t attrs)
 6203 {
 6204         pt_entry_t diff, mask;
 6205 
 6206         mask = pg_nx | PG_G | PG_RW | PG_U | PG_PDE_CACHE;
 6207         diff = (range->attrs ^ attrs) & mask;
 6208         if (diff == 0)
 6209                 return (true);
 6210         if ((diff & ~PG_PDE_PAT) == 0 &&
 6211             pmap_pat_index(kernel_pmap, range->attrs, true) ==
 6212             pmap_pat_index(kernel_pmap, attrs, true))
 6213                 return (true);
 6214         return (false);
 6215 }
 6216 
 6217 static void
 6218 sysctl_kmaps_reinit(struct pmap_kernel_map_range *range, vm_offset_t va,
 6219     pt_entry_t attrs)
 6220 {
 6221 
 6222         memset(range, 0, sizeof(*range));
 6223         range->sva = va;
 6224         range->attrs = attrs;
 6225 }
 6226 
 6227 /*
 6228  * Given a leaf PTE, derive the mapping's attributes.  If they do not match
 6229  * those of the current run, dump the address range and its attributes, and
 6230  * begin a new run.
 6231  */
 6232 static void
 6233 sysctl_kmaps_check(struct sbuf *sb, struct pmap_kernel_map_range *range,
 6234     vm_offset_t va, pd_entry_t pde, pt_entry_t pte)
 6235 {
 6236         pt_entry_t attrs;
 6237 
 6238         attrs = pde & (PG_RW | PG_U | pg_nx);
 6239 
 6240         if ((pde & PG_PS) != 0) {
 6241                 attrs |= pde & (PG_G | PG_PDE_CACHE);
 6242         } else if (pte != 0) {
 6243                 attrs |= pte & pg_nx;
 6244                 attrs &= pg_nx | (pte & (PG_RW | PG_U));
 6245                 attrs |= pte & (PG_G | PG_PTE_CACHE);
 6246 
 6247                 /* Canonicalize by always using the PDE PAT bit. */
 6248                 if ((attrs & PG_PTE_PAT) != 0)
 6249                         attrs ^= PG_PDE_PAT | PG_PTE_PAT;
 6250         }
 6251 
 6252         if (range->sva > va || !sysctl_kmaps_match(range, attrs)) {
 6253                 sysctl_kmaps_dump(sb, range, va);
 6254                 sysctl_kmaps_reinit(range, va, attrs);
 6255         }
 6256 }
 6257 
 6258 static int
 6259 __CONCAT(PMTYPE, sysctl_kmaps)(SYSCTL_HANDLER_ARGS)
 6260 {
 6261         struct pmap_kernel_map_range range;
 6262         struct sbuf sbuf, *sb;
 6263         pd_entry_t pde;
 6264         pt_entry_t *pt, pte;
 6265         vm_offset_t sva;
 6266         vm_paddr_t pa;
 6267         int error;
 6268         u_int i, k;
 6269 
 6270         error = sysctl_wire_old_buffer(req, 0);
 6271         if (error != 0)
 6272                 return (error);
 6273         sb = &sbuf;
 6274         sbuf_new_for_sysctl(sb, NULL, PAGE_SIZE, req);
 6275 
 6276         /* Sentinel value. */
 6277         range.sva = 0xffffffff;
 6278 
 6279         /*
 6280          * Iterate over the kernel page tables without holding the
 6281          * kernel pmap lock.  Kernel page table pages are never freed,
 6282          * so at worst we will observe inconsistencies in the output.
 6283          */
 6284         for (sva = 0, i = 0; i < NPTEPG * NPGPTD * NPDEPG ;) {
 6285                 if (i == 0)
 6286                         sbuf_printf(sb, "\nLow PDE:\n");
 6287                 else if (i == LOWPTDI * NPTEPG)
 6288                         sbuf_printf(sb, "Low PDE dup:\n");
 6289                 else if (i == PTDPTDI * NPTEPG)
 6290                         sbuf_printf(sb, "Recursive map:\n");
 6291                 else if (i == KERNPTDI * NPTEPG)
 6292                         sbuf_printf(sb, "Kernel base:\n");
 6293                 else if (i == TRPTDI * NPTEPG)
 6294                         sbuf_printf(sb, "Trampoline:\n");
 6295                 pde = IdlePTD[sva >> PDRSHIFT];
 6296                 if ((pde & PG_V) == 0) {
 6297                         sva = rounddown2(sva, NBPDR);
 6298                         sysctl_kmaps_dump(sb, &range, sva);
 6299                         sva += NBPDR;
 6300                         i += NPTEPG;
 6301                         continue;
 6302                 }
 6303                 pa = pde & PG_FRAME;
 6304                 if ((pde & PG_PS) != 0) {
 6305                         sysctl_kmaps_check(sb, &range, sva, pde, 0);
 6306                         range.pdes++;
 6307                         sva += NBPDR;
 6308                         i += NPTEPG;
 6309                         continue;
 6310                 }
 6311                 for (pt = vtopte(sva), k = 0; k < NPTEPG; i++, k++, pt++,
 6312                     sva += PAGE_SIZE) {
 6313                         pte = *pt;
 6314                         if ((pte & PG_V) == 0) {
 6315                                 sysctl_kmaps_dump(sb, &range, sva);
 6316                                 continue;
 6317                         }
 6318                         sysctl_kmaps_check(sb, &range, sva, pde, pte);
 6319                         range.ptes++;
 6320                 }
 6321         }
 6322 
 6323         error = sbuf_finish(sb);
 6324         sbuf_delete(sb);
 6325         return (error);
 6326 }
 6327 
 6328 #define PMM(a)                                  \
 6329         .pm_##a = __CONCAT(PMTYPE, a),
 6330 
 6331 struct pmap_methods __CONCAT(PMTYPE, methods) = {
 6332         PMM(ksetrw)
 6333         PMM(remap_lower)
 6334         PMM(remap_lowptdi)
 6335         PMM(align_superpage)
 6336         PMM(quick_enter_page)
 6337         PMM(quick_remove_page)
 6338         PMM(trm_alloc)
 6339         PMM(trm_free)
 6340         PMM(get_map_low)
 6341         PMM(get_vm_maxuser_address)
 6342         PMM(kextract)
 6343         PMM(pg_frame)
 6344         PMM(sf_buf_map)
 6345         PMM(cp_slow0_map)
 6346         PMM(get_kcr3)
 6347         PMM(get_cr3)
 6348         PMM(cmap3)
 6349         PMM(basemem_setup)
 6350         PMM(set_nx)
 6351         PMM(bios16_enter)
 6352         PMM(bios16_leave)
 6353         PMM(bootstrap)
 6354         PMM(is_valid_memattr)
 6355         PMM(cache_bits)
 6356         PMM(ps_enabled)
 6357         PMM(pinit0)
 6358         PMM(pinit)
 6359         PMM(activate)
 6360         PMM(activate_boot)
 6361         PMM(advise)
 6362         PMM(clear_modify)
 6363         PMM(change_attr)
 6364         PMM(mincore)
 6365         PMM(copy)
 6366         PMM(copy_page)
 6367         PMM(copy_pages)
 6368         PMM(zero_page)
 6369         PMM(zero_page_area)
 6370         PMM(enter)
 6371         PMM(enter_object)
 6372         PMM(enter_quick)
 6373         PMM(kenter_temporary)
 6374         PMM(object_init_pt)
 6375         PMM(unwire)
 6376         PMM(page_exists_quick)
 6377         PMM(page_wired_mappings)
 6378         PMM(page_is_mapped)
 6379         PMM(remove_pages)
 6380         PMM(is_modified)
 6381         PMM(is_prefaultable)
 6382         PMM(is_referenced)
 6383         PMM(remove_write)
 6384         PMM(ts_referenced)
 6385         PMM(mapdev_attr)
 6386         PMM(unmapdev)
 6387         PMM(page_set_memattr)
 6388         PMM(extract)
 6389         PMM(extract_and_hold)
 6390         PMM(map)
 6391         PMM(qenter)
 6392         PMM(qremove)
 6393         PMM(release)
 6394         PMM(remove)
 6395         PMM(protect)
 6396         PMM(remove_all)
 6397         PMM(init)
 6398         PMM(init_pat)
 6399         PMM(growkernel)
 6400         PMM(invalidate_page)
 6401         PMM(invalidate_range)
 6402         PMM(invalidate_all)
 6403         PMM(invalidate_cache)
 6404         PMM(flush_page)
 6405         PMM(kenter)
 6406         PMM(kremove)
 6407         PMM(sysctl_kmaps)
 6408 };

Cache object: 2d166e7b693f3a3f40edfe583372afdd


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.