The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/pmap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-4-Clause
    3  *
    4  * Copyright (c) 1991 Regents of the University of California.
    5  * All rights reserved.
    6  * Copyright (c) 1994 John S. Dyson
    7  * All rights reserved.
    8  * Copyright (c) 1994 David Greenman
    9  * All rights reserved.
   10  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
   11  * All rights reserved.
   12  *
   13  * This code is derived from software contributed to Berkeley by
   14  * the Systems Programming Group of the University of Utah Computer
   15  * Science Department and William Jolitz of UUNET Technologies Inc.
   16  *
   17  * Redistribution and use in source and binary forms, with or without
   18  * modification, are permitted provided that the following conditions
   19  * are met:
   20  * 1. Redistributions of source code must retain the above copyright
   21  *    notice, this list of conditions and the following disclaimer.
   22  * 2. Redistributions in binary form must reproduce the above copyright
   23  *    notice, this list of conditions and the following disclaimer in the
   24  *    documentation and/or other materials provided with the distribution.
   25  * 3. All advertising materials mentioning features or use of this software
   26  *    must display the following acknowledgement:
   27  *      This product includes software developed by the University of
   28  *      California, Berkeley and its contributors.
   29  * 4. Neither the name of the University nor the names of its contributors
   30  *    may be used to endorse or promote products derived from this software
   31  *    without specific prior written permission.
   32  *
   33  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   34  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   35  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   36  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   37  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   38  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   39  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   40  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   41  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   42  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   43  * SUCH DAMAGE.
   44  *
   45  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   46  */
   47 /*-
   48  * Copyright (c) 2003 Networks Associates Technology, Inc.
   49  * All rights reserved.
   50  * Copyright (c) 2018 The FreeBSD Foundation
   51  * All rights reserved.
   52  *
   53  * This software was developed for the FreeBSD Project by Jake Burkholder,
   54  * Safeport Network Services, and Network Associates Laboratories, the
   55  * Security Research Division of Network Associates, Inc. under
   56  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   57  * CHATS research program.
   58  *
   59  * Portions of this software were developed by
   60  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
   61  * the FreeBSD Foundation.
   62  *
   63  * Redistribution and use in source and binary forms, with or without
   64  * modification, are permitted provided that the following conditions
   65  * are met:
   66  * 1. Redistributions of source code must retain the above copyright
   67  *    notice, this list of conditions and the following disclaimer.
   68  * 2. Redistributions in binary form must reproduce the above copyright
   69  *    notice, this list of conditions and the following disclaimer in the
   70  *    documentation and/or other materials provided with the distribution.
   71  *
   72  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   73  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   74  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   75  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   76  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   77  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   78  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   79  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   80  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   81  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   82  * SUCH DAMAGE.
   83  */
   84 
   85 #include <sys/cdefs.h>
   86 __FBSDID("$FreeBSD$");
   87 
   88 /*
   89  *      Manages physical address maps.
   90  *
   91  *      Since the information managed by this module is
   92  *      also stored by the logical address mapping module,
   93  *      this module may throw away valid virtual-to-physical
   94  *      mappings at almost any time.  However, invalidations
   95  *      of virtual-to-physical mappings must be done as
   96  *      requested.
   97  *
   98  *      In order to cope with hardware architectures which
   99  *      make virtual-to-physical map invalidates expensive,
  100  *      this module may delay invalidate or reduced protection
  101  *      operations until such time as they are actually
  102  *      necessary.  This module is given full information as
  103  *      to which processors are currently using which maps,
  104  *      and to when physical maps must be made correct.
  105  */
  106 
  107 #include "opt_apic.h"
  108 #include "opt_cpu.h"
  109 #include "opt_pmap.h"
  110 #include "opt_smp.h"
  111 #include "opt_vm.h"
  112 
  113 #include <sys/param.h>
  114 #include <sys/systm.h>
  115 #include <sys/kernel.h>
  116 #include <sys/ktr.h>
  117 #include <sys/lock.h>
  118 #include <sys/malloc.h>
  119 #include <sys/mman.h>
  120 #include <sys/msgbuf.h>
  121 #include <sys/mutex.h>
  122 #include <sys/proc.h>
  123 #include <sys/rwlock.h>
  124 #include <sys/sf_buf.h>
  125 #include <sys/sx.h>
  126 #include <sys/vmmeter.h>
  127 #include <sys/sched.h>
  128 #include <sys/sysctl.h>
  129 #include <sys/smp.h>
  130 #include <sys/vmem.h>
  131 
  132 #include <vm/vm.h>
  133 #include <vm/vm_param.h>
  134 #include <vm/vm_kern.h>
  135 #include <vm/vm_page.h>
  136 #include <vm/vm_map.h>
  137 #include <vm/vm_object.h>
  138 #include <vm/vm_extern.h>
  139 #include <vm/vm_pageout.h>
  140 #include <vm/vm_pager.h>
  141 #include <vm/vm_phys.h>
  142 #include <vm/vm_radix.h>
  143 #include <vm/vm_reserv.h>
  144 #include <vm/uma.h>
  145 
  146 #ifdef DEV_APIC
  147 #include <sys/bus.h>
  148 #include <machine/intr_machdep.h>
  149 #include <x86/apicvar.h>
  150 #endif
  151 #include <x86/ifunc.h>
  152 #include <machine/bootinfo.h>
  153 #include <machine/cpu.h>
  154 #include <machine/cputypes.h>
  155 #include <machine/md_var.h>
  156 #include <machine/pcb.h>
  157 #include <machine/specialreg.h>
  158 #ifdef SMP
  159 #include <machine/smp.h>
  160 #endif
  161 
  162 #ifndef PMAP_SHPGPERPROC
  163 #define PMAP_SHPGPERPROC 200
  164 #endif
  165 
  166 #if !defined(DIAGNOSTIC)
  167 #ifdef __GNUC_GNU_INLINE__
  168 #define PMAP_INLINE     __attribute__((__gnu_inline__)) inline
  169 #else
  170 #define PMAP_INLINE     extern inline
  171 #endif
  172 #else
  173 #define PMAP_INLINE
  174 #endif
  175 
  176 #ifdef PV_STATS
  177 #define PV_STAT(x)      do { x ; } while (0)
  178 #else
  179 #define PV_STAT(x)      do { } while (0)
  180 #endif
  181 
  182 #define pa_index(pa)    ((pa) >> PDRSHIFT)
  183 #define pa_to_pvh(pa)   (&pv_table[pa_index(pa)])
  184 
  185 /*
  186  * Get PDEs and PTEs for user/kernel address space
  187  */
  188 #define pmap_pde(m, v)  (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
  189 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
  190 
  191 #define pmap_pde_v(pte)         ((*(int *)pte & PG_V) != 0)
  192 #define pmap_pte_w(pte)         ((*(int *)pte & PG_W) != 0)
  193 #define pmap_pte_m(pte)         ((*(int *)pte & PG_M) != 0)
  194 #define pmap_pte_u(pte)         ((*(int *)pte & PG_A) != 0)
  195 #define pmap_pte_v(pte)         ((*(int *)pte & PG_V) != 0)
  196 
  197 #define pmap_pte_set_w(pte, v)  ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \
  198     atomic_clear_int((u_int *)(pte), PG_W))
  199 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
  200 
  201 struct pmap kernel_pmap_store;
  202 
  203 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  204 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  205 static int pgeflag = 0;         /* PG_G or-in */
  206 static int pseflag = 0;         /* PG_PS or-in */
  207 
  208 static int nkpt = NKPT;
  209 vm_offset_t kernel_vm_end = /* 0 + */ NKPT * NBPDR;
  210 
  211 #if defined(PAE) || defined(PAE_TABLES)
  212 pt_entry_t pg_nx;
  213 static uma_zone_t pdptzone;
  214 #endif
  215 
  216 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  217 
  218 static int pat_works = 1;
  219 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 0,
  220     "Is page attribute table fully functional?");
  221 
  222 static int pg_ps_enabled = 1;
  223 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  224     &pg_ps_enabled, 0, "Are large page mappings enabled?");
  225 
  226 #define PAT_INDEX_SIZE  8
  227 static int pat_index[PAT_INDEX_SIZE];   /* cache mode to PAT index conversion */
  228 
  229 /*
  230  * pmap_mapdev support pre initialization (i.e. console)
  231  */
  232 #define PMAP_PREINIT_MAPPING_COUNT      8
  233 static struct pmap_preinit_mapping {
  234         vm_paddr_t      pa;
  235         vm_offset_t     va;
  236         vm_size_t       sz;
  237         int             mode;
  238 } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT];
  239 static int pmap_initialized;
  240 
  241 static struct rwlock_padalign pvh_global_lock;
  242 
  243 /*
  244  * Data for the pv entry allocation mechanism
  245  */
  246 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
  247 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  248 static struct md_page *pv_table;
  249 static int shpgperproc = PMAP_SHPGPERPROC;
  250 
  251 struct pv_chunk *pv_chunkbase;          /* KVA block for pv_chunks */
  252 int pv_maxchunks;                       /* How many chunks we have KVA for */
  253 vm_offset_t pv_vafree;                  /* freelist stored in the PTE */
  254 
  255 /*
  256  * All those kernel PT submaps that BSD is so fond of
  257  */
  258 pt_entry_t *CMAP3;
  259 static pd_entry_t *KPTD;
  260 caddr_t ptvmmap = 0;
  261 caddr_t CADDR3;
  262 
  263 /*
  264  * Crashdump maps.
  265  */
  266 static caddr_t crashdumpmap;
  267 
  268 static pt_entry_t *PMAP1 = NULL, *PMAP2, *PMAP3;
  269 static pt_entry_t *PADDR1 = NULL, *PADDR2, *PADDR3;
  270 #ifdef SMP
  271 static int PMAP1cpu, PMAP3cpu;
  272 static int PMAP1changedcpu;
  273 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 
  274            &PMAP1changedcpu, 0,
  275            "Number of times pmap_pte_quick changed CPU with same PMAP1");
  276 #endif
  277 static int PMAP1changed;
  278 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 
  279            &PMAP1changed, 0,
  280            "Number of times pmap_pte_quick changed PMAP1");
  281 static int PMAP1unchanged;
  282 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 
  283            &PMAP1unchanged, 0,
  284            "Number of times pmap_pte_quick didn't change PMAP1");
  285 static struct mtx PMAP2mutex;
  286 
  287 int pti;
  288 
  289 /*
  290  * Internal flags for pmap_enter()'s helper functions.
  291  */
  292 #define PMAP_ENTER_NORECLAIM    0x1000000       /* Don't reclaim PV entries. */
  293 #define PMAP_ENTER_NOREPLACE    0x2000000       /* Don't replace mappings. */
  294 
  295 /*
  296  * Internal flags for pmap_mapdev_internal().
  297  */
  298 #define MAPDEV_SETATTR          0x0000001       /* Modify existing attrs. */
  299 
  300 static void     free_pv_chunk(struct pv_chunk *pc);
  301 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  302 static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
  303 static void     pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  304 static bool     pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,
  305                     u_int flags);
  306 #if VM_NRESERVLEVEL > 0
  307 static void     pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  308 #endif
  309 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  310 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  311                     vm_offset_t va);
  312 static int      pmap_pvh_wired_mappings(struct md_page *pvh, int count);
  313 
  314 static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  315 static bool     pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
  316                     vm_prot_t prot);
  317 static int      pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
  318                     u_int flags, vm_page_t m);
  319 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  320     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  321 static void pmap_flush_page(vm_page_t m);
  322 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted);
  323 static void pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva,
  324     vm_offset_t eva);
  325 static void pmap_invalidate_cache_range_all(vm_offset_t sva,
  326     vm_offset_t eva);
  327 static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
  328                     pd_entry_t pde);
  329 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
  330 static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
  331 static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
  332 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
  333 static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
  334 static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
  335 #if VM_NRESERVLEVEL > 0
  336 static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  337 #endif
  338 static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
  339     vm_prot_t prot);
  340 static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
  341 static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
  342     struct spglist *free);
  343 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
  344     struct spglist *free);
  345 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
  346 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free);
  347 static bool     pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
  348                     struct spglist *free);
  349 static void pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va);
  350 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  351 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  352     vm_page_t m);
  353 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  354     pd_entry_t newpde);
  355 static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
  356 
  357 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);
  358 
  359 static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags);
  360 static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free);
  361 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
  362 static void pmap_pte_release(pt_entry_t *pte);
  363 static int pmap_unuse_pt(pmap_t, vm_offset_t, struct spglist *);
  364 #if defined(PAE) || defined(PAE_TABLES)
  365 static void *pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain,
  366     uint8_t *flags, int wait);
  367 #endif
  368 static void pmap_init_trm(void);
  369 
  370 static __inline void pagezero(void *page);
  371 
  372 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  373 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  374 
  375 void pmap_cold(void);
  376 extern char _end[];
  377 u_long physfree;        /* phys addr of next free page */
  378 u_long vm86phystk;      /* PA of vm86/bios stack */
  379 u_long vm86paddr;       /* address of vm86 region */
  380 int vm86pa;             /* phys addr of vm86 region */
  381 u_long KERNend;         /* phys addr end of kernel (just after bss) */
  382 pd_entry_t *IdlePTD;    /* phys addr of kernel PTD */
  383 #if defined(PAE) || defined(PAE_TABLES)
  384 pdpt_entry_t *IdlePDPT; /* phys addr of kernel PDPT */
  385 #endif
  386 pt_entry_t *KPTmap;     /* address of kernel page tables */
  387 u_long KPTphys;         /* phys addr of kernel page tables */
  388 extern u_long tramp_idleptd;
  389 
  390 static u_long
  391 allocpages(u_int cnt, u_long *physfree)
  392 {
  393         u_long res;
  394 
  395         res = *physfree;
  396         *physfree += PAGE_SIZE * cnt;
  397         bzero((void *)res, PAGE_SIZE * cnt);
  398         return (res);
  399 }
  400 
  401 static void
  402 pmap_cold_map(u_long pa, u_long va, u_long cnt)
  403 {
  404         pt_entry_t *pt;
  405 
  406         for (pt = (pt_entry_t *)KPTphys + atop(va); cnt > 0;
  407             cnt--, pt++, va += PAGE_SIZE, pa += PAGE_SIZE)
  408                 *pt = pa | PG_V | PG_RW | PG_A | PG_M;
  409 }
  410 
  411 static void
  412 pmap_cold_mapident(u_long pa, u_long cnt)
  413 {
  414 
  415         pmap_cold_map(pa, pa, cnt);
  416 }
  417 
  418 _Static_assert(2 * NBPDR == KERNBASE, "Broken double-map of zero PTD");
  419 
  420 /*
  421  * Called from locore.s before paging is enabled.  Sets up the first
  422  * kernel page table.  Since kernel is mapped with PA == VA, this code
  423  * does not require relocations.
  424  */
  425 void
  426 pmap_cold(void)
  427 {
  428         pt_entry_t *pt;
  429         u_long a;
  430         u_int cr3, ncr4;
  431 
  432         physfree = (u_long)&_end;
  433         if (bootinfo.bi_esymtab != 0)
  434                 physfree = bootinfo.bi_esymtab;
  435         if (bootinfo.bi_kernend != 0)
  436                 physfree = bootinfo.bi_kernend;
  437         physfree = roundup2(physfree, NBPDR);
  438         KERNend = physfree;
  439 
  440         /* Allocate Kernel Page Tables */
  441         KPTphys = allocpages(NKPT, &physfree);
  442         KPTmap = (pt_entry_t *)KPTphys;
  443 
  444         /* Allocate Page Table Directory */
  445 #if defined(PAE) || defined(PAE_TABLES)
  446         /* XXX only need 32 bytes (easier for now) */
  447         IdlePDPT = (pdpt_entry_t *)allocpages(1, &physfree);
  448 #endif
  449         IdlePTD = (pd_entry_t *)allocpages(NPGPTD, &physfree);
  450 
  451         /*
  452          * Allocate KSTACK.  Leave a guard page between IdlePTD and
  453          * proc0kstack, to control stack overflow for thread0 and
  454          * prevent corruption of the page table.  We leak the guard
  455          * physical memory due to 1:1 mappings.
  456          */
  457         allocpages(1, &physfree);
  458         proc0kstack = allocpages(TD0_KSTACK_PAGES, &physfree);
  459 
  460         /* vm86/bios stack */
  461         vm86phystk = allocpages(1, &physfree);
  462 
  463         /* pgtable + ext + IOPAGES */
  464         vm86paddr = vm86pa = allocpages(3, &physfree);
  465 
  466         /* Install page tables into PTD.  Page table page 1 is wasted. */
  467         for (a = 0; a < NKPT; a++)
  468                 IdlePTD[a] = (KPTphys + ptoa(a)) | PG_V | PG_RW | PG_A | PG_M;
  469 
  470 #if defined(PAE) || defined(PAE_TABLES)
  471         /* PAE install PTD pointers into PDPT */
  472         for (a = 0; a < NPGPTD; a++)
  473                 IdlePDPT[a] = ((u_int)IdlePTD + ptoa(a)) | PG_V;
  474 #endif
  475 
  476         /*
  477          * Install recursive mapping for kernel page tables into
  478          * itself.
  479          */
  480         for (a = 0; a < NPGPTD; a++)
  481                 IdlePTD[PTDPTDI + a] = ((u_int)IdlePTD + ptoa(a)) | PG_V |
  482                     PG_RW;
  483 
  484         /*
  485          * Initialize page table pages mapping physical address zero
  486          * through the (physical) end of the kernel.  Many of these
  487          * pages must be reserved, and we reserve them all and map
  488          * them linearly for convenience.  We do this even if we've
  489          * enabled PSE above; we'll just switch the corresponding
  490          * kernel PDEs before we turn on paging.
  491          *
  492          * This and all other page table entries allow read and write
  493          * access for various reasons.  Kernel mappings never have any
  494          * access restrictions.
  495          */
  496         pmap_cold_mapident(0, atop(NBPDR));
  497         pmap_cold_map(0, NBPDR, atop(NBPDR));
  498         pmap_cold_mapident(KERNBASE, atop(KERNend - KERNBASE));
  499 
  500         /* Map page table directory */
  501 #if defined(PAE) || defined(PAE_TABLES)
  502         pmap_cold_mapident((u_long)IdlePDPT, 1);
  503 #endif
  504         pmap_cold_mapident((u_long)IdlePTD, NPGPTD);
  505 
  506         /* Map early KPTmap.  It is really pmap_cold_mapident. */
  507         pmap_cold_map(KPTphys, (u_long)KPTmap, NKPT);
  508 
  509         /* Map proc0kstack */
  510         pmap_cold_mapident(proc0kstack, TD0_KSTACK_PAGES);
  511         /* ISA hole already mapped */
  512 
  513         pmap_cold_mapident(vm86phystk, 1);
  514         pmap_cold_mapident(vm86pa, 3);
  515 
  516         /* Map page 0 into the vm86 page table */
  517         *(pt_entry_t *)vm86pa = 0 | PG_RW | PG_U | PG_A | PG_M | PG_V;
  518 
  519         /* ...likewise for the ISA hole for vm86 */
  520         for (pt = (pt_entry_t *)vm86pa + atop(ISA_HOLE_START), a = 0;
  521             a < atop(ISA_HOLE_LENGTH); a++, pt++)
  522                 *pt = (ISA_HOLE_START + ptoa(a)) | PG_RW | PG_U | PG_A |
  523                     PG_M | PG_V;
  524 
  525         /* Enable PSE, PGE, VME, and PAE if configured. */
  526         ncr4 = 0;
  527         if ((cpu_feature & CPUID_PSE) != 0) {
  528                 ncr4 |= CR4_PSE;
  529                 pseflag = PG_PS;
  530                 /*
  531                  * Superpage mapping of the kernel text.  Existing 4k
  532                  * page table pages are wasted.
  533                  */
  534                 for (a = KERNBASE; a < KERNend; a += NBPDR)
  535                         IdlePTD[a >> PDRSHIFT] = a | PG_PS | PG_A | PG_M |
  536                             PG_RW | PG_V;
  537         }
  538         if ((cpu_feature & CPUID_PGE) != 0) {
  539                 ncr4 |= CR4_PGE;
  540                 pgeflag = PG_G;
  541         }
  542         ncr4 |= (cpu_feature & CPUID_VME) != 0 ? CR4_VME : 0;
  543 #if defined(PAE) || defined(PAE_TABLES)
  544         ncr4 |= CR4_PAE;
  545 #endif
  546         if (ncr4 != 0)
  547                 load_cr4(rcr4() | ncr4);
  548 
  549         /* Now enable paging */
  550 #if defined(PAE) || defined(PAE_TABLES)
  551         cr3 = (u_int)IdlePDPT;
  552 #else
  553         cr3 = (u_int)IdlePTD;
  554 #endif
  555         tramp_idleptd = cr3;
  556         load_cr3(cr3);
  557         load_cr0(rcr0() | CR0_PG);
  558 
  559         /*
  560          * Now running relocated at KERNBASE where the system is
  561          * linked to run.
  562          */
  563 
  564         /*
  565          * Remove the lowest part of the double mapping of low memory
  566          * to get some null pointer checks.
  567          */
  568         IdlePTD[0] = 0;
  569         load_cr3(cr3);          /* invalidate TLB */
  570 }
  571 
  572 /*
  573  *      Bootstrap the system enough to run with virtual memory.
  574  *
  575  *      On the i386 this is called after pmap_cold() created initial
  576  *      kernel page table and enabled paging, and just syncs the pmap
  577  *      module with what has already been done.
  578  */
  579 void
  580 pmap_bootstrap(vm_paddr_t firstaddr)
  581 {
  582         vm_offset_t va;
  583         pt_entry_t *pte, *unused;
  584         struct pcpu *pc;
  585         u_long res;
  586         int i;
  587 
  588         res = atop(firstaddr - (vm_paddr_t)KERNLOAD);
  589 
  590         /*
  591          * Add a physical memory segment (vm_phys_seg) corresponding to the
  592          * preallocated kernel page table pages so that vm_page structures
  593          * representing these pages will be created.  The vm_page structures
  594          * are required for promotion of the corresponding kernel virtual
  595          * addresses to superpage mappings.
  596          */
  597         vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt));
  598 
  599         /*
  600          * Initialize the first available kernel virtual address.
  601          * However, using "firstaddr" may waste a few pages of the
  602          * kernel virtual address space, because pmap_cold() may not
  603          * have mapped every physical page that it allocated.
  604          * Preferably, pmap_cold() would provide a first unused
  605          * virtual address in addition to "firstaddr".
  606          */
  607         virtual_avail = (vm_offset_t)firstaddr;
  608         virtual_end = VM_MAX_KERNEL_ADDRESS;
  609 
  610         /*
  611          * Initialize the kernel pmap (which is statically allocated).
  612          * Count bootstrap data as being resident in case any of this data is
  613          * later unmapped (using pmap_remove()) and freed.
  614          */
  615         PMAP_LOCK_INIT(kernel_pmap);
  616         kernel_pmap->pm_pdir = IdlePTD;
  617 #if defined(PAE) || defined(PAE_TABLES)
  618         kernel_pmap->pm_pdpt = IdlePDPT;
  619 #endif
  620         CPU_FILL(&kernel_pmap->pm_active);      /* don't allow deactivation */
  621         kernel_pmap->pm_stats.resident_count = res;
  622         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  623 
  624         /*
  625          * Initialize the global pv list lock.
  626          */
  627         rw_init(&pvh_global_lock, "pmap pv global");
  628 
  629         /*
  630          * Reserve some special page table entries/VA space for temporary
  631          * mapping of pages.
  632          */
  633 #define SYSMAP(c, p, v, n)      \
  634         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  635 
  636         va = virtual_avail;
  637         pte = vtopte(va);
  638 
  639 
  640         /*
  641          * Initialize temporary map objects on the current CPU for use
  642          * during early boot.
  643          * CMAP1/CMAP2 are used for zeroing and copying pages.
  644          * CMAP3 is used for the boot-time memory test.
  645          */
  646         pc = get_pcpu();
  647         mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
  648         SYSMAP(caddr_t, pc->pc_cmap_pte1, pc->pc_cmap_addr1, 1)
  649         SYSMAP(caddr_t, pc->pc_cmap_pte2, pc->pc_cmap_addr2, 1)
  650         SYSMAP(vm_offset_t, pte, pc->pc_qmap_addr, 1)
  651 
  652         SYSMAP(caddr_t, CMAP3, CADDR3, 1);
  653 
  654         /*
  655          * Crashdump maps.
  656          */
  657         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  658 
  659         /*
  660          * ptvmmap is used for reading arbitrary physical pages via /dev/mem.
  661          */
  662         SYSMAP(caddr_t, unused, ptvmmap, 1)
  663 
  664         /*
  665          * msgbufp is used to map the system message buffer.
  666          */
  667         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(msgbufsize)))
  668 
  669         /*
  670          * KPTmap is used by pmap_kextract().
  671          *
  672          * KPTmap is first initialized by pmap_cold().  However, that initial
  673          * KPTmap can only support NKPT page table pages.  Here, a larger
  674          * KPTmap is created that can support KVA_PAGES page table pages.
  675          */
  676         SYSMAP(pt_entry_t *, KPTD, KPTmap, KVA_PAGES)
  677 
  678         for (i = 0; i < NKPT; i++)
  679                 KPTD[i] = (KPTphys + ptoa(i)) | PG_RW | PG_V;
  680 
  681         /*
  682          * PADDR1 and PADDR2 are used by pmap_pte_quick() and pmap_pte(),
  683          * respectively.
  684          */
  685         SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1)
  686         SYSMAP(pt_entry_t *, PMAP2, PADDR2, 1)
  687         SYSMAP(pt_entry_t *, PMAP3, PADDR3, 1)
  688 
  689         mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
  690 
  691         virtual_avail = va;
  692 
  693         /*
  694          * Initialize the PAT MSR if present.
  695          * pmap_init_pat() clears and sets CR4_PGE, which, as a
  696          * side-effect, invalidates stale PG_G TLB entries that might
  697          * have been created in our pre-boot environment.  We assume
  698          * that PAT support implies PGE and in reverse, PGE presence
  699          * comes with PAT.  Both features were added for Pentium Pro.
  700          */
  701         pmap_init_pat();
  702 }
  703 
  704 static void
  705 pmap_init_reserved_pages(void)
  706 {
  707         struct pcpu *pc;
  708         vm_offset_t pages;
  709         int i;
  710 
  711         CPU_FOREACH(i) {
  712                 pc = pcpu_find(i);
  713                 mtx_init(&pc->pc_copyout_mlock, "cpmlk", NULL, MTX_DEF |
  714                     MTX_NEW);
  715                 pc->pc_copyout_maddr = kva_alloc(ptoa(2));
  716                 if (pc->pc_copyout_maddr == 0)
  717                         panic("unable to allocate non-sleepable copyout KVA");
  718                 sx_init(&pc->pc_copyout_slock, "cpslk");
  719                 pc->pc_copyout_saddr = kva_alloc(ptoa(2));
  720                 if (pc->pc_copyout_saddr == 0)
  721                         panic("unable to allocate sleepable copyout KVA");
  722                 pc->pc_pmap_eh_va = kva_alloc(ptoa(1));
  723                 if (pc->pc_pmap_eh_va == 0)
  724                         panic("unable to allocate pmap_extract_and_hold KVA");
  725                 pc->pc_pmap_eh_ptep = (char *)vtopte(pc->pc_pmap_eh_va);
  726 
  727                 /*
  728                  * Skip if the mappings have already been initialized,
  729                  * i.e. this is the BSP.
  730                  */
  731                 if (pc->pc_cmap_addr1 != 0)
  732                         continue;
  733 
  734                 mtx_init(&pc->pc_cmap_lock, "SYSMAPS", NULL, MTX_DEF);
  735                 pages = kva_alloc(PAGE_SIZE * 3);
  736                 if (pages == 0)
  737                         panic("unable to allocate CMAP KVA");
  738                 pc->pc_cmap_pte1 = vtopte(pages);
  739                 pc->pc_cmap_pte2 = vtopte(pages + PAGE_SIZE);
  740                 pc->pc_cmap_addr1 = (caddr_t)pages;
  741                 pc->pc_cmap_addr2 = (caddr_t)(pages + PAGE_SIZE);
  742                 pc->pc_qmap_addr = pages + ptoa(2);
  743         }
  744 }
  745  
  746 SYSINIT(rpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_reserved_pages, NULL);
  747 
  748 /*
  749  * Setup the PAT MSR.
  750  */
  751 void
  752 pmap_init_pat(void)
  753 {
  754         int pat_table[PAT_INDEX_SIZE];
  755         uint64_t pat_msr;
  756         u_long cr0, cr4;
  757         int i;
  758 
  759         /* Set default PAT index table. */
  760         for (i = 0; i < PAT_INDEX_SIZE; i++)
  761                 pat_table[i] = -1;
  762         pat_table[PAT_WRITE_BACK] = 0;
  763         pat_table[PAT_WRITE_THROUGH] = 1;
  764         pat_table[PAT_UNCACHEABLE] = 3;
  765         pat_table[PAT_WRITE_COMBINING] = 3;
  766         pat_table[PAT_WRITE_PROTECTED] = 3;
  767         pat_table[PAT_UNCACHED] = 3;
  768 
  769         /*
  770          * Bail if this CPU doesn't implement PAT.
  771          * We assume that PAT support implies PGE.
  772          */
  773         if ((cpu_feature & CPUID_PAT) == 0) {
  774                 for (i = 0; i < PAT_INDEX_SIZE; i++)
  775                         pat_index[i] = pat_table[i];
  776                 pat_works = 0;
  777                 return;
  778         }
  779 
  780         /*
  781          * Due to some Intel errata, we can only safely use the lower 4
  782          * PAT entries.
  783          *
  784          *   Intel Pentium III Processor Specification Update
  785          * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
  786          * or Mode C Paging)
  787          *
  788          *   Intel Pentium IV  Processor Specification Update
  789          * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
  790          */
  791         if (cpu_vendor_id == CPU_VENDOR_INTEL &&
  792             !(CPUID_TO_FAMILY(cpu_id) == 6 && CPUID_TO_MODEL(cpu_id) >= 0xe))
  793                 pat_works = 0;
  794 
  795         /* Initialize default PAT entries. */
  796         pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |
  797             PAT_VALUE(1, PAT_WRITE_THROUGH) |
  798             PAT_VALUE(2, PAT_UNCACHED) |
  799             PAT_VALUE(3, PAT_UNCACHEABLE) |
  800             PAT_VALUE(4, PAT_WRITE_BACK) |
  801             PAT_VALUE(5, PAT_WRITE_THROUGH) |
  802             PAT_VALUE(6, PAT_UNCACHED) |
  803             PAT_VALUE(7, PAT_UNCACHEABLE);
  804 
  805         if (pat_works) {
  806                 /*
  807                  * Leave the indices 0-3 at the default of WB, WT, UC-, and UC.
  808                  * Program 5 and 6 as WP and WC.
  809                  * Leave 4 and 7 as WB and UC.
  810                  */
  811                 pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6));
  812                 pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) |
  813                     PAT_VALUE(6, PAT_WRITE_COMBINING);
  814                 pat_table[PAT_UNCACHED] = 2;
  815                 pat_table[PAT_WRITE_PROTECTED] = 5;
  816                 pat_table[PAT_WRITE_COMBINING] = 6;
  817         } else {
  818                 /*
  819                  * Just replace PAT Index 2 with WC instead of UC-.
  820                  */
  821                 pat_msr &= ~PAT_MASK(2);
  822                 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  823                 pat_table[PAT_WRITE_COMBINING] = 2;
  824         }
  825 
  826         /* Disable PGE. */
  827         cr4 = rcr4();
  828         load_cr4(cr4 & ~CR4_PGE);
  829 
  830         /* Disable caches (CD = 1, NW = 0). */
  831         cr0 = rcr0();
  832         load_cr0((cr0 & ~CR0_NW) | CR0_CD);
  833 
  834         /* Flushes caches and TLBs. */
  835         wbinvd();
  836         invltlb();
  837 
  838         /* Update PAT and index table. */
  839         wrmsr(MSR_PAT, pat_msr);
  840         for (i = 0; i < PAT_INDEX_SIZE; i++)
  841                 pat_index[i] = pat_table[i];
  842 
  843         /* Flush caches and TLBs again. */
  844         wbinvd();
  845         invltlb();
  846 
  847         /* Restore caches and PGE. */
  848         load_cr0(cr0);
  849         load_cr4(cr4);
  850 }
  851 
  852 /*
  853  * Initialize a vm_page's machine-dependent fields.
  854  */
  855 void
  856 pmap_page_init(vm_page_t m)
  857 {
  858 
  859         TAILQ_INIT(&m->md.pv_list);
  860         m->md.pat_mode = PAT_WRITE_BACK;
  861 }
  862 
  863 #if defined(PAE) || defined(PAE_TABLES)
  864 static void *
  865 pmap_pdpt_allocf(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
  866     int wait)
  867 {
  868 
  869         /* Inform UMA that this allocator uses kernel_map/object. */
  870         *flags = UMA_SLAB_KERNEL;
  871         return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
  872             bytes, wait, 0x0ULL, 0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
  873 }
  874 #endif
  875 
  876 /*
  877  * Abuse the pte nodes for unmapped kva to thread a kva freelist through.
  878  * Requirements:
  879  *  - Must deal with pages in order to ensure that none of the PG_* bits
  880  *    are ever set, PG_V in particular.
  881  *  - Assumes we can write to ptes without pte_store() atomic ops, even
  882  *    on PAE systems.  This should be ok.
  883  *  - Assumes nothing will ever test these addresses for 0 to indicate
  884  *    no mapping instead of correctly checking PG_V.
  885  *  - Assumes a vm_offset_t will fit in a pte (true for i386).
  886  * Because PG_V is never set, there can be no mappings to invalidate.
  887  */
  888 static vm_offset_t
  889 pmap_ptelist_alloc(vm_offset_t *head)
  890 {
  891         pt_entry_t *pte;
  892         vm_offset_t va;
  893 
  894         va = *head;
  895         if (va == 0)
  896                 panic("pmap_ptelist_alloc: exhausted ptelist KVA");
  897         pte = vtopte(va);
  898         *head = *pte;
  899         if (*head & PG_V)
  900                 panic("pmap_ptelist_alloc: va with PG_V set!");
  901         *pte = 0;
  902         return (va);
  903 }
  904 
  905 static void
  906 pmap_ptelist_free(vm_offset_t *head, vm_offset_t va)
  907 {
  908         pt_entry_t *pte;
  909 
  910         if (va & PG_V)
  911                 panic("pmap_ptelist_free: freeing va with PG_V set!");
  912         pte = vtopte(va);
  913         *pte = *head;           /* virtual! PG_V is 0 though */
  914         *head = va;
  915 }
  916 
  917 static void
  918 pmap_ptelist_init(vm_offset_t *head, void *base, int npages)
  919 {
  920         int i;
  921         vm_offset_t va;
  922 
  923         *head = 0;
  924         for (i = npages - 1; i >= 0; i--) {
  925                 va = (vm_offset_t)base + i * PAGE_SIZE;
  926                 pmap_ptelist_free(head, va);
  927         }
  928 }
  929 
  930 
  931 /*
  932  *      Initialize the pmap module.
  933  *      Called by vm_init, to initialize any structures that the pmap
  934  *      system needs to map virtual memory.
  935  */
  936 void
  937 pmap_init(void)
  938 {
  939         struct pmap_preinit_mapping *ppim;
  940         vm_page_t mpte;
  941         vm_size_t s;
  942         int i, pv_npg;
  943 
  944         /*
  945          * Initialize the vm page array entries for the kernel pmap's
  946          * page table pages.
  947          */ 
  948         PMAP_LOCK(kernel_pmap);
  949         for (i = 0; i < NKPT; i++) {
  950                 mpte = PHYS_TO_VM_PAGE(KPTphys + ptoa(i));
  951                 KASSERT(mpte >= vm_page_array &&
  952                     mpte < &vm_page_array[vm_page_array_size],
  953                     ("pmap_init: page table page is out of range"));
  954                 mpte->pindex = i + KPTDI;
  955                 mpte->phys_addr = KPTphys + ptoa(i);
  956                 mpte->wire_count = 1;
  957 
  958                 /*
  959                  * Collect the page table pages that were replaced by a 2/4MB
  960                  * page.  They are filled with equivalent 4KB page mappings.
  961                  */
  962                 if (pseflag != 0 &&
  963                     KERNBASE <= i << PDRSHIFT && i << PDRSHIFT < KERNend &&
  964                     pmap_insert_pt_page(kernel_pmap, mpte, true))
  965                         panic("pmap_init: pmap_insert_pt_page failed");
  966         }
  967         PMAP_UNLOCK(kernel_pmap);
  968         vm_wire_add(NKPT);
  969 
  970         /*
  971          * Initialize the address space (zone) for the pv entries.  Set a
  972          * high water mark so that the system can recover from excessive
  973          * numbers of pv entries.
  974          */
  975         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  976         pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count;
  977         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  978         pv_entry_max = roundup(pv_entry_max, _NPCPV);
  979         pv_entry_high_water = 9 * (pv_entry_max / 10);
  980 
  981         /*
  982          * If the kernel is running on a virtual machine, then it must assume
  983          * that MCA is enabled by the hypervisor.  Moreover, the kernel must
  984          * be prepared for the hypervisor changing the vendor and family that
  985          * are reported by CPUID.  Consequently, the workaround for AMD Family
  986          * 10h Erratum 383 is enabled if the processor's feature set does not
  987          * include at least one feature that is only supported by older Intel
  988          * or newer AMD processors.
  989          */
  990         if (vm_guest != VM_GUEST_NO && (cpu_feature & CPUID_SS) == 0 &&
  991             (cpu_feature2 & (CPUID2_SSSE3 | CPUID2_SSE41 | CPUID2_AESNI |
  992             CPUID2_AVX | CPUID2_XSAVE)) == 0 && (amd_feature2 & (AMDID2_XOP |
  993             AMDID2_FMA4)) == 0)
  994                 workaround_erratum383 = 1;
  995 
  996         /*
  997          * Are large page mappings supported and enabled?
  998          */
  999         TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
 1000         if (pseflag == 0)
 1001                 pg_ps_enabled = 0;
 1002         else if (pg_ps_enabled) {
 1003                 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
 1004                     ("pmap_init: can't assign to pagesizes[1]"));
 1005                 pagesizes[1] = NBPDR;
 1006         }
 1007 
 1008         /*
 1009          * Calculate the size of the pv head table for superpages.
 1010          * Handle the possibility that "vm_phys_segs[...].end" is zero.
 1011          */
 1012         pv_npg = trunc_4mpage(vm_phys_segs[vm_phys_nsegs - 1].end -
 1013             PAGE_SIZE) / NBPDR + 1;
 1014 
 1015         /*
 1016          * Allocate memory for the pv head table for superpages.
 1017          */
 1018         s = (vm_size_t)(pv_npg * sizeof(struct md_page));
 1019         s = round_page(s);
 1020         pv_table = (struct md_page *)kmem_malloc(s, M_WAITOK | M_ZERO);
 1021         for (i = 0; i < pv_npg; i++)
 1022                 TAILQ_INIT(&pv_table[i].pv_list);
 1023 
 1024         pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc);
 1025         pv_chunkbase = (struct pv_chunk *)kva_alloc(PAGE_SIZE * pv_maxchunks);
 1026         if (pv_chunkbase == NULL)
 1027                 panic("pmap_init: not enough kvm for pv chunks");
 1028         pmap_ptelist_init(&pv_vafree, pv_chunkbase, pv_maxchunks);
 1029 #if defined(PAE) || defined(PAE_TABLES)
 1030         pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL,
 1031             NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1,
 1032             UMA_ZONE_VM | UMA_ZONE_NOFREE);
 1033         uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf);
 1034 #endif
 1035 
 1036         pmap_initialized = 1;
 1037         pmap_init_trm();
 1038 
 1039         if (!bootverbose)
 1040                 return;
 1041         for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 1042                 ppim = pmap_preinit_mapping + i;
 1043                 if (ppim->va == 0)
 1044                         continue;
 1045                 printf("PPIM %u: PA=%#jx, VA=%#x, size=%#x, mode=%#x\n", i,
 1046                     (uintmax_t)ppim->pa, ppim->va, ppim->sz, ppim->mode);
 1047         }
 1048 
 1049 }
 1050 
 1051 
 1052 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0,
 1053         "Max number of PV entries");
 1054 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0,
 1055         "Page share factor per proc");
 1056 
 1057 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
 1058     "2/4MB page mapping counters");
 1059 
 1060 static u_long pmap_pde_demotions;
 1061 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
 1062     &pmap_pde_demotions, 0, "2/4MB page demotions");
 1063 
 1064 static u_long pmap_pde_mappings;
 1065 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
 1066     &pmap_pde_mappings, 0, "2/4MB page mappings");
 1067 
 1068 static u_long pmap_pde_p_failures;
 1069 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
 1070     &pmap_pde_p_failures, 0, "2/4MB page promotion failures");
 1071 
 1072 static u_long pmap_pde_promotions;
 1073 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
 1074     &pmap_pde_promotions, 0, "2/4MB page promotions");
 1075 
 1076 /***************************************************
 1077  * Low level helper routines.....
 1078  ***************************************************/
 1079 
 1080 boolean_t
 1081 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)
 1082 {
 1083 
 1084         return (mode >= 0 && mode < PAT_INDEX_SIZE &&
 1085             pat_index[(int)mode] >= 0);
 1086 }
 1087 
 1088 /*
 1089  * Determine the appropriate bits to set in a PTE or PDE for a specified
 1090  * caching mode.
 1091  */
 1092 int
 1093 pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde)
 1094 {
 1095         int cache_bits, pat_flag, pat_idx;
 1096 
 1097         if (!pmap_is_valid_memattr(pmap, mode))
 1098                 panic("Unknown caching mode %d\n", mode);
 1099 
 1100         /* The PAT bit is different for PTE's and PDE's. */
 1101         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
 1102 
 1103         /* Map the caching mode to a PAT index. */
 1104         pat_idx = pat_index[mode];
 1105 
 1106         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
 1107         cache_bits = 0;
 1108         if (pat_idx & 0x4)
 1109                 cache_bits |= pat_flag;
 1110         if (pat_idx & 0x2)
 1111                 cache_bits |= PG_NC_PCD;
 1112         if (pat_idx & 0x1)
 1113                 cache_bits |= PG_NC_PWT;
 1114         return (cache_bits);
 1115 }
 1116 
 1117 bool
 1118 pmap_ps_enabled(pmap_t pmap __unused)
 1119 {
 1120 
 1121         return (pg_ps_enabled);
 1122 }
 1123 
 1124 /*
 1125  * The caller is responsible for maintaining TLB consistency.
 1126  */
 1127 static void
 1128 pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde)
 1129 {
 1130         pd_entry_t *pde;
 1131 
 1132         pde = pmap_pde(kernel_pmap, va);
 1133         pde_store(pde, newpde);
 1134 }
 1135 
 1136 /*
 1137  * After changing the page size for the specified virtual address in the page
 1138  * table, flush the corresponding entries from the processor's TLB.  Only the
 1139  * calling processor's TLB is affected.
 1140  *
 1141  * The calling thread must be pinned to a processor.
 1142  */
 1143 static void
 1144 pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
 1145 {
 1146 
 1147         if ((newpde & PG_PS) == 0)
 1148                 /* Demotion: flush a specific 2MB page mapping. */
 1149                 invlpg(va);
 1150         else /* if ((newpde & PG_G) == 0) */
 1151                 /*
 1152                  * Promotion: flush every 4KB page mapping from the TLB
 1153                  * because there are too many to flush individually.
 1154                  */
 1155                 invltlb();
 1156 }
 1157 
 1158 void
 1159 invltlb_glob(void)
 1160 {
 1161 
 1162         invltlb();
 1163 }
 1164 
 1165 
 1166 #ifdef SMP
 1167 
 1168 static void
 1169 pmap_curcpu_cb_dummy(pmap_t pmap __unused, vm_offset_t addr1 __unused,
 1170     vm_offset_t addr2 __unused)
 1171 {
 1172 }
 1173 
 1174 /*
 1175  * For SMP, these functions have to use the IPI mechanism for coherence.
 1176  *
 1177  * N.B.: Before calling any of the following TLB invalidation functions,
 1178  * the calling processor must ensure that all stores updating a non-
 1179  * kernel page table are globally performed.  Otherwise, another
 1180  * processor could cache an old, pre-update entry without being
 1181  * invalidated.  This can happen one of two ways: (1) The pmap becomes
 1182  * active on another processor after its pm_active field is checked by
 1183  * one of the following functions but before a store updating the page
 1184  * table is globally performed. (2) The pmap becomes active on another
 1185  * processor before its pm_active field is checked but due to
 1186  * speculative loads one of the following functions stills reads the
 1187  * pmap as inactive on the other processor.
 1188  * 
 1189  * The kernel page table is exempt because its pm_active field is
 1190  * immutable.  The kernel page table is always active on every
 1191  * processor.
 1192  */
 1193 void
 1194 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 1195 {
 1196         cpuset_t *mask, other_cpus;
 1197         u_int cpuid;
 1198 
 1199         sched_pin();
 1200         if (pmap == kernel_pmap) {
 1201                 invlpg(va);
 1202                 mask = &all_cpus;
 1203         } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1204                 mask = &all_cpus;
 1205         } else {
 1206                 cpuid = PCPU_GET(cpuid);
 1207                 other_cpus = all_cpus;
 1208                 CPU_CLR(cpuid, &other_cpus);
 1209                 CPU_AND(&other_cpus, &pmap->pm_active);
 1210                 mask = &other_cpus;
 1211         }
 1212         smp_masked_invlpg(*mask, va, pmap, pmap_curcpu_cb_dummy);
 1213         sched_unpin();
 1214 }
 1215 
 1216 /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */
 1217 #define PMAP_INVLPG_THRESHOLD   (4 * 1024 * PAGE_SIZE)
 1218 
 1219 void
 1220 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1221 {
 1222         cpuset_t *mask, other_cpus;
 1223         vm_offset_t addr;
 1224         u_int cpuid;
 1225 
 1226         if (eva - sva >= PMAP_INVLPG_THRESHOLD) {
 1227                 pmap_invalidate_all(pmap);
 1228                 return;
 1229         }
 1230 
 1231         sched_pin();
 1232         if (pmap == kernel_pmap) {
 1233                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1234                         invlpg(addr);
 1235                 mask = &all_cpus;
 1236         } else  if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1237                 mask = &all_cpus;
 1238         } else {
 1239                 cpuid = PCPU_GET(cpuid);
 1240                 other_cpus = all_cpus;
 1241                 CPU_CLR(cpuid, &other_cpus);
 1242                 CPU_AND(&other_cpus, &pmap->pm_active);
 1243                 mask = &other_cpus;
 1244         }
 1245         smp_masked_invlpg_range(*mask, sva, eva, pmap, pmap_curcpu_cb_dummy);
 1246         sched_unpin();
 1247 }
 1248 
 1249 void
 1250 pmap_invalidate_all(pmap_t pmap)
 1251 {
 1252         cpuset_t *mask, other_cpus;
 1253         u_int cpuid;
 1254 
 1255         sched_pin();
 1256         if (pmap == kernel_pmap) {
 1257                 invltlb();
 1258                 mask = &all_cpus;
 1259         } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1260                 mask = &all_cpus;
 1261         } else {
 1262                 cpuid = PCPU_GET(cpuid);
 1263                 other_cpus = all_cpus;
 1264                 CPU_CLR(cpuid, &other_cpus);
 1265                 CPU_AND(&other_cpus, &pmap->pm_active);
 1266                 mask = &other_cpus;
 1267         }
 1268         smp_masked_invltlb(*mask, pmap, pmap_curcpu_cb_dummy);
 1269         sched_unpin();
 1270 }
 1271 
 1272 static void
 1273 pmap_invalidate_cache_curcpu_cb(pmap_t pmap __unused,
 1274     vm_offset_t addr1 __unused, vm_offset_t addr2 __unused)
 1275 {
 1276         wbinvd();
 1277 }
 1278 
 1279 void
 1280 pmap_invalidate_cache(void)
 1281 {
 1282         smp_cache_flush(pmap_invalidate_cache_curcpu_cb);
 1283 }
 1284 
 1285 struct pde_action {
 1286         cpuset_t invalidate;    /* processors that invalidate their TLB */
 1287         vm_offset_t va;
 1288         pd_entry_t *pde;
 1289         pd_entry_t newpde;
 1290         u_int store;            /* processor that updates the PDE */
 1291 };
 1292 
 1293 static void
 1294 pmap_update_pde_kernel(void *arg)
 1295 {
 1296         struct pde_action *act = arg;
 1297         pd_entry_t *pde;
 1298 
 1299         if (act->store == PCPU_GET(cpuid)) {
 1300                 pde = pmap_pde(kernel_pmap, act->va);
 1301                 pde_store(pde, act->newpde);
 1302         }
 1303 }
 1304 
 1305 static void
 1306 pmap_update_pde_user(void *arg)
 1307 {
 1308         struct pde_action *act = arg;
 1309 
 1310         if (act->store == PCPU_GET(cpuid))
 1311                 pde_store(act->pde, act->newpde);
 1312 }
 1313 
 1314 static void
 1315 pmap_update_pde_teardown(void *arg)
 1316 {
 1317         struct pde_action *act = arg;
 1318 
 1319         if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
 1320                 pmap_update_pde_invalidate(act->va, act->newpde);
 1321 }
 1322 
 1323 /*
 1324  * Change the page size for the specified virtual address in a way that
 1325  * prevents any possibility of the TLB ever having two entries that map the
 1326  * same virtual address using different page sizes.  This is the recommended
 1327  * workaround for Erratum 383 on AMD Family 10h processors.  It prevents a
 1328  * machine check exception for a TLB state that is improperly diagnosed as a
 1329  * hardware error.
 1330  */
 1331 static void
 1332 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1333 {
 1334         struct pde_action act;
 1335         cpuset_t active, other_cpus;
 1336         u_int cpuid;
 1337 
 1338         sched_pin();
 1339         cpuid = PCPU_GET(cpuid);
 1340         other_cpus = all_cpus;
 1341         CPU_CLR(cpuid, &other_cpus);
 1342         if (pmap == kernel_pmap)
 1343                 active = all_cpus;
 1344         else
 1345                 active = pmap->pm_active;
 1346         if (CPU_OVERLAP(&active, &other_cpus)) {
 1347                 act.store = cpuid;
 1348                 act.invalidate = active;
 1349                 act.va = va;
 1350                 act.pde = pde;
 1351                 act.newpde = newpde;
 1352                 CPU_SET(cpuid, &active);
 1353                 smp_rendezvous_cpus(active,
 1354                     smp_no_rendezvous_barrier, pmap == kernel_pmap ?
 1355                     pmap_update_pde_kernel : pmap_update_pde_user,
 1356                     pmap_update_pde_teardown, &act);
 1357         } else {
 1358                 if (pmap == kernel_pmap)
 1359                         pmap_kenter_pde(va, newpde);
 1360                 else
 1361                         pde_store(pde, newpde);
 1362                 if (CPU_ISSET(cpuid, &active))
 1363                         pmap_update_pde_invalidate(va, newpde);
 1364         }
 1365         sched_unpin();
 1366 }
 1367 #else /* !SMP */
 1368 /*
 1369  * Normal, non-SMP, 486+ invalidation functions.
 1370  * We inline these within pmap.c for speed.
 1371  */
 1372 PMAP_INLINE void
 1373 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 1374 {
 1375 
 1376         if (pmap == kernel_pmap)
 1377                 invlpg(va);
 1378 }
 1379 
 1380 PMAP_INLINE void
 1381 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1382 {
 1383         vm_offset_t addr;
 1384 
 1385         if (pmap == kernel_pmap)
 1386                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1387                         invlpg(addr);
 1388 }
 1389 
 1390 PMAP_INLINE void
 1391 pmap_invalidate_all(pmap_t pmap)
 1392 {
 1393 
 1394         if (pmap == kernel_pmap)
 1395                 invltlb();
 1396 }
 1397 
 1398 PMAP_INLINE void
 1399 pmap_invalidate_cache(void)
 1400 {
 1401 
 1402         wbinvd();
 1403 }
 1404 
 1405 static void
 1406 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1407 {
 1408 
 1409         if (pmap == kernel_pmap)
 1410                 pmap_kenter_pde(va, newpde);
 1411         else
 1412                 pde_store(pde, newpde);
 1413         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1414                 pmap_update_pde_invalidate(va, newpde);
 1415 }
 1416 #endif /* !SMP */
 1417 
 1418 static void
 1419 pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
 1420 {
 1421 
 1422         /*
 1423          * When the PDE has PG_PROMOTED set, the 2- or 4MB page mapping was
 1424          * created by a promotion that did not invalidate the 512 or 1024 4KB
 1425          * page mappings that might exist in the TLB.  Consequently, at this
 1426          * point, the TLB may hold both 4KB and 2- or 4MB page mappings for
 1427          * the address range [va, va + NBPDR).  Therefore, the entire range
 1428          * must be invalidated here.  In contrast, when PG_PROMOTED is clear,
 1429          * the TLB will not hold any 4KB page mappings for the address range
 1430          * [va, va + NBPDR), and so a single INVLPG suffices to invalidate the
 1431          * 2- or 4MB page mapping from the TLB.
 1432          */
 1433         if ((pde & PG_PROMOTED) != 0)
 1434                 pmap_invalidate_range(pmap, va, va + NBPDR - 1);
 1435         else
 1436                 pmap_invalidate_page(pmap, va);
 1437 }
 1438 
 1439 DEFINE_IFUNC(, void, pmap_invalidate_cache_range, (vm_offset_t, vm_offset_t),
 1440     static)
 1441 {
 1442 
 1443         if ((cpu_feature & CPUID_SS) != 0)
 1444                 return (pmap_invalidate_cache_range_selfsnoop);
 1445         if ((cpu_feature & CPUID_CLFSH) != 0)
 1446                 return (pmap_force_invalidate_cache_range);
 1447         return (pmap_invalidate_cache_range_all);
 1448 }
 1449 
 1450 #define PMAP_CLFLUSH_THRESHOLD  (2 * 1024 * 1024)
 1451 
 1452 static void
 1453 pmap_invalidate_cache_range_check_align(vm_offset_t sva, vm_offset_t eva)
 1454 {
 1455 
 1456         KASSERT((sva & PAGE_MASK) == 0,
 1457             ("pmap_invalidate_cache_range: sva not page-aligned"));
 1458         KASSERT((eva & PAGE_MASK) == 0,
 1459             ("pmap_invalidate_cache_range: eva not page-aligned"));
 1460 }
 1461 
 1462 static void
 1463 pmap_invalidate_cache_range_selfsnoop(vm_offset_t sva, vm_offset_t eva)
 1464 {
 1465 
 1466         pmap_invalidate_cache_range_check_align(sva, eva);
 1467 }
 1468 
 1469 void
 1470 pmap_force_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
 1471 {
 1472 
 1473         sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
 1474         if (eva - sva >= PMAP_CLFLUSH_THRESHOLD) {
 1475                 /*
 1476                  * The supplied range is bigger than 2MB.
 1477                  * Globally invalidate cache.
 1478                  */
 1479                 pmap_invalidate_cache();
 1480                 return;
 1481         }
 1482 
 1483 #ifdef DEV_APIC
 1484         /*
 1485          * XXX: Some CPUs fault, hang, or trash the local APIC
 1486          * registers if we use CLFLUSH on the local APIC
 1487          * range.  The local APIC is always uncached, so we
 1488          * don't need to flush for that range anyway.
 1489          */
 1490         if (pmap_kextract(sva) == lapic_paddr)
 1491                 return;
 1492 #endif
 1493 
 1494         if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0) {
 1495                 /*
 1496                  * Do per-cache line flush.  Use the sfence
 1497                  * instruction to insure that previous stores are
 1498                  * included in the write-back.  The processor
 1499                  * propagates flush to other processors in the cache
 1500                  * coherence domain.
 1501                  */
 1502                 sfence();
 1503                 for (; sva < eva; sva += cpu_clflush_line_size)
 1504                         clflushopt(sva);
 1505                 sfence();
 1506         } else {
 1507                 /*
 1508                  * Writes are ordered by CLFLUSH on Intel CPUs.
 1509                  */
 1510                 if (cpu_vendor_id != CPU_VENDOR_INTEL)
 1511                         mfence();
 1512                 for (; sva < eva; sva += cpu_clflush_line_size)
 1513                         clflush(sva);
 1514                 if (cpu_vendor_id != CPU_VENDOR_INTEL)
 1515                         mfence();
 1516         }
 1517 }
 1518 
 1519 static void
 1520 pmap_invalidate_cache_range_all(vm_offset_t sva, vm_offset_t eva)
 1521 {
 1522 
 1523         pmap_invalidate_cache_range_check_align(sva, eva);
 1524         pmap_invalidate_cache();
 1525 }
 1526 
 1527 void
 1528 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
 1529 {
 1530         int i;
 1531 
 1532         if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
 1533             (cpu_feature & CPUID_CLFSH) == 0) {
 1534                 pmap_invalidate_cache();
 1535         } else {
 1536                 for (i = 0; i < count; i++)
 1537                         pmap_flush_page(pages[i]);
 1538         }
 1539 }
 1540 
 1541 /*
 1542  * Are we current address space or kernel?
 1543  */
 1544 static __inline int
 1545 pmap_is_current(pmap_t pmap)
 1546 {
 1547 
 1548         return (pmap == kernel_pmap);
 1549 }
 1550 
 1551 /*
 1552  * If the given pmap is not the current or kernel pmap, the returned pte must
 1553  * be released by passing it to pmap_pte_release().
 1554  */
 1555 pt_entry_t *
 1556 pmap_pte(pmap_t pmap, vm_offset_t va)
 1557 {
 1558         pd_entry_t newpf;
 1559         pd_entry_t *pde;
 1560 
 1561         pde = pmap_pde(pmap, va);
 1562         if (*pde & PG_PS)
 1563                 return (pde);
 1564         if (*pde != 0) {
 1565                 /* are we current address space or kernel? */
 1566                 if (pmap_is_current(pmap))
 1567                         return (vtopte(va));
 1568                 mtx_lock(&PMAP2mutex);
 1569                 newpf = *pde & PG_FRAME;
 1570                 if ((*PMAP2 & PG_FRAME) != newpf) {
 1571                         *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1572                         pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
 1573                 }
 1574                 return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
 1575         }
 1576         return (NULL);
 1577 }
 1578 
 1579 /*
 1580  * Releases a pte that was obtained from pmap_pte().  Be prepared for the pte
 1581  * being NULL.
 1582  */
 1583 static __inline void
 1584 pmap_pte_release(pt_entry_t *pte)
 1585 {
 1586 
 1587         if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2)
 1588                 mtx_unlock(&PMAP2mutex);
 1589 }
 1590 
 1591 /*
 1592  * NB:  The sequence of updating a page table followed by accesses to the
 1593  * corresponding pages is subject to the situation described in the "AMD64
 1594  * Architecture Programmer's Manual Volume 2: System Programming" rev. 3.23,
 1595  * "7.3.1 Special Coherency Considerations".  Therefore, issuing the INVLPG
 1596  * right after modifying the PTE bits is crucial.
 1597  */
 1598 static __inline void
 1599 invlcaddr(void *caddr)
 1600 {
 1601 
 1602         invlpg((u_int)caddr);
 1603 }
 1604 
 1605 /*
 1606  * Super fast pmap_pte routine best used when scanning
 1607  * the pv lists.  This eliminates many coarse-grained
 1608  * invltlb calls.  Note that many of the pv list
 1609  * scans are across different pmaps.  It is very wasteful
 1610  * to do an entire invltlb for checking a single mapping.
 1611  *
 1612  * If the given pmap is not the current pmap, pvh_global_lock
 1613  * must be held and curthread pinned to a CPU.
 1614  */
 1615 static pt_entry_t *
 1616 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
 1617 {
 1618         pd_entry_t newpf;
 1619         pd_entry_t *pde;
 1620 
 1621         pde = pmap_pde(pmap, va);
 1622         if (*pde & PG_PS)
 1623                 return (pde);
 1624         if (*pde != 0) {
 1625                 /* are we current address space or kernel? */
 1626                 if (pmap_is_current(pmap))
 1627                         return (vtopte(va));
 1628                 rw_assert(&pvh_global_lock, RA_WLOCKED);
 1629                 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 1630                 newpf = *pde & PG_FRAME;
 1631                 if ((*PMAP1 & PG_FRAME) != newpf) {
 1632                         *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1633 #ifdef SMP
 1634                         PMAP1cpu = PCPU_GET(cpuid);
 1635 #endif
 1636                         invlcaddr(PADDR1);
 1637                         PMAP1changed++;
 1638                 } else
 1639 #ifdef SMP
 1640                 if (PMAP1cpu != PCPU_GET(cpuid)) {
 1641                         PMAP1cpu = PCPU_GET(cpuid);
 1642                         invlcaddr(PADDR1);
 1643                         PMAP1changedcpu++;
 1644                 } else
 1645 #endif
 1646                         PMAP1unchanged++;
 1647                 return (PADDR1 + (i386_btop(va) & (NPTEPG - 1)));
 1648         }
 1649         return (0);
 1650 }
 1651 
 1652 static pt_entry_t *
 1653 pmap_pte_quick3(pmap_t pmap, vm_offset_t va)
 1654 {
 1655         pd_entry_t newpf;
 1656         pd_entry_t *pde;
 1657 
 1658         pde = pmap_pde(pmap, va);
 1659         if (*pde & PG_PS)
 1660                 return (pde);
 1661         if (*pde != 0) {
 1662                 rw_assert(&pvh_global_lock, RA_WLOCKED);
 1663                 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 1664                 newpf = *pde & PG_FRAME;
 1665                 if ((*PMAP3 & PG_FRAME) != newpf) {
 1666                         *PMAP3 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1667 #ifdef SMP
 1668                         PMAP3cpu = PCPU_GET(cpuid);
 1669 #endif
 1670                         invlcaddr(PADDR3);
 1671                         PMAP1changed++;
 1672                 } else
 1673 #ifdef SMP
 1674                 if (PMAP3cpu != PCPU_GET(cpuid)) {
 1675                         PMAP3cpu = PCPU_GET(cpuid);
 1676                         invlcaddr(PADDR3);
 1677                         PMAP1changedcpu++;
 1678                 } else
 1679 #endif
 1680                         PMAP1unchanged++;
 1681                 return (PADDR3 + (i386_btop(va) & (NPTEPG - 1)));
 1682         }
 1683         return (0);
 1684 }
 1685 
 1686 static pt_entry_t
 1687 pmap_pte_ufast(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
 1688 {
 1689         pt_entry_t *eh_ptep, pte, *ptep;
 1690 
 1691         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1692         pde &= PG_FRAME;
 1693         critical_enter();
 1694         eh_ptep = (pt_entry_t *)PCPU_GET(pmap_eh_ptep);
 1695         if ((*eh_ptep & PG_FRAME) != pde) {
 1696                 *eh_ptep = pde | PG_RW | PG_V | PG_A | PG_M;
 1697                 invlcaddr((void *)PCPU_GET(pmap_eh_va));
 1698         }
 1699         ptep = (pt_entry_t *)PCPU_GET(pmap_eh_va) + (i386_btop(va) &
 1700             (NPTEPG - 1));
 1701         pte = *ptep;
 1702         critical_exit();
 1703         return (pte);
 1704 }
 1705 
 1706 /*
 1707  *      Routine:        pmap_extract
 1708  *      Function:
 1709  *              Extract the physical page address associated
 1710  *              with the given map/virtual_address pair.
 1711  */
 1712 vm_paddr_t 
 1713 pmap_extract(pmap_t pmap, vm_offset_t va)
 1714 {
 1715         vm_paddr_t rtval;
 1716         pt_entry_t pte;
 1717         pd_entry_t pde;
 1718 
 1719         rtval = 0;
 1720         PMAP_LOCK(pmap);
 1721         pde = pmap->pm_pdir[va >> PDRSHIFT];
 1722         if (pde != 0) {
 1723                 if ((pde & PG_PS) != 0)
 1724                         rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
 1725                 else {
 1726                         pte = pmap_pte_ufast(pmap, va, pde);
 1727                         rtval = (pte & PG_FRAME) | (va & PAGE_MASK);
 1728                 }
 1729         }
 1730         PMAP_UNLOCK(pmap);
 1731         return (rtval);
 1732 }
 1733 
 1734 /*
 1735  *      Routine:        pmap_extract_and_hold
 1736  *      Function:
 1737  *              Atomically extract and hold the physical page
 1738  *              with the given pmap and virtual address pair
 1739  *              if that mapping permits the given protection.
 1740  */
 1741 vm_page_t
 1742 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 1743 {
 1744         pd_entry_t pde;
 1745         pt_entry_t pte;
 1746         vm_page_t m;
 1747         vm_paddr_t pa;
 1748 
 1749         pa = 0;
 1750         m = NULL;
 1751         PMAP_LOCK(pmap);
 1752 retry:
 1753         pde = *pmap_pde(pmap, va);
 1754         if (pde != 0) {
 1755                 if (pde & PG_PS) {
 1756                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
 1757                                 if (vm_page_pa_tryrelock(pmap, (pde &
 1758                                     PG_PS_FRAME) | (va & PDRMASK), &pa))
 1759                                         goto retry;
 1760                                 m = PHYS_TO_VM_PAGE(pa);
 1761                         }
 1762                 } else {
 1763                         pte = pmap_pte_ufast(pmap, va, pde);
 1764                         if (pte != 0 &&
 1765                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
 1766                                 if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME,
 1767                                     &pa))
 1768                                         goto retry;
 1769                                 m = PHYS_TO_VM_PAGE(pa);
 1770                         }
 1771                 }
 1772                 if (m != NULL)
 1773                         vm_page_hold(m);
 1774         }
 1775         PA_UNLOCK_COND(pa);
 1776         PMAP_UNLOCK(pmap);
 1777         return (m);
 1778 }
 1779 
 1780 /***************************************************
 1781  * Low level mapping routines.....
 1782  ***************************************************/
 1783 
 1784 /*
 1785  * Add a wired page to the kva.
 1786  * Note: not SMP coherent.
 1787  *
 1788  * This function may be used before pmap_bootstrap() is called.
 1789  */
 1790 PMAP_INLINE void 
 1791 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
 1792 {
 1793         pt_entry_t *pte;
 1794 
 1795         pte = vtopte(va);
 1796         pte_store(pte, pa | PG_RW | PG_V);
 1797 }
 1798 
 1799 static __inline void
 1800 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
 1801 {
 1802         pt_entry_t *pte;
 1803 
 1804         pte = vtopte(va);
 1805         pte_store(pte, pa | PG_RW | PG_V | pmap_cache_bits(kernel_pmap,
 1806             mode, 0));
 1807 }
 1808 
 1809 /*
 1810  * Remove a page from the kernel pagetables.
 1811  * Note: not SMP coherent.
 1812  *
 1813  * This function may be used before pmap_bootstrap() is called.
 1814  */
 1815 PMAP_INLINE void
 1816 pmap_kremove(vm_offset_t va)
 1817 {
 1818         pt_entry_t *pte;
 1819 
 1820         pte = vtopte(va);
 1821         pte_clear(pte);
 1822 }
 1823 
 1824 /*
 1825  *      Used to map a range of physical addresses into kernel
 1826  *      virtual address space.
 1827  *
 1828  *      The value passed in '*virt' is a suggested virtual address for
 1829  *      the mapping. Architectures which can support a direct-mapped
 1830  *      physical to virtual region can return the appropriate address
 1831  *      within that region, leaving '*virt' unchanged. Other
 1832  *      architectures should map the pages starting at '*virt' and
 1833  *      update '*virt' with the first usable address after the mapped
 1834  *      region.
 1835  */
 1836 vm_offset_t
 1837 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 1838 {
 1839         vm_offset_t va, sva;
 1840         vm_paddr_t superpage_offset;
 1841         pd_entry_t newpde;
 1842 
 1843         va = *virt;
 1844         /*
 1845          * Does the physical address range's size and alignment permit at
 1846          * least one superpage mapping to be created?
 1847          */ 
 1848         superpage_offset = start & PDRMASK;
 1849         if ((end - start) - ((NBPDR - superpage_offset) & PDRMASK) >= NBPDR) {
 1850                 /*
 1851                  * Increase the starting virtual address so that its alignment
 1852                  * does not preclude the use of superpage mappings.
 1853                  */
 1854                 if ((va & PDRMASK) < superpage_offset)
 1855                         va = (va & ~PDRMASK) + superpage_offset;
 1856                 else if ((va & PDRMASK) > superpage_offset)
 1857                         va = ((va + PDRMASK) & ~PDRMASK) + superpage_offset;
 1858         }
 1859         sva = va;
 1860         while (start < end) {
 1861                 if ((start & PDRMASK) == 0 && end - start >= NBPDR &&
 1862                     pseflag != 0) {
 1863                         KASSERT((va & PDRMASK) == 0,
 1864                             ("pmap_map: misaligned va %#x", va));
 1865                         newpde = start | PG_PS | PG_RW | PG_V;
 1866                         pmap_kenter_pde(va, newpde);
 1867                         va += NBPDR;
 1868                         start += NBPDR;
 1869                 } else {
 1870                         pmap_kenter(va, start);
 1871                         va += PAGE_SIZE;
 1872                         start += PAGE_SIZE;
 1873                 }
 1874         }
 1875         pmap_invalidate_range(kernel_pmap, sva, va);
 1876         *virt = va;
 1877         return (sva);
 1878 }
 1879 
 1880 
 1881 /*
 1882  * Add a list of wired pages to the kva
 1883  * this routine is only used for temporary
 1884  * kernel mappings that do not need to have
 1885  * page modification or references recorded.
 1886  * Note that old mappings are simply written
 1887  * over.  The page *must* be wired.
 1888  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1889  */
 1890 void
 1891 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 1892 {
 1893         pt_entry_t *endpte, oldpte, pa, *pte;
 1894         vm_page_t m;
 1895 
 1896         oldpte = 0;
 1897         pte = vtopte(sva);
 1898         endpte = pte + count;
 1899         while (pte < endpte) {
 1900                 m = *ma++;
 1901                 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(kernel_pmap,
 1902                     m->md.pat_mode, 0);
 1903                 if ((*pte & (PG_FRAME | PG_PTE_CACHE)) != pa) {
 1904                         oldpte |= *pte;
 1905 #if defined(PAE) || defined(PAE_TABLES)
 1906                         pte_store(pte, pa | pg_nx | PG_RW | PG_V);
 1907 #else
 1908                         pte_store(pte, pa | PG_RW | PG_V);
 1909 #endif
 1910                 }
 1911                 pte++;
 1912         }
 1913         if (__predict_false((oldpte & PG_V) != 0))
 1914                 pmap_invalidate_range(kernel_pmap, sva, sva + count *
 1915                     PAGE_SIZE);
 1916 }
 1917 
 1918 /*
 1919  * This routine tears out page mappings from the
 1920  * kernel -- it is meant only for temporary mappings.
 1921  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1922  */
 1923 void
 1924 pmap_qremove(vm_offset_t sva, int count)
 1925 {
 1926         vm_offset_t va;
 1927 
 1928         va = sva;
 1929         while (count-- > 0) {
 1930                 pmap_kremove(va);
 1931                 va += PAGE_SIZE;
 1932         }
 1933         pmap_invalidate_range(kernel_pmap, sva, va);
 1934 }
 1935 
 1936 /***************************************************
 1937  * Page table page management routines.....
 1938  ***************************************************/
 1939 /*
 1940  * Schedule the specified unused page table page to be freed.  Specifically,
 1941  * add the page to the specified list of pages that will be released to the
 1942  * physical memory manager after the TLB has been updated.
 1943  */
 1944 static __inline void
 1945 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
 1946     boolean_t set_PG_ZERO)
 1947 {
 1948 
 1949         if (set_PG_ZERO)
 1950                 m->flags |= PG_ZERO;
 1951         else
 1952                 m->flags &= ~PG_ZERO;
 1953         SLIST_INSERT_HEAD(free, m, plinks.s.ss);
 1954 }
 1955 
 1956 /*
 1957  * Inserts the specified page table page into the specified pmap's collection
 1958  * of idle page table pages.  Each of a pmap's page table pages is responsible
 1959  * for mapping a distinct range of virtual addresses.  The pmap's collection is
 1960  * ordered by this virtual address range.
 1961  *
 1962  * If "promoted" is false, then the page table page "mpte" must be zero filled.
 1963  */
 1964 static __inline int
 1965 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte, bool promoted)
 1966 {
 1967 
 1968         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1969         mpte->valid = promoted ? VM_PAGE_BITS_ALL : 0;
 1970         return (vm_radix_insert(&pmap->pm_root, mpte));
 1971 }
 1972 
 1973 /*
 1974  * Removes the page table page mapping the specified virtual address from the
 1975  * specified pmap's collection of idle page table pages, and returns it.
 1976  * Otherwise, returns NULL if there is no page table page corresponding to the
 1977  * specified virtual address.
 1978  */
 1979 static __inline vm_page_t
 1980 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
 1981 {
 1982 
 1983         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1984         return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT));
 1985 }
 1986 
 1987 /*
 1988  * Decrements a page table page's wire count, which is used to record the
 1989  * number of valid page table entries within the page.  If the wire count
 1990  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
 1991  * page table page was unmapped and FALSE otherwise.
 1992  */
 1993 static inline boolean_t
 1994 pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
 1995 {
 1996 
 1997         --m->wire_count;
 1998         if (m->wire_count == 0) {
 1999                 _pmap_unwire_ptp(pmap, m, free);
 2000                 return (TRUE);
 2001         } else
 2002                 return (FALSE);
 2003 }
 2004 
 2005 static void
 2006 _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
 2007 {
 2008 
 2009         /*
 2010          * unmap the page table page
 2011          */
 2012         pmap->pm_pdir[m->pindex] = 0;
 2013         --pmap->pm_stats.resident_count;
 2014 
 2015         /*
 2016          * There is not need to invalidate the recursive mapping since
 2017          * we never instantiate such mapping for the usermode pmaps,
 2018          * and never remove page table pages from the kernel pmap.
 2019          * Put page on a list so that it is released since all TLB
 2020          * shootdown is done.
 2021          */
 2022         MPASS(pmap != kernel_pmap);
 2023         pmap_add_delayed_free_list(m, free, TRUE);
 2024 }
 2025 
 2026 /*
 2027  * After removing a page table entry, this routine is used to
 2028  * conditionally free the page, and manage the hold/wire counts.
 2029  */
 2030 static int
 2031 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free)
 2032 {
 2033         pd_entry_t ptepde;
 2034         vm_page_t mpte;
 2035 
 2036         if (pmap == kernel_pmap)
 2037                 return (0);
 2038         ptepde = *pmap_pde(pmap, va);
 2039         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 2040         return (pmap_unwire_ptp(pmap, mpte, free));
 2041 }
 2042 
 2043 /*
 2044  * Initialize the pmap for the swapper process.
 2045  */
 2046 void
 2047 pmap_pinit0(pmap_t pmap)
 2048 {
 2049 
 2050         PMAP_LOCK_INIT(pmap);
 2051         pmap->pm_pdir = IdlePTD;
 2052 #if defined(PAE) || defined(PAE_TABLES)
 2053         pmap->pm_pdpt = IdlePDPT;
 2054 #endif
 2055         pmap->pm_root.rt_root = 0;
 2056         CPU_ZERO(&pmap->pm_active);
 2057         TAILQ_INIT(&pmap->pm_pvchunk);
 2058         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 2059         pmap_activate_boot(pmap);
 2060 }
 2061 
 2062 /*
 2063  * Initialize a preallocated and zeroed pmap structure,
 2064  * such as one in a vmspace structure.
 2065  */
 2066 int
 2067 pmap_pinit(pmap_t pmap)
 2068 {
 2069         vm_page_t m;
 2070         int i;
 2071 
 2072         /*
 2073          * No need to allocate page table space yet but we do need a valid
 2074          * page directory table.
 2075          */
 2076         if (pmap->pm_pdir == NULL) {
 2077                 pmap->pm_pdir = (pd_entry_t *)kva_alloc(NBPTD);
 2078                 if (pmap->pm_pdir == NULL)
 2079                         return (0);
 2080 #if defined(PAE) || defined(PAE_TABLES)
 2081                 pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
 2082                 KASSERT(((vm_offset_t)pmap->pm_pdpt &
 2083                     ((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0,
 2084                     ("pmap_pinit: pdpt misaligned"));
 2085                 KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
 2086                     ("pmap_pinit: pdpt above 4g"));
 2087 #endif
 2088                 pmap->pm_root.rt_root = 0;
 2089         }
 2090         KASSERT(vm_radix_is_empty(&pmap->pm_root),
 2091             ("pmap_pinit: pmap has reserved page table page(s)"));
 2092 
 2093         /*
 2094          * allocate the page directory page(s)
 2095          */
 2096         for (i = 0; i < NPGPTD;) {
 2097                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 2098                     VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 2099                 if (m == NULL) {
 2100                         vm_wait(NULL);
 2101                 } else {
 2102                         pmap->pm_ptdpg[i] = m;
 2103 #if defined(PAE) || defined(PAE_TABLES)
 2104                         pmap->pm_pdpt[i] = VM_PAGE_TO_PHYS(m) | PG_V;
 2105 #endif
 2106                         i++;
 2107                 }
 2108         }
 2109 
 2110         pmap_qenter((vm_offset_t)pmap->pm_pdir, pmap->pm_ptdpg, NPGPTD);
 2111 
 2112         for (i = 0; i < NPGPTD; i++)
 2113                 if ((pmap->pm_ptdpg[i]->flags & PG_ZERO) == 0)
 2114                         pagezero(pmap->pm_pdir + (i * NPDEPG));
 2115 
 2116         /* Install the trampoline mapping. */
 2117         pmap->pm_pdir[TRPTDI] = PTD[TRPTDI];
 2118 
 2119         CPU_ZERO(&pmap->pm_active);
 2120         TAILQ_INIT(&pmap->pm_pvchunk);
 2121         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 2122 
 2123         return (1);
 2124 }
 2125 
 2126 /*
 2127  * this routine is called if the page table page is not
 2128  * mapped correctly.
 2129  */
 2130 static vm_page_t
 2131 _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags)
 2132 {
 2133         vm_paddr_t ptepa;
 2134         vm_page_t m;
 2135 
 2136         /*
 2137          * Allocate a page table page.
 2138          */
 2139         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 2140             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 2141                 if ((flags & PMAP_ENTER_NOSLEEP) == 0) {
 2142                         PMAP_UNLOCK(pmap);
 2143                         rw_wunlock(&pvh_global_lock);
 2144                         vm_wait(NULL);
 2145                         rw_wlock(&pvh_global_lock);
 2146                         PMAP_LOCK(pmap);
 2147                 }
 2148 
 2149                 /*
 2150                  * Indicate the need to retry.  While waiting, the page table
 2151                  * page may have been allocated.
 2152                  */
 2153                 return (NULL);
 2154         }
 2155         if ((m->flags & PG_ZERO) == 0)
 2156                 pmap_zero_page(m);
 2157 
 2158         /*
 2159          * Map the pagetable page into the process address space, if
 2160          * it isn't already there.
 2161          */
 2162 
 2163         pmap->pm_stats.resident_count++;
 2164 
 2165         ptepa = VM_PAGE_TO_PHYS(m);
 2166         pmap->pm_pdir[ptepindex] =
 2167                 (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
 2168 
 2169         return (m);
 2170 }
 2171 
 2172 static vm_page_t
 2173 pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)
 2174 {
 2175         u_int ptepindex;
 2176         pd_entry_t ptepa;
 2177         vm_page_t m;
 2178 
 2179         /*
 2180          * Calculate pagetable page index
 2181          */
 2182         ptepindex = va >> PDRSHIFT;
 2183 retry:
 2184         /*
 2185          * Get the page directory entry
 2186          */
 2187         ptepa = pmap->pm_pdir[ptepindex];
 2188 
 2189         /*
 2190          * This supports switching from a 4MB page to a
 2191          * normal 4K page.
 2192          */
 2193         if (ptepa & PG_PS) {
 2194                 (void)pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va);
 2195                 ptepa = pmap->pm_pdir[ptepindex];
 2196         }
 2197 
 2198         /*
 2199          * If the page table page is mapped, we just increment the
 2200          * hold count, and activate it.
 2201          */
 2202         if (ptepa) {
 2203                 m = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 2204                 m->wire_count++;
 2205         } else {
 2206                 /*
 2207                  * Here if the pte page isn't mapped, or if it has
 2208                  * been deallocated. 
 2209                  */
 2210                 m = _pmap_allocpte(pmap, ptepindex, flags);
 2211                 if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0)
 2212                         goto retry;
 2213         }
 2214         return (m);
 2215 }
 2216 
 2217 
 2218 /***************************************************
 2219 * Pmap allocation/deallocation routines.
 2220  ***************************************************/
 2221 
 2222 /*
 2223  * Release any resources held by the given physical map.
 2224  * Called when a pmap initialized by pmap_pinit is being released.
 2225  * Should only be called if the map contains no valid mappings.
 2226  */
 2227 void
 2228 pmap_release(pmap_t pmap)
 2229 {
 2230         vm_page_t m;
 2231         int i;
 2232 
 2233         KASSERT(pmap->pm_stats.resident_count == 0,
 2234             ("pmap_release: pmap resident count %ld != 0",
 2235             pmap->pm_stats.resident_count));
 2236         KASSERT(vm_radix_is_empty(&pmap->pm_root),
 2237             ("pmap_release: pmap has reserved page table page(s)"));
 2238         KASSERT(CPU_EMPTY(&pmap->pm_active),
 2239             ("releasing active pmap %p", pmap));
 2240 
 2241         pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
 2242 
 2243         for (i = 0; i < NPGPTD; i++) {
 2244                 m = pmap->pm_ptdpg[i];
 2245 #if defined(PAE) || defined(PAE_TABLES)
 2246                 KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
 2247                     ("pmap_release: got wrong ptd page"));
 2248 #endif
 2249                 vm_page_unwire_noq(m);
 2250                 vm_page_free(m);
 2251         }
 2252 }
 2253 
 2254 static int
 2255 kvm_size(SYSCTL_HANDLER_ARGS)
 2256 {
 2257         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
 2258 
 2259         return (sysctl_handle_long(oidp, &ksize, 0, req));
 2260 }
 2261 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 2262     0, 0, kvm_size, "IU", "Size of KVM");
 2263 
 2264 static int
 2265 kvm_free(SYSCTL_HANDLER_ARGS)
 2266 {
 2267         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 2268 
 2269         return (sysctl_handle_long(oidp, &kfree, 0, req));
 2270 }
 2271 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 2272     0, 0, kvm_free, "IU", "Amount of KVM free");
 2273 
 2274 /*
 2275  * grow the number of kernel page table entries, if needed
 2276  */
 2277 void
 2278 pmap_growkernel(vm_offset_t addr)
 2279 {
 2280         vm_paddr_t ptppaddr;
 2281         vm_page_t nkpg;
 2282         pd_entry_t newpdir;
 2283 
 2284         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 2285         addr = roundup2(addr, NBPDR);
 2286         if (addr - 1 >= vm_map_max(kernel_map))
 2287                 addr = vm_map_max(kernel_map);
 2288         while (kernel_vm_end < addr) {
 2289                 if (pdir_pde(PTD, kernel_vm_end)) {
 2290                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2291                         if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
 2292                                 kernel_vm_end = vm_map_max(kernel_map);
 2293                                 break;
 2294                         }
 2295                         continue;
 2296                 }
 2297 
 2298                 nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDRSHIFT,
 2299                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 2300                     VM_ALLOC_ZERO);
 2301                 if (nkpg == NULL)
 2302                         panic("pmap_growkernel: no memory to grow kernel");
 2303 
 2304                 nkpt++;
 2305 
 2306                 if ((nkpg->flags & PG_ZERO) == 0)
 2307                         pmap_zero_page(nkpg);
 2308                 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
 2309                 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
 2310                 pdir_pde(KPTD, kernel_vm_end) = newpdir;
 2311 
 2312                 pmap_kenter_pde(kernel_vm_end, newpdir);
 2313                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2314                 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
 2315                         kernel_vm_end = vm_map_max(kernel_map);
 2316                         break;
 2317                 }
 2318         }
 2319 }
 2320 
 2321 
 2322 /***************************************************
 2323  * page management routines.
 2324  ***************************************************/
 2325 
 2326 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 2327 CTASSERT(_NPCM == 11);
 2328 CTASSERT(_NPCPV == 336);
 2329 
 2330 static __inline struct pv_chunk *
 2331 pv_to_chunk(pv_entry_t pv)
 2332 {
 2333 
 2334         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
 2335 }
 2336 
 2337 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 2338 
 2339 #define PC_FREE0_9      0xfffffffful    /* Free values for index 0 through 9 */
 2340 #define PC_FREE10       0x0000fffful    /* Free values for index 10 */
 2341 
 2342 static const uint32_t pc_freemask[_NPCM] = {
 2343         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2344         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2345         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2346         PC_FREE0_9, PC_FREE10
 2347 };
 2348 
 2349 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 2350         "Current number of pv entries");
 2351 
 2352 #ifdef PV_STATS
 2353 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 2354 
 2355 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 2356         "Current number of pv entry chunks");
 2357 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 2358         "Current number of pv entry chunks allocated");
 2359 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 2360         "Current number of pv entry chunks frees");
 2361 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 2362         "Number of times tried to get a chunk page but failed.");
 2363 
 2364 static long pv_entry_frees, pv_entry_allocs;
 2365 static int pv_entry_spare;
 2366 
 2367 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 2368         "Current number of pv entry frees");
 2369 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 2370         "Current number of pv entry allocs");
 2371 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 2372         "Current number of spare pv entries");
 2373 #endif
 2374 
 2375 /*
 2376  * We are in a serious low memory condition.  Resort to
 2377  * drastic measures to free some pages so we can allocate
 2378  * another pv entry chunk.
 2379  */
 2380 static vm_page_t
 2381 pmap_pv_reclaim(pmap_t locked_pmap)
 2382 {
 2383         struct pch newtail;
 2384         struct pv_chunk *pc;
 2385         struct md_page *pvh;
 2386         pd_entry_t *pde;
 2387         pmap_t pmap;
 2388         pt_entry_t *pte, tpte;
 2389         pv_entry_t pv;
 2390         vm_offset_t va;
 2391         vm_page_t m, m_pc;
 2392         struct spglist free;
 2393         uint32_t inuse;
 2394         int bit, field, freed;
 2395 
 2396         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
 2397         pmap = NULL;
 2398         m_pc = NULL;
 2399         SLIST_INIT(&free);
 2400         TAILQ_INIT(&newtail);
 2401         while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 ||
 2402             SLIST_EMPTY(&free))) {
 2403                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2404                 if (pmap != pc->pc_pmap) {
 2405                         if (pmap != NULL) {
 2406                                 pmap_invalidate_all(pmap);
 2407                                 if (pmap != locked_pmap)
 2408                                         PMAP_UNLOCK(pmap);
 2409                         }
 2410                         pmap = pc->pc_pmap;
 2411                         /* Avoid deadlock and lock recursion. */
 2412                         if (pmap > locked_pmap)
 2413                                 PMAP_LOCK(pmap);
 2414                         else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
 2415                                 pmap = NULL;
 2416                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2417                                 continue;
 2418                         }
 2419                 }
 2420 
 2421                 /*
 2422                  * Destroy every non-wired, 4 KB page mapping in the chunk.
 2423                  */
 2424                 freed = 0;
 2425                 for (field = 0; field < _NPCM; field++) {
 2426                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
 2427                             inuse != 0; inuse &= ~(1UL << bit)) {
 2428                                 bit = bsfl(inuse);
 2429                                 pv = &pc->pc_pventry[field * 32 + bit];
 2430                                 va = pv->pv_va;
 2431                                 pde = pmap_pde(pmap, va);
 2432                                 if ((*pde & PG_PS) != 0)
 2433                                         continue;
 2434                                 pte = pmap_pte(pmap, va);
 2435                                 tpte = *pte;
 2436                                 if ((tpte & PG_W) == 0)
 2437                                         tpte = pte_load_clear(pte);
 2438                                 pmap_pte_release(pte);
 2439                                 if ((tpte & PG_W) != 0)
 2440                                         continue;
 2441                                 KASSERT(tpte != 0,
 2442                                     ("pmap_pv_reclaim: pmap %p va %x zero pte",
 2443                                     pmap, va));
 2444                                 if ((tpte & PG_G) != 0)
 2445                                         pmap_invalidate_page(pmap, va);
 2446                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2447                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2448                                         vm_page_dirty(m);
 2449                                 if ((tpte & PG_A) != 0)
 2450                                         vm_page_aflag_set(m, PGA_REFERENCED);
 2451                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 2452                                 if (TAILQ_EMPTY(&m->md.pv_list) &&
 2453                                     (m->flags & PG_FICTITIOUS) == 0) {
 2454                                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2455                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 2456                                                 vm_page_aflag_clear(m,
 2457                                                     PGA_WRITEABLE);
 2458                                         }
 2459                                 }
 2460                                 pc->pc_map[field] |= 1UL << bit;
 2461                                 pmap_unuse_pt(pmap, va, &free);
 2462                                 freed++;
 2463                         }
 2464                 }
 2465                 if (freed == 0) {
 2466                         TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2467                         continue;
 2468                 }
 2469                 /* Every freed mapping is for a 4 KB page. */
 2470                 pmap->pm_stats.resident_count -= freed;
 2471                 PV_STAT(pv_entry_frees += freed);
 2472                 PV_STAT(pv_entry_spare += freed);
 2473                 pv_entry_count -= freed;
 2474                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2475                 for (field = 0; field < _NPCM; field++)
 2476                         if (pc->pc_map[field] != pc_freemask[field]) {
 2477                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2478                                     pc_list);
 2479                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2480 
 2481                                 /*
 2482                                  * One freed pv entry in locked_pmap is
 2483                                  * sufficient.
 2484                                  */
 2485                                 if (pmap == locked_pmap)
 2486                                         goto out;
 2487                                 break;
 2488                         }
 2489                 if (field == _NPCM) {
 2490                         PV_STAT(pv_entry_spare -= _NPCPV);
 2491                         PV_STAT(pc_chunk_count--);
 2492                         PV_STAT(pc_chunk_frees++);
 2493                         /* Entire chunk is free; return it. */
 2494                         m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2495                         pmap_qremove((vm_offset_t)pc, 1);
 2496                         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2497                         break;
 2498                 }
 2499         }
 2500 out:
 2501         TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
 2502         if (pmap != NULL) {
 2503                 pmap_invalidate_all(pmap);
 2504                 if (pmap != locked_pmap)
 2505                         PMAP_UNLOCK(pmap);
 2506         }
 2507         if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) {
 2508                 m_pc = SLIST_FIRST(&free);
 2509                 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
 2510                 /* Recycle a freed page table page. */
 2511                 m_pc->wire_count = 1;
 2512         }
 2513         vm_page_free_pages_toq(&free, true);
 2514         return (m_pc);
 2515 }
 2516 
 2517 /*
 2518  * free the pv_entry back to the free list
 2519  */
 2520 static void
 2521 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 2522 {
 2523         struct pv_chunk *pc;
 2524         int idx, field, bit;
 2525 
 2526         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2527         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2528         PV_STAT(pv_entry_frees++);
 2529         PV_STAT(pv_entry_spare++);
 2530         pv_entry_count--;
 2531         pc = pv_to_chunk(pv);
 2532         idx = pv - &pc->pc_pventry[0];
 2533         field = idx / 32;
 2534         bit = idx % 32;
 2535         pc->pc_map[field] |= 1ul << bit;
 2536         for (idx = 0; idx < _NPCM; idx++)
 2537                 if (pc->pc_map[idx] != pc_freemask[idx]) {
 2538                         /*
 2539                          * 98% of the time, pc is already at the head of the
 2540                          * list.  If it isn't already, move it to the head.
 2541                          */
 2542                         if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
 2543                             pc)) {
 2544                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2545                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2546                                     pc_list);
 2547                         }
 2548                         return;
 2549                 }
 2550         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2551         free_pv_chunk(pc);
 2552 }
 2553 
 2554 static void
 2555 free_pv_chunk(struct pv_chunk *pc)
 2556 {
 2557         vm_page_t m;
 2558 
 2559         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2560         PV_STAT(pv_entry_spare -= _NPCPV);
 2561         PV_STAT(pc_chunk_count--);
 2562         PV_STAT(pc_chunk_frees++);
 2563         /* entire chunk is free, return it */
 2564         m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2565         pmap_qremove((vm_offset_t)pc, 1);
 2566         vm_page_unwire_noq(m);
 2567         vm_page_free(m);
 2568         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2569 }
 2570 
 2571 /*
 2572  * get a new pv_entry, allocating a block from the system
 2573  * when needed.
 2574  */
 2575 static pv_entry_t
 2576 get_pv_entry(pmap_t pmap, boolean_t try)
 2577 {
 2578         static const struct timeval printinterval = { 60, 0 };
 2579         static struct timeval lastprint;
 2580         int bit, field;
 2581         pv_entry_t pv;
 2582         struct pv_chunk *pc;
 2583         vm_page_t m;
 2584 
 2585         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2586         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2587         PV_STAT(pv_entry_allocs++);
 2588         pv_entry_count++;
 2589         if (pv_entry_count > pv_entry_high_water)
 2590                 if (ratecheck(&lastprint, &printinterval))
 2591                         printf("Approaching the limit on PV entries, consider "
 2592                             "increasing either the vm.pmap.shpgperproc or the "
 2593                             "vm.pmap.pv_entries tunable.\n");
 2594 retry:
 2595         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 2596         if (pc != NULL) {
 2597                 for (field = 0; field < _NPCM; field++) {
 2598                         if (pc->pc_map[field]) {
 2599                                 bit = bsfl(pc->pc_map[field]);
 2600                                 break;
 2601                         }
 2602                 }
 2603                 if (field < _NPCM) {
 2604                         pv = &pc->pc_pventry[field * 32 + bit];
 2605                         pc->pc_map[field] &= ~(1ul << bit);
 2606                         /* If this was the last item, move it to tail */
 2607                         for (field = 0; field < _NPCM; field++)
 2608                                 if (pc->pc_map[field] != 0) {
 2609                                         PV_STAT(pv_entry_spare--);
 2610                                         return (pv);    /* not full, return */
 2611                                 }
 2612                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2613                         TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 2614                         PV_STAT(pv_entry_spare--);
 2615                         return (pv);
 2616                 }
 2617         }
 2618         /*
 2619          * Access to the ptelist "pv_vafree" is synchronized by the pvh
 2620          * global lock.  If "pv_vafree" is currently non-empty, it will
 2621          * remain non-empty until pmap_ptelist_alloc() completes.
 2622          */
 2623         if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 2624             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 2625                 if (try) {
 2626                         pv_entry_count--;
 2627                         PV_STAT(pc_chunk_tryfail++);
 2628                         return (NULL);
 2629                 }
 2630                 m = pmap_pv_reclaim(pmap);
 2631                 if (m == NULL)
 2632                         goto retry;
 2633         }
 2634         PV_STAT(pc_chunk_count++);
 2635         PV_STAT(pc_chunk_allocs++);
 2636         pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree);
 2637         pmap_qenter((vm_offset_t)pc, &m, 1);
 2638         pc->pc_pmap = pmap;
 2639         pc->pc_map[0] = pc_freemask[0] & ~1ul;  /* preallocated bit 0 */
 2640         for (field = 1; field < _NPCM; field++)
 2641                 pc->pc_map[field] = pc_freemask[field];
 2642         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 2643         pv = &pc->pc_pventry[0];
 2644         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2645         PV_STAT(pv_entry_spare += _NPCPV - 1);
 2646         return (pv);
 2647 }
 2648 
 2649 static __inline pv_entry_t
 2650 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2651 {
 2652         pv_entry_t pv;
 2653 
 2654         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2655         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 2656                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 2657                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 2658                         break;
 2659                 }
 2660         }
 2661         return (pv);
 2662 }
 2663 
 2664 static void
 2665 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2666 {
 2667         struct md_page *pvh;
 2668         pv_entry_t pv;
 2669         vm_offset_t va_last;
 2670         vm_page_t m;
 2671 
 2672         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2673         KASSERT((pa & PDRMASK) == 0,
 2674             ("pmap_pv_demote_pde: pa is not 4mpage aligned"));
 2675 
 2676         /*
 2677          * Transfer the 4mpage's pv entry for this mapping to the first
 2678          * page's pv list.
 2679          */
 2680         pvh = pa_to_pvh(pa);
 2681         va = trunc_4mpage(va);
 2682         pv = pmap_pvh_remove(pvh, pmap, va);
 2683         KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
 2684         m = PHYS_TO_VM_PAGE(pa);
 2685         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2686         /* Instantiate the remaining NPTEPG - 1 pv entries. */
 2687         va_last = va + NBPDR - PAGE_SIZE;
 2688         do {
 2689                 m++;
 2690                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 2691                     ("pmap_pv_demote_pde: page %p is not managed", m));
 2692                 va += PAGE_SIZE;
 2693                 pmap_insert_entry(pmap, va, m);
 2694         } while (va < va_last);
 2695 }
 2696 
 2697 #if VM_NRESERVLEVEL > 0
 2698 static void
 2699 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2700 {
 2701         struct md_page *pvh;
 2702         pv_entry_t pv;
 2703         vm_offset_t va_last;
 2704         vm_page_t m;
 2705 
 2706         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2707         KASSERT((pa & PDRMASK) == 0,
 2708             ("pmap_pv_promote_pde: pa is not 4mpage aligned"));
 2709 
 2710         /*
 2711          * Transfer the first page's pv entry for this mapping to the
 2712          * 4mpage's pv list.  Aside from avoiding the cost of a call
 2713          * to get_pv_entry(), a transfer avoids the possibility that
 2714          * get_pv_entry() calls pmap_collect() and that pmap_collect()
 2715          * removes one of the mappings that is being promoted.
 2716          */
 2717         m = PHYS_TO_VM_PAGE(pa);
 2718         va = trunc_4mpage(va);
 2719         pv = pmap_pvh_remove(&m->md, pmap, va);
 2720         KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
 2721         pvh = pa_to_pvh(pa);
 2722         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 2723         /* Free the remaining NPTEPG - 1 pv entries. */
 2724         va_last = va + NBPDR - PAGE_SIZE;
 2725         do {
 2726                 m++;
 2727                 va += PAGE_SIZE;
 2728                 pmap_pvh_free(&m->md, pmap, va);
 2729         } while (va < va_last);
 2730 }
 2731 #endif /* VM_NRESERVLEVEL > 0 */
 2732 
 2733 static void
 2734 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2735 {
 2736         pv_entry_t pv;
 2737 
 2738         pv = pmap_pvh_remove(pvh, pmap, va);
 2739         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 2740         free_pv_entry(pmap, pv);
 2741 }
 2742 
 2743 static void
 2744 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 2745 {
 2746         struct md_page *pvh;
 2747 
 2748         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2749         pmap_pvh_free(&m->md, pmap, va);
 2750         if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) {
 2751                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2752                 if (TAILQ_EMPTY(&pvh->pv_list))
 2753                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 2754         }
 2755 }
 2756 
 2757 /*
 2758  * Create a pv entry for page at pa for
 2759  * (pmap, va).
 2760  */
 2761 static void
 2762 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2763 {
 2764         pv_entry_t pv;
 2765 
 2766         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2767         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2768         pv = get_pv_entry(pmap, FALSE);
 2769         pv->pv_va = va;
 2770         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2771 }
 2772 
 2773 /*
 2774  * Conditionally create a pv entry.
 2775  */
 2776 static boolean_t
 2777 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2778 {
 2779         pv_entry_t pv;
 2780 
 2781         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2782         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2783         if (pv_entry_count < pv_entry_high_water && 
 2784             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 2785                 pv->pv_va = va;
 2786                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2787                 return (TRUE);
 2788         } else
 2789                 return (FALSE);
 2790 }
 2791 
 2792 /*
 2793  * Create the pv entries for each of the pages within a superpage.
 2794  */
 2795 static bool
 2796 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags)
 2797 {
 2798         struct md_page *pvh;
 2799         pv_entry_t pv;
 2800         bool noreclaim;
 2801 
 2802         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2803         noreclaim = (flags & PMAP_ENTER_NORECLAIM) != 0;
 2804         if ((noreclaim && pv_entry_count >= pv_entry_high_water) ||
 2805             (pv = get_pv_entry(pmap, noreclaim)) == NULL)
 2806                 return (false);
 2807         pv->pv_va = va;
 2808         pvh = pa_to_pvh(pde & PG_PS_FRAME);
 2809         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 2810         return (true);
 2811 }
 2812 
 2813 /*
 2814  * Fills a page table page with mappings to consecutive physical pages.
 2815  */
 2816 static void
 2817 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
 2818 {
 2819         pt_entry_t *pte;
 2820 
 2821         for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
 2822                 *pte = newpte;  
 2823                 newpte += PAGE_SIZE;
 2824         }
 2825 }
 2826 
 2827 /*
 2828  * Tries to demote a 2- or 4MB page mapping.  If demotion fails, the
 2829  * 2- or 4MB page mapping is invalidated.
 2830  */
 2831 static boolean_t
 2832 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2833 {
 2834         pd_entry_t newpde, oldpde;
 2835         pt_entry_t *firstpte, newpte;
 2836         vm_paddr_t mptepa;
 2837         vm_page_t mpte;
 2838         struct spglist free;
 2839         vm_offset_t sva;
 2840 
 2841         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2842         oldpde = *pde;
 2843         KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
 2844             ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
 2845         if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
 2846             NULL) {
 2847                 KASSERT((oldpde & PG_W) == 0,
 2848                     ("pmap_demote_pde: page table page for a wired mapping"
 2849                     " is missing"));
 2850 
 2851                 /*
 2852                  * Invalidate the 2- or 4MB page mapping and return
 2853                  * "failure" if the mapping was never accessed or the
 2854                  * allocation of the new page table page fails.
 2855                  */
 2856                 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
 2857                     va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL |
 2858                     VM_ALLOC_WIRED)) == NULL) {
 2859                         SLIST_INIT(&free);
 2860                         sva = trunc_4mpage(va);
 2861                         pmap_remove_pde(pmap, pde, sva, &free);
 2862                         if ((oldpde & PG_G) == 0)
 2863                                 pmap_invalidate_pde_page(pmap, sva, oldpde);
 2864                         vm_page_free_pages_toq(&free, true);
 2865                         CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
 2866                             " in pmap %p", va, pmap);
 2867                         return (FALSE);
 2868                 }
 2869                 if (pmap != kernel_pmap) {
 2870                         mpte->wire_count = NPTEPG;
 2871                         pmap->pm_stats.resident_count++;
 2872                 }
 2873         }
 2874         mptepa = VM_PAGE_TO_PHYS(mpte);
 2875 
 2876         /*
 2877          * If the page mapping is in the kernel's address space, then the
 2878          * KPTmap can provide access to the page table page.  Otherwise,
 2879          * temporarily map the page table page (mpte) into the kernel's
 2880          * address space at either PADDR1 or PADDR2. 
 2881          */
 2882         if (pmap == kernel_pmap)
 2883                 firstpte = &KPTmap[i386_btop(trunc_4mpage(va))];
 2884         else if (curthread->td_pinned > 0 && rw_wowned(&pvh_global_lock)) {
 2885                 if ((*PMAP1 & PG_FRAME) != mptepa) {
 2886                         *PMAP1 = mptepa | PG_RW | PG_V | PG_A | PG_M;
 2887 #ifdef SMP
 2888                         PMAP1cpu = PCPU_GET(cpuid);
 2889 #endif
 2890                         invlcaddr(PADDR1);
 2891                         PMAP1changed++;
 2892                 } else
 2893 #ifdef SMP
 2894                 if (PMAP1cpu != PCPU_GET(cpuid)) {
 2895                         PMAP1cpu = PCPU_GET(cpuid);
 2896                         invlcaddr(PADDR1);
 2897                         PMAP1changedcpu++;
 2898                 } else
 2899 #endif
 2900                         PMAP1unchanged++;
 2901                 firstpte = PADDR1;
 2902         } else {
 2903                 mtx_lock(&PMAP2mutex);
 2904                 if ((*PMAP2 & PG_FRAME) != mptepa) {
 2905                         *PMAP2 = mptepa | PG_RW | PG_V | PG_A | PG_M;
 2906                         pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
 2907                 }
 2908                 firstpte = PADDR2;
 2909         }
 2910         newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
 2911         KASSERT((oldpde & PG_A) != 0,
 2912             ("pmap_demote_pde: oldpde is missing PG_A"));
 2913         KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
 2914             ("pmap_demote_pde: oldpde is missing PG_M"));
 2915         newpte = oldpde & ~PG_PS;
 2916         if ((newpte & PG_PDE_PAT) != 0)
 2917                 newpte ^= PG_PDE_PAT | PG_PTE_PAT;
 2918 
 2919         /*
 2920          * If the page table page is not leftover from an earlier promotion,
 2921          * initialize it.
 2922          */
 2923         if (mpte->valid == 0)
 2924                 pmap_fill_ptp(firstpte, newpte);
 2925 
 2926         KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
 2927             ("pmap_demote_pde: firstpte and newpte map different physical"
 2928             " addresses"));
 2929 
 2930         /*
 2931          * If the mapping has changed attributes, update the page table
 2932          * entries.
 2933          */ 
 2934         if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
 2935                 pmap_fill_ptp(firstpte, newpte);
 2936         
 2937         /*
 2938          * Demote the mapping.  This pmap is locked.  The old PDE has
 2939          * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
 2940          * set.  Thus, there is no danger of a race with another
 2941          * processor changing the setting of PG_A and/or PG_M between
 2942          * the read above and the store below. 
 2943          */
 2944         if (workaround_erratum383)
 2945                 pmap_update_pde(pmap, va, pde, newpde);
 2946         else if (pmap == kernel_pmap)
 2947                 pmap_kenter_pde(va, newpde);
 2948         else
 2949                 pde_store(pde, newpde); 
 2950         if (firstpte == PADDR2)
 2951                 mtx_unlock(&PMAP2mutex);
 2952 
 2953         /*
 2954          * Invalidate the recursive mapping of the page table page.
 2955          */
 2956         pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 2957 
 2958         /*
 2959          * Demote the pv entry.  This depends on the earlier demotion
 2960          * of the mapping.  Specifically, the (re)creation of a per-
 2961          * page pv entry might trigger the execution of pmap_collect(),
 2962          * which might reclaim a newly (re)created per-page pv entry
 2963          * and destroy the associated mapping.  In order to destroy
 2964          * the mapping, the PDE must have already changed from mapping
 2965          * the 2mpage to referencing the page table page.
 2966          */
 2967         if ((oldpde & PG_MANAGED) != 0)
 2968                 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);
 2969 
 2970         pmap_pde_demotions++;
 2971         CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#x"
 2972             " in pmap %p", va, pmap);
 2973         return (TRUE);
 2974 }
 2975 
 2976 /*
 2977  * Removes a 2- or 4MB page mapping from the kernel pmap.
 2978  */
 2979 static void
 2980 pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2981 {
 2982         pd_entry_t newpde;
 2983         vm_paddr_t mptepa;
 2984         vm_page_t mpte;
 2985 
 2986         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2987         mpte = pmap_remove_pt_page(pmap, va);
 2988         if (mpte == NULL)
 2989                 panic("pmap_remove_kernel_pde: Missing pt page.");
 2990 
 2991         mptepa = VM_PAGE_TO_PHYS(mpte);
 2992         newpde = mptepa | PG_M | PG_A | PG_RW | PG_V;
 2993 
 2994         /*
 2995          * If this page table page was unmapped by a promotion, then it
 2996          * contains valid mappings.  Zero it to invalidate those mappings.
 2997          */
 2998         if (mpte->valid != 0)
 2999                 pagezero((void *)&KPTmap[i386_btop(trunc_4mpage(va))]);
 3000 
 3001         /*
 3002          * Remove the mapping.
 3003          */
 3004         if (workaround_erratum383)
 3005                 pmap_update_pde(pmap, va, pde, newpde);
 3006         else 
 3007                 pmap_kenter_pde(va, newpde);
 3008 
 3009         /*
 3010          * Invalidate the recursive mapping of the page table page.
 3011          */
 3012         pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 3013 }
 3014 
 3015 /*
 3016  * pmap_remove_pde: do the things to unmap a superpage in a process
 3017  */
 3018 static void
 3019 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 3020     struct spglist *free)
 3021 {
 3022         struct md_page *pvh;
 3023         pd_entry_t oldpde;
 3024         vm_offset_t eva, va;
 3025         vm_page_t m, mpte;
 3026 
 3027         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3028         KASSERT((sva & PDRMASK) == 0,
 3029             ("pmap_remove_pde: sva is not 4mpage aligned"));
 3030         oldpde = pte_load_clear(pdq);
 3031         if (oldpde & PG_W)
 3032                 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
 3033 
 3034         /*
 3035          * Machines that don't support invlpg, also don't support
 3036          * PG_G.
 3037          */
 3038         if ((oldpde & PG_G) != 0)
 3039                 pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
 3040 
 3041         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 3042         if (oldpde & PG_MANAGED) {
 3043                 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
 3044                 pmap_pvh_free(pvh, pmap, sva);
 3045                 eva = sva + NBPDR;
 3046                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 3047                     va < eva; va += PAGE_SIZE, m++) {
 3048                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3049                                 vm_page_dirty(m);
 3050                         if (oldpde & PG_A)
 3051                                 vm_page_aflag_set(m, PGA_REFERENCED);
 3052                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 3053                             TAILQ_EMPTY(&pvh->pv_list))
 3054                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 3055                 }
 3056         }
 3057         if (pmap == kernel_pmap) {
 3058                 pmap_remove_kernel_pde(pmap, pdq, sva);
 3059         } else {
 3060                 mpte = pmap_remove_pt_page(pmap, sva);
 3061                 if (mpte != NULL) {
 3062                         KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
 3063                             ("pmap_remove_pde: pte page not promoted"));
 3064                         pmap->pm_stats.resident_count--;
 3065                         KASSERT(mpte->wire_count == NPTEPG,
 3066                             ("pmap_remove_pde: pte page wire count error"));
 3067                         mpte->wire_count = 0;
 3068                         pmap_add_delayed_free_list(mpte, free, FALSE);
 3069                 }
 3070         }
 3071 }
 3072 
 3073 /*
 3074  * pmap_remove_pte: do the things to unmap a page in a process
 3075  */
 3076 static int
 3077 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
 3078     struct spglist *free)
 3079 {
 3080         pt_entry_t oldpte;
 3081         vm_page_t m;
 3082 
 3083         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3084         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3085         oldpte = pte_load_clear(ptq);
 3086         KASSERT(oldpte != 0,
 3087             ("pmap_remove_pte: pmap %p va %x zero pte", pmap, va));
 3088         if (oldpte & PG_W)
 3089                 pmap->pm_stats.wired_count -= 1;
 3090         /*
 3091          * Machines that don't support invlpg, also don't support
 3092          * PG_G.
 3093          */
 3094         if (oldpte & PG_G)
 3095                 pmap_invalidate_page(kernel_pmap, va);
 3096         pmap->pm_stats.resident_count -= 1;
 3097         if (oldpte & PG_MANAGED) {
 3098                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 3099                 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3100                         vm_page_dirty(m);
 3101                 if (oldpte & PG_A)
 3102                         vm_page_aflag_set(m, PGA_REFERENCED);
 3103                 pmap_remove_entry(pmap, m, va);
 3104         }
 3105         return (pmap_unuse_pt(pmap, va, free));
 3106 }
 3107 
 3108 /*
 3109  * Remove a single page from a process address space
 3110  */
 3111 static void
 3112 pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)
 3113 {
 3114         pt_entry_t *pte;
 3115 
 3116         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3117         KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 3118         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3119         if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
 3120                 return;
 3121         pmap_remove_pte(pmap, pte, va, free);
 3122         pmap_invalidate_page(pmap, va);
 3123 }
 3124 
 3125 /*
 3126  * Removes the specified range of addresses from the page table page.
 3127  */
 3128 static bool
 3129 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
 3130     struct spglist *free)
 3131 {
 3132         pt_entry_t *pte;
 3133         bool anyvalid;
 3134 
 3135         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3136         KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 3137         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3138         anyvalid = false;
 3139         for (pte = pmap_pte_quick(pmap, sva); sva != eva; pte++,
 3140             sva += PAGE_SIZE) {
 3141                 if (*pte == 0)
 3142                         continue;
 3143 
 3144                 /*
 3145                  * The TLB entry for a PG_G mapping is invalidated by
 3146                  * pmap_remove_pte().
 3147                  */
 3148                 if ((*pte & PG_G) == 0)
 3149                         anyvalid = true;
 3150 
 3151                 if (pmap_remove_pte(pmap, pte, sva, free))
 3152                         break;
 3153         }
 3154         return (anyvalid);
 3155 }
 3156 
 3157 /*
 3158  *      Remove the given range of addresses from the specified map.
 3159  *
 3160  *      It is assumed that the start and end are properly
 3161  *      rounded to the page size.
 3162  */
 3163 void
 3164 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 3165 {
 3166         vm_offset_t pdnxt;
 3167         pd_entry_t ptpaddr;
 3168         struct spglist free;
 3169         int anyvalid;
 3170 
 3171         /*
 3172          * Perform an unsynchronized read.  This is, however, safe.
 3173          */
 3174         if (pmap->pm_stats.resident_count == 0)
 3175                 return;
 3176 
 3177         anyvalid = 0;
 3178         SLIST_INIT(&free);
 3179 
 3180         rw_wlock(&pvh_global_lock);
 3181         sched_pin();
 3182         PMAP_LOCK(pmap);
 3183 
 3184         /*
 3185          * special handling of removing one page.  a very
 3186          * common operation and easy to short circuit some
 3187          * code.
 3188          */
 3189         if ((sva + PAGE_SIZE == eva) && 
 3190             ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
 3191                 pmap_remove_page(pmap, sva, &free);
 3192                 goto out;
 3193         }
 3194 
 3195         for (; sva < eva; sva = pdnxt) {
 3196                 u_int pdirindex;
 3197 
 3198                 /*
 3199                  * Calculate index for next page table.
 3200                  */
 3201                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 3202                 if (pdnxt < sva)
 3203                         pdnxt = eva;
 3204                 if (pmap->pm_stats.resident_count == 0)
 3205                         break;
 3206 
 3207                 pdirindex = sva >> PDRSHIFT;
 3208                 ptpaddr = pmap->pm_pdir[pdirindex];
 3209 
 3210                 /*
 3211                  * Weed out invalid mappings. Note: we assume that the page
 3212                  * directory table is always allocated, and in kernel virtual.
 3213                  */
 3214                 if (ptpaddr == 0)
 3215                         continue;
 3216 
 3217                 /*
 3218                  * Check for large page.
 3219                  */
 3220                 if ((ptpaddr & PG_PS) != 0) {
 3221                         /*
 3222                          * Are we removing the entire large page?  If not,
 3223                          * demote the mapping and fall through.
 3224                          */
 3225                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 3226                                 /*
 3227                                  * The TLB entry for a PG_G mapping is
 3228                                  * invalidated by pmap_remove_pde().
 3229                                  */
 3230                                 if ((ptpaddr & PG_G) == 0)
 3231                                         anyvalid = 1;
 3232                                 pmap_remove_pde(pmap,
 3233                                     &pmap->pm_pdir[pdirindex], sva, &free);
 3234                                 continue;
 3235                         } else if (!pmap_demote_pde(pmap,
 3236                             &pmap->pm_pdir[pdirindex], sva)) {
 3237                                 /* The large page mapping was destroyed. */
 3238                                 continue;
 3239                         }
 3240                 }
 3241 
 3242                 /*
 3243                  * Limit our scan to either the end of the va represented
 3244                  * by the current page table page, or to the end of the
 3245                  * range being removed.
 3246                  */
 3247                 if (pdnxt > eva)
 3248                         pdnxt = eva;
 3249 
 3250                 if (pmap_remove_ptes(pmap, sva, pdnxt, &free))
 3251                         anyvalid = 1;
 3252         }
 3253 out:
 3254         sched_unpin();
 3255         if (anyvalid)
 3256                 pmap_invalidate_all(pmap);
 3257         rw_wunlock(&pvh_global_lock);
 3258         PMAP_UNLOCK(pmap);
 3259         vm_page_free_pages_toq(&free, true);
 3260 }
 3261 
 3262 /*
 3263  *      Routine:        pmap_remove_all
 3264  *      Function:
 3265  *              Removes this physical page from
 3266  *              all physical maps in which it resides.
 3267  *              Reflects back modify bits to the pager.
 3268  *
 3269  *      Notes:
 3270  *              Original versions of this routine were very
 3271  *              inefficient because they iteratively called
 3272  *              pmap_remove (slow...)
 3273  */
 3274 
 3275 void
 3276 pmap_remove_all(vm_page_t m)
 3277 {
 3278         struct md_page *pvh;
 3279         pv_entry_t pv;
 3280         pmap_t pmap;
 3281         pt_entry_t *pte, tpte;
 3282         pd_entry_t *pde;
 3283         vm_offset_t va;
 3284         struct spglist free;
 3285 
 3286         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3287             ("pmap_remove_all: page %p is not managed", m));
 3288         SLIST_INIT(&free);
 3289         rw_wlock(&pvh_global_lock);
 3290         sched_pin();
 3291         if ((m->flags & PG_FICTITIOUS) != 0)
 3292                 goto small_mappings;
 3293         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 3294         while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
 3295                 va = pv->pv_va;
 3296                 pmap = PV_PMAP(pv);
 3297                 PMAP_LOCK(pmap);
 3298                 pde = pmap_pde(pmap, va);
 3299                 (void)pmap_demote_pde(pmap, pde, va);
 3300                 PMAP_UNLOCK(pmap);
 3301         }
 3302 small_mappings:
 3303         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 3304                 pmap = PV_PMAP(pv);
 3305                 PMAP_LOCK(pmap);
 3306                 pmap->pm_stats.resident_count--;
 3307                 pde = pmap_pde(pmap, pv->pv_va);
 3308                 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
 3309                     " a 4mpage in page %p's pv list", m));
 3310                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3311                 tpte = pte_load_clear(pte);
 3312                 KASSERT(tpte != 0, ("pmap_remove_all: pmap %p va %x zero pte",
 3313                     pmap, pv->pv_va));
 3314                 if (tpte & PG_W)
 3315                         pmap->pm_stats.wired_count--;
 3316                 if (tpte & PG_A)
 3317                         vm_page_aflag_set(m, PGA_REFERENCED);
 3318 
 3319                 /*
 3320                  * Update the vm_page_t clean and reference bits.
 3321                  */
 3322                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3323                         vm_page_dirty(m);
 3324                 pmap_unuse_pt(pmap, pv->pv_va, &free);
 3325                 pmap_invalidate_page(pmap, pv->pv_va);
 3326                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 3327                 free_pv_entry(pmap, pv);
 3328                 PMAP_UNLOCK(pmap);
 3329         }
 3330         vm_page_aflag_clear(m, PGA_WRITEABLE);
 3331         sched_unpin();
 3332         rw_wunlock(&pvh_global_lock);
 3333         vm_page_free_pages_toq(&free, true);
 3334 }
 3335 
 3336 /*
 3337  * pmap_protect_pde: do the things to protect a 4mpage in a process
 3338  */
 3339 static boolean_t
 3340 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
 3341 {
 3342         pd_entry_t newpde, oldpde;
 3343         vm_page_t m, mt;
 3344         boolean_t anychanged;
 3345 
 3346         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3347         KASSERT((sva & PDRMASK) == 0,
 3348             ("pmap_protect_pde: sva is not 4mpage aligned"));
 3349         anychanged = FALSE;
 3350 retry:
 3351         oldpde = newpde = *pde;
 3352         if ((prot & VM_PROT_WRITE) == 0) {
 3353                 if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
 3354                     (PG_MANAGED | PG_M | PG_RW)) {
 3355                         m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 3356                         for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 3357                                 vm_page_dirty(mt);
 3358                 }
 3359                 newpde &= ~(PG_RW | PG_M);
 3360         }
 3361 #if defined(PAE) || defined(PAE_TABLES)
 3362         if ((prot & VM_PROT_EXECUTE) == 0)
 3363                 newpde |= pg_nx;
 3364 #endif
 3365         if (newpde != oldpde) {
 3366                 /*
 3367                  * As an optimization to future operations on this PDE, clear
 3368                  * PG_PROMOTED.  The impending invalidation will remove any
 3369                  * lingering 4KB page mappings from the TLB.
 3370                  */
 3371                 if (!pde_cmpset(pde, oldpde, newpde & ~PG_PROMOTED))
 3372                         goto retry;
 3373                 if ((oldpde & PG_G) != 0)
 3374                         pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
 3375                 else
 3376                         anychanged = TRUE;
 3377         }
 3378         return (anychanged);
 3379 }
 3380 
 3381 /*
 3382  *      Set the physical protection on the
 3383  *      specified range of this map as requested.
 3384  */
 3385 void
 3386 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 3387 {
 3388         vm_offset_t pdnxt;
 3389         pd_entry_t ptpaddr;
 3390         pt_entry_t *pte;
 3391         boolean_t anychanged, pv_lists_locked;
 3392 
 3393         KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
 3394         if (prot == VM_PROT_NONE) {
 3395                 pmap_remove(pmap, sva, eva);
 3396                 return;
 3397         }
 3398 
 3399 #if defined(PAE) || defined(PAE_TABLES)
 3400         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 3401             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 3402                 return;
 3403 #else
 3404         if (prot & VM_PROT_WRITE)
 3405                 return;
 3406 #endif
 3407 
 3408         if (pmap_is_current(pmap))
 3409                 pv_lists_locked = FALSE;
 3410         else {
 3411                 pv_lists_locked = TRUE;
 3412 resume:
 3413                 rw_wlock(&pvh_global_lock);
 3414                 sched_pin();
 3415         }
 3416         anychanged = FALSE;
 3417 
 3418         PMAP_LOCK(pmap);
 3419         for (; sva < eva; sva = pdnxt) {
 3420                 pt_entry_t obits, pbits;
 3421                 u_int pdirindex;
 3422 
 3423                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 3424                 if (pdnxt < sva)
 3425                         pdnxt = eva;
 3426 
 3427                 pdirindex = sva >> PDRSHIFT;
 3428                 ptpaddr = pmap->pm_pdir[pdirindex];
 3429 
 3430                 /*
 3431                  * Weed out invalid mappings. Note: we assume that the page
 3432                  * directory table is always allocated, and in kernel virtual.
 3433                  */
 3434                 if (ptpaddr == 0)
 3435                         continue;
 3436 
 3437                 /*
 3438                  * Check for large page.
 3439                  */
 3440                 if ((ptpaddr & PG_PS) != 0) {
 3441                         /*
 3442                          * Are we protecting the entire large page?  If not,
 3443                          * demote the mapping and fall through.
 3444                          */
 3445                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 3446                                 /*
 3447                                  * The TLB entry for a PG_G mapping is
 3448                                  * invalidated by pmap_protect_pde().
 3449                                  */
 3450                                 if (pmap_protect_pde(pmap,
 3451                                     &pmap->pm_pdir[pdirindex], sva, prot))
 3452                                         anychanged = TRUE;
 3453                                 continue;
 3454                         } else {
 3455                                 if (!pv_lists_locked) {
 3456                                         pv_lists_locked = TRUE;
 3457                                         if (!rw_try_wlock(&pvh_global_lock)) {
 3458                                                 if (anychanged)
 3459                                                         pmap_invalidate_all(
 3460                                                             pmap);
 3461                                                 PMAP_UNLOCK(pmap);
 3462                                                 goto resume;
 3463                                         }
 3464                                         sched_pin();
 3465                                 }
 3466                                 if (!pmap_demote_pde(pmap,
 3467                                     &pmap->pm_pdir[pdirindex], sva)) {
 3468                                         /*
 3469                                          * The large page mapping was
 3470                                          * destroyed.
 3471                                          */
 3472                                         continue;
 3473                                 }
 3474                         }
 3475                 }
 3476 
 3477                 if (pdnxt > eva)
 3478                         pdnxt = eva;
 3479 
 3480                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 3481                     sva += PAGE_SIZE) {
 3482                         vm_page_t m;
 3483 
 3484 retry:
 3485                         /*
 3486                          * Regardless of whether a pte is 32 or 64 bits in
 3487                          * size, PG_RW, PG_A, and PG_M are among the least
 3488                          * significant 32 bits.
 3489                          */
 3490                         obits = pbits = *pte;
 3491                         if ((pbits & PG_V) == 0)
 3492                                 continue;
 3493 
 3494                         if ((prot & VM_PROT_WRITE) == 0) {
 3495                                 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
 3496                                     (PG_MANAGED | PG_M | PG_RW)) {
 3497                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 3498                                         vm_page_dirty(m);
 3499                                 }
 3500                                 pbits &= ~(PG_RW | PG_M);
 3501                         }
 3502 #if defined(PAE) || defined(PAE_TABLES)
 3503                         if ((prot & VM_PROT_EXECUTE) == 0)
 3504                                 pbits |= pg_nx;
 3505 #endif
 3506 
 3507                         if (pbits != obits) {
 3508 #if defined(PAE) || defined(PAE_TABLES)
 3509                                 if (!atomic_cmpset_64(pte, obits, pbits))
 3510                                         goto retry;
 3511 #else
 3512                                 if (!atomic_cmpset_int((u_int *)pte, obits,
 3513                                     pbits))
 3514                                         goto retry;
 3515 #endif
 3516                                 if (obits & PG_G)
 3517                                         pmap_invalidate_page(pmap, sva);
 3518                                 else
 3519                                         anychanged = TRUE;
 3520                         }
 3521                 }
 3522         }
 3523         if (anychanged)
 3524                 pmap_invalidate_all(pmap);
 3525         if (pv_lists_locked) {
 3526                 sched_unpin();
 3527                 rw_wunlock(&pvh_global_lock);
 3528         }
 3529         PMAP_UNLOCK(pmap);
 3530 }
 3531 
 3532 #if VM_NRESERVLEVEL > 0
 3533 /*
 3534  * Tries to promote the 512 or 1024, contiguous 4KB page mappings that are
 3535  * within a single page table page (PTP) to a single 2- or 4MB page mapping.
 3536  * For promotion to occur, two conditions must be met: (1) the 4KB page
 3537  * mappings must map aligned, contiguous physical memory and (2) the 4KB page
 3538  * mappings must have identical characteristics.
 3539  *
 3540  * Managed (PG_MANAGED) mappings within the kernel address space are not
 3541  * promoted.  The reason is that kernel PDEs are replicated in each pmap but
 3542  * pmap_clear_ptes() and pmap_ts_referenced() only read the PDE from the kernel
 3543  * pmap.
 3544  */
 3545 static void
 3546 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 3547 {
 3548         pd_entry_t newpde;
 3549         pt_entry_t *firstpte, oldpte, pa, *pte;
 3550         vm_offset_t oldpteva;
 3551         vm_page_t mpte;
 3552 
 3553         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3554 
 3555         /*
 3556          * Examine the first PTE in the specified PTP.  Abort if this PTE is
 3557          * either invalid, unused, or does not map the first 4KB physical page
 3558          * within a 2- or 4MB page.
 3559          */
 3560         firstpte = pmap_pte_quick(pmap, trunc_4mpage(va));
 3561 setpde:
 3562         newpde = *firstpte;
 3563         if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
 3564                 pmap_pde_p_failures++;
 3565                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3566                     " in pmap %p", va, pmap);
 3567                 return;
 3568         }
 3569         if ((*firstpte & PG_MANAGED) != 0 && pmap == kernel_pmap) {
 3570                 pmap_pde_p_failures++;
 3571                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3572                     " in pmap %p", va, pmap);
 3573                 return;
 3574         }
 3575         if ((newpde & (PG_M | PG_RW)) == PG_RW) {
 3576                 /*
 3577                  * When PG_M is already clear, PG_RW can be cleared without
 3578                  * a TLB invalidation.
 3579                  */
 3580                 if (!atomic_cmpset_int((u_int *)firstpte, newpde, newpde &
 3581                     ~PG_RW))  
 3582                         goto setpde;
 3583                 newpde &= ~PG_RW;
 3584         }
 3585 
 3586         /* 
 3587          * Examine each of the other PTEs in the specified PTP.  Abort if this
 3588          * PTE maps an unexpected 4KB physical page or does not have identical
 3589          * characteristics to the first PTE.
 3590          */
 3591         pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + NBPDR - PAGE_SIZE;
 3592         for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
 3593 setpte:
 3594                 oldpte = *pte;
 3595                 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
 3596                         pmap_pde_p_failures++;
 3597                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3598                             " in pmap %p", va, pmap);
 3599                         return;
 3600                 }
 3601                 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
 3602                         /*
 3603                          * When PG_M is already clear, PG_RW can be cleared
 3604                          * without a TLB invalidation.
 3605                          */
 3606                         if (!atomic_cmpset_int((u_int *)pte, oldpte,
 3607                             oldpte & ~PG_RW))
 3608                                 goto setpte;
 3609                         oldpte &= ~PG_RW;
 3610                         oldpteva = (oldpte & PG_FRAME & PDRMASK) |
 3611                             (va & ~PDRMASK);
 3612                         CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#x"
 3613                             " in pmap %p", oldpteva, pmap);
 3614                 }
 3615                 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
 3616                         pmap_pde_p_failures++;
 3617                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3618                             " in pmap %p", va, pmap);
 3619                         return;
 3620                 }
 3621                 pa -= PAGE_SIZE;
 3622         }
 3623 
 3624         /*
 3625          * Save the page table page in its current state until the PDE
 3626          * mapping the superpage is demoted by pmap_demote_pde() or
 3627          * destroyed by pmap_remove_pde(). 
 3628          */
 3629         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 3630         KASSERT(mpte >= vm_page_array &&
 3631             mpte < &vm_page_array[vm_page_array_size],
 3632             ("pmap_promote_pde: page table page is out of range"));
 3633         KASSERT(mpte->pindex == va >> PDRSHIFT,
 3634             ("pmap_promote_pde: page table page's pindex is wrong"));
 3635         if (pmap_insert_pt_page(pmap, mpte, true)) {
 3636                 pmap_pde_p_failures++;
 3637                 CTR2(KTR_PMAP,
 3638                     "pmap_promote_pde: failure for va %#x in pmap %p", va,
 3639                     pmap);
 3640                 return;
 3641         }
 3642 
 3643         /*
 3644          * Promote the pv entries.
 3645          */
 3646         if ((newpde & PG_MANAGED) != 0)
 3647                 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
 3648 
 3649         /*
 3650          * Propagate the PAT index to its proper position.
 3651          */
 3652         if ((newpde & PG_PTE_PAT) != 0)
 3653                 newpde ^= PG_PDE_PAT | PG_PTE_PAT;
 3654 
 3655         /*
 3656          * Map the superpage.
 3657          */
 3658         if (workaround_erratum383)
 3659                 pmap_update_pde(pmap, va, pde, PG_PS | newpde);
 3660         else if (pmap == kernel_pmap)
 3661                 pmap_kenter_pde(va, PG_PROMOTED | PG_PS | newpde);
 3662         else
 3663                 pde_store(pde, PG_PROMOTED | PG_PS | newpde);
 3664 
 3665         pmap_pde_promotions++;
 3666         CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x"
 3667             " in pmap %p", va, pmap);
 3668 }
 3669 #endif /* VM_NRESERVLEVEL > 0 */
 3670 
 3671 /*
 3672  *      Insert the given physical page (p) at
 3673  *      the specified virtual address (v) in the
 3674  *      target physical map with the protection requested.
 3675  *
 3676  *      If specified, the page will be wired down, meaning
 3677  *      that the related pte can not be reclaimed.
 3678  *
 3679  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 3680  *      or lose information.  That is, this routine must actually
 3681  *      insert this page into the given map NOW.
 3682  */
 3683 int
 3684 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 3685     u_int flags, int8_t psind)
 3686 {
 3687         pd_entry_t *pde;
 3688         pt_entry_t *pte;
 3689         pt_entry_t newpte, origpte;
 3690         pv_entry_t pv;
 3691         vm_paddr_t opa, pa;
 3692         vm_page_t mpte, om;
 3693         int rv;
 3694 
 3695         va = trunc_page(va);
 3696         KASSERT((pmap == kernel_pmap && va < VM_MAX_KERNEL_ADDRESS) ||
 3697             (pmap != kernel_pmap && va < VM_MAXUSER_ADDRESS),
 3698             ("pmap_enter: toobig k%d %#x", pmap == kernel_pmap, va));
 3699         KASSERT(va < PMAP_TRM_MIN_ADDRESS,
 3700             ("pmap_enter: invalid to pmap_enter into trampoline (va: 0x%x)",
 3701             va));
 3702         KASSERT(pmap != kernel_pmap || (m->oflags & VPO_UNMANAGED) != 0 ||
 3703             va < kmi.clean_sva || va >= kmi.clean_eva,
 3704             ("pmap_enter: managed mapping within the clean submap"));
 3705         if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
 3706                 VM_OBJECT_ASSERT_LOCKED(m->object);
 3707         KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
 3708             ("pmap_enter: flags %u has reserved bits set", flags));
 3709         pa = VM_PAGE_TO_PHYS(m);
 3710         newpte = (pt_entry_t)(pa | PG_A | PG_V);
 3711         if ((flags & VM_PROT_WRITE) != 0)
 3712                 newpte |= PG_M;
 3713         if ((prot & VM_PROT_WRITE) != 0)
 3714                 newpte |= PG_RW;
 3715         KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
 3716             ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
 3717 #if defined(PAE) || defined(PAE_TABLES)
 3718         if ((prot & VM_PROT_EXECUTE) == 0)
 3719                 newpte |= pg_nx;
 3720 #endif
 3721         if ((flags & PMAP_ENTER_WIRED) != 0)
 3722                 newpte |= PG_W;
 3723         if (pmap != kernel_pmap)
 3724                 newpte |= PG_U;
 3725         newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);
 3726         if ((m->oflags & VPO_UNMANAGED) == 0)
 3727                 newpte |= PG_MANAGED;
 3728 
 3729         rw_wlock(&pvh_global_lock);
 3730         PMAP_LOCK(pmap);
 3731         sched_pin();
 3732         if (psind == 1) {
 3733                 /* Assert the required virtual and physical alignment. */ 
 3734                 KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned"));
 3735                 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
 3736                 rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m);
 3737                 goto out;
 3738         }
 3739 
 3740         pde = pmap_pde(pmap, va);
 3741         if (pmap != kernel_pmap) {
 3742                 /*
 3743                  * va is for UVA.
 3744                  * In the case that a page table page is not resident,
 3745                  * we are creating it here.  pmap_allocpte() handles
 3746                  * demotion.
 3747                  */
 3748                 mpte = pmap_allocpte(pmap, va, flags);
 3749                 if (mpte == NULL) {
 3750                         KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0,
 3751                             ("pmap_allocpte failed with sleep allowed"));
 3752                         rv = KERN_RESOURCE_SHORTAGE;
 3753                         goto out;
 3754                 }
 3755         } else {
 3756                 /*
 3757                  * va is for KVA, so pmap_demote_pde() will never fail
 3758                  * to install a page table page.  PG_V is also
 3759                  * asserted by pmap_demote_pde().
 3760                  */
 3761                 mpte = NULL;
 3762                 KASSERT(pde != NULL && (*pde & PG_V) != 0,
 3763                     ("KVA %#x invalid pde pdir %#jx", va,
 3764                     (uintmax_t)pmap->pm_pdir[PTDPTDI]));
 3765                 if ((*pde & PG_PS) != 0)
 3766                         pmap_demote_pde(pmap, pde, va);
 3767         }
 3768         pte = pmap_pte_quick(pmap, va);
 3769 
 3770         /*
 3771          * Page Directory table entry is not valid, which should not
 3772          * happen.  We should have either allocated the page table
 3773          * page or demoted the existing mapping above.
 3774          */
 3775         if (pte == NULL) {
 3776                 panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x",
 3777                     (uintmax_t)pmap->pm_pdir[PTDPTDI], va);
 3778         }
 3779 
 3780         origpte = *pte;
 3781         pv = NULL;
 3782 
 3783         /*
 3784          * Is the specified virtual address already mapped?
 3785          */
 3786         if ((origpte & PG_V) != 0) {
 3787                 /*
 3788                  * Wiring change, just update stats. We don't worry about
 3789                  * wiring PT pages as they remain resident as long as there
 3790                  * are valid mappings in them. Hence, if a user page is wired,
 3791                  * the PT page will be also.
 3792                  */
 3793                 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
 3794                         pmap->pm_stats.wired_count++;
 3795                 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
 3796                         pmap->pm_stats.wired_count--;
 3797 
 3798                 /*
 3799                  * Remove the extra PT page reference.
 3800                  */
 3801                 if (mpte != NULL) {
 3802                         mpte->wire_count--;
 3803                         KASSERT(mpte->wire_count > 0,
 3804                             ("pmap_enter: missing reference to page table page,"
 3805                              " va: 0x%x", va));
 3806                 }
 3807 
 3808                 /*
 3809                  * Has the physical page changed?
 3810                  */
 3811                 opa = origpte & PG_FRAME;
 3812                 if (opa == pa) {
 3813                         /*
 3814                          * No, might be a protection or wiring change.
 3815                          */
 3816                         if ((origpte & PG_MANAGED) != 0 &&
 3817                             (newpte & PG_RW) != 0)
 3818                                 vm_page_aflag_set(m, PGA_WRITEABLE);
 3819                         if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
 3820                                 goto unchanged;
 3821                         goto validate;
 3822                 }
 3823 
 3824                 /*
 3825                  * The physical page has changed.  Temporarily invalidate
 3826                  * the mapping.  This ensures that all threads sharing the
 3827                  * pmap keep a consistent view of the mapping, which is
 3828                  * necessary for the correct handling of COW faults.  It
 3829                  * also permits reuse of the old mapping's PV entry,
 3830                  * avoiding an allocation.
 3831                  *
 3832                  * For consistency, handle unmanaged mappings the same way.
 3833                  */
 3834                 origpte = pte_load_clear(pte);
 3835                 KASSERT((origpte & PG_FRAME) == opa,
 3836                     ("pmap_enter: unexpected pa update for %#x", va));
 3837                 if ((origpte & PG_MANAGED) != 0) {
 3838                         om = PHYS_TO_VM_PAGE(opa);
 3839 
 3840                         /*
 3841                          * The pmap lock is sufficient to synchronize with
 3842                          * concurrent calls to pmap_page_test_mappings() and
 3843                          * pmap_ts_referenced().
 3844                          */
 3845                         if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3846                                 vm_page_dirty(om);
 3847                         if ((origpte & PG_A) != 0)
 3848                                 vm_page_aflag_set(om, PGA_REFERENCED);
 3849                         pv = pmap_pvh_remove(&om->md, pmap, va);
 3850                         KASSERT(pv != NULL,
 3851                             ("pmap_enter: no PV entry for %#x", va));
 3852                         if ((newpte & PG_MANAGED) == 0)
 3853                                 free_pv_entry(pmap, pv);
 3854                         if ((om->aflags & PGA_WRITEABLE) != 0 &&
 3855                             TAILQ_EMPTY(&om->md.pv_list) &&
 3856                             ((om->flags & PG_FICTITIOUS) != 0 ||
 3857                             TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
 3858                                 vm_page_aflag_clear(om, PGA_WRITEABLE);
 3859                 }
 3860                 if ((origpte & PG_A) != 0)
 3861                         pmap_invalidate_page(pmap, va);
 3862                 origpte = 0;
 3863         } else {
 3864                 /*
 3865                  * Increment the counters.
 3866                  */
 3867                 if ((newpte & PG_W) != 0)
 3868                         pmap->pm_stats.wired_count++;
 3869                 pmap->pm_stats.resident_count++;
 3870         }
 3871 
 3872         /*
 3873          * Enter on the PV list if part of our managed memory.
 3874          */
 3875         if ((newpte & PG_MANAGED) != 0) {
 3876                 if (pv == NULL) {
 3877                         pv = get_pv_entry(pmap, FALSE);
 3878                         pv->pv_va = va;
 3879                 }
 3880                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 3881                 if ((newpte & PG_RW) != 0)
 3882                         vm_page_aflag_set(m, PGA_WRITEABLE);
 3883         }
 3884 
 3885         /*
 3886          * Update the PTE.
 3887          */
 3888         if ((origpte & PG_V) != 0) {
 3889 validate:
 3890                 origpte = pte_load_store(pte, newpte);
 3891                 KASSERT((origpte & PG_FRAME) == pa,
 3892                     ("pmap_enter: unexpected pa update for %#x", va));
 3893                 if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) ==
 3894                     (PG_M | PG_RW)) {
 3895                         if ((origpte & PG_MANAGED) != 0)
 3896                                 vm_page_dirty(m);
 3897 
 3898                         /*
 3899                          * Although the PTE may still have PG_RW set, TLB
 3900                          * invalidation may nonetheless be required because
 3901                          * the PTE no longer has PG_M set.
 3902                          */
 3903                 }
 3904 #if defined(PAE) || defined(PAE_TABLES)
 3905                 else if ((origpte & PG_NX) != 0 || (newpte & PG_NX) == 0) {
 3906                         /*
 3907                          * This PTE change does not require TLB invalidation.
 3908                          */
 3909                         goto unchanged;
 3910                 }
 3911 #endif
 3912                 if ((origpte & PG_A) != 0)
 3913                         pmap_invalidate_page(pmap, va);
 3914         } else
 3915                 pte_store(pte, newpte);
 3916 
 3917 unchanged:
 3918 
 3919 #if VM_NRESERVLEVEL > 0
 3920         /*
 3921          * If both the page table page and the reservation are fully
 3922          * populated, then attempt promotion.
 3923          */
 3924         if ((mpte == NULL || mpte->wire_count == NPTEPG) &&
 3925             pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 &&
 3926             vm_reserv_level_iffullpop(m) == 0)
 3927                 pmap_promote_pde(pmap, pde, va);
 3928 #endif
 3929 
 3930         rv = KERN_SUCCESS;
 3931 out:
 3932         sched_unpin();
 3933         rw_wunlock(&pvh_global_lock);
 3934         PMAP_UNLOCK(pmap);
 3935         return (rv);
 3936 }
 3937 
 3938 /*
 3939  * Tries to create a read- and/or execute-only 2 or 4 MB page mapping.  Returns
 3940  * true if successful.  Returns false if (1) a mapping already exists at the
 3941  * specified virtual address or (2) a PV entry cannot be allocated without
 3942  * reclaiming another PV entry.
 3943  */
 3944 static bool
 3945 pmap_enter_4mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 3946 {
 3947         pd_entry_t newpde;
 3948 
 3949         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3950         newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 1) |
 3951             PG_PS | PG_V;
 3952         if ((m->oflags & VPO_UNMANAGED) == 0)
 3953                 newpde |= PG_MANAGED;
 3954 #if defined(PAE) || defined(PAE_TABLES)
 3955         if ((prot & VM_PROT_EXECUTE) == 0)
 3956                 newpde |= pg_nx;
 3957 #endif
 3958         if (pmap != kernel_pmap)
 3959                 newpde |= PG_U;
 3960         return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
 3961             PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL) ==
 3962             KERN_SUCCESS);
 3963 }
 3964 
 3965 /*
 3966  * Tries to create the specified 2 or 4 MB page mapping.  Returns KERN_SUCCESS
 3967  * if the mapping was created, and either KERN_FAILURE or
 3968  * KERN_RESOURCE_SHORTAGE otherwise.  Returns KERN_FAILURE if
 3969  * PMAP_ENTER_NOREPLACE was specified and a mapping already exists at the
 3970  * specified virtual address.  Returns KERN_RESOURCE_SHORTAGE if
 3971  * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
 3972  *
 3973  * The parameter "m" is only used when creating a managed, writeable mapping.
 3974  */
 3975 static int
 3976 pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,
 3977     vm_page_t m)
 3978 {
 3979         struct spglist free;
 3980         pd_entry_t oldpde, *pde;
 3981         vm_page_t mt;
 3982 
 3983         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3984         KASSERT((newpde & (PG_M | PG_RW)) != PG_RW,
 3985             ("pmap_enter_pde: newpde is missing PG_M"));
 3986         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3987         pde = pmap_pde(pmap, va);
 3988         oldpde = *pde;
 3989         if ((oldpde & PG_V) != 0) {
 3990                 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
 3991                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3992                             " in pmap %p", va, pmap);
 3993                         return (KERN_FAILURE);
 3994                 }
 3995                 /* Break the existing mapping(s). */
 3996                 SLIST_INIT(&free);
 3997                 if ((oldpde & PG_PS) != 0) {
 3998                         /*
 3999                          * If the PDE resulted from a promotion, then a
 4000                          * reserved PT page could be freed.
 4001                          */
 4002                         (void)pmap_remove_pde(pmap, pde, va, &free);
 4003                         if ((oldpde & PG_G) == 0)
 4004                                 pmap_invalidate_pde_page(pmap, va, oldpde);
 4005                 } else {
 4006                         if (pmap_remove_ptes(pmap, va, va + NBPDR, &free))
 4007                                pmap_invalidate_all(pmap);
 4008                 }
 4009                 vm_page_free_pages_toq(&free, true);
 4010                 if (pmap == kernel_pmap) {
 4011                         /*
 4012                          * Both pmap_remove_pde() and pmap_remove_ptes() will
 4013                          * leave the kernel page table page zero filled.
 4014                          */
 4015                         mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 4016                         if (pmap_insert_pt_page(pmap, mt, false))
 4017                                 panic("pmap_enter_pde: trie insert failed");
 4018                 } else
 4019                         KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
 4020                             pde));
 4021         }
 4022         if ((newpde & PG_MANAGED) != 0) {
 4023                 /*
 4024                  * Abort this mapping if its PV entry could not be created.
 4025                  */
 4026                 if (!pmap_pv_insert_pde(pmap, va, newpde, flags)) {
 4027                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 4028                             " in pmap %p", va, pmap);
 4029                         return (KERN_RESOURCE_SHORTAGE);
 4030                 }
 4031                 if ((newpde & PG_RW) != 0) {
 4032                         for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4033                                 vm_page_aflag_set(mt, PGA_WRITEABLE);
 4034                 }
 4035         }
 4036 
 4037         /*
 4038          * Increment counters.
 4039          */
 4040         if ((newpde & PG_W) != 0)
 4041                 pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;
 4042         pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
 4043 
 4044         /*
 4045          * Map the superpage.  (This is not a promoted mapping; there will not
 4046          * be any lingering 4KB page mappings in the TLB.)
 4047          */
 4048         pde_store(pde, newpde);
 4049 
 4050         pmap_pde_mappings++;
 4051         CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
 4052             " in pmap %p", va, pmap);
 4053         return (KERN_SUCCESS);
 4054 }
 4055 
 4056 /*
 4057  * Maps a sequence of resident pages belonging to the same object.
 4058  * The sequence begins with the given page m_start.  This page is
 4059  * mapped at the given virtual address start.  Each subsequent page is
 4060  * mapped at a virtual address that is offset from start by the same
 4061  * amount as the page is offset from m_start within the object.  The
 4062  * last page in the sequence is the page with the largest offset from
 4063  * m_start that can be mapped at a virtual address less than the given
 4064  * virtual address end.  Not every virtual page between start and end
 4065  * is mapped; only those for which a resident page exists with the
 4066  * corresponding offset from m_start are mapped.
 4067  */
 4068 void
 4069 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 4070     vm_page_t m_start, vm_prot_t prot)
 4071 {
 4072         vm_offset_t va;
 4073         vm_page_t m, mpte;
 4074         vm_pindex_t diff, psize;
 4075 
 4076         VM_OBJECT_ASSERT_LOCKED(m_start->object);
 4077 
 4078         psize = atop(end - start);
 4079         mpte = NULL;
 4080         m = m_start;
 4081         rw_wlock(&pvh_global_lock);
 4082         PMAP_LOCK(pmap);
 4083         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 4084                 va = start + ptoa(diff);
 4085                 if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
 4086                     m->psind == 1 && pg_ps_enabled &&
 4087                     pmap_enter_4mpage(pmap, va, m, prot))
 4088                         m = &m[NBPDR / PAGE_SIZE - 1];
 4089                 else
 4090                         mpte = pmap_enter_quick_locked(pmap, va, m, prot,
 4091                             mpte);
 4092                 m = TAILQ_NEXT(m, listq);
 4093         }
 4094         rw_wunlock(&pvh_global_lock);
 4095         PMAP_UNLOCK(pmap);
 4096 }
 4097 
 4098 /*
 4099  * this code makes some *MAJOR* assumptions:
 4100  * 1. Current pmap & pmap exists.
 4101  * 2. Not wired.
 4102  * 3. Read access.
 4103  * 4. No page table pages.
 4104  * but is *MUCH* faster than pmap_enter...
 4105  */
 4106 
 4107 void
 4108 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 4109 {
 4110 
 4111         rw_wlock(&pvh_global_lock);
 4112         PMAP_LOCK(pmap);
 4113         (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);
 4114         rw_wunlock(&pvh_global_lock);
 4115         PMAP_UNLOCK(pmap);
 4116 }
 4117 
 4118 static vm_page_t
 4119 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 4120     vm_prot_t prot, vm_page_t mpte)
 4121 {
 4122         pt_entry_t newpte, *pte;
 4123         struct spglist free;
 4124 
 4125         KASSERT(pmap != kernel_pmap || va < kmi.clean_sva ||
 4126             va >= kmi.clean_eva || (m->oflags & VPO_UNMANAGED) != 0,
 4127             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 4128         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4129         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4130 
 4131         /*
 4132          * In the case that a page table page is not
 4133          * resident, we are creating it here.
 4134          */
 4135         if (pmap != kernel_pmap) {
 4136                 u_int ptepindex;
 4137                 pd_entry_t ptepa;
 4138 
 4139                 /*
 4140                  * Calculate pagetable page index
 4141                  */
 4142                 ptepindex = va >> PDRSHIFT;
 4143                 if (mpte && (mpte->pindex == ptepindex)) {
 4144                         mpte->wire_count++;
 4145                 } else {
 4146                         /*
 4147                          * Get the page directory entry
 4148                          */
 4149                         ptepa = pmap->pm_pdir[ptepindex];
 4150 
 4151                         /*
 4152                          * If the page table page is mapped, we just increment
 4153                          * the hold count, and activate it.
 4154                          */
 4155                         if (ptepa) {
 4156                                 if (ptepa & PG_PS)
 4157                                         return (NULL);
 4158                                 mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 4159                                 mpte->wire_count++;
 4160                         } else {
 4161                                 mpte = _pmap_allocpte(pmap, ptepindex,
 4162                                     PMAP_ENTER_NOSLEEP);
 4163                                 if (mpte == NULL)
 4164                                         return (mpte);
 4165                         }
 4166                 }
 4167         } else {
 4168                 mpte = NULL;
 4169         }
 4170 
 4171         sched_pin();
 4172         pte = pmap_pte_quick(pmap, va);
 4173         if (*pte) {
 4174                 if (mpte != NULL) {
 4175                         mpte->wire_count--;
 4176                         mpte = NULL;
 4177                 }
 4178                 sched_unpin();
 4179                 return (mpte);
 4180         }
 4181 
 4182         /*
 4183          * Enter on the PV list if part of our managed memory.
 4184          */
 4185         if ((m->oflags & VPO_UNMANAGED) == 0 &&
 4186             !pmap_try_insert_pv_entry(pmap, va, m)) {
 4187                 if (mpte != NULL) {
 4188                         SLIST_INIT(&free);
 4189                         if (pmap_unwire_ptp(pmap, mpte, &free)) {
 4190                                 pmap_invalidate_page(pmap, va);
 4191                                 vm_page_free_pages_toq(&free, true);
 4192                         }
 4193                         
 4194                         mpte = NULL;
 4195                 }
 4196                 sched_unpin();
 4197                 return (mpte);
 4198         }
 4199 
 4200         /*
 4201          * Increment counters
 4202          */
 4203         pmap->pm_stats.resident_count++;
 4204 
 4205         newpte = VM_PAGE_TO_PHYS(m) | PG_V |
 4206             pmap_cache_bits(pmap, m->md.pat_mode, 0);
 4207         if ((m->oflags & VPO_UNMANAGED) == 0)
 4208                 newpte |= PG_MANAGED;
 4209 #if defined(PAE) || defined(PAE_TABLES)
 4210         if ((prot & VM_PROT_EXECUTE) == 0)
 4211                 newpte |= pg_nx;
 4212 #endif
 4213         if (pmap != kernel_pmap)
 4214                 newpte |= PG_U;
 4215         pte_store(pte, newpte);
 4216         sched_unpin();
 4217         return (mpte);
 4218 }
 4219 
 4220 /*
 4221  * Make a temporary mapping for a physical address.  This is only intended
 4222  * to be used for panic dumps.
 4223  */
 4224 void *
 4225 pmap_kenter_temporary(vm_paddr_t pa, int i)
 4226 {
 4227         vm_offset_t va;
 4228 
 4229         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 4230         pmap_kenter(va, pa);
 4231         invlpg(va);
 4232         return ((void *)crashdumpmap);
 4233 }
 4234 
 4235 /*
 4236  * This code maps large physical mmap regions into the
 4237  * processor address space.  Note that some shortcuts
 4238  * are taken, but the code works.
 4239  */
 4240 void
 4241 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
 4242     vm_pindex_t pindex, vm_size_t size)
 4243 {
 4244         pd_entry_t *pde;
 4245         vm_paddr_t pa, ptepa;
 4246         vm_page_t p;
 4247         int pat_mode;
 4248 
 4249         VM_OBJECT_ASSERT_WLOCKED(object);
 4250         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 4251             ("pmap_object_init_pt: non-device object"));
 4252         if (pg_ps_enabled &&
 4253             (addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
 4254                 if (!vm_object_populate(object, pindex, pindex + atop(size)))
 4255                         return;
 4256                 p = vm_page_lookup(object, pindex);
 4257                 KASSERT(p->valid == VM_PAGE_BITS_ALL,
 4258                     ("pmap_object_init_pt: invalid page %p", p));
 4259                 pat_mode = p->md.pat_mode;
 4260 
 4261                 /*
 4262                  * Abort the mapping if the first page is not physically
 4263                  * aligned to a 2/4MB page boundary.
 4264                  */
 4265                 ptepa = VM_PAGE_TO_PHYS(p);
 4266                 if (ptepa & (NBPDR - 1))
 4267                         return;
 4268 
 4269                 /*
 4270                  * Skip the first page.  Abort the mapping if the rest of
 4271                  * the pages are not physically contiguous or have differing
 4272                  * memory attributes.
 4273                  */
 4274                 p = TAILQ_NEXT(p, listq);
 4275                 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
 4276                     pa += PAGE_SIZE) {
 4277                         KASSERT(p->valid == VM_PAGE_BITS_ALL,
 4278                             ("pmap_object_init_pt: invalid page %p", p));
 4279                         if (pa != VM_PAGE_TO_PHYS(p) ||
 4280                             pat_mode != p->md.pat_mode)
 4281                                 return;
 4282                         p = TAILQ_NEXT(p, listq);
 4283                 }
 4284 
 4285                 /*
 4286                  * Map using 2/4MB pages.  Since "ptepa" is 2/4M aligned and
 4287                  * "size" is a multiple of 2/4M, adding the PAT setting to
 4288                  * "pa" will not affect the termination of this loop.
 4289                  */
 4290                 PMAP_LOCK(pmap);
 4291                 for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, 1);
 4292                     pa < ptepa + size; pa += NBPDR) {
 4293                         pde = pmap_pde(pmap, addr);
 4294                         if (*pde == 0) {
 4295                                 pde_store(pde, pa | PG_PS | PG_M | PG_A |
 4296                                     PG_U | PG_RW | PG_V);
 4297                                 pmap->pm_stats.resident_count += NBPDR /
 4298                                     PAGE_SIZE;
 4299                                 pmap_pde_mappings++;
 4300                         }
 4301                         /* Else continue on if the PDE is already valid. */
 4302                         addr += NBPDR;
 4303                 }
 4304                 PMAP_UNLOCK(pmap);
 4305         }
 4306 }
 4307 
 4308 /*
 4309  *      Clear the wired attribute from the mappings for the specified range of
 4310  *      addresses in the given pmap.  Every valid mapping within that range
 4311  *      must have the wired attribute set.  In contrast, invalid mappings
 4312  *      cannot have the wired attribute set, so they are ignored.
 4313  *
 4314  *      The wired attribute of the page table entry is not a hardware feature,
 4315  *      so there is no need to invalidate any TLB entries.
 4316  */
 4317 void
 4318 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 4319 {
 4320         vm_offset_t pdnxt;
 4321         pd_entry_t *pde;
 4322         pt_entry_t *pte;
 4323         boolean_t pv_lists_locked;
 4324 
 4325         if (pmap_is_current(pmap))
 4326                 pv_lists_locked = FALSE;
 4327         else {
 4328                 pv_lists_locked = TRUE;
 4329 resume:
 4330                 rw_wlock(&pvh_global_lock);
 4331                 sched_pin();
 4332         }
 4333         PMAP_LOCK(pmap);
 4334         for (; sva < eva; sva = pdnxt) {
 4335                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 4336                 if (pdnxt < sva)
 4337                         pdnxt = eva;
 4338                 pde = pmap_pde(pmap, sva);
 4339                 if ((*pde & PG_V) == 0)
 4340                         continue;
 4341                 if ((*pde & PG_PS) != 0) {
 4342                         if ((*pde & PG_W) == 0)
 4343                                 panic("pmap_unwire: pde %#jx is missing PG_W",
 4344                                     (uintmax_t)*pde);
 4345 
 4346                         /*
 4347                          * Are we unwiring the entire large page?  If not,
 4348                          * demote the mapping and fall through.
 4349                          */
 4350                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 4351                                 /*
 4352                                  * Regardless of whether a pde (or pte) is 32
 4353                                  * or 64 bits in size, PG_W is among the least
 4354                                  * significant 32 bits.
 4355                                  */
 4356                                 atomic_clear_int((u_int *)pde, PG_W);
 4357                                 pmap->pm_stats.wired_count -= NBPDR /
 4358                                     PAGE_SIZE;
 4359                                 continue;
 4360                         } else {
 4361                                 if (!pv_lists_locked) {
 4362                                         pv_lists_locked = TRUE;
 4363                                         if (!rw_try_wlock(&pvh_global_lock)) {
 4364                                                 PMAP_UNLOCK(pmap);
 4365                                                 /* Repeat sva. */
 4366                                                 goto resume;
 4367                                         }
 4368                                         sched_pin();
 4369                                 }
 4370                                 if (!pmap_demote_pde(pmap, pde, sva))
 4371                                         panic("pmap_unwire: demotion failed");
 4372                         }
 4373                 }
 4374                 if (pdnxt > eva)
 4375                         pdnxt = eva;
 4376                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 4377                     sva += PAGE_SIZE) {
 4378                         if ((*pte & PG_V) == 0)
 4379                                 continue;
 4380                         if ((*pte & PG_W) == 0)
 4381                                 panic("pmap_unwire: pte %#jx is missing PG_W",
 4382                                     (uintmax_t)*pte);
 4383 
 4384                         /*
 4385                          * PG_W must be cleared atomically.  Although the pmap
 4386                          * lock synchronizes access to PG_W, another processor
 4387                          * could be setting PG_M and/or PG_A concurrently.
 4388                          *
 4389                          * PG_W is among the least significant 32 bits.
 4390                          */
 4391                         atomic_clear_int((u_int *)pte, PG_W);
 4392                         pmap->pm_stats.wired_count--;
 4393                 }
 4394         }
 4395         if (pv_lists_locked) {
 4396                 sched_unpin();
 4397                 rw_wunlock(&pvh_global_lock);
 4398         }
 4399         PMAP_UNLOCK(pmap);
 4400 }
 4401 
 4402 
 4403 /*
 4404  *      Copy the range specified by src_addr/len
 4405  *      from the source map to the range dst_addr/len
 4406  *      in the destination map.
 4407  *
 4408  *      This routine is only advisory and need not do anything.  Since
 4409  *      current pmap is always the kernel pmap when executing in
 4410  *      kernel, and we do not copy from the kernel pmap to a user
 4411  *      pmap, this optimization is not usable in 4/4G full split i386
 4412  *      world.
 4413  */
 4414 
 4415 void
 4416 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 4417     vm_offset_t src_addr)
 4418 {
 4419         struct spglist free;
 4420         pt_entry_t *src_pte, *dst_pte, ptetemp;
 4421         pd_entry_t srcptepaddr;
 4422         vm_page_t dstmpte, srcmpte;
 4423         vm_offset_t addr, end_addr, pdnxt;
 4424         u_int ptepindex;
 4425 
 4426         if (dst_addr != src_addr)
 4427                 return;
 4428 
 4429         end_addr = src_addr + len;
 4430 
 4431         rw_wlock(&pvh_global_lock);
 4432         if (dst_pmap < src_pmap) {
 4433                 PMAP_LOCK(dst_pmap);
 4434                 PMAP_LOCK(src_pmap);
 4435         } else {
 4436                 PMAP_LOCK(src_pmap);
 4437                 PMAP_LOCK(dst_pmap);
 4438         }
 4439         sched_pin();
 4440         for (addr = src_addr; addr < end_addr; addr = pdnxt) {
 4441                 KASSERT(addr < PMAP_TRM_MIN_ADDRESS,
 4442                     ("pmap_copy: invalid to pmap_copy the trampoline"));
 4443 
 4444                 pdnxt = (addr + NBPDR) & ~PDRMASK;
 4445                 if (pdnxt < addr)
 4446                         pdnxt = end_addr;
 4447                 ptepindex = addr >> PDRSHIFT;
 4448 
 4449                 srcptepaddr = src_pmap->pm_pdir[ptepindex];
 4450                 if (srcptepaddr == 0)
 4451                         continue;
 4452 
 4453                 if (srcptepaddr & PG_PS) {
 4454                         if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr)
 4455                                 continue;
 4456                         if (dst_pmap->pm_pdir[ptepindex] == 0 &&
 4457                             ((srcptepaddr & PG_MANAGED) == 0 ||
 4458                             pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr,
 4459                             PMAP_ENTER_NORECLAIM))) {
 4460                                 dst_pmap->pm_pdir[ptepindex] = srcptepaddr &
 4461                                     ~PG_W;
 4462                                 dst_pmap->pm_stats.resident_count +=
 4463                                     NBPDR / PAGE_SIZE;
 4464                                 pmap_pde_mappings++;
 4465                         }
 4466                         continue;
 4467                 }
 4468 
 4469                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 4470                 KASSERT(srcmpte->wire_count > 0,
 4471                     ("pmap_copy: source page table page is unused"));
 4472 
 4473                 if (pdnxt > end_addr)
 4474                         pdnxt = end_addr;
 4475 
 4476                 src_pte = pmap_pte_quick3(src_pmap, addr);
 4477                 while (addr < pdnxt) {
 4478                         ptetemp = *src_pte;
 4479                         /*
 4480                          * we only virtual copy managed pages
 4481                          */
 4482                         if ((ptetemp & PG_MANAGED) != 0) {
 4483                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 4484                                     PMAP_ENTER_NOSLEEP);
 4485                                 if (dstmpte == NULL)
 4486                                         goto out;
 4487                                 dst_pte = pmap_pte_quick(dst_pmap, addr);
 4488                                 if (*dst_pte == 0 &&
 4489                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 4490                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
 4491                                         /*
 4492                                          * Clear the wired, modified, and
 4493                                          * accessed (referenced) bits
 4494                                          * during the copy.
 4495                                          */
 4496                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 4497                                             PG_A);
 4498                                         dst_pmap->pm_stats.resident_count++;
 4499                                 } else {
 4500                                         SLIST_INIT(&free);
 4501                                         if (pmap_unwire_ptp(dst_pmap, dstmpte,
 4502                                             &free)) {
 4503                                                 pmap_invalidate_page(dst_pmap,
 4504                                                     addr);
 4505                                                 vm_page_free_pages_toq(&free,
 4506                                                     true);
 4507                                         }
 4508                                         goto out;
 4509                                 }
 4510                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 4511                                         break;
 4512                         }
 4513                         addr += PAGE_SIZE;
 4514                         src_pte++;
 4515                 }
 4516         }
 4517 out:
 4518         sched_unpin();
 4519         rw_wunlock(&pvh_global_lock);
 4520         PMAP_UNLOCK(src_pmap);
 4521         PMAP_UNLOCK(dst_pmap);
 4522 }
 4523 
 4524 /*
 4525  * Zero 1 page of virtual memory mapped from a hardware page by the caller.
 4526  */
 4527 static __inline void
 4528 pagezero(void *page)
 4529 {
 4530 #if defined(I686_CPU)
 4531         if (cpu_class == CPUCLASS_686) {
 4532                 if (cpu_feature & CPUID_SSE2)
 4533                         sse2_pagezero(page);
 4534                 else
 4535                         i686_pagezero(page);
 4536         } else
 4537 #endif
 4538                 bzero(page, PAGE_SIZE);
 4539 }
 4540 
 4541 /*
 4542  * Zero the specified hardware page.
 4543  */
 4544 void
 4545 pmap_zero_page(vm_page_t m)
 4546 {
 4547         pt_entry_t *cmap_pte2;
 4548         struct pcpu *pc;
 4549 
 4550         sched_pin();
 4551         pc = get_pcpu();
 4552         cmap_pte2 = pc->pc_cmap_pte2;
 4553         mtx_lock(&pc->pc_cmap_lock);
 4554         if (*cmap_pte2)
 4555                 panic("pmap_zero_page: CMAP2 busy");
 4556         *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 4557             pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0);
 4558         invlcaddr(pc->pc_cmap_addr2);
 4559         pagezero(pc->pc_cmap_addr2);
 4560         *cmap_pte2 = 0;
 4561 
 4562         /*
 4563          * Unpin the thread before releasing the lock.  Otherwise the thread
 4564          * could be rescheduled while still bound to the current CPU, only
 4565          * to unpin itself immediately upon resuming execution.
 4566          */
 4567         sched_unpin();
 4568         mtx_unlock(&pc->pc_cmap_lock);
 4569 }
 4570 
 4571 /*
 4572  * Zero an area within a single hardware page.  off and size must not
 4573  * cover an area beyond a single hardware page.
 4574  */
 4575 void
 4576 pmap_zero_page_area(vm_page_t m, int off, int size)
 4577 {
 4578         pt_entry_t *cmap_pte2;
 4579         struct pcpu *pc;
 4580 
 4581         sched_pin();
 4582         pc = get_pcpu();
 4583         cmap_pte2 = pc->pc_cmap_pte2;
 4584         mtx_lock(&pc->pc_cmap_lock);
 4585         if (*cmap_pte2)
 4586                 panic("pmap_zero_page_area: CMAP2 busy");
 4587         *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 4588             pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0);
 4589         invlcaddr(pc->pc_cmap_addr2);
 4590         if (off == 0 && size == PAGE_SIZE) 
 4591                 pagezero(pc->pc_cmap_addr2);
 4592         else
 4593                 bzero(pc->pc_cmap_addr2 + off, size);
 4594         *cmap_pte2 = 0;
 4595         sched_unpin();
 4596         mtx_unlock(&pc->pc_cmap_lock);
 4597 }
 4598 
 4599 /*
 4600  * Copy 1 specified hardware page to another.
 4601  */
 4602 void
 4603 pmap_copy_page(vm_page_t src, vm_page_t dst)
 4604 {
 4605         pt_entry_t *cmap_pte1, *cmap_pte2;
 4606         struct pcpu *pc;
 4607 
 4608         sched_pin();
 4609         pc = get_pcpu();
 4610         cmap_pte1 = pc->pc_cmap_pte1; 
 4611         cmap_pte2 = pc->pc_cmap_pte2;
 4612         mtx_lock(&pc->pc_cmap_lock);
 4613         if (*cmap_pte1)
 4614                 panic("pmap_copy_page: CMAP1 busy");
 4615         if (*cmap_pte2)
 4616                 panic("pmap_copy_page: CMAP2 busy");
 4617         *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A |
 4618             pmap_cache_bits(kernel_pmap, src->md.pat_mode, 0);
 4619         invlcaddr(pc->pc_cmap_addr1);
 4620         *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M |
 4621             pmap_cache_bits(kernel_pmap, dst->md.pat_mode, 0);
 4622         invlcaddr(pc->pc_cmap_addr2);
 4623         bcopy(pc->pc_cmap_addr1, pc->pc_cmap_addr2, PAGE_SIZE);
 4624         *cmap_pte1 = 0;
 4625         *cmap_pte2 = 0;
 4626         sched_unpin();
 4627         mtx_unlock(&pc->pc_cmap_lock);
 4628 }
 4629 
 4630 int unmapped_buf_allowed = 1;
 4631 
 4632 void
 4633 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 4634     vm_offset_t b_offset, int xfersize)
 4635 {
 4636         vm_page_t a_pg, b_pg;
 4637         char *a_cp, *b_cp;
 4638         vm_offset_t a_pg_offset, b_pg_offset;
 4639         pt_entry_t *cmap_pte1, *cmap_pte2;
 4640         struct pcpu *pc;
 4641         int cnt;
 4642 
 4643         sched_pin();
 4644         pc = get_pcpu();
 4645         cmap_pte1 = pc->pc_cmap_pte1; 
 4646         cmap_pte2 = pc->pc_cmap_pte2;
 4647         mtx_lock(&pc->pc_cmap_lock);
 4648         if (*cmap_pte1 != 0)
 4649                 panic("pmap_copy_pages: CMAP1 busy");
 4650         if (*cmap_pte2 != 0)
 4651                 panic("pmap_copy_pages: CMAP2 busy");
 4652         while (xfersize > 0) {
 4653                 a_pg = ma[a_offset >> PAGE_SHIFT];
 4654                 a_pg_offset = a_offset & PAGE_MASK;
 4655                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
 4656                 b_pg = mb[b_offset >> PAGE_SHIFT];
 4657                 b_pg_offset = b_offset & PAGE_MASK;
 4658                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 4659                 *cmap_pte1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A |
 4660                     pmap_cache_bits(kernel_pmap, a_pg->md.pat_mode, 0);
 4661                 invlcaddr(pc->pc_cmap_addr1);
 4662                 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A |
 4663                     PG_M | pmap_cache_bits(kernel_pmap, b_pg->md.pat_mode, 0);
 4664                 invlcaddr(pc->pc_cmap_addr2);
 4665                 a_cp = pc->pc_cmap_addr1 + a_pg_offset;
 4666                 b_cp = pc->pc_cmap_addr2 + b_pg_offset;
 4667                 bcopy(a_cp, b_cp, cnt);
 4668                 a_offset += cnt;
 4669                 b_offset += cnt;
 4670                 xfersize -= cnt;
 4671         }
 4672         *cmap_pte1 = 0;
 4673         *cmap_pte2 = 0;
 4674         sched_unpin();
 4675         mtx_unlock(&pc->pc_cmap_lock);
 4676 }
 4677 
 4678 /*
 4679  * Returns true if the pmap's pv is one of the first
 4680  * 16 pvs linked to from this page.  This count may
 4681  * be changed upwards or downwards in the future; it
 4682  * is only necessary that true be returned for a small
 4683  * subset of pmaps for proper page aging.
 4684  */
 4685 boolean_t
 4686 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 4687 {
 4688         struct md_page *pvh;
 4689         pv_entry_t pv;
 4690         int loops = 0;
 4691         boolean_t rv;
 4692 
 4693         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4694             ("pmap_page_exists_quick: page %p is not managed", m));
 4695         rv = FALSE;
 4696         rw_wlock(&pvh_global_lock);
 4697         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 4698                 if (PV_PMAP(pv) == pmap) {
 4699                         rv = TRUE;
 4700                         break;
 4701                 }
 4702                 loops++;
 4703                 if (loops >= 16)
 4704                         break;
 4705         }
 4706         if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
 4707                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4708                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4709                         if (PV_PMAP(pv) == pmap) {
 4710                                 rv = TRUE;
 4711                                 break;
 4712                         }
 4713                         loops++;
 4714                         if (loops >= 16)
 4715                                 break;
 4716                 }
 4717         }
 4718         rw_wunlock(&pvh_global_lock);
 4719         return (rv);
 4720 }
 4721 
 4722 /*
 4723  *      pmap_page_wired_mappings:
 4724  *
 4725  *      Return the number of managed mappings to the given physical page
 4726  *      that are wired.
 4727  */
 4728 int
 4729 pmap_page_wired_mappings(vm_page_t m)
 4730 {
 4731         int count;
 4732 
 4733         count = 0;
 4734         if ((m->oflags & VPO_UNMANAGED) != 0)
 4735                 return (count);
 4736         rw_wlock(&pvh_global_lock);
 4737         count = pmap_pvh_wired_mappings(&m->md, count);
 4738         if ((m->flags & PG_FICTITIOUS) == 0) {
 4739             count = pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)),
 4740                 count);
 4741         }
 4742         rw_wunlock(&pvh_global_lock);
 4743         return (count);
 4744 }
 4745 
 4746 /*
 4747  *      pmap_pvh_wired_mappings:
 4748  *
 4749  *      Return the updated number "count" of managed mappings that are wired.
 4750  */
 4751 static int
 4752 pmap_pvh_wired_mappings(struct md_page *pvh, int count)
 4753 {
 4754         pmap_t pmap;
 4755         pt_entry_t *pte;
 4756         pv_entry_t pv;
 4757 
 4758         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4759         sched_pin();
 4760         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4761                 pmap = PV_PMAP(pv);
 4762                 PMAP_LOCK(pmap);
 4763                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4764                 if ((*pte & PG_W) != 0)
 4765                         count++;
 4766                 PMAP_UNLOCK(pmap);
 4767         }
 4768         sched_unpin();
 4769         return (count);
 4770 }
 4771 
 4772 /*
 4773  * Returns TRUE if the given page is mapped individually or as part of
 4774  * a 4mpage.  Otherwise, returns FALSE.
 4775  */
 4776 boolean_t
 4777 pmap_page_is_mapped(vm_page_t m)
 4778 {
 4779         boolean_t rv;
 4780 
 4781         if ((m->oflags & VPO_UNMANAGED) != 0)
 4782                 return (FALSE);
 4783         rw_wlock(&pvh_global_lock);
 4784         rv = !TAILQ_EMPTY(&m->md.pv_list) ||
 4785             ((m->flags & PG_FICTITIOUS) == 0 &&
 4786             !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
 4787         rw_wunlock(&pvh_global_lock);
 4788         return (rv);
 4789 }
 4790 
 4791 /*
 4792  * Remove all pages from specified address space
 4793  * this aids process exit speeds.  Also, this code
 4794  * is special cased for current process only, but
 4795  * can have the more generic (and slightly slower)
 4796  * mode enabled.  This is much faster than pmap_remove
 4797  * in the case of running down an entire address space.
 4798  */
 4799 void
 4800 pmap_remove_pages(pmap_t pmap)
 4801 {
 4802         pt_entry_t *pte, tpte;
 4803         vm_page_t m, mpte, mt;
 4804         pv_entry_t pv;
 4805         struct md_page *pvh;
 4806         struct pv_chunk *pc, *npc;
 4807         struct spglist free;
 4808         int field, idx;
 4809         int32_t bit;
 4810         uint32_t inuse, bitmask;
 4811         int allfree;
 4812 
 4813         if (pmap != PCPU_GET(curpmap)) {
 4814                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 4815                 return;
 4816         }
 4817         SLIST_INIT(&free);
 4818         rw_wlock(&pvh_global_lock);
 4819         PMAP_LOCK(pmap);
 4820         sched_pin();
 4821         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 4822                 KASSERT(pc->pc_pmap == pmap, ("Wrong pmap %p %p", pmap,
 4823                     pc->pc_pmap));
 4824                 allfree = 1;
 4825                 for (field = 0; field < _NPCM; field++) {
 4826                         inuse = ~pc->pc_map[field] & pc_freemask[field];
 4827                         while (inuse != 0) {
 4828                                 bit = bsfl(inuse);
 4829                                 bitmask = 1UL << bit;
 4830                                 idx = field * 32 + bit;
 4831                                 pv = &pc->pc_pventry[idx];
 4832                                 inuse &= ~bitmask;
 4833 
 4834                                 pte = pmap_pde(pmap, pv->pv_va);
 4835                                 tpte = *pte;
 4836                                 if ((tpte & PG_PS) == 0) {
 4837                                         pte = pmap_pte_quick(pmap, pv->pv_va);
 4838                                         tpte = *pte & ~PG_PTE_PAT;
 4839                                 }
 4840 
 4841                                 if (tpte == 0) {
 4842                                         printf(
 4843                                             "TPTE at %p  IS ZERO @ VA %08x\n",
 4844                                             pte, pv->pv_va);
 4845                                         panic("bad pte");
 4846                                 }
 4847 
 4848 /*
 4849  * We cannot remove wired pages from a process' mapping at this time
 4850  */
 4851                                 if (tpte & PG_W) {
 4852                                         allfree = 0;
 4853                                         continue;
 4854                                 }
 4855 
 4856                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 4857                                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 4858                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 4859                                     m, (uintmax_t)m->phys_addr,
 4860                                     (uintmax_t)tpte));
 4861 
 4862                                 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
 4863                                     m < &vm_page_array[vm_page_array_size],
 4864                                     ("pmap_remove_pages: bad tpte %#jx",
 4865                                     (uintmax_t)tpte));
 4866 
 4867                                 pte_clear(pte);
 4868 
 4869                                 /*
 4870                                  * Update the vm_page_t clean/reference bits.
 4871                                  */
 4872                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4873                                         if ((tpte & PG_PS) != 0) {
 4874                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4875                                                         vm_page_dirty(mt);
 4876                                         } else
 4877                                                 vm_page_dirty(m);
 4878                                 }
 4879 
 4880                                 /* Mark free */
 4881                                 PV_STAT(pv_entry_frees++);
 4882                                 PV_STAT(pv_entry_spare++);
 4883                                 pv_entry_count--;
 4884                                 pc->pc_map[field] |= bitmask;
 4885                                 if ((tpte & PG_PS) != 0) {
 4886                                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 4887                                         pvh = pa_to_pvh(tpte & PG_PS_FRAME);
 4888                                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 4889                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 4890                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4891                                                         if (TAILQ_EMPTY(&mt->md.pv_list))
 4892                                                                 vm_page_aflag_clear(mt, PGA_WRITEABLE);
 4893                                         }
 4894                                         mpte = pmap_remove_pt_page(pmap, pv->pv_va);
 4895                                         if (mpte != NULL) {
 4896                                                 KASSERT(mpte->valid == VM_PAGE_BITS_ALL,
 4897                                                     ("pmap_remove_pages: pte page not promoted"));
 4898                                                 pmap->pm_stats.resident_count--;
 4899                                                 KASSERT(mpte->wire_count == NPTEPG,
 4900                                                     ("pmap_remove_pages: pte page wire count error"));
 4901                                                 mpte->wire_count = 0;
 4902                                                 pmap_add_delayed_free_list(mpte, &free, FALSE);
 4903                                         }
 4904                                 } else {
 4905                                         pmap->pm_stats.resident_count--;
 4906                                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 4907                                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 4908                                             (m->flags & PG_FICTITIOUS) == 0) {
 4909                                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4910                                                 if (TAILQ_EMPTY(&pvh->pv_list))
 4911                                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 4912                                         }
 4913                                         pmap_unuse_pt(pmap, pv->pv_va, &free);
 4914                                 }
 4915                         }
 4916                 }
 4917                 if (allfree) {
 4918                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 4919                         free_pv_chunk(pc);
 4920                 }
 4921         }
 4922         sched_unpin();
 4923         pmap_invalidate_all(pmap);
 4924         rw_wunlock(&pvh_global_lock);
 4925         PMAP_UNLOCK(pmap);
 4926         vm_page_free_pages_toq(&free, true);
 4927 }
 4928 
 4929 /*
 4930  *      pmap_is_modified:
 4931  *
 4932  *      Return whether or not the specified physical page was modified
 4933  *      in any physical maps.
 4934  */
 4935 boolean_t
 4936 pmap_is_modified(vm_page_t m)
 4937 {
 4938         boolean_t rv;
 4939 
 4940         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4941             ("pmap_is_modified: page %p is not managed", m));
 4942 
 4943         /*
 4944          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
 4945          * concurrently set while the object is locked.  Thus, if PGA_WRITEABLE
 4946          * is clear, no PTEs can have PG_M set.
 4947          */
 4948         VM_OBJECT_ASSERT_WLOCKED(m->object);
 4949         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
 4950                 return (FALSE);
 4951         rw_wlock(&pvh_global_lock);
 4952         rv = pmap_is_modified_pvh(&m->md) ||
 4953             ((m->flags & PG_FICTITIOUS) == 0 &&
 4954             pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4955         rw_wunlock(&pvh_global_lock);
 4956         return (rv);
 4957 }
 4958 
 4959 /*
 4960  * Returns TRUE if any of the given mappings were used to modify
 4961  * physical memory.  Otherwise, returns FALSE.  Both page and 2mpage
 4962  * mappings are supported.
 4963  */
 4964 static boolean_t
 4965 pmap_is_modified_pvh(struct md_page *pvh)
 4966 {
 4967         pv_entry_t pv;
 4968         pt_entry_t *pte;
 4969         pmap_t pmap;
 4970         boolean_t rv;
 4971 
 4972         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4973         rv = FALSE;
 4974         sched_pin();
 4975         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4976                 pmap = PV_PMAP(pv);
 4977                 PMAP_LOCK(pmap);
 4978                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4979                 rv = (*pte & (PG_M | PG_RW)) == (PG_M | PG_RW);
 4980                 PMAP_UNLOCK(pmap);
 4981                 if (rv)
 4982                         break;
 4983         }
 4984         sched_unpin();
 4985         return (rv);
 4986 }
 4987 
 4988 /*
 4989  *      pmap_is_prefaultable:
 4990  *
 4991  *      Return whether or not the specified virtual address is elgible
 4992  *      for prefault.
 4993  */
 4994 boolean_t
 4995 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 4996 {
 4997         pd_entry_t pde;
 4998         boolean_t rv;
 4999 
 5000         rv = FALSE;
 5001         PMAP_LOCK(pmap);
 5002         pde = *pmap_pde(pmap, addr);
 5003         if (pde != 0 && (pde & PG_PS) == 0)
 5004                 rv = pmap_pte_ufast(pmap, addr, pde) == 0;
 5005         PMAP_UNLOCK(pmap);
 5006         return (rv);
 5007 }
 5008 
 5009 /*
 5010  *      pmap_is_referenced:
 5011  *
 5012  *      Return whether or not the specified physical page was referenced
 5013  *      in any physical maps.
 5014  */
 5015 boolean_t
 5016 pmap_is_referenced(vm_page_t m)
 5017 {
 5018         boolean_t rv;
 5019 
 5020         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5021             ("pmap_is_referenced: page %p is not managed", m));
 5022         rw_wlock(&pvh_global_lock);
 5023         rv = pmap_is_referenced_pvh(&m->md) ||
 5024             ((m->flags & PG_FICTITIOUS) == 0 &&
 5025             pmap_is_referenced_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 5026         rw_wunlock(&pvh_global_lock);
 5027         return (rv);
 5028 }
 5029 
 5030 /*
 5031  * Returns TRUE if any of the given mappings were referenced and FALSE
 5032  * otherwise.  Both page and 4mpage mappings are supported.
 5033  */
 5034 static boolean_t
 5035 pmap_is_referenced_pvh(struct md_page *pvh)
 5036 {
 5037         pv_entry_t pv;
 5038         pt_entry_t *pte;
 5039         pmap_t pmap;
 5040         boolean_t rv;
 5041 
 5042         rw_assert(&pvh_global_lock, RA_WLOCKED);
 5043         rv = FALSE;
 5044         sched_pin();
 5045         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 5046                 pmap = PV_PMAP(pv);
 5047                 PMAP_LOCK(pmap);
 5048                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5049                 rv = (*pte & (PG_A | PG_V)) == (PG_A | PG_V);
 5050                 PMAP_UNLOCK(pmap);
 5051                 if (rv)
 5052                         break;
 5053         }
 5054         sched_unpin();
 5055         return (rv);
 5056 }
 5057 
 5058 /*
 5059  * Clear the write and modified bits in each of the given page's mappings.
 5060  */
 5061 void
 5062 pmap_remove_write(vm_page_t m)
 5063 {
 5064         struct md_page *pvh;
 5065         pv_entry_t next_pv, pv;
 5066         pmap_t pmap;
 5067         pd_entry_t *pde;
 5068         pt_entry_t oldpte, *pte;
 5069         vm_offset_t va;
 5070 
 5071         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5072             ("pmap_remove_write: page %p is not managed", m));
 5073 
 5074         /*
 5075          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
 5076          * set by another thread while the object is locked.  Thus,
 5077          * if PGA_WRITEABLE is clear, no page table entries need updating.
 5078          */
 5079         VM_OBJECT_ASSERT_WLOCKED(m->object);
 5080         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
 5081                 return;
 5082         rw_wlock(&pvh_global_lock);
 5083         sched_pin();
 5084         if ((m->flags & PG_FICTITIOUS) != 0)
 5085                 goto small_mappings;
 5086         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5087         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
 5088                 va = pv->pv_va;
 5089                 pmap = PV_PMAP(pv);
 5090                 PMAP_LOCK(pmap);
 5091                 pde = pmap_pde(pmap, va);
 5092                 if ((*pde & PG_RW) != 0)
 5093                         (void)pmap_demote_pde(pmap, pde, va);
 5094                 PMAP_UNLOCK(pmap);
 5095         }
 5096 small_mappings:
 5097         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5098                 pmap = PV_PMAP(pv);
 5099                 PMAP_LOCK(pmap);
 5100                 pde = pmap_pde(pmap, pv->pv_va);
 5101                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_write: found"
 5102                     " a 4mpage in page %p's pv list", m));
 5103                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5104 retry:
 5105                 oldpte = *pte;
 5106                 if ((oldpte & PG_RW) != 0) {
 5107                         /*
 5108                          * Regardless of whether a pte is 32 or 64 bits
 5109                          * in size, PG_RW and PG_M are among the least
 5110                          * significant 32 bits.
 5111                          */
 5112                         if (!atomic_cmpset_int((u_int *)pte, oldpte,
 5113                             oldpte & ~(PG_RW | PG_M)))
 5114                                 goto retry;
 5115                         if ((oldpte & PG_M) != 0)
 5116                                 vm_page_dirty(m);
 5117                         pmap_invalidate_page(pmap, pv->pv_va);
 5118                 }
 5119                 PMAP_UNLOCK(pmap);
 5120         }
 5121         vm_page_aflag_clear(m, PGA_WRITEABLE);
 5122         sched_unpin();
 5123         rw_wunlock(&pvh_global_lock);
 5124 }
 5125 
 5126 /*
 5127  *      pmap_ts_referenced:
 5128  *
 5129  *      Return a count of reference bits for a page, clearing those bits.
 5130  *      It is not necessary for every reference bit to be cleared, but it
 5131  *      is necessary that 0 only be returned when there are truly no
 5132  *      reference bits set.
 5133  *
 5134  *      As an optimization, update the page's dirty field if a modified bit is
 5135  *      found while counting reference bits.  This opportunistic update can be
 5136  *      performed at low cost and can eliminate the need for some future calls
 5137  *      to pmap_is_modified().  However, since this function stops after
 5138  *      finding PMAP_TS_REFERENCED_MAX reference bits, it may not detect some
 5139  *      dirty pages.  Those dirty pages will only be detected by a future call
 5140  *      to pmap_is_modified().
 5141  */
 5142 int
 5143 pmap_ts_referenced(vm_page_t m)
 5144 {
 5145         struct md_page *pvh;
 5146         pv_entry_t pv, pvf;
 5147         pmap_t pmap;
 5148         pd_entry_t *pde;
 5149         pt_entry_t *pte;
 5150         vm_paddr_t pa;
 5151         int rtval = 0;
 5152 
 5153         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5154             ("pmap_ts_referenced: page %p is not managed", m));
 5155         pa = VM_PAGE_TO_PHYS(m);
 5156         pvh = pa_to_pvh(pa);
 5157         rw_wlock(&pvh_global_lock);
 5158         sched_pin();
 5159         if ((m->flags & PG_FICTITIOUS) != 0 ||
 5160             (pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
 5161                 goto small_mappings;
 5162         pv = pvf;
 5163         do {
 5164                 pmap = PV_PMAP(pv);
 5165                 PMAP_LOCK(pmap);
 5166                 pde = pmap_pde(pmap, pv->pv_va);
 5167                 if ((*pde & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5168                         /*
 5169                          * Although "*pde" is mapping a 2/4MB page, because
 5170                          * this function is called at a 4KB page granularity,
 5171                          * we only update the 4KB page under test.
 5172                          */
 5173                         vm_page_dirty(m);
 5174                 }
 5175                 if ((*pde & PG_A) != 0) {
 5176                         /*
 5177                          * Since this reference bit is shared by either 1024
 5178                          * or 512 4KB pages, it should not be cleared every
 5179                          * time it is tested.  Apply a simple "hash" function
 5180                          * on the physical page number, the virtual superpage
 5181                          * number, and the pmap address to select one 4KB page
 5182                          * out of the 1024 or 512 on which testing the
 5183                          * reference bit will result in clearing that bit.
 5184                          * This function is designed to avoid the selection of
 5185                          * the same 4KB page for every 2- or 4MB page mapping.
 5186                          *
 5187                          * On demotion, a mapping that hasn't been referenced
 5188                          * is simply destroyed.  To avoid the possibility of a
 5189                          * subsequent page fault on a demoted wired mapping,
 5190                          * always leave its reference bit set.  Moreover,
 5191                          * since the superpage is wired, the current state of
 5192                          * its reference bit won't affect page replacement.
 5193                          */
 5194                         if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PDRSHIFT) ^
 5195                             (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
 5196                             (*pde & PG_W) == 0) {
 5197                                 atomic_clear_int((u_int *)pde, PG_A);
 5198                                 pmap_invalidate_page(pmap, pv->pv_va);
 5199                         }
 5200                         rtval++;
 5201                 }
 5202                 PMAP_UNLOCK(pmap);
 5203                 /* Rotate the PV list if it has more than one entry. */
 5204                 if (TAILQ_NEXT(pv, pv_next) != NULL) {
 5205                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 5206                         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 5207                 }
 5208                 if (rtval >= PMAP_TS_REFERENCED_MAX)
 5209                         goto out;
 5210         } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
 5211 small_mappings:
 5212         if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
 5213                 goto out;
 5214         pv = pvf;
 5215         do {
 5216                 pmap = PV_PMAP(pv);
 5217                 PMAP_LOCK(pmap);
 5218                 pde = pmap_pde(pmap, pv->pv_va);
 5219                 KASSERT((*pde & PG_PS) == 0,
 5220                     ("pmap_ts_referenced: found a 4mpage in page %p's pv list",
 5221                     m));
 5222                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5223                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 5224                         vm_page_dirty(m);
 5225                 if ((*pte & PG_A) != 0) {
 5226                         atomic_clear_int((u_int *)pte, PG_A);
 5227                         pmap_invalidate_page(pmap, pv->pv_va);
 5228                         rtval++;
 5229                 }
 5230                 PMAP_UNLOCK(pmap);
 5231                 /* Rotate the PV list if it has more than one entry. */
 5232                 if (TAILQ_NEXT(pv, pv_next) != NULL) {
 5233                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 5234                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 5235                 }
 5236         } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && rtval <
 5237             PMAP_TS_REFERENCED_MAX);
 5238 out:
 5239         sched_unpin();
 5240         rw_wunlock(&pvh_global_lock);
 5241         return (rtval);
 5242 }
 5243 
 5244 /*
 5245  *      Apply the given advice to the specified range of addresses within the
 5246  *      given pmap.  Depending on the advice, clear the referenced and/or
 5247  *      modified flags in each mapping and set the mapped page's dirty field.
 5248  */
 5249 void
 5250 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
 5251 {
 5252         pd_entry_t oldpde, *pde;
 5253         pt_entry_t *pte;
 5254         vm_offset_t va, pdnxt;
 5255         vm_page_t m;
 5256         bool anychanged, pv_lists_locked;
 5257 
 5258         if (advice != MADV_DONTNEED && advice != MADV_FREE)
 5259                 return;
 5260         if (pmap_is_current(pmap))
 5261                 pv_lists_locked = false;
 5262         else {
 5263                 pv_lists_locked = true;
 5264 resume:
 5265                 rw_wlock(&pvh_global_lock);
 5266                 sched_pin();
 5267         }
 5268         anychanged = false;
 5269         PMAP_LOCK(pmap);
 5270         for (; sva < eva; sva = pdnxt) {
 5271                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 5272                 if (pdnxt < sva)
 5273                         pdnxt = eva;
 5274                 pde = pmap_pde(pmap, sva);
 5275                 oldpde = *pde;
 5276                 if ((oldpde & PG_V) == 0)
 5277                         continue;
 5278                 else if ((oldpde & PG_PS) != 0) {
 5279                         if ((oldpde & PG_MANAGED) == 0)
 5280                                 continue;
 5281                         if (!pv_lists_locked) {
 5282                                 pv_lists_locked = true;
 5283                                 if (!rw_try_wlock(&pvh_global_lock)) {
 5284                                         if (anychanged)
 5285                                                 pmap_invalidate_all(pmap);
 5286                                         PMAP_UNLOCK(pmap);
 5287                                         goto resume;
 5288                                 }
 5289                                 sched_pin();
 5290                         }
 5291                         if (!pmap_demote_pde(pmap, pde, sva)) {
 5292                                 /*
 5293                                  * The large page mapping was destroyed.
 5294                                  */
 5295                                 continue;
 5296                         }
 5297 
 5298                         /*
 5299                          * Unless the page mappings are wired, remove the
 5300                          * mapping to a single page so that a subsequent
 5301                          * access may repromote.  Choosing the last page
 5302                          * within the address range [sva, min(pdnxt, eva))
 5303                          * generally results in more repromotions.  Since the
 5304                          * underlying page table page is fully populated, this
 5305                          * removal never frees a page table page.
 5306                          */
 5307                         if ((oldpde & PG_W) == 0) {
 5308                                 va = eva;
 5309                                 if (va > pdnxt)
 5310                                         va = pdnxt;
 5311                                 va -= PAGE_SIZE;
 5312                                 KASSERT(va >= sva,
 5313                                     ("pmap_advise: no address gap"));
 5314                                 pte = pmap_pte_quick(pmap, va);
 5315                                 KASSERT((*pte & PG_V) != 0,
 5316                                     ("pmap_advise: invalid PTE"));
 5317                                 pmap_remove_pte(pmap, pte, va, NULL);
 5318                                 anychanged = true;
 5319                         }
 5320                 }
 5321                 if (pdnxt > eva)
 5322                         pdnxt = eva;
 5323                 va = pdnxt;
 5324                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 5325                     sva += PAGE_SIZE) {
 5326                         if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V))
 5327                                 goto maybe_invlrng;
 5328                         else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5329                                 if (advice == MADV_DONTNEED) {
 5330                                         /*
 5331                                          * Future calls to pmap_is_modified()
 5332                                          * can be avoided by making the page
 5333                                          * dirty now.
 5334                                          */
 5335                                         m = PHYS_TO_VM_PAGE(*pte & PG_FRAME);
 5336                                         vm_page_dirty(m);
 5337                                 }
 5338                                 atomic_clear_int((u_int *)pte, PG_M | PG_A);
 5339                         } else if ((*pte & PG_A) != 0)
 5340                                 atomic_clear_int((u_int *)pte, PG_A);
 5341                         else
 5342                                 goto maybe_invlrng;
 5343                         if ((*pte & PG_G) != 0) {
 5344                                 if (va == pdnxt)
 5345                                         va = sva;
 5346                         } else
 5347                                 anychanged = true;
 5348                         continue;
 5349 maybe_invlrng:
 5350                         if (va != pdnxt) {
 5351                                 pmap_invalidate_range(pmap, va, sva);
 5352                                 va = pdnxt;
 5353                         }
 5354                 }
 5355                 if (va != pdnxt)
 5356                         pmap_invalidate_range(pmap, va, sva);
 5357         }
 5358         if (anychanged)
 5359                 pmap_invalidate_all(pmap);
 5360         if (pv_lists_locked) {
 5361                 sched_unpin();
 5362                 rw_wunlock(&pvh_global_lock);
 5363         }
 5364         PMAP_UNLOCK(pmap);
 5365 }
 5366 
 5367 /*
 5368  *      Clear the modify bits on the specified physical page.
 5369  */
 5370 void
 5371 pmap_clear_modify(vm_page_t m)
 5372 {
 5373         struct md_page *pvh;
 5374         pv_entry_t next_pv, pv;
 5375         pmap_t pmap;
 5376         pd_entry_t oldpde, *pde;
 5377         pt_entry_t *pte;
 5378         vm_offset_t va;
 5379 
 5380         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5381             ("pmap_clear_modify: page %p is not managed", m));
 5382         VM_OBJECT_ASSERT_WLOCKED(m->object);
 5383         KASSERT(!vm_page_xbusied(m),
 5384             ("pmap_clear_modify: page %p is exclusive busied", m));
 5385 
 5386         /*
 5387          * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
 5388          * If the object containing the page is locked and the page is not
 5389          * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
 5390          */
 5391         if ((m->aflags & PGA_WRITEABLE) == 0)
 5392                 return;
 5393         rw_wlock(&pvh_global_lock);
 5394         sched_pin();
 5395         if ((m->flags & PG_FICTITIOUS) != 0)
 5396                 goto small_mappings;
 5397         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5398         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
 5399                 va = pv->pv_va;
 5400                 pmap = PV_PMAP(pv);
 5401                 PMAP_LOCK(pmap);
 5402                 pde = pmap_pde(pmap, va);
 5403                 oldpde = *pde;
 5404                 /* If oldpde has PG_RW set, then it also has PG_M set. */
 5405                 if ((oldpde & PG_RW) != 0 &&
 5406                     pmap_demote_pde(pmap, pde, va) &&
 5407                     (oldpde & PG_W) == 0) {
 5408                         /*
 5409                          * Write protect the mapping to a single page so that
 5410                          * a subsequent write access may repromote.
 5411                          */
 5412                         va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME);
 5413                         pte = pmap_pte_quick(pmap, va);
 5414                         /*
 5415                          * Regardless of whether a pte is 32 or 64 bits
 5416                          * in size, PG_RW and PG_M are among the least
 5417                          * significant 32 bits.
 5418                          */
 5419                         atomic_clear_int((u_int *)pte, PG_M | PG_RW);
 5420                         vm_page_dirty(m);
 5421                         pmap_invalidate_page(pmap, va);
 5422                 }
 5423                 PMAP_UNLOCK(pmap);
 5424         }
 5425 small_mappings:
 5426         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5427                 pmap = PV_PMAP(pv);
 5428                 PMAP_LOCK(pmap);
 5429                 pde = pmap_pde(pmap, pv->pv_va);
 5430                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
 5431                     " a 4mpage in page %p's pv list", m));
 5432                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5433                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5434                         /*
 5435                          * Regardless of whether a pte is 32 or 64 bits
 5436                          * in size, PG_M is among the least significant
 5437                          * 32 bits. 
 5438                          */
 5439                         atomic_clear_int((u_int *)pte, PG_M);
 5440                         pmap_invalidate_page(pmap, pv->pv_va);
 5441                 }
 5442                 PMAP_UNLOCK(pmap);
 5443         }
 5444         sched_unpin();
 5445         rw_wunlock(&pvh_global_lock);
 5446 }
 5447 
 5448 /*
 5449  * Miscellaneous support routines follow
 5450  */
 5451 
 5452 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
 5453 static __inline void
 5454 pmap_pte_attr(pt_entry_t *pte, int cache_bits)
 5455 {
 5456         u_int opte, npte;
 5457 
 5458         /*
 5459          * The cache mode bits are all in the low 32-bits of the
 5460          * PTE, so we can just spin on updating the low 32-bits.
 5461          */
 5462         do {
 5463                 opte = *(u_int *)pte;
 5464                 npte = opte & ~PG_PTE_CACHE;
 5465                 npte |= cache_bits;
 5466         } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
 5467 }
 5468 
 5469 /* Adjust the cache mode for a 2/4MB page mapped via a PDE. */
 5470 static __inline void
 5471 pmap_pde_attr(pd_entry_t *pde, int cache_bits)
 5472 {
 5473         u_int opde, npde;
 5474 
 5475         /*
 5476          * The cache mode bits are all in the low 32-bits of the
 5477          * PDE, so we can just spin on updating the low 32-bits.
 5478          */
 5479         do {
 5480                 opde = *(u_int *)pde;
 5481                 npde = opde & ~PG_PDE_CACHE;
 5482                 npde |= cache_bits;
 5483         } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
 5484 }
 5485 
 5486 /*
 5487  * Map a set of physical memory pages into the kernel virtual
 5488  * address space. Return a pointer to where it is mapped. This
 5489  * routine is intended to be used for mapping device memory,
 5490  * NOT real memory.
 5491  */
 5492 static void *
 5493 pmap_mapdev_internal(vm_paddr_t pa, vm_size_t size, int mode, int flags)
 5494 {
 5495         struct pmap_preinit_mapping *ppim;
 5496         vm_offset_t va, offset;
 5497         vm_page_t m;
 5498         vm_size_t tmpsize;
 5499         int i;
 5500 
 5501         offset = pa & PAGE_MASK;
 5502         size = round_page(offset + size);
 5503         pa = pa & PG_FRAME;
 5504 
 5505         if (pa < PMAP_MAP_LOW && pa + size <= PMAP_MAP_LOW) {
 5506                 va = pa + PMAP_MAP_LOW;
 5507                 if ((flags & MAPDEV_SETATTR) == 0)
 5508                         return ((void *)(va + offset));
 5509         } else if (!pmap_initialized) {
 5510                 va = 0;
 5511                 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 5512                         ppim = pmap_preinit_mapping + i;
 5513                         if (ppim->va == 0) {
 5514                                 ppim->pa = pa;
 5515                                 ppim->sz = size;
 5516                                 ppim->mode = mode;
 5517                                 ppim->va = virtual_avail;
 5518                                 virtual_avail += size;
 5519                                 va = ppim->va;
 5520                                 break;
 5521                         }
 5522                 }
 5523                 if (va == 0)
 5524                         panic("%s: too many preinit mappings", __func__);
 5525         } else {
 5526                 /*
 5527                  * If we have a preinit mapping, re-use it.
 5528                  */
 5529                 for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 5530                         ppim = pmap_preinit_mapping + i;
 5531                         if (ppim->pa == pa && ppim->sz == size &&
 5532                             (ppim->mode == mode ||
 5533                             (flags & MAPDEV_SETATTR) == 0))
 5534                                 return ((void *)(ppim->va + offset));
 5535                 }
 5536                 va = kva_alloc(size);
 5537                 if (va == 0)
 5538                         panic("%s: Couldn't allocate KVA", __func__);
 5539         }
 5540         for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) {
 5541                 if ((flags & MAPDEV_SETATTR) == 0 && pmap_initialized) {
 5542                         m = PHYS_TO_VM_PAGE(pa);
 5543                         if (m != NULL && VM_PAGE_TO_PHYS(m) == pa) {
 5544                                 pmap_kenter_attr(va + tmpsize, pa + tmpsize,
 5545                                     m->md.pat_mode);
 5546                                 continue;
 5547                         }
 5548                 }
 5549                 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
 5550         }
 5551         pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
 5552         pmap_invalidate_cache_range(va, va + size);
 5553         return ((void *)(va + offset));
 5554 }
 5555 
 5556 void *
 5557 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 5558 {
 5559 
 5560         return (pmap_mapdev_internal(pa, size, mode, MAPDEV_SETATTR));
 5561 }
 5562 
 5563 void *
 5564 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 5565 {
 5566 
 5567         return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
 5568 }
 5569 
 5570 void *
 5571 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
 5572 {
 5573 
 5574         return (pmap_mapdev_internal(pa, size, PAT_WRITE_BACK, 0));
 5575 }
 5576 
 5577 void
 5578 pmap_unmapdev(vm_offset_t va, vm_size_t size)
 5579 {
 5580         struct pmap_preinit_mapping *ppim;
 5581         vm_offset_t offset;
 5582         int i;
 5583 
 5584         if (va >= PMAP_MAP_LOW && va <= KERNBASE && va + size <= KERNBASE)
 5585                 return;
 5586         offset = va & PAGE_MASK;
 5587         size = round_page(offset + size);
 5588         va = trunc_page(va);
 5589         for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 5590                 ppim = pmap_preinit_mapping + i;
 5591                 if (ppim->va == va && ppim->sz == size) {
 5592                         if (pmap_initialized)
 5593                                 return;
 5594                         ppim->pa = 0;
 5595                         ppim->va = 0;
 5596                         ppim->sz = 0;
 5597                         ppim->mode = 0;
 5598                         if (va + size == virtual_avail)
 5599                                 virtual_avail = va;
 5600                         return;
 5601                 }
 5602         }
 5603         if (pmap_initialized) {
 5604                 pmap_qremove(va, atop(size));
 5605                 kva_free(va, size);
 5606         }
 5607 }
 5608 
 5609 /*
 5610  * Sets the memory attribute for the specified page.
 5611  */
 5612 void
 5613 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
 5614 {
 5615 
 5616         m->md.pat_mode = ma;
 5617         if ((m->flags & PG_FICTITIOUS) != 0)
 5618                 return;
 5619 
 5620         /*
 5621          * If "m" is a normal page, flush it from the cache.
 5622          * See pmap_invalidate_cache_range().
 5623          *
 5624          * First, try to find an existing mapping of the page by sf
 5625          * buffer. sf_buf_invalidate_cache() modifies mapping and
 5626          * flushes the cache.
 5627          */    
 5628         if (sf_buf_invalidate_cache(m))
 5629                 return;
 5630 
 5631         /*
 5632          * If page is not mapped by sf buffer, but CPU does not
 5633          * support self snoop, map the page transient and do
 5634          * invalidation. In the worst case, whole cache is flushed by
 5635          * pmap_invalidate_cache_range().
 5636          */
 5637         if ((cpu_feature & CPUID_SS) == 0)
 5638                 pmap_flush_page(m);
 5639 }
 5640 
 5641 static void
 5642 pmap_flush_page(vm_page_t m)
 5643 {
 5644         pt_entry_t *cmap_pte2;
 5645         struct pcpu *pc;
 5646         vm_offset_t sva, eva;
 5647         bool useclflushopt;
 5648 
 5649         useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0;
 5650         if (useclflushopt || (cpu_feature & CPUID_CLFSH) != 0) {
 5651                 sched_pin();
 5652                 pc = get_pcpu();
 5653                 cmap_pte2 = pc->pc_cmap_pte2; 
 5654                 mtx_lock(&pc->pc_cmap_lock);
 5655                 if (*cmap_pte2)
 5656                         panic("pmap_flush_page: CMAP2 busy");
 5657                 *cmap_pte2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
 5658                     PG_A | PG_M | pmap_cache_bits(kernel_pmap, m->md.pat_mode,
 5659                     0);
 5660                 invlcaddr(pc->pc_cmap_addr2);
 5661                 sva = (vm_offset_t)pc->pc_cmap_addr2;
 5662                 eva = sva + PAGE_SIZE;
 5663 
 5664                 /*
 5665                  * Use mfence or sfence despite the ordering implied by
 5666                  * mtx_{un,}lock() because clflush on non-Intel CPUs
 5667                  * and clflushopt are not guaranteed to be ordered by
 5668                  * any other instruction.
 5669                  */
 5670                 if (useclflushopt)
 5671                         sfence();
 5672                 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 5673                         mfence();
 5674                 for (; sva < eva; sva += cpu_clflush_line_size) {
 5675                         if (useclflushopt)
 5676                                 clflushopt(sva);
 5677                         else
 5678                                 clflush(sva);
 5679                 }
 5680                 if (useclflushopt)
 5681                         sfence();
 5682                 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 5683                         mfence();
 5684                 *cmap_pte2 = 0;
 5685                 sched_unpin();
 5686                 mtx_unlock(&pc->pc_cmap_lock);
 5687         } else
 5688                 pmap_invalidate_cache();
 5689 }
 5690 
 5691 /*
 5692  * Changes the specified virtual address range's memory type to that given by
 5693  * the parameter "mode".  The specified virtual address range must be
 5694  * completely contained within either the kernel map.
 5695  *
 5696  * Returns zero if the change completed successfully, and either EINVAL or
 5697  * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
 5698  * of the virtual address range was not mapped, and ENOMEM is returned if
 5699  * there was insufficient memory available to complete the change.
 5700  */
 5701 int
 5702 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
 5703 {
 5704         vm_offset_t base, offset, tmpva;
 5705         pd_entry_t *pde;
 5706         pt_entry_t *pte;
 5707         int cache_bits_pte, cache_bits_pde;
 5708         boolean_t changed;
 5709 
 5710         base = trunc_page(va);
 5711         offset = va & PAGE_MASK;
 5712         size = round_page(offset + size);
 5713 
 5714         /*
 5715          * Only supported on kernel virtual addresses above the recursive map.
 5716          */
 5717         if (base < VM_MIN_KERNEL_ADDRESS)
 5718                 return (EINVAL);
 5719 
 5720         cache_bits_pde = pmap_cache_bits(kernel_pmap, mode, 1);
 5721         cache_bits_pte = pmap_cache_bits(kernel_pmap, mode, 0);
 5722         changed = FALSE;
 5723 
 5724         /*
 5725          * Pages that aren't mapped aren't supported.  Also break down
 5726          * 2/4MB pages into 4KB pages if required.
 5727          */
 5728         PMAP_LOCK(kernel_pmap);
 5729         for (tmpva = base; tmpva < base + size; ) {
 5730                 pde = pmap_pde(kernel_pmap, tmpva);
 5731                 if (*pde == 0) {
 5732                         PMAP_UNLOCK(kernel_pmap);
 5733                         return (EINVAL);
 5734                 }
 5735                 if (*pde & PG_PS) {
 5736                         /*
 5737                          * If the current 2/4MB page already has
 5738                          * the required memory type, then we need not
 5739                          * demote this page.  Just increment tmpva to
 5740                          * the next 2/4MB page frame.
 5741                          */
 5742                         if ((*pde & PG_PDE_CACHE) == cache_bits_pde) {
 5743                                 tmpva = trunc_4mpage(tmpva) + NBPDR;
 5744                                 continue;
 5745                         }
 5746 
 5747                         /*
 5748                          * If the current offset aligns with a 2/4MB
 5749                          * page frame and there is at least 2/4MB left
 5750                          * within the range, then we need not break
 5751                          * down this page into 4KB pages.
 5752                          */
 5753                         if ((tmpva & PDRMASK) == 0 &&
 5754                             tmpva + PDRMASK < base + size) {
 5755                                 tmpva += NBPDR;
 5756                                 continue;
 5757                         }
 5758                         if (!pmap_demote_pde(kernel_pmap, pde, tmpva)) {
 5759                                 PMAP_UNLOCK(kernel_pmap);
 5760                                 return (ENOMEM);
 5761                         }
 5762                 }
 5763                 pte = vtopte(tmpva);
 5764                 if (*pte == 0) {
 5765                         PMAP_UNLOCK(kernel_pmap);
 5766                         return (EINVAL);
 5767                 }
 5768                 tmpva += PAGE_SIZE;
 5769         }
 5770         PMAP_UNLOCK(kernel_pmap);
 5771 
 5772         /*
 5773          * Ok, all the pages exist, so run through them updating their
 5774          * cache mode if required.
 5775          */
 5776         for (tmpva = base; tmpva < base + size; ) {
 5777                 pde = pmap_pde(kernel_pmap, tmpva);
 5778                 if (*pde & PG_PS) {
 5779                         if ((*pde & PG_PDE_CACHE) != cache_bits_pde) {
 5780                                 pmap_pde_attr(pde, cache_bits_pde);
 5781                                 changed = TRUE;
 5782                         }
 5783                         tmpva = trunc_4mpage(tmpva) + NBPDR;
 5784                 } else {
 5785                         pte = vtopte(tmpva);
 5786                         if ((*pte & PG_PTE_CACHE) != cache_bits_pte) {
 5787                                 pmap_pte_attr(pte, cache_bits_pte);
 5788                                 changed = TRUE;
 5789                         }
 5790                         tmpva += PAGE_SIZE;
 5791                 }
 5792         }
 5793 
 5794         /*
 5795          * Flush CPU caches to make sure any data isn't cached that
 5796          * shouldn't be, etc.
 5797          */
 5798         if (changed) {
 5799                 pmap_invalidate_range(kernel_pmap, base, tmpva);
 5800                 pmap_invalidate_cache_range(base, tmpva);
 5801         }
 5802         return (0);
 5803 }
 5804 
 5805 /*
 5806  * perform the pmap work for mincore
 5807  */
 5808 int
 5809 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
 5810 {
 5811         pd_entry_t pde;
 5812         pt_entry_t pte;
 5813         vm_paddr_t pa;
 5814         int val;
 5815 
 5816         PMAP_LOCK(pmap);
 5817 retry:
 5818         pde = *pmap_pde(pmap, addr);
 5819         if (pde != 0) {
 5820                 if ((pde & PG_PS) != 0) {
 5821                         pte = pde;
 5822                         /* Compute the physical address of the 4KB page. */
 5823                         pa = ((pde & PG_PS_FRAME) | (addr & PDRMASK)) &
 5824                             PG_FRAME;
 5825                         val = MINCORE_SUPER;
 5826                 } else {
 5827                         pte = pmap_pte_ufast(pmap, addr, pde);
 5828                         pa = pte & PG_FRAME;
 5829                         val = 0;