The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/pmap.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2003 Peter Wemm
    9  * All rights reserved.
   10  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
   11  * All rights reserved.
   12  * Copyright (c) 2014-2018 The FreeBSD Foundation
   13  * All rights reserved.
   14  *
   15  * This code is derived from software contributed to Berkeley by
   16  * the Systems Programming Group of the University of Utah Computer
   17  * Science Department and William Jolitz of UUNET Technologies Inc.
   18  *
   19  * Portions of this software were developed by
   20  * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
   21  * the FreeBSD Foundation.
   22  *
   23  * Redistribution and use in source and binary forms, with or without
   24  * modification, are permitted provided that the following conditions
   25  * are met:
   26  * 1. Redistributions of source code must retain the above copyright
   27  *    notice, this list of conditions and the following disclaimer.
   28  * 2. Redistributions in binary form must reproduce the above copyright
   29  *    notice, this list of conditions and the following disclaimer in the
   30  *    documentation and/or other materials provided with the distribution.
   31  * 3. All advertising materials mentioning features or use of this software
   32  *    must display the following acknowledgement:
   33  *      This product includes software developed by the University of
   34  *      California, Berkeley and its contributors.
   35  * 4. Neither the name of the University nor the names of its contributors
   36  *    may be used to endorse or promote products derived from this software
   37  *    without specific prior written permission.
   38  *
   39  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   40  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   42  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   43  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   44  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   45  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   46  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   47  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   48  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   49  * SUCH DAMAGE.
   50  *
   51  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   52  */
   53 /*-
   54  * Copyright (c) 2003 Networks Associates Technology, Inc.
   55  * All rights reserved.
   56  *
   57  * This software was developed for the FreeBSD Project by Jake Burkholder,
   58  * Safeport Network Services, and Network Associates Laboratories, the
   59  * Security Research Division of Network Associates, Inc. under
   60  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   61  * CHATS research program.
   62  *
   63  * Redistribution and use in source and binary forms, with or without
   64  * modification, are permitted provided that the following conditions
   65  * are met:
   66  * 1. Redistributions of source code must retain the above copyright
   67  *    notice, this list of conditions and the following disclaimer.
   68  * 2. Redistributions in binary form must reproduce the above copyright
   69  *    notice, this list of conditions and the following disclaimer in the
   70  *    documentation and/or other materials provided with the distribution.
   71  *
   72  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   73  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   74  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   75  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   76  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   77  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   78  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   79  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   80  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   81  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   82  * SUCH DAMAGE.
   83  */
   84 
   85 #define AMD64_NPT_AWARE
   86 
   87 #include <sys/cdefs.h>
   88 __FBSDID("$FreeBSD: releng/11.2/sys/amd64/amd64/pmap.c 337828 2018-08-15 02:30:11Z delphij $");
   89 
   90 /*
   91  *      Manages physical address maps.
   92  *
   93  *      Since the information managed by this module is
   94  *      also stored by the logical address mapping module,
   95  *      this module may throw away valid virtual-to-physical
   96  *      mappings at almost any time.  However, invalidations
   97  *      of virtual-to-physical mappings must be done as
   98  *      requested.
   99  *
  100  *      In order to cope with hardware architectures which
  101  *      make virtual-to-physical map invalidates expensive,
  102  *      this module may delay invalidate or reduced protection
  103  *      operations until such time as they are actually
  104  *      necessary.  This module is given full information as
  105  *      to which processors are currently using which maps,
  106  *      and to when physical maps must be made correct.
  107  */
  108 
  109 #include "opt_pmap.h"
  110 #include "opt_vm.h"
  111 
  112 #include <sys/param.h>
  113 #include <sys/bitstring.h>
  114 #include <sys/bus.h>
  115 #include <sys/systm.h>
  116 #include <sys/kernel.h>
  117 #include <sys/ktr.h>
  118 #include <sys/lock.h>
  119 #include <sys/malloc.h>
  120 #include <sys/mman.h>
  121 #include <sys/mutex.h>
  122 #include <sys/proc.h>
  123 #include <sys/rwlock.h>
  124 #include <sys/sx.h>
  125 #include <sys/turnstile.h>
  126 #include <sys/vmem.h>
  127 #include <sys/vmmeter.h>
  128 #include <sys/sched.h>
  129 #include <sys/sysctl.h>
  130 #include <sys/smp.h>
  131 
  132 #include <vm/vm.h>
  133 #include <vm/vm_param.h>
  134 #include <vm/vm_kern.h>
  135 #include <vm/vm_page.h>
  136 #include <vm/vm_map.h>
  137 #include <vm/vm_object.h>
  138 #include <vm/vm_extern.h>
  139 #include <vm/vm_pageout.h>
  140 #include <vm/vm_pager.h>
  141 #include <vm/vm_phys.h>
  142 #include <vm/vm_radix.h>
  143 #include <vm/vm_reserv.h>
  144 #include <vm/uma.h>
  145 
  146 #include <machine/intr_machdep.h>
  147 #include <x86/apicvar.h>
  148 #include <machine/cpu.h>
  149 #include <machine/cputypes.h>
  150 #include <machine/md_var.h>
  151 #include <machine/pcb.h>
  152 #include <machine/specialreg.h>
  153 #ifdef SMP
  154 #include <machine/smp.h>
  155 #endif
  156 #include <machine/tss.h>
  157 
  158 static __inline boolean_t
  159 pmap_type_guest(pmap_t pmap)
  160 {
  161 
  162         return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI));
  163 }
  164 
  165 static __inline boolean_t
  166 pmap_emulate_ad_bits(pmap_t pmap)
  167 {
  168 
  169         return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0);
  170 }
  171 
  172 static __inline pt_entry_t
  173 pmap_valid_bit(pmap_t pmap)
  174 {
  175         pt_entry_t mask;
  176 
  177         switch (pmap->pm_type) {
  178         case PT_X86:
  179         case PT_RVI:
  180                 mask = X86_PG_V;
  181                 break;
  182         case PT_EPT:
  183                 if (pmap_emulate_ad_bits(pmap))
  184                         mask = EPT_PG_EMUL_V;
  185                 else
  186                         mask = EPT_PG_READ;
  187                 break;
  188         default:
  189                 panic("pmap_valid_bit: invalid pm_type %d", pmap->pm_type);
  190         }
  191 
  192         return (mask);
  193 }
  194 
  195 static __inline pt_entry_t
  196 pmap_rw_bit(pmap_t pmap)
  197 {
  198         pt_entry_t mask;
  199 
  200         switch (pmap->pm_type) {
  201         case PT_X86:
  202         case PT_RVI:
  203                 mask = X86_PG_RW;
  204                 break;
  205         case PT_EPT:
  206                 if (pmap_emulate_ad_bits(pmap))
  207                         mask = EPT_PG_EMUL_RW;
  208                 else
  209                         mask = EPT_PG_WRITE;
  210                 break;
  211         default:
  212                 panic("pmap_rw_bit: invalid pm_type %d", pmap->pm_type);
  213         }
  214 
  215         return (mask);
  216 }
  217 
  218 static pt_entry_t pg_g;
  219 
  220 static __inline pt_entry_t
  221 pmap_global_bit(pmap_t pmap)
  222 {
  223         pt_entry_t mask;
  224 
  225         switch (pmap->pm_type) {
  226         case PT_X86:
  227                 mask = pg_g;
  228                 break;
  229         case PT_RVI:
  230         case PT_EPT:
  231                 mask = 0;
  232                 break;
  233         default:
  234                 panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type);
  235         }
  236 
  237         return (mask);
  238 }
  239 
  240 static __inline pt_entry_t
  241 pmap_accessed_bit(pmap_t pmap)
  242 {
  243         pt_entry_t mask;
  244 
  245         switch (pmap->pm_type) {
  246         case PT_X86:
  247         case PT_RVI:
  248                 mask = X86_PG_A;
  249                 break;
  250         case PT_EPT:
  251                 if (pmap_emulate_ad_bits(pmap))
  252                         mask = EPT_PG_READ;
  253                 else
  254                         mask = EPT_PG_A;
  255                 break;
  256         default:
  257                 panic("pmap_accessed_bit: invalid pm_type %d", pmap->pm_type);
  258         }
  259 
  260         return (mask);
  261 }
  262 
  263 static __inline pt_entry_t
  264 pmap_modified_bit(pmap_t pmap)
  265 {
  266         pt_entry_t mask;
  267 
  268         switch (pmap->pm_type) {
  269         case PT_X86:
  270         case PT_RVI:
  271                 mask = X86_PG_M;
  272                 break;
  273         case PT_EPT:
  274                 if (pmap_emulate_ad_bits(pmap))
  275                         mask = EPT_PG_WRITE;
  276                 else
  277                         mask = EPT_PG_M;
  278                 break;
  279         default:
  280                 panic("pmap_modified_bit: invalid pm_type %d", pmap->pm_type);
  281         }
  282 
  283         return (mask);
  284 }
  285 
  286 extern  struct pcpu __pcpu[];
  287 
  288 #if !defined(DIAGNOSTIC)
  289 #ifdef __GNUC_GNU_INLINE__
  290 #define PMAP_INLINE     __attribute__((__gnu_inline__)) inline
  291 #else
  292 #define PMAP_INLINE     extern inline
  293 #endif
  294 #else
  295 #define PMAP_INLINE
  296 #endif
  297 
  298 #ifdef PV_STATS
  299 #define PV_STAT(x)      do { x ; } while (0)
  300 #else
  301 #define PV_STAT(x)      do { } while (0)
  302 #endif
  303 
  304 #define pa_index(pa)    ((pa) >> PDRSHIFT)
  305 #define pa_to_pvh(pa)   (&pv_table[pa_index(pa)])
  306 
  307 #define NPV_LIST_LOCKS  MAXCPU
  308 
  309 #define PHYS_TO_PV_LIST_LOCK(pa)        \
  310                         (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS])
  311 
  312 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa)  do {    \
  313         struct rwlock **_lockp = (lockp);               \
  314         struct rwlock *_new_lock;                       \
  315                                                         \
  316         _new_lock = PHYS_TO_PV_LIST_LOCK(pa);           \
  317         if (_new_lock != *_lockp) {                     \
  318                 if (*_lockp != NULL)                    \
  319                         rw_wunlock(*_lockp);            \
  320                 *_lockp = _new_lock;                    \
  321                 rw_wlock(*_lockp);                      \
  322         }                                               \
  323 } while (0)
  324 
  325 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m)        \
  326                         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
  327 
  328 #define RELEASE_PV_LIST_LOCK(lockp)             do {    \
  329         struct rwlock **_lockp = (lockp);               \
  330                                                         \
  331         if (*_lockp != NULL) {                          \
  332                 rw_wunlock(*_lockp);                    \
  333                 *_lockp = NULL;                         \
  334         }                                               \
  335 } while (0)
  336 
  337 #define VM_PAGE_TO_PV_LIST_LOCK(m)      \
  338                         PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
  339 
  340 struct pmap kernel_pmap_store;
  341 
  342 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  343 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  344 
  345 int nkpt;
  346 SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0,
  347     "Number of kernel page table pages allocated on bootup");
  348 
  349 static int ndmpdp;
  350 vm_paddr_t dmaplimit;
  351 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
  352 pt_entry_t pg_nx;
  353 
  354 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  355 
  356 static int pat_works = 1;
  357 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1,
  358     "Is page attribute table fully functional?");
  359 
  360 static int pg_ps_enabled = 1;
  361 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  362     &pg_ps_enabled, 0, "Are large page mappings enabled?");
  363 
  364 #define PAT_INDEX_SIZE  8
  365 static int pat_index[PAT_INDEX_SIZE];   /* cache mode to PAT index conversion */
  366 
  367 static u_int64_t        KPTphys;        /* phys addr of kernel level 1 */
  368 static u_int64_t        KPDphys;        /* phys addr of kernel level 2 */
  369 u_int64_t               KPDPphys;       /* phys addr of kernel level 3 */
  370 u_int64_t               KPML4phys;      /* phys addr of kernel level 4 */
  371 
  372 static u_int64_t        DMPDphys;       /* phys addr of direct mapped level 2 */
  373 static u_int64_t        DMPDPphys;      /* phys addr of direct mapped level 3 */
  374 static int              ndmpdpphys;     /* number of DMPDPphys pages */
  375 
  376 /*
  377  * pmap_mapdev support pre initialization (i.e. console)
  378  */
  379 #define PMAP_PREINIT_MAPPING_COUNT      8
  380 static struct pmap_preinit_mapping {
  381         vm_paddr_t      pa;
  382         vm_offset_t     va;
  383         vm_size_t       sz;
  384         int             mode;
  385 } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT];
  386 static int pmap_initialized;
  387 
  388 /*
  389  * Data for the pv entry allocation mechanism.
  390  * Updates to pv_invl_gen are protected by the pv_list_locks[]
  391  * elements, but reads are not.
  392  */
  393 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
  394 static struct mtx pv_chunks_mutex;
  395 static struct rwlock pv_list_locks[NPV_LIST_LOCKS];
  396 static u_long pv_invl_gen[NPV_LIST_LOCKS];
  397 static struct md_page *pv_table;
  398 static struct md_page pv_dummy;
  399 
  400 /*
  401  * All those kernel PT submaps that BSD is so fond of
  402  */
  403 pt_entry_t *CMAP1 = NULL;
  404 caddr_t CADDR1 = 0;
  405 static vm_offset_t qframe = 0;
  406 static struct mtx qframe_mtx;
  407 
  408 static int pmap_flags = PMAP_PDE_SUPERPAGE;     /* flags for x86 pmaps */
  409 
  410 int pmap_pcid_enabled = 1;
  411 SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  412     &pmap_pcid_enabled, 0, "Is TLB Context ID enabled ?");
  413 int invpcid_works = 0;
  414 SYSCTL_INT(_vm_pmap, OID_AUTO, invpcid_works, CTLFLAG_RD, &invpcid_works, 0,
  415     "Is the invpcid instruction available ?");
  416 
  417 int pti = 0;
  418 SYSCTL_INT(_vm_pmap, OID_AUTO, pti, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
  419     &pti, 0,
  420     "Page Table Isolation enabled");
  421 static vm_object_t pti_obj;
  422 static pml4_entry_t *pti_pml4;
  423 static vm_pindex_t pti_pg_idx;
  424 static bool pti_finalized;
  425 
  426 static int
  427 pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS)
  428 {
  429         int i;
  430         uint64_t res;
  431 
  432         res = 0;
  433         CPU_FOREACH(i) {
  434                 res += cpuid_to_pcpu[i]->pc_pm_save_cnt;
  435         }
  436         return (sysctl_handle_64(oidp, &res, 0, req));
  437 }
  438 SYSCTL_PROC(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLTYPE_U64 | CTLFLAG_RW |
  439     CTLFLAG_MPSAFE, NULL, 0, pmap_pcid_save_cnt_proc, "QU",
  440     "Count of saved TLB context on switch");
  441 
  442 static LIST_HEAD(, pmap_invl_gen) pmap_invl_gen_tracker =
  443     LIST_HEAD_INITIALIZER(&pmap_invl_gen_tracker);
  444 static struct mtx invl_gen_mtx;
  445 static u_long pmap_invl_gen = 0;
  446 /* Fake lock object to satisfy turnstiles interface. */
  447 static struct lock_object invl_gen_ts = {
  448         .lo_name = "invlts",
  449 };
  450 
  451 static bool
  452 pmap_not_in_di(void)
  453 {
  454 
  455         return (curthread->td_md.md_invl_gen.gen == 0);
  456 }
  457 
  458 #define PMAP_ASSERT_NOT_IN_DI() \
  459     KASSERT(pmap_not_in_di(), ("DI already started"))
  460 
  461 /*
  462  * Start a new Delayed Invalidation (DI) block of code, executed by
  463  * the current thread.  Within a DI block, the current thread may
  464  * destroy both the page table and PV list entries for a mapping and
  465  * then release the corresponding PV list lock before ensuring that
  466  * the mapping is flushed from the TLBs of any processors with the
  467  * pmap active.
  468  */
  469 static void
  470 pmap_delayed_invl_started(void)
  471 {
  472         struct pmap_invl_gen *invl_gen;
  473         u_long currgen;
  474 
  475         invl_gen = &curthread->td_md.md_invl_gen;
  476         PMAP_ASSERT_NOT_IN_DI();
  477         mtx_lock(&invl_gen_mtx);
  478         if (LIST_EMPTY(&pmap_invl_gen_tracker))
  479                 currgen = pmap_invl_gen;
  480         else
  481                 currgen = LIST_FIRST(&pmap_invl_gen_tracker)->gen;
  482         invl_gen->gen = currgen + 1;
  483         LIST_INSERT_HEAD(&pmap_invl_gen_tracker, invl_gen, link);
  484         mtx_unlock(&invl_gen_mtx);
  485 }
  486 
  487 /*
  488  * Finish the DI block, previously started by the current thread.  All
  489  * required TLB flushes for the pages marked by
  490  * pmap_delayed_invl_page() must be finished before this function is
  491  * called.
  492  *
  493  * This function works by bumping the global DI generation number to
  494  * the generation number of the current thread's DI, unless there is a
  495  * pending DI that started earlier.  In the latter case, bumping the
  496  * global DI generation number would incorrectly signal that the
  497  * earlier DI had finished.  Instead, this function bumps the earlier
  498  * DI's generation number to match the generation number of the
  499  * current thread's DI.
  500  */
  501 static void
  502 pmap_delayed_invl_finished(void)
  503 {
  504         struct pmap_invl_gen *invl_gen, *next;
  505         struct turnstile *ts;
  506 
  507         invl_gen = &curthread->td_md.md_invl_gen;
  508         KASSERT(invl_gen->gen != 0, ("missed invl_started"));
  509         mtx_lock(&invl_gen_mtx);
  510         next = LIST_NEXT(invl_gen, link);
  511         if (next == NULL) {
  512                 turnstile_chain_lock(&invl_gen_ts);
  513                 ts = turnstile_lookup(&invl_gen_ts);
  514                 pmap_invl_gen = invl_gen->gen;
  515                 if (ts != NULL) {
  516                         turnstile_broadcast(ts, TS_SHARED_QUEUE);
  517                         turnstile_unpend(ts, TS_SHARED_LOCK);
  518                 }
  519                 turnstile_chain_unlock(&invl_gen_ts);
  520         } else {
  521                 next->gen = invl_gen->gen;
  522         }
  523         LIST_REMOVE(invl_gen, link);
  524         mtx_unlock(&invl_gen_mtx);
  525         invl_gen->gen = 0;
  526 }
  527 
  528 #ifdef PV_STATS
  529 static long invl_wait;
  530 SYSCTL_LONG(_vm_pmap, OID_AUTO, invl_wait, CTLFLAG_RD, &invl_wait, 0,
  531     "Number of times DI invalidation blocked pmap_remove_all/write");
  532 #endif
  533 
  534 static u_long *
  535 pmap_delayed_invl_genp(vm_page_t m)
  536 {
  537 
  538         return (&pv_invl_gen[pa_index(VM_PAGE_TO_PHYS(m)) % NPV_LIST_LOCKS]);
  539 }
  540 
  541 /*
  542  * Ensure that all currently executing DI blocks, that need to flush
  543  * TLB for the given page m, actually flushed the TLB at the time the
  544  * function returned.  If the page m has an empty PV list and we call
  545  * pmap_delayed_invl_wait(), upon its return we know that no CPU has a
  546  * valid mapping for the page m in either its page table or TLB.
  547  *
  548  * This function works by blocking until the global DI generation
  549  * number catches up with the generation number associated with the
  550  * given page m and its PV list.  Since this function's callers
  551  * typically own an object lock and sometimes own a page lock, it
  552  * cannot sleep.  Instead, it blocks on a turnstile to relinquish the
  553  * processor.
  554  */
  555 static void
  556 pmap_delayed_invl_wait(vm_page_t m)
  557 {
  558         struct turnstile *ts;
  559         u_long *m_gen;
  560 #ifdef PV_STATS
  561         bool accounted = false;
  562 #endif
  563 
  564         m_gen = pmap_delayed_invl_genp(m);
  565         while (*m_gen > pmap_invl_gen) {
  566 #ifdef PV_STATS
  567                 if (!accounted) {
  568                         atomic_add_long(&invl_wait, 1);
  569                         accounted = true;
  570                 }
  571 #endif
  572                 ts = turnstile_trywait(&invl_gen_ts);
  573                 if (*m_gen > pmap_invl_gen)
  574                         turnstile_wait(ts, NULL, TS_SHARED_QUEUE);
  575                 else
  576                         turnstile_cancel(ts);
  577         }
  578 }
  579 
  580 /*
  581  * Mark the page m's PV list as participating in the current thread's
  582  * DI block.  Any threads concurrently using m's PV list to remove or
  583  * restrict all mappings to m will wait for the current thread's DI
  584  * block to complete before proceeding.
  585  *
  586  * The function works by setting the DI generation number for m's PV
  587  * list to at least the DI generation number of the current thread.
  588  * This forces a caller of pmap_delayed_invl_wait() to block until
  589  * current thread calls pmap_delayed_invl_finished().
  590  */
  591 static void
  592 pmap_delayed_invl_page(vm_page_t m)
  593 {
  594         u_long gen, *m_gen;
  595 
  596         rw_assert(VM_PAGE_TO_PV_LIST_LOCK(m), RA_WLOCKED);
  597         gen = curthread->td_md.md_invl_gen.gen;
  598         if (gen == 0)
  599                 return;
  600         m_gen = pmap_delayed_invl_genp(m);
  601         if (*m_gen < gen)
  602                 *m_gen = gen;
  603 }
  604 
  605 /*
  606  * Crashdump maps.
  607  */
  608 static caddr_t crashdumpmap;
  609 
  610 /*
  611  * Internal flags for pmap_enter()'s helper functions.
  612  */
  613 #define PMAP_ENTER_NORECLAIM    0x1000000       /* Don't reclaim PV entries. */
  614 #define PMAP_ENTER_NOREPLACE    0x2000000       /* Don't replace mappings. */
  615 
  616 static void     free_pv_chunk(struct pv_chunk *pc);
  617 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  618 static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);
  619 static int      popcnt_pc_map_pq(uint64_t *map);
  620 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
  621 static void     reserve_pv_entries(pmap_t pmap, int needed,
  622                     struct rwlock **lockp);
  623 static void     pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
  624                     struct rwlock **lockp);
  625 static bool     pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde,
  626                     u_int flags, struct rwlock **lockp);
  627 #if VM_NRESERVLEVEL > 0
  628 static void     pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
  629                     struct rwlock **lockp);
  630 #endif
  631 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  632 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  633                     vm_offset_t va);
  634 
  635 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode);
  636 static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  637 static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde,
  638     vm_offset_t va, struct rwlock **lockp);
  639 static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,
  640     vm_offset_t va);
  641 static bool     pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m,
  642                     vm_prot_t prot, struct rwlock **lockp);
  643 static int      pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde,
  644                     u_int flags, vm_page_t m, struct rwlock **lockp);
  645 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  646     vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
  647 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
  648 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
  649 static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va,
  650                     pd_entry_t pde);
  651 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
  652 static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask);
  653 #if VM_NRESERVLEVEL > 0
  654 static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
  655     struct rwlock **lockp);
  656 #endif
  657 static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
  658     vm_prot_t prot);
  659 static void pmap_pte_attr(pt_entry_t *pte, int cache_bits, int mask);
  660 static void pmap_pti_add_kva_locked(vm_offset_t sva, vm_offset_t eva,
  661     bool exec);
  662 static pdp_entry_t *pmap_pti_pdpe(vm_offset_t va);
  663 static pd_entry_t *pmap_pti_pde(vm_offset_t va);
  664 static void pmap_pti_wire_pte(void *pte);
  665 static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
  666     struct spglist *free, struct rwlock **lockp);
  667 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
  668     pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp);
  669 static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va);
  670 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  671     struct spglist *free);
  672 static bool     pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
  673                     pd_entry_t *pde, struct spglist *free,
  674                     struct rwlock **lockp);
  675 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  676     vm_page_t m, struct rwlock **lockp);
  677 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  678     pd_entry_t newpde);
  679 static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde);
  680 
  681 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex,
  682                 struct rwlock **lockp);
  683 static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va,
  684                 struct rwlock **lockp);
  685 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
  686                 struct rwlock **lockp);
  687 
  688 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
  689     struct spglist *free);
  690 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *);
  691 
  692 /********************/
  693 /* Inline functions */
  694 /********************/
  695 
  696 /* Return a non-clipped PD index for a given VA */
  697 static __inline vm_pindex_t
  698 pmap_pde_pindex(vm_offset_t va)
  699 {
  700         return (va >> PDRSHIFT);
  701 }
  702 
  703 
  704 /* Return a pointer to the PML4 slot that corresponds to a VA */
  705 static __inline pml4_entry_t *
  706 pmap_pml4e(pmap_t pmap, vm_offset_t va)
  707 {
  708 
  709         return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
  710 }
  711 
  712 /* Return a pointer to the PDP slot that corresponds to a VA */
  713 static __inline pdp_entry_t *
  714 pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
  715 {
  716         pdp_entry_t *pdpe;
  717 
  718         pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
  719         return (&pdpe[pmap_pdpe_index(va)]);
  720 }
  721 
  722 /* Return a pointer to the PDP slot that corresponds to a VA */
  723 static __inline pdp_entry_t *
  724 pmap_pdpe(pmap_t pmap, vm_offset_t va)
  725 {
  726         pml4_entry_t *pml4e;
  727         pt_entry_t PG_V;
  728 
  729         PG_V = pmap_valid_bit(pmap);
  730         pml4e = pmap_pml4e(pmap, va);
  731         if ((*pml4e & PG_V) == 0)
  732                 return (NULL);
  733         return (pmap_pml4e_to_pdpe(pml4e, va));
  734 }
  735 
  736 /* Return a pointer to the PD slot that corresponds to a VA */
  737 static __inline pd_entry_t *
  738 pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
  739 {
  740         pd_entry_t *pde;
  741 
  742         pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
  743         return (&pde[pmap_pde_index(va)]);
  744 }
  745 
  746 /* Return a pointer to the PD slot that corresponds to a VA */
  747 static __inline pd_entry_t *
  748 pmap_pde(pmap_t pmap, vm_offset_t va)
  749 {
  750         pdp_entry_t *pdpe;
  751         pt_entry_t PG_V;
  752 
  753         PG_V = pmap_valid_bit(pmap);
  754         pdpe = pmap_pdpe(pmap, va);
  755         if (pdpe == NULL || (*pdpe & PG_V) == 0)
  756                 return (NULL);
  757         return (pmap_pdpe_to_pde(pdpe, va));
  758 }
  759 
  760 /* Return a pointer to the PT slot that corresponds to a VA */
  761 static __inline pt_entry_t *
  762 pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
  763 {
  764         pt_entry_t *pte;
  765 
  766         pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
  767         return (&pte[pmap_pte_index(va)]);
  768 }
  769 
  770 /* Return a pointer to the PT slot that corresponds to a VA */
  771 static __inline pt_entry_t *
  772 pmap_pte(pmap_t pmap, vm_offset_t va)
  773 {
  774         pd_entry_t *pde;
  775         pt_entry_t PG_V;
  776 
  777         PG_V = pmap_valid_bit(pmap);
  778         pde = pmap_pde(pmap, va);
  779         if (pde == NULL || (*pde & PG_V) == 0)
  780                 return (NULL);
  781         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  782                 return ((pt_entry_t *)pde);
  783         return (pmap_pde_to_pte(pde, va));
  784 }
  785 
  786 static __inline void
  787 pmap_resident_count_inc(pmap_t pmap, int count)
  788 {
  789 
  790         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
  791         pmap->pm_stats.resident_count += count;
  792 }
  793 
  794 static __inline void
  795 pmap_resident_count_dec(pmap_t pmap, int count)
  796 {
  797 
  798         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
  799         KASSERT(pmap->pm_stats.resident_count >= count,
  800             ("pmap %p resident count underflow %ld %d", pmap,
  801             pmap->pm_stats.resident_count, count));
  802         pmap->pm_stats.resident_count -= count;
  803 }
  804 
  805 PMAP_INLINE pt_entry_t *
  806 vtopte(vm_offset_t va)
  807 {
  808         u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  809 
  810         KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopte on a uva/gpa 0x%0lx", va));
  811 
  812         return (PTmap + ((va >> PAGE_SHIFT) & mask));
  813 }
  814 
  815 static __inline pd_entry_t *
  816 vtopde(vm_offset_t va)
  817 {
  818         u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  819 
  820         KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopde on a uva/gpa 0x%0lx", va));
  821 
  822         return (PDmap + ((va >> PDRSHIFT) & mask));
  823 }
  824 
  825 static u_int64_t
  826 allocpages(vm_paddr_t *firstaddr, int n)
  827 {
  828         u_int64_t ret;
  829 
  830         ret = *firstaddr;
  831         bzero((void *)ret, n * PAGE_SIZE);
  832         *firstaddr += n * PAGE_SIZE;
  833         return (ret);
  834 }
  835 
  836 CTASSERT(powerof2(NDMPML4E));
  837 
  838 /* number of kernel PDP slots */
  839 #define NKPDPE(ptpgs)           howmany(ptpgs, NPDEPG)
  840 
  841 static void
  842 nkpt_init(vm_paddr_t addr)
  843 {
  844         int pt_pages;
  845         
  846 #ifdef NKPT
  847         pt_pages = NKPT;
  848 #else
  849         pt_pages = howmany(addr, 1 << PDRSHIFT);
  850         pt_pages += NKPDPE(pt_pages);
  851 
  852         /*
  853          * Add some slop beyond the bare minimum required for bootstrapping
  854          * the kernel.
  855          *
  856          * This is quite important when allocating KVA for kernel modules.
  857          * The modules are required to be linked in the negative 2GB of
  858          * the address space.  If we run out of KVA in this region then
  859          * pmap_growkernel() will need to allocate page table pages to map
  860          * the entire 512GB of KVA space which is an unnecessary tax on
  861          * physical memory.
  862          *
  863          * Secondly, device memory mapped as part of setting up the low-
  864          * level console(s) is taken from KVA, starting at virtual_avail.
  865          * This is because cninit() is called after pmap_bootstrap() but
  866          * before vm_init() and pmap_init(). 20MB for a frame buffer is
  867          * not uncommon.
  868          */
  869         pt_pages += 32;         /* 64MB additional slop. */
  870 #endif
  871         nkpt = pt_pages;
  872 }
  873 
  874 static void
  875 create_pagetables(vm_paddr_t *firstaddr)
  876 {
  877         int i, j, ndm1g, nkpdpe;
  878         pt_entry_t *pt_p;
  879         pd_entry_t *pd_p;
  880         pdp_entry_t *pdp_p;
  881         pml4_entry_t *p4_p;
  882 
  883         /* Allocate page table pages for the direct map */
  884         ndmpdp = howmany(ptoa(Maxmem), NBPDP);
  885         if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
  886                 ndmpdp = 4;
  887         ndmpdpphys = howmany(ndmpdp, NPDPEPG);
  888         if (ndmpdpphys > NDMPML4E) {
  889                 /*
  890                  * Each NDMPML4E allows 512 GB, so limit to that,
  891                  * and then readjust ndmpdp and ndmpdpphys.
  892                  */
  893                 printf("NDMPML4E limits system to %d GB\n", NDMPML4E * 512);
  894                 Maxmem = atop(NDMPML4E * NBPML4);
  895                 ndmpdpphys = NDMPML4E;
  896                 ndmpdp = NDMPML4E * NPDEPG;
  897         }
  898         DMPDPphys = allocpages(firstaddr, ndmpdpphys);
  899         ndm1g = 0;
  900         if ((amd_feature & AMDID_PAGE1GB) != 0)
  901                 ndm1g = ptoa(Maxmem) >> PDPSHIFT;
  902         if (ndm1g < ndmpdp)
  903                 DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g);
  904         dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
  905 
  906         /* Allocate pages */
  907         KPML4phys = allocpages(firstaddr, 1);
  908         KPDPphys = allocpages(firstaddr, NKPML4E);
  909 
  910         /*
  911          * Allocate the initial number of kernel page table pages required to
  912          * bootstrap.  We defer this until after all memory-size dependent
  913          * allocations are done (e.g. direct map), so that we don't have to
  914          * build in too much slop in our estimate.
  915          *
  916          * Note that when NKPML4E > 1, we have an empty page underneath
  917          * all but the KPML4I'th one, so we need NKPML4E-1 extra (zeroed)
  918          * pages.  (pmap_enter requires a PD page to exist for each KPML4E.)
  919          */
  920         nkpt_init(*firstaddr);
  921         nkpdpe = NKPDPE(nkpt);
  922 
  923         KPTphys = allocpages(firstaddr, nkpt);
  924         KPDphys = allocpages(firstaddr, nkpdpe);
  925 
  926         /* Fill in the underlying page table pages */
  927         /* Nominally read-only (but really R/W) from zero to physfree */
  928         /* XXX not fully used, underneath 2M pages */
  929         pt_p = (pt_entry_t *)KPTphys;
  930         for (i = 0; ptoa(i) < *firstaddr; i++)
  931                 pt_p[i] = ptoa(i) | X86_PG_RW | X86_PG_V | pg_g;
  932 
  933         /* Now map the page tables at their location within PTmap */
  934         pd_p = (pd_entry_t *)KPDphys;
  935         for (i = 0; i < nkpt; i++)
  936                 pd_p[i] = (KPTphys + ptoa(i)) | X86_PG_RW | X86_PG_V;
  937 
  938         /* Map from zero to end of allocations under 2M pages */
  939         /* This replaces some of the KPTphys entries above */
  940         for (i = 0; (i << PDRSHIFT) < *firstaddr; i++)
  941                 pd_p[i] = (i << PDRSHIFT) | X86_PG_RW | X86_PG_V | PG_PS |
  942                     pg_g;
  943 
  944         /*
  945          * Because we map the physical blocks in 2M pages, adjust firstaddr
  946          * to record the physical blocks we've actually mapped into kernel
  947          * virtual address space.
  948          */
  949         *firstaddr = round_2mpage(*firstaddr);
  950 
  951         /* And connect up the PD to the PDP (leaving room for L4 pages) */
  952         pdp_p = (pdp_entry_t *)(KPDPphys + ptoa(KPML4I - KPML4BASE));
  953         for (i = 0; i < nkpdpe; i++)
  954                 pdp_p[i + KPDPI] = (KPDphys + ptoa(i)) | X86_PG_RW | X86_PG_V;
  955 
  956         /*
  957          * Now, set up the direct map region using 2MB and/or 1GB pages.  If
  958          * the end of physical memory is not aligned to a 1GB page boundary,
  959          * then the residual physical memory is mapped with 2MB pages.  Later,
  960          * if pmap_mapdev{_attr}() uses the direct map for non-write-back
  961          * memory, pmap_change_attr() will demote any 2MB or 1GB page mappings
  962          * that are partially used. 
  963          */
  964         pd_p = (pd_entry_t *)DMPDphys;
  965         for (i = NPDEPG * ndm1g, j = 0; i < NPDEPG * ndmpdp; i++, j++) {
  966                 pd_p[j] = (vm_paddr_t)i << PDRSHIFT;
  967                 /* Preset PG_M and PG_A because demotion expects it. */
  968                 pd_p[j] |= X86_PG_RW | X86_PG_V | PG_PS | pg_g |
  969                     X86_PG_M | X86_PG_A;
  970         }
  971         pdp_p = (pdp_entry_t *)DMPDPphys;
  972         for (i = 0; i < ndm1g; i++) {
  973                 pdp_p[i] = (vm_paddr_t)i << PDPSHIFT;
  974                 /* Preset PG_M and PG_A because demotion expects it. */
  975                 pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_PS | pg_g |
  976                     X86_PG_M | X86_PG_A;
  977         }
  978         for (j = 0; i < ndmpdp; i++, j++) {
  979                 pdp_p[i] = DMPDphys + ptoa(j);
  980                 pdp_p[i] |= X86_PG_RW | X86_PG_V;
  981         }
  982 
  983         /* And recursively map PML4 to itself in order to get PTmap */
  984         p4_p = (pml4_entry_t *)KPML4phys;
  985         p4_p[PML4PML4I] = KPML4phys;
  986         p4_p[PML4PML4I] |= X86_PG_RW | X86_PG_V | pg_nx;
  987 
  988         /* Connect the Direct Map slot(s) up to the PML4. */
  989         for (i = 0; i < ndmpdpphys; i++) {
  990                 p4_p[DMPML4I + i] = DMPDPphys + ptoa(i);
  991                 p4_p[DMPML4I + i] |= X86_PG_RW | X86_PG_V;
  992         }
  993 
  994         /* Connect the KVA slots up to the PML4 */
  995         for (i = 0; i < NKPML4E; i++) {
  996                 p4_p[KPML4BASE + i] = KPDPphys + ptoa(i);
  997                 p4_p[KPML4BASE + i] |= X86_PG_RW | X86_PG_V;
  998         }
  999 }
 1000 
 1001 /*
 1002  *      Bootstrap the system enough to run with virtual memory.
 1003  *
 1004  *      On amd64 this is called after mapping has already been enabled
 1005  *      and just syncs the pmap module with what has already been done.
 1006  *      [We can't call it easily with mapping off since the kernel is not
 1007  *      mapped with PA == VA, hence we would have to relocate every address
 1008  *      from the linked base (virtual) address "KERNBASE" to the actual
 1009  *      (physical) address starting relative to 0]
 1010  */
 1011 void
 1012 pmap_bootstrap(vm_paddr_t *firstaddr)
 1013 {
 1014         vm_offset_t va;
 1015         pt_entry_t *pte;
 1016         int i;
 1017 
 1018         if (!pti)
 1019                 pg_g = X86_PG_G;
 1020 
 1021         /*
 1022          * Create an initial set of page tables to run the kernel in.
 1023          */
 1024         create_pagetables(firstaddr);
 1025 
 1026         /*
 1027          * Add a physical memory segment (vm_phys_seg) corresponding to the
 1028          * preallocated kernel page table pages so that vm_page structures
 1029          * representing these pages will be created.  The vm_page structures
 1030          * are required for promotion of the corresponding kernel virtual
 1031          * addresses to superpage mappings.
 1032          */
 1033         vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt));
 1034 
 1035         virtual_avail = (vm_offset_t) KERNBASE + *firstaddr;
 1036 
 1037         virtual_end = VM_MAX_KERNEL_ADDRESS;
 1038 
 1039 
 1040         /* XXX do %cr0 as well */
 1041         load_cr4(rcr4() | CR4_PGE);
 1042         load_cr3(KPML4phys);
 1043         if (cpu_stdext_feature & CPUID_STDEXT_SMEP)
 1044                 load_cr4(rcr4() | CR4_SMEP);
 1045 
 1046         /*
 1047          * Initialize the kernel pmap (which is statically allocated).
 1048          */
 1049         PMAP_LOCK_INIT(kernel_pmap);
 1050         kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys);
 1051         kernel_pmap->pm_cr3 = KPML4phys;
 1052         kernel_pmap->pm_ucr3 = PMAP_NO_CR3;
 1053         CPU_FILL(&kernel_pmap->pm_active);      /* don't allow deactivation */
 1054         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
 1055         kernel_pmap->pm_flags = pmap_flags;
 1056 
 1057         /*
 1058          * Initialize the TLB invalidations generation number lock.
 1059          */
 1060         mtx_init(&invl_gen_mtx, "invlgn", NULL, MTX_DEF);
 1061 
 1062         /*
 1063          * Reserve some special page table entries/VA space for temporary
 1064          * mapping of pages.
 1065          */
 1066 #define SYSMAP(c, p, v, n)      \
 1067         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
 1068 
 1069         va = virtual_avail;
 1070         pte = vtopte(va);
 1071 
 1072         /*
 1073          * Crashdump maps.  The first page is reused as CMAP1 for the
 1074          * memory test.
 1075          */
 1076         SYSMAP(caddr_t, CMAP1, crashdumpmap, MAXDUMPPGS)
 1077         CADDR1 = crashdumpmap;
 1078 
 1079         virtual_avail = va;
 1080 
 1081         /*
 1082          * Initialize the PAT MSR.
 1083          * pmap_init_pat() clears and sets CR4_PGE, which, as a
 1084          * side-effect, invalidates stale PG_G TLB entries that might
 1085          * have been created in our pre-boot environment.
 1086          */
 1087         pmap_init_pat();
 1088 
 1089         /* Initialize TLB Context Id. */
 1090         TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled);
 1091         if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) {
 1092                 /* Check for INVPCID support */
 1093                 invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID)
 1094                     != 0;
 1095                 for (i = 0; i < MAXCPU; i++) {
 1096                         kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN;
 1097                         kernel_pmap->pm_pcids[i].pm_gen = 1;
 1098                 }
 1099                 __pcpu[0].pc_pcid_next = PMAP_PCID_KERN + 1;
 1100                 __pcpu[0].pc_pcid_gen = 1;
 1101                 /*
 1102                  * pcpu area for APs is zeroed during AP startup.
 1103                  * pc_pcid_next and pc_pcid_gen are initialized by AP
 1104                  * during pcpu setup.
 1105                  */
 1106                 load_cr4(rcr4() | CR4_PCIDE);
 1107         } else {
 1108                 pmap_pcid_enabled = 0;
 1109         }
 1110 }
 1111 
 1112 /*
 1113  * Setup the PAT MSR.
 1114  */
 1115 void
 1116 pmap_init_pat(void)
 1117 {
 1118         int pat_table[PAT_INDEX_SIZE];
 1119         uint64_t pat_msr;
 1120         u_long cr0, cr4;
 1121         int i;
 1122 
 1123         /* Bail if this CPU doesn't implement PAT. */
 1124         if ((cpu_feature & CPUID_PAT) == 0)
 1125                 panic("no PAT??");
 1126 
 1127         /* Set default PAT index table. */
 1128         for (i = 0; i < PAT_INDEX_SIZE; i++)
 1129                 pat_table[i] = -1;
 1130         pat_table[PAT_WRITE_BACK] = 0;
 1131         pat_table[PAT_WRITE_THROUGH] = 1;
 1132         pat_table[PAT_UNCACHEABLE] = 3;
 1133         pat_table[PAT_WRITE_COMBINING] = 3;
 1134         pat_table[PAT_WRITE_PROTECTED] = 3;
 1135         pat_table[PAT_UNCACHED] = 3;
 1136 
 1137         /* Initialize default PAT entries. */
 1138         pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |
 1139             PAT_VALUE(1, PAT_WRITE_THROUGH) |
 1140             PAT_VALUE(2, PAT_UNCACHED) |
 1141             PAT_VALUE(3, PAT_UNCACHEABLE) |
 1142             PAT_VALUE(4, PAT_WRITE_BACK) |
 1143             PAT_VALUE(5, PAT_WRITE_THROUGH) |
 1144             PAT_VALUE(6, PAT_UNCACHED) |
 1145             PAT_VALUE(7, PAT_UNCACHEABLE);
 1146 
 1147         if (pat_works) {
 1148                 /*
 1149                  * Leave the indices 0-3 at the default of WB, WT, UC-, and UC.
 1150                  * Program 5 and 6 as WP and WC.
 1151                  * Leave 4 and 7 as WB and UC.
 1152                  */
 1153                 pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6));
 1154                 pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) |
 1155                     PAT_VALUE(6, PAT_WRITE_COMBINING);
 1156                 pat_table[PAT_UNCACHED] = 2;
 1157                 pat_table[PAT_WRITE_PROTECTED] = 5;
 1158                 pat_table[PAT_WRITE_COMBINING] = 6;
 1159         } else {
 1160                 /*
 1161                  * Just replace PAT Index 2 with WC instead of UC-.
 1162                  */
 1163                 pat_msr &= ~PAT_MASK(2);
 1164                 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
 1165                 pat_table[PAT_WRITE_COMBINING] = 2;
 1166         }
 1167 
 1168         /* Disable PGE. */
 1169         cr4 = rcr4();
 1170         load_cr4(cr4 & ~CR4_PGE);
 1171 
 1172         /* Disable caches (CD = 1, NW = 0). */
 1173         cr0 = rcr0();
 1174         load_cr0((cr0 & ~CR0_NW) | CR0_CD);
 1175 
 1176         /* Flushes caches and TLBs. */
 1177         wbinvd();
 1178         invltlb();
 1179 
 1180         /* Update PAT and index table. */
 1181         wrmsr(MSR_PAT, pat_msr);
 1182         for (i = 0; i < PAT_INDEX_SIZE; i++)
 1183                 pat_index[i] = pat_table[i];
 1184 
 1185         /* Flush caches and TLBs again. */
 1186         wbinvd();
 1187         invltlb();
 1188 
 1189         /* Restore caches and PGE. */
 1190         load_cr0(cr0);
 1191         load_cr4(cr4);
 1192 }
 1193 
 1194 /*
 1195  *      Initialize a vm_page's machine-dependent fields.
 1196  */
 1197 void
 1198 pmap_page_init(vm_page_t m)
 1199 {
 1200 
 1201         TAILQ_INIT(&m->md.pv_list);
 1202         m->md.pat_mode = PAT_WRITE_BACK;
 1203 }
 1204 
 1205 /*
 1206  *      Initialize the pmap module.
 1207  *      Called by vm_init, to initialize any structures that the pmap
 1208  *      system needs to map virtual memory.
 1209  */
 1210 void
 1211 pmap_init(void)
 1212 {
 1213         struct pmap_preinit_mapping *ppim;
 1214         vm_page_t mpte;
 1215         vm_size_t s;
 1216         int error, i, pv_npg, ret, skz63;
 1217 
 1218         /* L1TF, reserve page @0 unconditionally */
 1219         vm_page_blacklist_add(0, bootverbose);
 1220 
 1221         /* Detect bare-metal Skylake Server and Skylake-X. */
 1222         if (vm_guest == VM_GUEST_NO && cpu_vendor_id == CPU_VENDOR_INTEL &&
 1223             CPUID_TO_FAMILY(cpu_id) == 0x6 && CPUID_TO_MODEL(cpu_id) == 0x55) {
 1224                 /*
 1225                  * Skylake-X errata SKZ63. Processor May Hang When
 1226                  * Executing Code In an HLE Transaction Region between
 1227                  * 40000000H and 403FFFFFH.
 1228                  *
 1229                  * Mark the pages in the range as preallocated.  It
 1230                  * seems to be impossible to distinguish between
 1231                  * Skylake Server and Skylake X.
 1232                  */
 1233                 skz63 = 1;
 1234                 TUNABLE_INT_FETCH("hw.skz63_enable", &skz63);
 1235                 if (skz63 != 0) {
 1236                         if (bootverbose)
 1237                                 printf("SKZ63: skipping 4M RAM starting "
 1238                                     "at physical 1G\n");
 1239                         for (i = 0; i < atop(0x400000); i++) {
 1240                                 ret = vm_page_blacklist_add(0x40000000 +
 1241                                     ptoa(i), FALSE);
 1242                                 if (!ret && bootverbose)
 1243                                         printf("page at %#lx already used\n",
 1244                                             0x40000000 + ptoa(i));
 1245                         }
 1246                 }
 1247         }
 1248 
 1249         /*
 1250          * Initialize the vm page array entries for the kernel pmap's
 1251          * page table pages.
 1252          */ 
 1253         for (i = 0; i < nkpt; i++) {
 1254                 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
 1255                 KASSERT(mpte >= vm_page_array &&
 1256                     mpte < &vm_page_array[vm_page_array_size],
 1257                     ("pmap_init: page table page is out of range"));
 1258                 mpte->pindex = pmap_pde_pindex(KERNBASE) + i;
 1259                 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
 1260                 mpte->wire_count = 1;
 1261         }
 1262         atomic_add_int(&vm_cnt.v_wire_count, nkpt);
 1263 
 1264         /*
 1265          * If the kernel is running on a virtual machine, then it must assume
 1266          * that MCA is enabled by the hypervisor.  Moreover, the kernel must
 1267          * be prepared for the hypervisor changing the vendor and family that
 1268          * are reported by CPUID.  Consequently, the workaround for AMD Family
 1269          * 10h Erratum 383 is enabled if the processor's feature set does not
 1270          * include at least one feature that is only supported by older Intel
 1271          * or newer AMD processors.
 1272          */
 1273         if (vm_guest != VM_GUEST_NO && (cpu_feature & CPUID_SS) == 0 &&
 1274             (cpu_feature2 & (CPUID2_SSSE3 | CPUID2_SSE41 | CPUID2_AESNI |
 1275             CPUID2_AVX | CPUID2_XSAVE)) == 0 && (amd_feature2 & (AMDID2_XOP |
 1276             AMDID2_FMA4)) == 0)
 1277                 workaround_erratum383 = 1;
 1278 
 1279         /*
 1280          * Are large page mappings enabled?
 1281          */
 1282         TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
 1283         if (pg_ps_enabled) {
 1284                 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
 1285                     ("pmap_init: can't assign to pagesizes[1]"));
 1286                 pagesizes[1] = NBPDR;
 1287         }
 1288 
 1289         /*
 1290          * Initialize the pv chunk list mutex.
 1291          */
 1292         mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
 1293 
 1294         /*
 1295          * Initialize the pool of pv list locks.
 1296          */
 1297         for (i = 0; i < NPV_LIST_LOCKS; i++)
 1298                 rw_init(&pv_list_locks[i], "pmap pv list");
 1299 
 1300         /*
 1301          * Calculate the size of the pv head table for superpages.
 1302          */
 1303         pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, NBPDR);
 1304 
 1305         /*
 1306          * Allocate memory for the pv head table for superpages.
 1307          */
 1308         s = (vm_size_t)(pv_npg * sizeof(struct md_page));
 1309         s = round_page(s);
 1310         pv_table = (struct md_page *)kmem_malloc(kernel_arena, s,
 1311             M_WAITOK | M_ZERO);
 1312         for (i = 0; i < pv_npg; i++)
 1313                 TAILQ_INIT(&pv_table[i].pv_list);
 1314         TAILQ_INIT(&pv_dummy.pv_list);
 1315 
 1316         pmap_initialized = 1;
 1317         for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) {
 1318                 ppim = pmap_preinit_mapping + i;
 1319                 if (ppim->va == 0)
 1320                         continue;
 1321                 /* Make the direct map consistent */
 1322                 if (ppim->pa < dmaplimit && ppim->pa + ppim->sz < dmaplimit) {
 1323                         (void)pmap_change_attr(PHYS_TO_DMAP(ppim->pa),
 1324                             ppim->sz, ppim->mode);
 1325                 }
 1326                 if (!bootverbose)
 1327                         continue;
 1328                 printf("PPIM %u: PA=%#lx, VA=%#lx, size=%#lx, mode=%#x\n", i,
 1329                     ppim->pa, ppim->va, ppim->sz, ppim->mode);
 1330         }
 1331 
 1332         mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN);
 1333         error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK,
 1334             (vmem_addr_t *)&qframe);
 1335         if (error != 0)
 1336                 panic("qframe allocation failed");
 1337 }
 1338 
 1339 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
 1340     "2MB page mapping counters");
 1341 
 1342 static u_long pmap_pde_demotions;
 1343 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
 1344     &pmap_pde_demotions, 0, "2MB page demotions");
 1345 
 1346 static u_long pmap_pde_mappings;
 1347 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
 1348     &pmap_pde_mappings, 0, "2MB page mappings");
 1349 
 1350 static u_long pmap_pde_p_failures;
 1351 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
 1352     &pmap_pde_p_failures, 0, "2MB page promotion failures");
 1353 
 1354 static u_long pmap_pde_promotions;
 1355 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
 1356     &pmap_pde_promotions, 0, "2MB page promotions");
 1357 
 1358 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD, 0,
 1359     "1GB page mapping counters");
 1360 
 1361 static u_long pmap_pdpe_demotions;
 1362 SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
 1363     &pmap_pdpe_demotions, 0, "1GB page demotions");
 1364 
 1365 /***************************************************
 1366  * Low level helper routines.....
 1367  ***************************************************/
 1368 
 1369 static pt_entry_t
 1370 pmap_swap_pat(pmap_t pmap, pt_entry_t entry)
 1371 {
 1372         int x86_pat_bits = X86_PG_PTE_PAT | X86_PG_PDE_PAT;
 1373 
 1374         switch (pmap->pm_type) {
 1375         case PT_X86:
 1376         case PT_RVI:
 1377                 /* Verify that both PAT bits are not set at the same time */
 1378                 KASSERT((entry & x86_pat_bits) != x86_pat_bits,
 1379                     ("Invalid PAT bits in entry %#lx", entry));
 1380 
 1381                 /* Swap the PAT bits if one of them is set */
 1382                 if ((entry & x86_pat_bits) != 0)
 1383                         entry ^= x86_pat_bits;
 1384                 break;
 1385         case PT_EPT:
 1386                 /*
 1387                  * Nothing to do - the memory attributes are represented
 1388                  * the same way for regular pages and superpages.
 1389                  */
 1390                 break;
 1391         default:
 1392                 panic("pmap_switch_pat_bits: bad pm_type %d", pmap->pm_type);
 1393         }
 1394 
 1395         return (entry);
 1396 }
 1397 
 1398 /*
 1399  * Determine the appropriate bits to set in a PTE or PDE for a specified
 1400  * caching mode.
 1401  */
 1402 int
 1403 pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde)
 1404 {
 1405         int cache_bits, pat_flag, pat_idx;
 1406 
 1407         if (mode < 0 || mode >= PAT_INDEX_SIZE || pat_index[mode] < 0)
 1408                 panic("Unknown caching mode %d\n", mode);
 1409 
 1410         switch (pmap->pm_type) {
 1411         case PT_X86:
 1412         case PT_RVI:
 1413                 /* The PAT bit is different for PTE's and PDE's. */
 1414                 pat_flag = is_pde ? X86_PG_PDE_PAT : X86_PG_PTE_PAT;
 1415 
 1416                 /* Map the caching mode to a PAT index. */
 1417                 pat_idx = pat_index[mode];
 1418 
 1419                 /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
 1420                 cache_bits = 0;
 1421                 if (pat_idx & 0x4)
 1422                         cache_bits |= pat_flag;
 1423                 if (pat_idx & 0x2)
 1424                         cache_bits |= PG_NC_PCD;
 1425                 if (pat_idx & 0x1)
 1426                         cache_bits |= PG_NC_PWT;
 1427                 break;
 1428 
 1429         case PT_EPT:
 1430                 cache_bits = EPT_PG_IGNORE_PAT | EPT_PG_MEMORY_TYPE(mode);
 1431                 break;
 1432 
 1433         default:
 1434                 panic("unsupported pmap type %d", pmap->pm_type);
 1435         }
 1436 
 1437         return (cache_bits);
 1438 }
 1439 
 1440 static int
 1441 pmap_cache_mask(pmap_t pmap, boolean_t is_pde)
 1442 {
 1443         int mask;
 1444 
 1445         switch (pmap->pm_type) {
 1446         case PT_X86:
 1447         case PT_RVI:
 1448                 mask = is_pde ? X86_PG_PDE_CACHE : X86_PG_PTE_CACHE;
 1449                 break;
 1450         case PT_EPT:
 1451                 mask = EPT_PG_IGNORE_PAT | EPT_PG_MEMORY_TYPE(0x7);
 1452                 break;
 1453         default:
 1454                 panic("pmap_cache_mask: invalid pm_type %d", pmap->pm_type);
 1455         }
 1456 
 1457         return (mask);
 1458 }
 1459 
 1460 bool
 1461 pmap_ps_enabled(pmap_t pmap)
 1462 {
 1463 
 1464         return (pg_ps_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
 1465 }
 1466 
 1467 static void
 1468 pmap_update_pde_store(pmap_t pmap, pd_entry_t *pde, pd_entry_t newpde)
 1469 {
 1470 
 1471         switch (pmap->pm_type) {
 1472         case PT_X86:
 1473                 break;
 1474         case PT_RVI:
 1475         case PT_EPT:
 1476                 /*
 1477                  * XXX
 1478                  * This is a little bogus since the generation number is
 1479                  * supposed to be bumped up when a region of the address
 1480                  * space is invalidated in the page tables.
 1481                  *
 1482                  * In this case the old PDE entry is valid but yet we want
 1483                  * to make sure that any mappings using the old entry are
 1484                  * invalidated in the TLB.
 1485                  *
 1486                  * The reason this works as expected is because we rendezvous
 1487                  * "all" host cpus and force any vcpu context to exit as a
 1488                  * side-effect.
 1489                  */
 1490                 atomic_add_acq_long(&pmap->pm_eptgen, 1);
 1491                 break;
 1492         default:
 1493                 panic("pmap_update_pde_store: bad pm_type %d", pmap->pm_type);
 1494         }
 1495         pde_store(pde, newpde);
 1496 }
 1497 
 1498 /*
 1499  * After changing the page size for the specified virtual address in the page
 1500  * table, flush the corresponding entries from the processor's TLB.  Only the
 1501  * calling processor's TLB is affected.
 1502  *
 1503  * The calling thread must be pinned to a processor.
 1504  */
 1505 static void
 1506 pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde)
 1507 {
 1508         pt_entry_t PG_G;
 1509 
 1510         if (pmap_type_guest(pmap))
 1511                 return;
 1512 
 1513         KASSERT(pmap->pm_type == PT_X86,
 1514             ("pmap_update_pde_invalidate: invalid type %d", pmap->pm_type));
 1515 
 1516         PG_G = pmap_global_bit(pmap);
 1517 
 1518         if ((newpde & PG_PS) == 0)
 1519                 /* Demotion: flush a specific 2MB page mapping. */
 1520                 invlpg(va);
 1521         else if ((newpde & PG_G) == 0)
 1522                 /*
 1523                  * Promotion: flush every 4KB page mapping from the TLB
 1524                  * because there are too many to flush individually.
 1525                  */
 1526                 invltlb();
 1527         else {
 1528                 /*
 1529                  * Promotion: flush every 4KB page mapping from the TLB,
 1530                  * including any global (PG_G) mappings.
 1531                  */
 1532                 invltlb_glob();
 1533         }
 1534 }
 1535 #ifdef SMP
 1536 
 1537 /*
 1538  * For SMP, these functions have to use the IPI mechanism for coherence.
 1539  *
 1540  * N.B.: Before calling any of the following TLB invalidation functions,
 1541  * the calling processor must ensure that all stores updating a non-
 1542  * kernel page table are globally performed.  Otherwise, another
 1543  * processor could cache an old, pre-update entry without being
 1544  * invalidated.  This can happen one of two ways: (1) The pmap becomes
 1545  * active on another processor after its pm_active field is checked by
 1546  * one of the following functions but before a store updating the page
 1547  * table is globally performed. (2) The pmap becomes active on another
 1548  * processor before its pm_active field is checked but due to
 1549  * speculative loads one of the following functions stills reads the
 1550  * pmap as inactive on the other processor.
 1551  * 
 1552  * The kernel page table is exempt because its pm_active field is
 1553  * immutable.  The kernel page table is always active on every
 1554  * processor.
 1555  */
 1556 
 1557 /*
 1558  * Interrupt the cpus that are executing in the guest context.
 1559  * This will force the vcpu to exit and the cached EPT mappings
 1560  * will be invalidated by the host before the next vmresume.
 1561  */
 1562 static __inline void
 1563 pmap_invalidate_ept(pmap_t pmap)
 1564 {
 1565         int ipinum;
 1566 
 1567         sched_pin();
 1568         KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),
 1569             ("pmap_invalidate_ept: absurd pm_active"));
 1570 
 1571         /*
 1572          * The TLB mappings associated with a vcpu context are not
 1573          * flushed each time a different vcpu is chosen to execute.
 1574          *
 1575          * This is in contrast with a process's vtop mappings that
 1576          * are flushed from the TLB on each context switch.
 1577          *
 1578          * Therefore we need to do more than just a TLB shootdown on
 1579          * the active cpus in 'pmap->pm_active'. To do this we keep
 1580          * track of the number of invalidations performed on this pmap.
 1581          *
 1582          * Each vcpu keeps a cache of this counter and compares it
 1583          * just before a vmresume. If the counter is out-of-date an
 1584          * invept will be done to flush stale mappings from the TLB.
 1585          */
 1586         atomic_add_acq_long(&pmap->pm_eptgen, 1);
 1587 
 1588         /*
 1589          * Force the vcpu to exit and trap back into the hypervisor.
 1590          */
 1591         ipinum = pmap->pm_flags & PMAP_NESTED_IPIMASK;
 1592         ipi_selected(pmap->pm_active, ipinum);
 1593         sched_unpin();
 1594 }
 1595 
 1596 void
 1597 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 1598 {
 1599         cpuset_t *mask;
 1600         struct invpcid_descr d;
 1601         uint64_t kcr3, ucr3;
 1602         uint32_t pcid;
 1603         u_int cpuid, i;
 1604 
 1605         if (pmap_type_guest(pmap)) {
 1606                 pmap_invalidate_ept(pmap);
 1607                 return;
 1608         }
 1609 
 1610         KASSERT(pmap->pm_type == PT_X86,
 1611             ("pmap_invalidate_page: invalid type %d", pmap->pm_type));
 1612 
 1613         sched_pin();
 1614         if (pmap == kernel_pmap) {
 1615                 invlpg(va);
 1616                 mask = &all_cpus;
 1617         } else {
 1618                 cpuid = PCPU_GET(cpuid);
 1619                 if (pmap == PCPU_GET(curpmap)) {
 1620                         invlpg(va);
 1621                         if (pmap_pcid_enabled && pmap->pm_ucr3 != PMAP_NO_CR3) {
 1622                                 /*
 1623                                  * Disable context switching. pm_pcid
 1624                                  * is recalculated on switch, which
 1625                                  * might make us use wrong pcid below.
 1626                                  */
 1627                                 critical_enter();
 1628                                 pcid = pmap->pm_pcids[cpuid].pm_pcid;
 1629 
 1630                                 if (invpcid_works) {
 1631                                         d.pcid = pcid | PMAP_PCID_USER_PT;
 1632                                         d.pad = 0;
 1633                                         d.addr = va;
 1634                                         invpcid(&d, INVPCID_ADDR);
 1635                                 } else {
 1636                                         kcr3 = pmap->pm_cr3 | pcid |
 1637                                             CR3_PCID_SAVE;
 1638                                         ucr3 = pmap->pm_ucr3 | pcid |
 1639                                             PMAP_PCID_USER_PT | CR3_PCID_SAVE;
 1640                                         pmap_pti_pcid_invlpg(ucr3, kcr3, va);
 1641                                 }
 1642                                 critical_exit();
 1643                         }
 1644                 } else if (pmap_pcid_enabled)
 1645                         pmap->pm_pcids[cpuid].pm_gen = 0;
 1646                 if (pmap_pcid_enabled) {
 1647                         CPU_FOREACH(i) {
 1648                                 if (cpuid != i)
 1649                                         pmap->pm_pcids[i].pm_gen = 0;
 1650                         }
 1651 
 1652                         /*
 1653                          * The fence is between stores to pm_gen and the read of
 1654                          * the pm_active mask.  We need to ensure that it is
 1655                          * impossible for us to miss the bit update in pm_active
 1656                          * and simultaneously observe a non-zero pm_gen in
 1657                          * pmap_activate_sw(), otherwise TLB update is missed.
 1658                          * Without the fence, IA32 allows such an outcome.
 1659                          * Note that pm_active is updated by a locked operation,
 1660                          * which provides the reciprocal fence.
 1661                          */
 1662                         atomic_thread_fence_seq_cst();
 1663                 }
 1664                 mask = &pmap->pm_active;
 1665         }
 1666         smp_masked_invlpg(*mask, va, pmap);
 1667         sched_unpin();
 1668 }
 1669 
 1670 /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */
 1671 #define PMAP_INVLPG_THRESHOLD   (4 * 1024 * PAGE_SIZE)
 1672 
 1673 void
 1674 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1675 {
 1676         cpuset_t *mask;
 1677         struct invpcid_descr d;
 1678         vm_offset_t addr;
 1679         uint64_t kcr3, ucr3;
 1680         uint32_t pcid;
 1681         u_int cpuid, i;
 1682 
 1683         if (eva - sva >= PMAP_INVLPG_THRESHOLD) {
 1684                 pmap_invalidate_all(pmap);
 1685                 return;
 1686         }
 1687 
 1688         if (pmap_type_guest(pmap)) {
 1689                 pmap_invalidate_ept(pmap);
 1690                 return;
 1691         }
 1692 
 1693         KASSERT(pmap->pm_type == PT_X86,
 1694             ("pmap_invalidate_range: invalid type %d", pmap->pm_type));
 1695 
 1696         sched_pin();
 1697         cpuid = PCPU_GET(cpuid);
 1698         if (pmap == kernel_pmap) {
 1699                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1700                         invlpg(addr);
 1701                 mask = &all_cpus;
 1702         } else {
 1703                 if (pmap == PCPU_GET(curpmap)) {
 1704                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1705                                 invlpg(addr);
 1706                         if (pmap_pcid_enabled && pmap->pm_ucr3 != PMAP_NO_CR3) {
 1707                                 critical_enter();
 1708                                 pcid = pmap->pm_pcids[cpuid].pm_pcid;
 1709                                 if (invpcid_works) {
 1710                                         d.pcid = pcid | PMAP_PCID_USER_PT;
 1711                                         d.pad = 0;
 1712                                         d.addr = sva;
 1713                                         for (; d.addr < eva; d.addr +=
 1714                                             PAGE_SIZE)
 1715                                                 invpcid(&d, INVPCID_ADDR);
 1716                                 } else {
 1717                                         kcr3 = pmap->pm_cr3 | pcid |
 1718                                             CR3_PCID_SAVE;
 1719                                         ucr3 = pmap->pm_ucr3 | pcid |
 1720                                             PMAP_PCID_USER_PT | CR3_PCID_SAVE;
 1721                                         pmap_pti_pcid_invlrng(ucr3, kcr3, sva,
 1722                                             eva);
 1723                                 }
 1724                                 critical_exit();
 1725                         }
 1726                 } else if (pmap_pcid_enabled) {
 1727                         pmap->pm_pcids[cpuid].pm_gen = 0;
 1728                 }
 1729                 if (pmap_pcid_enabled) {
 1730                         CPU_FOREACH(i) {
 1731                                 if (cpuid != i)
 1732                                         pmap->pm_pcids[i].pm_gen = 0;
 1733                         }
 1734                         /* See the comment in pmap_invalidate_page(). */
 1735                         atomic_thread_fence_seq_cst();
 1736                 }
 1737                 mask = &pmap->pm_active;
 1738         }
 1739         smp_masked_invlpg_range(*mask, sva, eva, pmap);
 1740         sched_unpin();
 1741 }
 1742 
 1743 void
 1744 pmap_invalidate_all(pmap_t pmap)
 1745 {
 1746         cpuset_t *mask;
 1747         struct invpcid_descr d;
 1748         uint64_t kcr3, ucr3;
 1749         uint32_t pcid;
 1750         u_int cpuid, i;
 1751 
 1752         if (pmap_type_guest(pmap)) {
 1753                 pmap_invalidate_ept(pmap);
 1754                 return;
 1755         }
 1756 
 1757         KASSERT(pmap->pm_type == PT_X86,
 1758             ("pmap_invalidate_all: invalid type %d", pmap->pm_type));
 1759 
 1760         sched_pin();
 1761         if (pmap == kernel_pmap) {
 1762                 if (pmap_pcid_enabled && invpcid_works) {
 1763                         bzero(&d, sizeof(d));
 1764                         invpcid(&d, INVPCID_CTXGLOB);
 1765                 } else {
 1766                         invltlb_glob();
 1767                 }
 1768                 mask = &all_cpus;
 1769         } else {
 1770                 cpuid = PCPU_GET(cpuid);
 1771                 if (pmap == PCPU_GET(curpmap)) {
 1772                         if (pmap_pcid_enabled) {
 1773                                 critical_enter();
 1774                                 pcid = pmap->pm_pcids[cpuid].pm_pcid;
 1775                                 if (invpcid_works) {
 1776                                         d.pcid = pcid;
 1777                                         d.pad = 0;
 1778                                         d.addr = 0;
 1779                                         invpcid(&d, INVPCID_CTX);
 1780                                         if (pmap->pm_ucr3 != PMAP_NO_CR3) {
 1781                                                 d.pcid |= PMAP_PCID_USER_PT;
 1782                                                 invpcid(&d, INVPCID_CTX);
 1783                                         }
 1784                                 } else {
 1785                                         kcr3 = pmap->pm_cr3 | pcid;
 1786                                         ucr3 = pmap->pm_ucr3;
 1787                                         if (ucr3 != PMAP_NO_CR3) {
 1788                                                 ucr3 |= pcid | PMAP_PCID_USER_PT;
 1789                                                 pmap_pti_pcid_invalidate(ucr3,
 1790                                                     kcr3);
 1791                                         } else {
 1792                                                 load_cr3(kcr3);
 1793                                         }
 1794                                 }
 1795                                 critical_exit();
 1796                         } else {
 1797                                 invltlb();
 1798                         }
 1799                 } else if (pmap_pcid_enabled) {
 1800                         pmap->pm_pcids[cpuid].pm_gen = 0;
 1801                 }
 1802                 if (pmap_pcid_enabled) {
 1803                         CPU_FOREACH(i) {
 1804                                 if (cpuid != i)
 1805                                         pmap->pm_pcids[i].pm_gen = 0;
 1806                         }
 1807                         /* See the comment in pmap_invalidate_page(). */
 1808                         atomic_thread_fence_seq_cst();
 1809                 }
 1810                 mask = &pmap->pm_active;
 1811         }
 1812         smp_masked_invltlb(*mask, pmap);
 1813         sched_unpin();
 1814 }
 1815 
 1816 void
 1817 pmap_invalidate_cache(void)
 1818 {
 1819 
 1820         sched_pin();
 1821         wbinvd();
 1822         smp_cache_flush();
 1823         sched_unpin();
 1824 }
 1825 
 1826 struct pde_action {
 1827         cpuset_t invalidate;    /* processors that invalidate their TLB */
 1828         pmap_t pmap;
 1829         vm_offset_t va;
 1830         pd_entry_t *pde;
 1831         pd_entry_t newpde;
 1832         u_int store;            /* processor that updates the PDE */
 1833 };
 1834 
 1835 static void
 1836 pmap_update_pde_action(void *arg)
 1837 {
 1838         struct pde_action *act = arg;
 1839 
 1840         if (act->store == PCPU_GET(cpuid))
 1841                 pmap_update_pde_store(act->pmap, act->pde, act->newpde);
 1842 }
 1843 
 1844 static void
 1845 pmap_update_pde_teardown(void *arg)
 1846 {
 1847         struct pde_action *act = arg;
 1848 
 1849         if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
 1850                 pmap_update_pde_invalidate(act->pmap, act->va, act->newpde);
 1851 }
 1852 
 1853 /*
 1854  * Change the page size for the specified virtual address in a way that
 1855  * prevents any possibility of the TLB ever having two entries that map the
 1856  * same virtual address using different page sizes.  This is the recommended
 1857  * workaround for Erratum 383 on AMD Family 10h processors.  It prevents a
 1858  * machine check exception for a TLB state that is improperly diagnosed as a
 1859  * hardware error.
 1860  */
 1861 static void
 1862 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1863 {
 1864         struct pde_action act;
 1865         cpuset_t active, other_cpus;
 1866         u_int cpuid;
 1867 
 1868         sched_pin();
 1869         cpuid = PCPU_GET(cpuid);
 1870         other_cpus = all_cpus;
 1871         CPU_CLR(cpuid, &other_cpus);
 1872         if (pmap == kernel_pmap || pmap_type_guest(pmap)) 
 1873                 active = all_cpus;
 1874         else {
 1875                 active = pmap->pm_active;
 1876         }
 1877         if (CPU_OVERLAP(&active, &other_cpus)) { 
 1878                 act.store = cpuid;
 1879                 act.invalidate = active;
 1880                 act.va = va;
 1881                 act.pmap = pmap;
 1882                 act.pde = pde;
 1883                 act.newpde = newpde;
 1884                 CPU_SET(cpuid, &active);
 1885                 smp_rendezvous_cpus(active,
 1886                     smp_no_rendezvous_barrier, pmap_update_pde_action,
 1887                     pmap_update_pde_teardown, &act);
 1888         } else {
 1889                 pmap_update_pde_store(pmap, pde, newpde);
 1890                 if (CPU_ISSET(cpuid, &active))
 1891                         pmap_update_pde_invalidate(pmap, va, newpde);
 1892         }
 1893         sched_unpin();
 1894 }
 1895 #else /* !SMP */
 1896 /*
 1897  * Normal, non-SMP, invalidation functions.
 1898  */
 1899 void
 1900 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 1901 {
 1902         struct invpcid_descr d;
 1903         uint64_t kcr3, ucr3;
 1904         uint32_t pcid;
 1905 
 1906         if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) {
 1907                 pmap->pm_eptgen++;
 1908                 return;
 1909         }
 1910         KASSERT(pmap->pm_type == PT_X86,
 1911             ("pmap_invalidate_range: unknown type %d", pmap->pm_type));
 1912 
 1913         if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) {
 1914                 invlpg(va);
 1915                 if (pmap == PCPU_GET(curpmap) && pmap_pcid_enabled &&
 1916                     pmap->pm_ucr3 != PMAP_NO_CR3) {
 1917                         critical_enter();
 1918                         pcid = pmap->pm_pcids[0].pm_pcid;
 1919                         if (invpcid_works) {
 1920                                 d.pcid = pcid | PMAP_PCID_USER_PT;
 1921                                 d.pad = 0;
 1922                                 d.addr = va;
 1923                                 invpcid(&d, INVPCID_ADDR);
 1924                         } else {
 1925                                 kcr3 = pmap->pm_cr3 | pcid | CR3_PCID_SAVE;
 1926                                 ucr3 = pmap->pm_ucr3 | pcid |
 1927                                     PMAP_PCID_USER_PT | CR3_PCID_SAVE;
 1928                                 pmap_pti_pcid_invlpg(ucr3, kcr3, va);
 1929                         }
 1930                         critical_exit();
 1931                 }
 1932         } else if (pmap_pcid_enabled)
 1933                 pmap->pm_pcids[0].pm_gen = 0;
 1934 }
 1935 
 1936 void
 1937 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1938 {
 1939         struct invpcid_descr d;
 1940         vm_offset_t addr;
 1941         uint64_t kcr3, ucr3;
 1942 
 1943         if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) {
 1944                 pmap->pm_eptgen++;
 1945                 return;
 1946         }
 1947         KASSERT(pmap->pm_type == PT_X86,
 1948             ("pmap_invalidate_range: unknown type %d", pmap->pm_type));
 1949 
 1950         if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) {
 1951                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1952                         invlpg(addr);
 1953                 if (pmap == PCPU_GET(curpmap) && pmap_pcid_enabled &&
 1954                     pmap->pm_ucr3 != PMAP_NO_CR3) {
 1955                         critical_enter();
 1956                         if (invpcid_works) {
 1957                                 d.pcid = pmap->pm_pcids[0].pm_pcid |
 1958                                     PMAP_PCID_USER_PT;
 1959                                 d.pad = 0;
 1960                                 d.addr = sva;
 1961                                 for (; d.addr < eva; d.addr += PAGE_SIZE)
 1962                                         invpcid(&d, INVPCID_ADDR);
 1963                         } else {
 1964                                 kcr3 = pmap->pm_cr3 | pmap->pm_pcids[0].
 1965                                     pm_pcid | CR3_PCID_SAVE;
 1966                                 ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[0].
 1967                                     pm_pcid | PMAP_PCID_USER_PT | CR3_PCID_SAVE;
 1968                                 pmap_pti_pcid_invlrng(ucr3, kcr3, sva, eva);
 1969                         }
 1970                         critical_exit();
 1971                 }
 1972         } else if (pmap_pcid_enabled) {
 1973                 pmap->pm_pcids[0].pm_gen = 0;
 1974         }
 1975 }
 1976 
 1977 void
 1978 pmap_invalidate_all(pmap_t pmap)
 1979 {
 1980         struct invpcid_descr d;
 1981         uint64_t kcr3, ucr3;
 1982 
 1983         if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) {
 1984                 pmap->pm_eptgen++;
 1985                 return;
 1986         }
 1987         KASSERT(pmap->pm_type == PT_X86,
 1988             ("pmap_invalidate_all: unknown type %d", pmap->pm_type));
 1989 
 1990         if (pmap == kernel_pmap) {
 1991                 if (pmap_pcid_enabled && invpcid_works) {
 1992                         bzero(&d, sizeof(d));
 1993                         invpcid(&d, INVPCID_CTXGLOB);
 1994                 } else {
 1995                         invltlb_glob();
 1996                 }
 1997         } else if (pmap == PCPU_GET(curpmap)) {
 1998                 if (pmap_pcid_enabled) {
 1999                         critical_enter();
 2000                         if (invpcid_works) {
 2001                                 d.pcid = pmap->pm_pcids[0].pm_pcid;
 2002                                 d.pad = 0;
 2003                                 d.addr = 0;
 2004                                 invpcid(&d, INVPCID_CTX);
 2005                                 if (pmap->pm_ucr3 != PMAP_NO_CR3) {
 2006                                         d.pcid |= PMAP_PCID_USER_PT;
 2007                                         invpcid(&d, INVPCID_CTX);
 2008                                 }
 2009                         } else {
 2010                                 kcr3 = pmap->pm_cr3 | pmap->pm_pcids[0].pm_pcid;
 2011                                 if (pmap->pm_ucr3 != PMAP_NO_CR3) {
 2012                                         ucr3 = pmap->pm_ucr3 | pmap->pm_pcids[
 2013                                             0].pm_pcid | PMAP_PCID_USER_PT;
 2014                                         pmap_pti_pcid_invalidate(ucr3, kcr3);
 2015                                 } else
 2016                                         load_cr3(kcr3);
 2017                         }
 2018                         critical_exit();
 2019                 } else {
 2020                         invltlb();
 2021                 }
 2022         } else if (pmap_pcid_enabled) {
 2023                 pmap->pm_pcids[0].pm_gen = 0;
 2024         }
 2025 }
 2026 
 2027 PMAP_INLINE void
 2028 pmap_invalidate_cache(void)
 2029 {
 2030 
 2031         wbinvd();
 2032 }
 2033 
 2034 static void
 2035 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 2036 {
 2037 
 2038         pmap_update_pde_store(pmap, pde, newpde);
 2039         if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap))
 2040                 pmap_update_pde_invalidate(pmap, va, newpde);
 2041         else
 2042                 pmap->pm_pcids[0].pm_gen = 0;
 2043 }
 2044 #endif /* !SMP */
 2045 
 2046 static void
 2047 pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
 2048 {
 2049 
 2050         /*
 2051          * When the PDE has PG_PROMOTED set, the 2MB page mapping was created
 2052          * by a promotion that did not invalidate the 512 4KB page mappings
 2053          * that might exist in the TLB.  Consequently, at this point, the TLB
 2054          * may hold both 4KB and 2MB page mappings for the address range [va,
 2055          * va + NBPDR).  Therefore, the entire range must be invalidated here.
 2056          * In contrast, when PG_PROMOTED is clear, the TLB will not hold any
 2057          * 4KB page mappings for the address range [va, va + NBPDR), and so a
 2058          * single INVLPG suffices to invalidate the 2MB page mapping from the
 2059          * TLB.
 2060          */
 2061         if ((pde & PG_PROMOTED) != 0)
 2062                 pmap_invalidate_range(pmap, va, va + NBPDR - 1);
 2063         else
 2064                 pmap_invalidate_page(pmap, va);
 2065 }
 2066 
 2067 #define PMAP_CLFLUSH_THRESHOLD   (2 * 1024 * 1024)
 2068 
 2069 void
 2070 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force)
 2071 {
 2072 
 2073         if (force) {
 2074                 sva &= ~(vm_offset_t)(cpu_clflush_line_size - 1);
 2075         } else {
 2076                 KASSERT((sva & PAGE_MASK) == 0,
 2077                     ("pmap_invalidate_cache_range: sva not page-aligned"));
 2078                 KASSERT((eva & PAGE_MASK) == 0,
 2079                     ("pmap_invalidate_cache_range: eva not page-aligned"));
 2080         }
 2081 
 2082         if ((cpu_feature & CPUID_SS) != 0 && !force)
 2083                 ; /* If "Self Snoop" is supported and allowed, do nothing. */
 2084         else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0 &&
 2085             eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 2086                 /*
 2087                  * XXX: Some CPUs fault, hang, or trash the local APIC
 2088                  * registers if we use CLFLUSH on the local APIC
 2089                  * range.  The local APIC is always uncached, so we
 2090                  * don't need to flush for that range anyway.
 2091                  */
 2092                 if (pmap_kextract(sva) == lapic_paddr)
 2093                         return;
 2094 
 2095                 /*
 2096                  * Otherwise, do per-cache line flush.  Use the sfence
 2097                  * instruction to insure that previous stores are
 2098                  * included in the write-back.  The processor
 2099                  * propagates flush to other processors in the cache
 2100                  * coherence domain.
 2101                  */
 2102                 sfence();
 2103                 for (; sva < eva; sva += cpu_clflush_line_size)
 2104                         clflushopt(sva);
 2105                 sfence();
 2106         } else if ((cpu_feature & CPUID_CLFSH) != 0 &&
 2107             eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 2108                 if (pmap_kextract(sva) == lapic_paddr)
 2109                         return;
 2110                 /*
 2111                  * Writes are ordered by CLFLUSH on Intel CPUs.
 2112                  */
 2113                 if (cpu_vendor_id != CPU_VENDOR_INTEL)
 2114                         mfence();
 2115                 for (; sva < eva; sva += cpu_clflush_line_size)
 2116                         clflush(sva);
 2117                 if (cpu_vendor_id != CPU_VENDOR_INTEL)
 2118                         mfence();
 2119         } else {
 2120 
 2121                 /*
 2122                  * No targeted cache flush methods are supported by CPU,
 2123                  * or the supplied range is bigger than 2MB.
 2124                  * Globally invalidate cache.
 2125                  */
 2126                 pmap_invalidate_cache();
 2127         }
 2128 }
 2129 
 2130 /*
 2131  * Remove the specified set of pages from the data and instruction caches.
 2132  *
 2133  * In contrast to pmap_invalidate_cache_range(), this function does not
 2134  * rely on the CPU's self-snoop feature, because it is intended for use
 2135  * when moving pages into a different cache domain.
 2136  */
 2137 void
 2138 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
 2139 {
 2140         vm_offset_t daddr, eva;
 2141         int i;
 2142         bool useclflushopt;
 2143 
 2144         useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0;
 2145         if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
 2146             ((cpu_feature & CPUID_CLFSH) == 0 && !useclflushopt))
 2147                 pmap_invalidate_cache();
 2148         else {
 2149                 if (useclflushopt)
 2150                         sfence();
 2151                 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 2152                         mfence();
 2153                 for (i = 0; i < count; i++) {
 2154                         daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i]));
 2155                         eva = daddr + PAGE_SIZE;
 2156                         for (; daddr < eva; daddr += cpu_clflush_line_size) {
 2157                                 if (useclflushopt)
 2158                                         clflushopt(daddr);
 2159                                 else
 2160                                         clflush(daddr);
 2161                         }
 2162                 }
 2163                 if (useclflushopt)
 2164                         sfence();
 2165                 else if (cpu_vendor_id != CPU_VENDOR_INTEL)
 2166                         mfence();
 2167         }
 2168 }
 2169 
 2170 /*
 2171  *      Routine:        pmap_extract
 2172  *      Function:
 2173  *              Extract the physical page address associated
 2174  *              with the given map/virtual_address pair.
 2175  */
 2176 vm_paddr_t 
 2177 pmap_extract(pmap_t pmap, vm_offset_t va)
 2178 {
 2179         pdp_entry_t *pdpe;
 2180         pd_entry_t *pde;
 2181         pt_entry_t *pte, PG_V;
 2182         vm_paddr_t pa;
 2183 
 2184         pa = 0;
 2185         PG_V = pmap_valid_bit(pmap);
 2186         PMAP_LOCK(pmap);
 2187         pdpe = pmap_pdpe(pmap, va);
 2188         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 2189                 if ((*pdpe & PG_PS) != 0)
 2190                         pa = (*pdpe & PG_PS_FRAME) | (va & PDPMASK);
 2191                 else {
 2192                         pde = pmap_pdpe_to_pde(pdpe, va);
 2193                         if ((*pde & PG_V) != 0) {
 2194                                 if ((*pde & PG_PS) != 0) {
 2195                                         pa = (*pde & PG_PS_FRAME) |
 2196                                             (va & PDRMASK);
 2197                                 } else {
 2198                                         pte = pmap_pde_to_pte(pde, va);
 2199                                         pa = (*pte & PG_FRAME) |
 2200                                             (va & PAGE_MASK);
 2201                                 }
 2202                         }
 2203                 }
 2204         }
 2205         PMAP_UNLOCK(pmap);
 2206         return (pa);
 2207 }
 2208 
 2209 /*
 2210  *      Routine:        pmap_extract_and_hold
 2211  *      Function:
 2212  *              Atomically extract and hold the physical page
 2213  *              with the given pmap and virtual address pair
 2214  *              if that mapping permits the given protection.
 2215  */
 2216 vm_page_t
 2217 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 2218 {
 2219         pd_entry_t pde, *pdep;
 2220         pt_entry_t pte, PG_RW, PG_V;
 2221         vm_paddr_t pa;
 2222         vm_page_t m;
 2223 
 2224         pa = 0;
 2225         m = NULL;
 2226         PG_RW = pmap_rw_bit(pmap);
 2227         PG_V = pmap_valid_bit(pmap);
 2228         PMAP_LOCK(pmap);
 2229 retry:
 2230         pdep = pmap_pde(pmap, va);
 2231         if (pdep != NULL && (pde = *pdep)) {
 2232                 if (pde & PG_PS) {
 2233                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
 2234                                 if (vm_page_pa_tryrelock(pmap, (pde &
 2235                                     PG_PS_FRAME) | (va & PDRMASK), &pa))
 2236                                         goto retry;
 2237                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
 2238                                     (va & PDRMASK));
 2239                                 vm_page_hold(m);
 2240                         }
 2241                 } else {
 2242                         pte = *pmap_pde_to_pte(pdep, va);
 2243                         if ((pte & PG_V) &&
 2244                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
 2245                                 if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME,
 2246                                     &pa))
 2247                                         goto retry;
 2248                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
 2249                                 vm_page_hold(m);
 2250                         }
 2251                 }
 2252         }
 2253         PA_UNLOCK_COND(pa);
 2254         PMAP_UNLOCK(pmap);
 2255         return (m);
 2256 }
 2257 
 2258 vm_paddr_t
 2259 pmap_kextract(vm_offset_t va)
 2260 {
 2261         pd_entry_t pde;
 2262         vm_paddr_t pa;
 2263 
 2264         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
 2265                 pa = DMAP_TO_PHYS(va);
 2266         } else {
 2267                 pde = *vtopde(va);
 2268                 if (pde & PG_PS) {
 2269                         pa = (pde & PG_PS_FRAME) | (va & PDRMASK);
 2270                 } else {
 2271                         /*
 2272                          * Beware of a concurrent promotion that changes the
 2273                          * PDE at this point!  For example, vtopte() must not
 2274                          * be used to access the PTE because it would use the
 2275                          * new PDE.  It is, however, safe to use the old PDE
 2276                          * because the page table page is preserved by the
 2277                          * promotion.
 2278                          */
 2279                         pa = *pmap_pde_to_pte(&pde, va);
 2280                         pa = (pa & PG_FRAME) | (va & PAGE_MASK);
 2281                 }
 2282         }
 2283         return (pa);
 2284 }
 2285 
 2286 /***************************************************
 2287  * Low level mapping routines.....
 2288  ***************************************************/
 2289 
 2290 /*
 2291  * Add a wired page to the kva.
 2292  * Note: not SMP coherent.
 2293  */
 2294 PMAP_INLINE void 
 2295 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
 2296 {
 2297         pt_entry_t *pte;
 2298 
 2299         pte = vtopte(va);
 2300         pte_store(pte, pa | X86_PG_RW | X86_PG_V | pg_g);
 2301 }
 2302 
 2303 static __inline void
 2304 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
 2305 {
 2306         pt_entry_t *pte;
 2307         int cache_bits;
 2308 
 2309         pte = vtopte(va);
 2310         cache_bits = pmap_cache_bits(kernel_pmap, mode, 0);
 2311         pte_store(pte, pa | X86_PG_RW | X86_PG_V | pg_g | cache_bits);
 2312 }
 2313 
 2314 /*
 2315  * Remove a page from the kernel pagetables.
 2316  * Note: not SMP coherent.
 2317  */
 2318 PMAP_INLINE void
 2319 pmap_kremove(vm_offset_t va)
 2320 {
 2321         pt_entry_t *pte;
 2322 
 2323         pte = vtopte(va);
 2324         pte_clear(pte);
 2325 }
 2326 
 2327 /*
 2328  *      Used to map a range of physical addresses into kernel
 2329  *      virtual address space.
 2330  *
 2331  *      The value passed in '*virt' is a suggested virtual address for
 2332  *      the mapping. Architectures which can support a direct-mapped
 2333  *      physical to virtual region can return the appropriate address
 2334  *      within that region, leaving '*virt' unchanged. Other
 2335  *      architectures should map the pages starting at '*virt' and
 2336  *      update '*virt' with the first usable address after the mapped
 2337  *      region.
 2338  */
 2339 vm_offset_t
 2340 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 2341 {
 2342         return PHYS_TO_DMAP(start);
 2343 }
 2344 
 2345 
 2346 /*
 2347  * Add a list of wired pages to the kva
 2348  * this routine is only used for temporary
 2349  * kernel mappings that do not need to have
 2350  * page modification or references recorded.
 2351  * Note that old mappings are simply written
 2352  * over.  The page *must* be wired.
 2353  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 2354  */
 2355 void
 2356 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 2357 {
 2358         pt_entry_t *endpte, oldpte, pa, *pte;
 2359         vm_page_t m;
 2360         int cache_bits;
 2361 
 2362         oldpte = 0;
 2363         pte = vtopte(sva);
 2364         endpte = pte + count;
 2365         while (pte < endpte) {
 2366                 m = *ma++;
 2367                 cache_bits = pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0);
 2368                 pa = VM_PAGE_TO_PHYS(m) | cache_bits;
 2369                 if ((*pte & (PG_FRAME | X86_PG_PTE_CACHE)) != pa) {
 2370                         oldpte |= *pte;
 2371                         pte_store(pte, pa | pg_g | X86_PG_RW | X86_PG_V);
 2372                 }
 2373                 pte++;
 2374         }
 2375         if (__predict_false((oldpte & X86_PG_V) != 0))
 2376                 pmap_invalidate_range(kernel_pmap, sva, sva + count *
 2377                     PAGE_SIZE);
 2378 }
 2379 
 2380 /*
 2381  * This routine tears out page mappings from the
 2382  * kernel -- it is meant only for temporary mappings.
 2383  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 2384  */
 2385 void
 2386 pmap_qremove(vm_offset_t sva, int count)
 2387 {
 2388         vm_offset_t va;
 2389 
 2390         va = sva;
 2391         while (count-- > 0) {
 2392                 KASSERT(va >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", va));
 2393                 pmap_kremove(va);
 2394                 va += PAGE_SIZE;
 2395         }
 2396         pmap_invalidate_range(kernel_pmap, sva, va);
 2397 }
 2398 
 2399 /***************************************************
 2400  * Page table page management routines.....
 2401  ***************************************************/
 2402 static __inline void
 2403 pmap_free_zero_pages(struct spglist *free)
 2404 {
 2405         vm_page_t m;
 2406         int count;
 2407 
 2408         for (count = 0; (m = SLIST_FIRST(free)) != NULL; count++) {
 2409                 SLIST_REMOVE_HEAD(free, plinks.s.ss);
 2410                 /* Preserve the page's PG_ZERO setting. */
 2411                 vm_page_free_toq(m);
 2412         }
 2413         atomic_subtract_int(&vm_cnt.v_wire_count, count);
 2414 }
 2415 
 2416 /*
 2417  * Schedule the specified unused page table page to be freed.  Specifically,
 2418  * add the page to the specified list of pages that will be released to the
 2419  * physical memory manager after the TLB has been updated.
 2420  */
 2421 static __inline void
 2422 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
 2423     boolean_t set_PG_ZERO)
 2424 {
 2425 
 2426         if (set_PG_ZERO)
 2427                 m->flags |= PG_ZERO;
 2428         else
 2429                 m->flags &= ~PG_ZERO;
 2430         SLIST_INSERT_HEAD(free, m, plinks.s.ss);
 2431 }
 2432         
 2433 /*
 2434  * Inserts the specified page table page into the specified pmap's collection
 2435  * of idle page table pages.  Each of a pmap's page table pages is responsible
 2436  * for mapping a distinct range of virtual addresses.  The pmap's collection is
 2437  * ordered by this virtual address range.
 2438  */
 2439 static __inline int
 2440 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
 2441 {
 2442 
 2443         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2444         return (vm_radix_insert(&pmap->pm_root, mpte));
 2445 }
 2446 
 2447 /*
 2448  * Removes the page table page mapping the specified virtual address from the
 2449  * specified pmap's collection of idle page table pages, and returns it.
 2450  * Otherwise, returns NULL if there is no page table page corresponding to the
 2451  * specified virtual address.
 2452  */
 2453 static __inline vm_page_t
 2454 pmap_remove_pt_page(pmap_t pmap, vm_offset_t va)
 2455 {
 2456 
 2457         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2458         return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va)));
 2459 }
 2460 
 2461 /*
 2462  * Decrements a page table page's wire count, which is used to record the
 2463  * number of valid page table entries within the page.  If the wire count
 2464  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
 2465  * page table page was unmapped and FALSE otherwise.
 2466  */
 2467 static inline boolean_t
 2468 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
 2469 {
 2470 
 2471         --m->wire_count;
 2472         if (m->wire_count == 0) {
 2473                 _pmap_unwire_ptp(pmap, va, m, free);
 2474                 return (TRUE);
 2475         } else
 2476                 return (FALSE);
 2477 }
 2478 
 2479 static void
 2480 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free)
 2481 {
 2482 
 2483         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2484         /*
 2485          * unmap the page table page
 2486          */
 2487         if (m->pindex >= (NUPDE + NUPDPE)) {
 2488                 /* PDP page */
 2489                 pml4_entry_t *pml4;
 2490                 pml4 = pmap_pml4e(pmap, va);
 2491                 *pml4 = 0;
 2492                 if (pmap->pm_pml4u != NULL && va <= VM_MAXUSER_ADDRESS) {
 2493                         pml4 = &pmap->pm_pml4u[pmap_pml4e_index(va)];
 2494                         *pml4 = 0;
 2495                 }
 2496         } else if (m->pindex >= NUPDE) {
 2497                 /* PD page */
 2498                 pdp_entry_t *pdp;
 2499                 pdp = pmap_pdpe(pmap, va);
 2500                 *pdp = 0;
 2501         } else {
 2502                 /* PTE page */
 2503                 pd_entry_t *pd;
 2504                 pd = pmap_pde(pmap, va);
 2505                 *pd = 0;
 2506         }
 2507         pmap_resident_count_dec(pmap, 1);
 2508         if (m->pindex < NUPDE) {
 2509                 /* We just released a PT, unhold the matching PD */
 2510                 vm_page_t pdpg;
 2511 
 2512                 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
 2513                 pmap_unwire_ptp(pmap, va, pdpg, free);
 2514         }
 2515         if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 2516                 /* We just released a PD, unhold the matching PDP */
 2517                 vm_page_t pdppg;
 2518 
 2519                 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
 2520                 pmap_unwire_ptp(pmap, va, pdppg, free);
 2521         }
 2522 
 2523         /* 
 2524          * Put page on a list so that it is released after
 2525          * *ALL* TLB shootdown is done
 2526          */
 2527         pmap_add_delayed_free_list(m, free, TRUE);
 2528 }
 2529 
 2530 /*
 2531  * After removing a page table entry, this routine is used to
 2532  * conditionally free the page, and manage the hold/wire counts.
 2533  */
 2534 static int
 2535 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde,
 2536     struct spglist *free)
 2537 {
 2538         vm_page_t mpte;
 2539 
 2540         if (va >= VM_MAXUSER_ADDRESS)
 2541                 return (0);
 2542         KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 2543         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 2544         return (pmap_unwire_ptp(pmap, va, mpte, free));
 2545 }
 2546 
 2547 void
 2548 pmap_pinit0(pmap_t pmap)
 2549 {
 2550         int i;
 2551 
 2552         PMAP_LOCK_INIT(pmap);
 2553         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
 2554         pmap->pm_pml4u = NULL;
 2555         pmap->pm_cr3 = KPML4phys;
 2556         /* hack to keep pmap_pti_pcid_invalidate() alive */
 2557         pmap->pm_ucr3 = PMAP_NO_CR3;
 2558         pmap->pm_root.rt_root = 0;
 2559         CPU_ZERO(&pmap->pm_active);
 2560         TAILQ_INIT(&pmap->pm_pvchunk);
 2561         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 2562         pmap->pm_flags = pmap_flags;
 2563         CPU_FOREACH(i) {
 2564                 pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
 2565                 pmap->pm_pcids[i].pm_gen = 0;
 2566                 if (!pti)
 2567                         __pcpu[i].pc_kcr3 = PMAP_NO_CR3;
 2568         }
 2569         PCPU_SET(curpmap, kernel_pmap);
 2570         pmap_activate(curthread);
 2571         CPU_FILL(&kernel_pmap->pm_active);
 2572 }
 2573 
 2574 void
 2575 pmap_pinit_pml4(vm_page_t pml4pg)
 2576 {
 2577         pml4_entry_t *pm_pml4;
 2578         int i;
 2579 
 2580         pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
 2581 
 2582         /* Wire in kernel global address entries. */
 2583         for (i = 0; i < NKPML4E; i++) {
 2584                 pm_pml4[KPML4BASE + i] = (KPDPphys + ptoa(i)) | X86_PG_RW |
 2585                     X86_PG_V;
 2586         }
 2587         for (i = 0; i < ndmpdpphys; i++) {
 2588                 pm_pml4[DMPML4I + i] = (DMPDPphys + ptoa(i)) | X86_PG_RW |
 2589                     X86_PG_V;
 2590         }
 2591 
 2592         /* install self-referential address mapping entry(s) */
 2593         pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | X86_PG_V | X86_PG_RW |
 2594             X86_PG_A | X86_PG_M;
 2595 }
 2596 
 2597 static void
 2598 pmap_pinit_pml4_pti(vm_page_t pml4pg)
 2599 {
 2600         pml4_entry_t *pm_pml4;
 2601         int i;
 2602 
 2603         pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
 2604         for (i = 0; i < NPML4EPG; i++)
 2605                 pm_pml4[i] = pti_pml4[i];
 2606 }
 2607 
 2608 /*
 2609  * Initialize a preallocated and zeroed pmap structure,
 2610  * such as one in a vmspace structure.
 2611  */
 2612 int
 2613 pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags)
 2614 {
 2615         vm_page_t pml4pg, pml4pgu;
 2616         vm_paddr_t pml4phys;
 2617         int i;
 2618 
 2619         /*
 2620          * allocate the page directory page
 2621          */
 2622         pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 2623             VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK);
 2624 
 2625         pml4phys = VM_PAGE_TO_PHYS(pml4pg);
 2626         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys);
 2627         CPU_FOREACH(i) {
 2628                 pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
 2629                 pmap->pm_pcids[i].pm_gen = 0;
 2630         }
 2631         pmap->pm_cr3 = PMAP_NO_CR3;     /* initialize to an invalid value */
 2632         pmap->pm_ucr3 = PMAP_NO_CR3;
 2633         pmap->pm_pml4u = NULL;
 2634 
 2635         pmap->pm_type = pm_type;
 2636         if ((pml4pg->flags & PG_ZERO) == 0)
 2637                 pagezero(pmap->pm_pml4);
 2638 
 2639         /*
 2640          * Do not install the host kernel mappings in the nested page
 2641          * tables. These mappings are meaningless in the guest physical
 2642          * address space.
 2643          * Install minimal kernel mappings in PTI case.
 2644          */
 2645         if (pm_type == PT_X86) {
 2646                 pmap->pm_cr3 = pml4phys;
 2647                 pmap_pinit_pml4(pml4pg);
 2648                 if (pti) {
 2649                         pml4pgu = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 2650                             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_WAITOK);
 2651                         pmap->pm_pml4u = (pml4_entry_t *)PHYS_TO_DMAP(
 2652                             VM_PAGE_TO_PHYS(pml4pgu));
 2653                         pmap_pinit_pml4_pti(pml4pgu);
 2654                         pmap->pm_ucr3 = VM_PAGE_TO_PHYS(pml4pgu);
 2655                 }
 2656         }
 2657 
 2658         pmap->pm_root.rt_root = 0;
 2659         CPU_ZERO(&pmap->pm_active);
 2660         TAILQ_INIT(&pmap->pm_pvchunk);
 2661         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 2662         pmap->pm_flags = flags;
 2663         pmap->pm_eptgen = 0;
 2664 
 2665         return (1);
 2666 }
 2667 
 2668 int
 2669 pmap_pinit(pmap_t pmap)
 2670 {
 2671 
 2672         return (pmap_pinit_type(pmap, PT_X86, pmap_flags));
 2673 }
 2674 
 2675 /*
 2676  * This routine is called if the desired page table page does not exist.
 2677  *
 2678  * If page table page allocation fails, this routine may sleep before
 2679  * returning NULL.  It sleeps only if a lock pointer was given.
 2680  *
 2681  * Note: If a page allocation fails at page table level two or three,
 2682  * one or two pages may be held during the wait, only to be released
 2683  * afterwards.  This conservative approach is easily argued to avoid
 2684  * race conditions.
 2685  */
 2686 static vm_page_t
 2687 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
 2688 {
 2689         vm_page_t m, pdppg, pdpg;
 2690         pt_entry_t PG_A, PG_M, PG_RW, PG_V;
 2691 
 2692         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2693 
 2694         PG_A = pmap_accessed_bit(pmap);
 2695         PG_M = pmap_modified_bit(pmap);
 2696         PG_V = pmap_valid_bit(pmap);
 2697         PG_RW = pmap_rw_bit(pmap);
 2698 
 2699         /*
 2700          * Allocate a page table page.
 2701          */
 2702         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 2703             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 2704                 if (lockp != NULL) {
 2705                         RELEASE_PV_LIST_LOCK(lockp);
 2706                         PMAP_UNLOCK(pmap);
 2707                         PMAP_ASSERT_NOT_IN_DI();
 2708                         VM_WAIT;
 2709                         PMAP_LOCK(pmap);
 2710                 }
 2711 
 2712                 /*
 2713                  * Indicate the need to retry.  While waiting, the page table
 2714                  * page may have been allocated.
 2715                  */
 2716                 return (NULL);
 2717         }
 2718         if ((m->flags & PG_ZERO) == 0)
 2719                 pmap_zero_page(m);
 2720 
 2721         /*
 2722          * Map the pagetable page into the process address space, if
 2723          * it isn't already there.
 2724          */
 2725 
 2726         if (ptepindex >= (NUPDE + NUPDPE)) {
 2727                 pml4_entry_t *pml4, *pml4u;
 2728                 vm_pindex_t pml4index;
 2729 
 2730                 /* Wire up a new PDPE page */
 2731                 pml4index = ptepindex - (NUPDE + NUPDPE);
 2732                 pml4 = &pmap->pm_pml4[pml4index];
 2733                 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 2734                 if (pmap->pm_pml4u != NULL && pml4index < NUPML4E) {
 2735                         /*
 2736                          * PTI: Make all user-space mappings in the
 2737                          * kernel-mode page table no-execute so that
 2738                          * we detect any programming errors that leave
 2739                          * the kernel-mode page table active on return
 2740                          * to user space.
 2741                          */
 2742                         *pml4 |= pg_nx;
 2743 
 2744                         pml4u = &pmap->pm_pml4u[pml4index];
 2745                         *pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V |
 2746                             PG_A | PG_M;
 2747                 }
 2748 
 2749         } else if (ptepindex >= NUPDE) {
 2750                 vm_pindex_t pml4index;
 2751                 vm_pindex_t pdpindex;
 2752                 pml4_entry_t *pml4;
 2753                 pdp_entry_t *pdp;
 2754 
 2755                 /* Wire up a new PDE page */
 2756                 pdpindex = ptepindex - NUPDE;
 2757                 pml4index = pdpindex >> NPML4EPGSHIFT;
 2758 
 2759                 pml4 = &pmap->pm_pml4[pml4index];
 2760                 if ((*pml4 & PG_V) == 0) {
 2761                         /* Have to allocate a new pdp, recurse */
 2762                         if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
 2763                             lockp) == NULL) {
 2764                                 --m->wire_count;
 2765                                 atomic_subtract_int(&vm_cnt.v_wire_count, 1);
 2766                                 vm_page_free_zero(m);
 2767                                 return (NULL);
 2768                         }
 2769                 } else {
 2770                         /* Add reference to pdp page */
 2771                         pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
 2772                         pdppg->wire_count++;
 2773                 }
 2774                 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 2775 
 2776                 /* Now find the pdp page */
 2777                 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 2778                 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 2779 
 2780         } else {
 2781                 vm_pindex_t pml4index;
 2782                 vm_pindex_t pdpindex;
 2783                 pml4_entry_t *pml4;
 2784                 pdp_entry_t *pdp;
 2785                 pd_entry_t *pd;
 2786 
 2787                 /* Wire up a new PTE page */
 2788                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 2789                 pml4index = pdpindex >> NPML4EPGSHIFT;
 2790 
 2791                 /* First, find the pdp and check that its valid. */
 2792                 pml4 = &pmap->pm_pml4[pml4index];
 2793                 if ((*pml4 & PG_V) == 0) {
 2794                         /* Have to allocate a new pd, recurse */
 2795                         if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 2796                             lockp) == NULL) {
 2797                                 --m->wire_count;
 2798                                 atomic_subtract_int(&vm_cnt.v_wire_count, 1);
 2799                                 vm_page_free_zero(m);
 2800                                 return (NULL);
 2801                         }
 2802                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 2803                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 2804                 } else {
 2805                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 2806                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 2807                         if ((*pdp & PG_V) == 0) {
 2808                                 /* Have to allocate a new pd, recurse */
 2809                                 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 2810                                     lockp) == NULL) {
 2811                                         --m->wire_count;
 2812                                         atomic_subtract_int(&vm_cnt.v_wire_count,
 2813                                             1);
 2814                                         vm_page_free_zero(m);
 2815                                         return (NULL);
 2816                                 }
 2817                         } else {
 2818                                 /* Add reference to the pd page */
 2819                                 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
 2820                                 pdpg->wire_count++;
 2821                         }
 2822                 }
 2823                 pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
 2824 
 2825                 /* Now we know where the page directory page is */
 2826                 pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
 2827                 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 2828         }
 2829 
 2830         pmap_resident_count_inc(pmap, 1);
 2831 
 2832         return (m);
 2833 }
 2834 
 2835 static vm_page_t
 2836 pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
 2837 {
 2838         vm_pindex_t pdpindex, ptepindex;
 2839         pdp_entry_t *pdpe, PG_V;
 2840         vm_page_t pdpg;
 2841 
 2842         PG_V = pmap_valid_bit(pmap);
 2843 
 2844 retry:
 2845         pdpe = pmap_pdpe(pmap, va);
 2846         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 2847                 /* Add a reference to the pd page. */
 2848                 pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
 2849                 pdpg->wire_count++;
 2850         } else {
 2851                 /* Allocate a pd page. */
 2852                 ptepindex = pmap_pde_pindex(va);
 2853                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 2854                 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
 2855                 if (pdpg == NULL && lockp != NULL)
 2856                         goto retry;
 2857         }
 2858         return (pdpg);
 2859 }
 2860 
 2861 static vm_page_t
 2862 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
 2863 {
 2864         vm_pindex_t ptepindex;
 2865         pd_entry_t *pd, PG_V;
 2866         vm_page_t m;
 2867 
 2868         PG_V = pmap_valid_bit(pmap);
 2869 
 2870         /*
 2871          * Calculate pagetable page index
 2872          */
 2873         ptepindex = pmap_pde_pindex(va);
 2874 retry:
 2875         /*
 2876          * Get the page directory entry
 2877          */
 2878         pd = pmap_pde(pmap, va);
 2879 
 2880         /*
 2881          * This supports switching from a 2MB page to a
 2882          * normal 4K page.
 2883          */
 2884         if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
 2885                 if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) {
 2886                         /*
 2887                          * Invalidation of the 2MB page mapping may have caused
 2888                          * the deallocation of the underlying PD page.
 2889                          */
 2890                         pd = NULL;
 2891                 }
 2892         }
 2893 
 2894         /*
 2895          * If the page table page is mapped, we just increment the
 2896          * hold count, and activate it.
 2897          */
 2898         if (pd != NULL && (*pd & PG_V) != 0) {
 2899                 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
 2900                 m->wire_count++;
 2901         } else {
 2902                 /*
 2903                  * Here if the pte page isn't mapped, or if it has been
 2904                  * deallocated.
 2905                  */
 2906                 m = _pmap_allocpte(pmap, ptepindex, lockp);
 2907                 if (m == NULL && lockp != NULL)
 2908                         goto retry;
 2909         }
 2910         return (m);
 2911 }
 2912 
 2913 
 2914 /***************************************************
 2915  * Pmap allocation/deallocation routines.
 2916  ***************************************************/
 2917 
 2918 /*
 2919  * Release any resources held by the given physical map.
 2920  * Called when a pmap initialized by pmap_pinit is being released.
 2921  * Should only be called if the map contains no valid mappings.
 2922  */
 2923 void
 2924 pmap_release(pmap_t pmap)
 2925 {
 2926         vm_page_t m;
 2927         int i;
 2928 
 2929         KASSERT(pmap->pm_stats.resident_count == 0,
 2930             ("pmap_release: pmap resident count %ld != 0",
 2931             pmap->pm_stats.resident_count));
 2932         KASSERT(vm_radix_is_empty(&pmap->pm_root),
 2933             ("pmap_release: pmap has reserved page table page(s)"));
 2934         KASSERT(CPU_EMPTY(&pmap->pm_active),
 2935             ("releasing active pmap %p", pmap));
 2936 
 2937         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4));
 2938 
 2939         for (i = 0; i < NKPML4E; i++)   /* KVA */
 2940                 pmap->pm_pml4[KPML4BASE + i] = 0;
 2941         for (i = 0; i < ndmpdpphys; i++)/* Direct Map */
 2942                 pmap->pm_pml4[DMPML4I + i] = 0;
 2943         pmap->pm_pml4[PML4PML4I] = 0;   /* Recursive Mapping */
 2944 
 2945         m->wire_count--;
 2946         atomic_subtract_int(&vm_cnt.v_wire_count, 1);
 2947         vm_page_free_zero(m);
 2948 
 2949         if (pmap->pm_pml4u != NULL) {
 2950                 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4u));
 2951                 m->wire_count--;
 2952                 atomic_subtract_int(&vm_cnt.v_wire_count, 1);
 2953                 vm_page_free(m);
 2954         }
 2955 }
 2956 
 2957 static int
 2958 kvm_size(SYSCTL_HANDLER_ARGS)
 2959 {
 2960         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
 2961 
 2962         return sysctl_handle_long(oidp, &ksize, 0, req);
 2963 }
 2964 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 2965     0, 0, kvm_size, "LU", "Size of KVM");
 2966 
 2967 static int
 2968 kvm_free(SYSCTL_HANDLER_ARGS)
 2969 {
 2970         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 2971 
 2972         return sysctl_handle_long(oidp, &kfree, 0, req);
 2973 }
 2974 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 2975     0, 0, kvm_free, "LU", "Amount of KVM free");
 2976 
 2977 /*
 2978  * grow the number of kernel page table entries, if needed
 2979  */
 2980 void
 2981 pmap_growkernel(vm_offset_t addr)
 2982 {
 2983         vm_paddr_t paddr;
 2984         vm_page_t nkpg;
 2985         pd_entry_t *pde, newpdir;
 2986         pdp_entry_t *pdpe;
 2987 
 2988         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 2989 
 2990         /*
 2991          * Return if "addr" is within the range of kernel page table pages
 2992          * that were preallocated during pmap bootstrap.  Moreover, leave
 2993          * "kernel_vm_end" and the kernel page table as they were.
 2994          *
 2995          * The correctness of this action is based on the following
 2996          * argument: vm_map_insert() allocates contiguous ranges of the
 2997          * kernel virtual address space.  It calls this function if a range
 2998          * ends after "kernel_vm_end".  If the kernel is mapped between
 2999          * "kernel_vm_end" and "addr", then the range cannot begin at
 3000          * "kernel_vm_end".  In fact, its beginning address cannot be less
 3001          * than the kernel.  Thus, there is no immediate need to allocate
 3002          * any new kernel page table pages between "kernel_vm_end" and
 3003          * "KERNBASE".
 3004          */
 3005         if (KERNBASE < addr && addr <= KERNBASE + nkpt * NBPDR)
 3006                 return;
 3007 
 3008         addr = roundup2(addr, NBPDR);
 3009         if (addr - 1 >= kernel_map->max_offset)
 3010                 addr = kernel_map->max_offset;
 3011         while (kernel_vm_end < addr) {
 3012                 pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end);
 3013                 if ((*pdpe & X86_PG_V) == 0) {
 3014                         /* We need a new PDP entry */
 3015                         nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT,
 3016                             VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
 3017                             VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 3018                         if (nkpg == NULL)
 3019                                 panic("pmap_growkernel: no memory to grow kernel");
 3020                         if ((nkpg->flags & PG_ZERO) == 0)
 3021                                 pmap_zero_page(nkpg);
 3022                         paddr = VM_PAGE_TO_PHYS(nkpg);
 3023                         *pdpe = (pdp_entry_t)(paddr | X86_PG_V | X86_PG_RW |
 3024                             X86_PG_A | X86_PG_M);
 3025                         continue; /* try again */
 3026                 }
 3027                 pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end);
 3028                 if ((*pde & X86_PG_V) != 0) {
 3029                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 3030                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 3031                                 kernel_vm_end = kernel_map->max_offset;
 3032                                 break;                       
 3033                         }
 3034                         continue;
 3035                 }
 3036 
 3037                 nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end),
 3038                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 3039                     VM_ALLOC_ZERO);
 3040                 if (nkpg == NULL)
 3041                         panic("pmap_growkernel: no memory to grow kernel");
 3042                 if ((nkpg->flags & PG_ZERO) == 0)
 3043                         pmap_zero_page(nkpg);
 3044                 paddr = VM_PAGE_TO_PHYS(nkpg);
 3045                 newpdir = paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M;
 3046                 pde_store(pde, newpdir);
 3047 
 3048                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 3049                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 3050                         kernel_vm_end = kernel_map->max_offset;
 3051                         break;                       
 3052                 }
 3053         }
 3054 }
 3055 
 3056 
 3057 /***************************************************
 3058  * page management routines.
 3059  ***************************************************/
 3060 
 3061 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 3062 CTASSERT(_NPCM == 3);
 3063 CTASSERT(_NPCPV == 168);
 3064 
 3065 static __inline struct pv_chunk *
 3066 pv_to_chunk(pv_entry_t pv)
 3067 {
 3068 
 3069         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
 3070 }
 3071 
 3072 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 3073 
 3074 #define PC_FREE0        0xfffffffffffffffful
 3075 #define PC_FREE1        0xfffffffffffffffful
 3076 #define PC_FREE2        0x000000fffffffffful
 3077 
 3078 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
 3079 
 3080 #ifdef PV_STATS
 3081 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 3082 
 3083 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 3084         "Current number of pv entry chunks");
 3085 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 3086         "Current number of pv entry chunks allocated");
 3087 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 3088         "Current number of pv entry chunks frees");
 3089 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 3090         "Number of times tried to get a chunk page but failed.");
 3091 
 3092 static long pv_entry_frees, pv_entry_allocs, pv_entry_count;
 3093 static int pv_entry_spare;
 3094 
 3095 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 3096         "Current number of pv entry frees");
 3097 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 3098         "Current number of pv entry allocs");
 3099 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 3100         "Current number of pv entries");
 3101 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 3102         "Current number of spare pv entries");
 3103 #endif
 3104 
 3105 static void
 3106 reclaim_pv_chunk_leave_pmap(pmap_t pmap, pmap_t locked_pmap, bool start_di)
 3107 {
 3108 
 3109         if (pmap == NULL)
 3110                 return;
 3111         pmap_invalidate_all(pmap);
 3112         if (pmap != locked_pmap)
 3113                 PMAP_UNLOCK(pmap);
 3114         if (start_di)
 3115                 pmap_delayed_invl_finished();
 3116 }
 3117 
 3118 /*
 3119  * We are in a serious low memory condition.  Resort to
 3120  * drastic measures to free some pages so we can allocate
 3121  * another pv entry chunk.
 3122  *
 3123  * Returns NULL if PV entries were reclaimed from the specified pmap.
 3124  *
 3125  * We do not, however, unmap 2mpages because subsequent accesses will
 3126  * allocate per-page pv entries until repromotion occurs, thereby
 3127  * exacerbating the shortage of free pv entries.
 3128  */
 3129 static vm_page_t
 3130 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
 3131 {
 3132         struct pv_chunk *pc, *pc_marker, *pc_marker_end;
 3133         struct pv_chunk_header pc_marker_b, pc_marker_end_b;
 3134         struct md_page *pvh;
 3135         pd_entry_t *pde;
 3136         pmap_t next_pmap, pmap;
 3137         pt_entry_t *pte, tpte;
 3138         pt_entry_t PG_G, PG_A, PG_M, PG_RW;
 3139         pv_entry_t pv;
 3140         vm_offset_t va;
 3141         vm_page_t m, m_pc;
 3142         struct spglist free;
 3143         uint64_t inuse;
 3144         int bit, field, freed;
 3145         bool start_di;
 3146         static int active_reclaims = 0;
 3147 
 3148         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
 3149         KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
 3150         pmap = NULL;
 3151         m_pc = NULL;
 3152         PG_G = PG_A = PG_M = PG_RW = 0;
 3153         SLIST_INIT(&free);
 3154         bzero(&pc_marker_b, sizeof(pc_marker_b));
 3155         bzero(&pc_marker_end_b, sizeof(pc_marker_end_b));
 3156         pc_marker = (struct pv_chunk *)&pc_marker_b;
 3157         pc_marker_end = (struct pv_chunk *)&pc_marker_end_b;
 3158 
 3159         /*
 3160          * A delayed invalidation block should already be active if
 3161          * pmap_advise() or pmap_remove() called this function by way
 3162          * of pmap_demote_pde_locked().
 3163          */
 3164         start_di = pmap_not_in_di();
 3165 
 3166         mtx_lock(&pv_chunks_mutex);
 3167         active_reclaims++;
 3168         TAILQ_INSERT_HEAD(&pv_chunks, pc_marker, pc_lru);
 3169         TAILQ_INSERT_TAIL(&pv_chunks, pc_marker_end, pc_lru);
 3170         while ((pc = TAILQ_NEXT(pc_marker, pc_lru)) != pc_marker_end &&
 3171             SLIST_EMPTY(&free)) {
 3172                 next_pmap = pc->pc_pmap;
 3173                 if (next_pmap == NULL) {
 3174                         /*
 3175                          * The next chunk is a marker.  However, it is
 3176                          * not our marker, so active_reclaims must be
 3177                          * > 1.  Consequently, the next_chunk code
 3178                          * will not rotate the pv_chunks list.
 3179                          */
 3180                         goto next_chunk;
 3181                 }
 3182                 mtx_unlock(&pv_chunks_mutex);
 3183 
 3184                 /*
 3185                  * A pv_chunk can only be removed from the pc_lru list
 3186                  * when both pc_chunks_mutex is owned and the
 3187                  * corresponding pmap is locked.
 3188                  */
 3189                 if (pmap != next_pmap) {
 3190                         reclaim_pv_chunk_leave_pmap(pmap, locked_pmap,
 3191                             start_di);
 3192                         pmap = next_pmap;
 3193                         /* Avoid deadlock and lock recursion. */
 3194                         if (pmap > locked_pmap) {
 3195                                 RELEASE_PV_LIST_LOCK(lockp);
 3196                                 PMAP_LOCK(pmap);
 3197                                 if (start_di)
 3198                                         pmap_delayed_invl_started();
 3199                                 mtx_lock(&pv_chunks_mutex);
 3200                                 continue;
 3201                         } else if (pmap != locked_pmap) {
 3202                                 if (PMAP_TRYLOCK(pmap)) {
 3203                                         if (start_di)
 3204                                                 pmap_delayed_invl_started();
 3205                                         mtx_lock(&pv_chunks_mutex);
 3206                                         continue;
 3207                                 } else {
 3208                                         pmap = NULL; /* pmap is not locked */
 3209                                         mtx_lock(&pv_chunks_mutex);
 3210                                         pc = TAILQ_NEXT(pc_marker, pc_lru);
 3211                                         if (pc == NULL ||
 3212                                             pc->pc_pmap != next_pmap)
 3213                                                 continue;
 3214                                         goto next_chunk;
 3215                                 }
 3216                         } else if (start_di)
 3217                                 pmap_delayed_invl_started();
 3218                         PG_G = pmap_global_bit(pmap);
 3219                         PG_A = pmap_accessed_bit(pmap);
 3220                         PG_M = pmap_modified_bit(pmap);
 3221                         PG_RW = pmap_rw_bit(pmap);
 3222                 }
 3223 
 3224                 /*
 3225                  * Destroy every non-wired, 4 KB page mapping in the chunk.
 3226                  */
 3227                 freed = 0;
 3228                 for (field = 0; field < _NPCM; field++) {
 3229                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
 3230                             inuse != 0; inuse &= ~(1UL << bit)) {
 3231                                 bit = bsfq(inuse);
 3232                                 pv = &pc->pc_pventry[field * 64 + bit];
 3233                                 va = pv->pv_va;
 3234                                 pde = pmap_pde(pmap, va);
 3235                                 if ((*pde & PG_PS) != 0)
 3236                                         continue;
 3237                                 pte = pmap_pde_to_pte(pde, va);
 3238                                 if ((*pte & PG_W) != 0)
 3239                                         continue;
 3240                                 tpte = pte_load_clear(pte);
 3241                                 if ((tpte & PG_G) != 0)
 3242                                         pmap_invalidate_page(pmap, va);
 3243                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 3244                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3245                                         vm_page_dirty(m);
 3246                                 if ((tpte & PG_A) != 0)
 3247                                         vm_page_aflag_set(m, PGA_REFERENCED);
 3248                                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 3249                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 3250                                 m->md.pv_gen++;
 3251                                 if (TAILQ_EMPTY(&m->md.pv_list) &&
 3252                                     (m->flags & PG_FICTITIOUS) == 0) {
 3253                                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 3254                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 3255                                                 vm_page_aflag_clear(m,
 3256                                                     PGA_WRITEABLE);
 3257                                         }
 3258                                 }
 3259                                 pmap_delayed_invl_page(m);
 3260                                 pc->pc_map[field] |= 1UL << bit;
 3261                                 pmap_unuse_pt(pmap, va, *pde, &free);
 3262                                 freed++;
 3263                         }
 3264                 }
 3265                 if (freed == 0) {
 3266                         mtx_lock(&pv_chunks_mutex);
 3267                         goto next_chunk;
 3268                 }
 3269                 /* Every freed mapping is for a 4 KB page. */
 3270                 pmap_resident_count_dec(pmap, freed);
 3271                 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
 3272                 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
 3273                 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
 3274                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3275                 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 &&
 3276                     pc->pc_map[2] == PC_FREE2) {
 3277                         PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
 3278                         PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
 3279                         PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
 3280                         /* Entire chunk is free; return it. */
 3281                         m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 3282                         dump_drop_page(m_pc->phys_addr);
 3283                         mtx_lock(&pv_chunks_mutex);
 3284                         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 3285                         break;
 3286                 }
 3287                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 3288                 mtx_lock(&pv_chunks_mutex);
 3289                 /* One freed pv entry in locked_pmap is sufficient. */
 3290                 if (pmap == locked_pmap)
 3291                         break;
 3292 next_chunk:
 3293                 TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
 3294                 TAILQ_INSERT_AFTER(&pv_chunks, pc, pc_marker, pc_lru);
 3295                 if (active_reclaims == 1 && pmap != NULL) {
 3296                         /*
 3297                          * Rotate the pv chunks list so that we do not
 3298                          * scan the same pv chunks that could not be
 3299                          * freed (because they contained a wired
 3300                          * and/or superpage mapping) on every
 3301                          * invocation of reclaim_pv_chunk().
 3302                          */
 3303                         while ((pc = TAILQ_FIRST(&pv_chunks)) != pc_marker) {
 3304                                 MPASS(pc->pc_pmap != NULL);
 3305                                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 3306                                 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 3307                         }
 3308                 }
 3309         }
 3310         TAILQ_REMOVE(&pv_chunks, pc_marker, pc_lru);
 3311         TAILQ_REMOVE(&pv_chunks, pc_marker_end, pc_lru);
 3312         active_reclaims--;
 3313         mtx_unlock(&pv_chunks_mutex);
 3314         reclaim_pv_chunk_leave_pmap(pmap, locked_pmap, start_di);
 3315         if (m_pc == NULL && !SLIST_EMPTY(&free)) {
 3316                 m_pc = SLIST_FIRST(&free);
 3317                 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
 3318                 /* Recycle a freed page table page. */
 3319                 m_pc->wire_count = 1;
 3320         }
 3321         pmap_free_zero_pages(&free);
 3322         return (m_pc);
 3323 }
 3324 
 3325 /*
 3326  * free the pv_entry back to the free list
 3327  */
 3328 static void
 3329 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 3330 {
 3331         struct pv_chunk *pc;
 3332         int idx, field, bit;
 3333 
 3334         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3335         PV_STAT(atomic_add_long(&pv_entry_frees, 1));
 3336         PV_STAT(atomic_add_int(&pv_entry_spare, 1));
 3337         PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
 3338         pc = pv_to_chunk(pv);
 3339         idx = pv - &pc->pc_pventry[0];
 3340         field = idx / 64;
 3341         bit = idx % 64;
 3342         pc->pc_map[field] |= 1ul << bit;
 3343         if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 ||
 3344             pc->pc_map[2] != PC_FREE2) {
 3345                 /* 98% of the time, pc is already at the head of the list. */
 3346                 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
 3347                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3348                         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 3349                 }
 3350                 return;
 3351         }
 3352         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3353         free_pv_chunk(pc);
 3354 }
 3355 
 3356 static void
 3357 free_pv_chunk(struct pv_chunk *pc)
 3358 {
 3359         vm_page_t m;
 3360 
 3361         mtx_lock(&pv_chunks_mutex);
 3362         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 3363         mtx_unlock(&pv_chunks_mutex);
 3364         PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
 3365         PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
 3366         PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
 3367         /* entire chunk is free, return it */
 3368         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 3369         dump_drop_page(m->phys_addr);
 3370         vm_page_unwire(m, PQ_NONE);
 3371         vm_page_free(m);
 3372 }
 3373 
 3374 /*
 3375  * Returns a new PV entry, allocating a new PV chunk from the system when
 3376  * needed.  If this PV chunk allocation fails and a PV list lock pointer was
 3377  * given, a PV chunk is reclaimed from an arbitrary pmap.  Otherwise, NULL is
 3378  * returned.
 3379  *
 3380  * The given PV list lock may be released.
 3381  */
 3382 static pv_entry_t
 3383 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
 3384 {
 3385         int bit, field;
 3386         pv_entry_t pv;
 3387         struct pv_chunk *pc;
 3388         vm_page_t m;
 3389 
 3390         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3391         PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
 3392 retry:
 3393         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 3394         if (pc != NULL) {
 3395                 for (field = 0; field < _NPCM; field++) {
 3396                         if (pc->pc_map[field]) {
 3397                                 bit = bsfq(pc->pc_map[field]);
 3398                                 break;
 3399                         }
 3400                 }
 3401                 if (field < _NPCM) {
 3402                         pv = &pc->pc_pventry[field * 64 + bit];
 3403                         pc->pc_map[field] &= ~(1ul << bit);
 3404                         /* If this was the last item, move it to tail */
 3405                         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 &&
 3406                             pc->pc_map[2] == 0) {
 3407                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3408                                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
 3409                                     pc_list);
 3410                         }
 3411                         PV_STAT(atomic_add_long(&pv_entry_count, 1));
 3412                         PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
 3413                         return (pv);
 3414                 }
 3415         }
 3416         /* No free items, allocate another chunk */
 3417         m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 3418             VM_ALLOC_WIRED);
 3419         if (m == NULL) {
 3420                 if (lockp == NULL) {
 3421                         PV_STAT(pc_chunk_tryfail++);
 3422                         return (NULL);
 3423                 }
 3424                 m = reclaim_pv_chunk(pmap, lockp);
 3425                 if (m == NULL)
 3426                         goto retry;
 3427         }
 3428         PV_STAT(atomic_add_int(&pc_chunk_count, 1));
 3429         PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
 3430         dump_add_page(m->phys_addr);
 3431         pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 3432         pc->pc_pmap = pmap;
 3433         pc->pc_map[0] = PC_FREE0 & ~1ul;        /* preallocated bit 0 */
 3434         pc->pc_map[1] = PC_FREE1;
 3435         pc->pc_map[2] = PC_FREE2;
 3436         mtx_lock(&pv_chunks_mutex);
 3437         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 3438         mtx_unlock(&pv_chunks_mutex);
 3439         pv = &pc->pc_pventry[0];
 3440         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 3441         PV_STAT(atomic_add_long(&pv_entry_count, 1));
 3442         PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
 3443         return (pv);
 3444 }
 3445 
 3446 /*
 3447  * Returns the number of one bits within the given PV chunk map.
 3448  *
 3449  * The erratas for Intel processors state that "POPCNT Instruction May
 3450  * Take Longer to Execute Than Expected".  It is believed that the
 3451  * issue is the spurious dependency on the destination register.
 3452  * Provide a hint to the register rename logic that the destination
 3453  * value is overwritten, by clearing it, as suggested in the
 3454  * optimization manual.  It should be cheap for unaffected processors
 3455  * as well.
 3456  *
 3457  * Reference numbers for erratas are
 3458  * 4th Gen Core: HSD146
 3459  * 5th Gen Core: BDM85
 3460  * 6th Gen Core: SKL029
 3461  */
 3462 static int
 3463 popcnt_pc_map_pq(uint64_t *map)
 3464 {
 3465         u_long result, tmp;
 3466 
 3467         __asm __volatile("xorl %k0,%k0;popcntq %2,%0;"
 3468             "xorl %k1,%k1;popcntq %3,%1;addl %k1,%k0;"
 3469             "xorl %k1,%k1;popcntq %4,%1;addl %k1,%k0"
 3470             : "=&r" (result), "=&r" (tmp)
 3471             : "m" (map[0]), "m" (map[1]), "m" (map[2]));
 3472         return (result);
 3473 }
 3474 
 3475 /*
 3476  * Ensure that the number of spare PV entries in the specified pmap meets or
 3477  * exceeds the given count, "needed".
 3478  *
 3479  * The given PV list lock may be released.
 3480  */
 3481 static void
 3482 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
 3483 {
 3484         struct pch new_tail;
 3485         struct pv_chunk *pc;
 3486         int avail, free;
 3487         vm_page_t m;
 3488 
 3489         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3490         KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
 3491 
 3492         /*
 3493          * Newly allocated PV chunks must be stored in a private list until
 3494          * the required number of PV chunks have been allocated.  Otherwise,
 3495          * reclaim_pv_chunk() could recycle one of these chunks.  In
 3496          * contrast, these chunks must be added to the pmap upon allocation.
 3497          */
 3498         TAILQ_INIT(&new_tail);
 3499 retry:
 3500         avail = 0;
 3501         TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
 3502 #ifndef __POPCNT__
 3503                 if ((cpu_feature2 & CPUID2_POPCNT) == 0)
 3504                         bit_count((bitstr_t *)pc->pc_map, 0,
 3505                             sizeof(pc->pc_map) * NBBY, &free);
 3506                 else
 3507 #endif
 3508                 free = popcnt_pc_map_pq(pc->pc_map);
 3509                 if (free == 0)
 3510                         break;
 3511                 avail += free;
 3512                 if (avail >= needed)
 3513                         break;
 3514         }
 3515         for (; avail < needed; avail += _NPCPV) {
 3516                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 3517                     VM_ALLOC_WIRED);
 3518                 if (m == NULL) {
 3519                         m = reclaim_pv_chunk(pmap, lockp);
 3520                         if (m == NULL)
 3521                                 goto retry;
 3522                 }
 3523                 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
 3524                 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
 3525                 dump_add_page(m->phys_addr);
 3526                 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 3527                 pc->pc_pmap = pmap;
 3528                 pc->pc_map[0] = PC_FREE0;
 3529                 pc->pc_map[1] = PC_FREE1;
 3530                 pc->pc_map[2] = PC_FREE2;
 3531                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 3532                 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
 3533                 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
 3534         }
 3535         if (!TAILQ_EMPTY(&new_tail)) {
 3536                 mtx_lock(&pv_chunks_mutex);
 3537                 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
 3538                 mtx_unlock(&pv_chunks_mutex);
 3539         }
 3540 }
 3541 
 3542 /*
 3543  * First find and then remove the pv entry for the specified pmap and virtual
 3544  * address from the specified pv list.  Returns the pv entry if found and NULL
 3545  * otherwise.  This operation can be performed on pv lists for either 4KB or
 3546  * 2MB page mappings.
 3547  */
 3548 static __inline pv_entry_t
 3549 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 3550 {
 3551         pv_entry_t pv;
 3552 
 3553         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 3554                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 3555                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 3556                         pvh->pv_gen++;
 3557                         break;
 3558                 }
 3559         }
 3560         return (pv);
 3561 }
 3562 
 3563 /*
 3564  * After demotion from a 2MB page mapping to 512 4KB page mappings,
 3565  * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
 3566  * entries for each of the 4KB page mappings.
 3567  */
 3568 static void
 3569 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
 3570     struct rwlock **lockp)
 3571 {
 3572         struct md_page *pvh;
 3573         struct pv_chunk *pc;
 3574         pv_entry_t pv;
 3575         vm_offset_t va_last;
 3576         vm_page_t m;
 3577         int bit, field;
 3578 
 3579         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3580         KASSERT((pa & PDRMASK) == 0,
 3581             ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
 3582         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
 3583 
 3584         /*
 3585          * Transfer the 2mpage's pv entry for this mapping to the first
 3586          * page's pv list.  Once this transfer begins, the pv list lock
 3587          * must not be released until the last pv entry is reinstantiated.
 3588          */
 3589         pvh = pa_to_pvh(pa);
 3590         va = trunc_2mpage(va);
 3591         pv = pmap_pvh_remove(pvh, pmap, va);
 3592         KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
 3593         m = PHYS_TO_VM_PAGE(pa);
 3594         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 3595         m->md.pv_gen++;
 3596         /* Instantiate the remaining NPTEPG - 1 pv entries. */
 3597         PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
 3598         va_last = va + NBPDR - PAGE_SIZE;
 3599         for (;;) {
 3600                 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 3601                 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 ||
 3602                     pc->pc_map[2] != 0, ("pmap_pv_demote_pde: missing spare"));
 3603                 for (field = 0; field < _NPCM; field++) {
 3604                         while (pc->pc_map[field]) {
 3605                                 bit = bsfq(pc->pc_map[field]);
 3606                                 pc->pc_map[field] &= ~(1ul << bit);
 3607                                 pv = &pc->pc_pventry[field * 64 + bit];
 3608                                 va += PAGE_SIZE;
 3609                                 pv->pv_va = va;
 3610                                 m++;
 3611                                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3612                             ("pmap_pv_demote_pde: page %p is not managed", m));
 3613                                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 3614                                 m->md.pv_gen++;
 3615                                 if (va == va_last)
 3616                                         goto out;
 3617                         }
 3618                 }
 3619                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3620                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 3621         }
 3622 out:
 3623         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) {
 3624                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3625                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 3626         }
 3627         PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
 3628         PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
 3629 }
 3630 
 3631 #if VM_NRESERVLEVEL > 0
 3632 /*
 3633  * After promotion from 512 4KB page mappings to a single 2MB page mapping,
 3634  * replace the many pv entries for the 4KB page mappings by a single pv entry
 3635  * for the 2MB page mapping.
 3636  */
 3637 static void
 3638 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
 3639     struct rwlock **lockp)
 3640 {
 3641         struct md_page *pvh;
 3642         pv_entry_t pv;
 3643         vm_offset_t va_last;
 3644         vm_page_t m;
 3645 
 3646         KASSERT((pa & PDRMASK) == 0,
 3647             ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
 3648         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
 3649 
 3650         /*
 3651          * Transfer the first page's pv entry for this mapping to the 2mpage's
 3652          * pv list.  Aside from avoiding the cost of a call to get_pv_entry(),
 3653          * a transfer avoids the possibility that get_pv_entry() calls
 3654          * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
 3655          * mappings that is being promoted.
 3656          */
 3657         m = PHYS_TO_VM_PAGE(pa);
 3658         va = trunc_2mpage(va);
 3659         pv = pmap_pvh_remove(&m->md, pmap, va);
 3660         KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
 3661         pvh = pa_to_pvh(pa);
 3662         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 3663         pvh->pv_gen++;
 3664         /* Free the remaining NPTEPG - 1 pv entries. */
 3665         va_last = va + NBPDR - PAGE_SIZE;
 3666         do {
 3667                 m++;
 3668                 va += PAGE_SIZE;
 3669                 pmap_pvh_free(&m->md, pmap, va);
 3670         } while (va < va_last);
 3671 }
 3672 #endif /* VM_NRESERVLEVEL > 0 */
 3673 
 3674 /*
 3675  * First find and then destroy the pv entry for the specified pmap and virtual
 3676  * address.  This operation can be performed on pv lists for either 4KB or 2MB
 3677  * page mappings.
 3678  */
 3679 static void
 3680 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 3681 {
 3682         pv_entry_t pv;
 3683 
 3684         pv = pmap_pvh_remove(pvh, pmap, va);
 3685         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 3686         free_pv_entry(pmap, pv);
 3687 }
 3688 
 3689 /*
 3690  * Conditionally create the PV entry for a 4KB page mapping if the required
 3691  * memory can be allocated without resorting to reclamation.
 3692  */
 3693 static boolean_t
 3694 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
 3695     struct rwlock **lockp)
 3696 {
 3697         pv_entry_t pv;
 3698 
 3699         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3700         /* Pass NULL instead of the lock pointer to disable reclamation. */
 3701         if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
 3702                 pv->pv_va = va;
 3703                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 3704                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 3705                 m->md.pv_gen++;
 3706                 return (TRUE);
 3707         } else
 3708                 return (FALSE);
 3709 }
 3710 
 3711 /*
 3712  * Create the PV entry for a 2MB page mapping.  Always returns true unless the
 3713  * flag PMAP_ENTER_NORECLAIM is specified.  If that flag is specified, returns
 3714  * false if the PV entry cannot be allocated without resorting to reclamation.
 3715  */
 3716 static bool
 3717 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, pd_entry_t pde, u_int flags,
 3718     struct rwlock **lockp)
 3719 {
 3720         struct md_page *pvh;
 3721         pv_entry_t pv;
 3722         vm_paddr_t pa;
 3723 
 3724         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3725         /* Pass NULL instead of the lock pointer to disable reclamation. */
 3726         if ((pv = get_pv_entry(pmap, (flags & PMAP_ENTER_NORECLAIM) != 0 ?
 3727             NULL : lockp)) == NULL)
 3728                 return (false);
 3729         pv->pv_va = va;
 3730         pa = pde & PG_PS_FRAME;
 3731         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
 3732         pvh = pa_to_pvh(pa);
 3733         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 3734         pvh->pv_gen++;
 3735         return (true);
 3736 }
 3737 
 3738 /*
 3739  * Fills a page table page with mappings to consecutive physical pages.
 3740  */
 3741 static void
 3742 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
 3743 {
 3744         pt_entry_t *pte;
 3745 
 3746         for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
 3747                 *pte = newpte;
 3748                 newpte += PAGE_SIZE;
 3749         }
 3750 }
 3751 
 3752 /*
 3753  * Tries to demote a 2MB page mapping.  If demotion fails, the 2MB page
 3754  * mapping is invalidated.
 3755  */
 3756 static boolean_t
 3757 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 3758 {
 3759         struct rwlock *lock;
 3760         boolean_t rv;
 3761 
 3762         lock = NULL;
 3763         rv = pmap_demote_pde_locked(pmap, pde, va, &lock);
 3764         if (lock != NULL)
 3765                 rw_wunlock(lock);
 3766         return (rv);
 3767 }
 3768 
 3769 static boolean_t
 3770 pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
 3771     struct rwlock **lockp)
 3772 {
 3773         pd_entry_t newpde, oldpde;
 3774         pt_entry_t *firstpte, newpte;
 3775         pt_entry_t PG_A, PG_G, PG_M, PG_RW, PG_V;
 3776         vm_paddr_t mptepa;
 3777         vm_page_t mpte;
 3778         struct spglist free;
 3779         vm_offset_t sva;
 3780         int PG_PTE_CACHE;
 3781 
 3782         PG_G = pmap_global_bit(pmap);
 3783         PG_A = pmap_accessed_bit(pmap);
 3784         PG_M = pmap_modified_bit(pmap);
 3785         PG_RW = pmap_rw_bit(pmap);
 3786         PG_V = pmap_valid_bit(pmap);
 3787         PG_PTE_CACHE = pmap_cache_mask(pmap, 0);
 3788 
 3789         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3790         oldpde = *pde;
 3791         KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
 3792             ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
 3793         if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) ==
 3794             NULL) {
 3795                 KASSERT((oldpde & PG_W) == 0,
 3796                     ("pmap_demote_pde: page table page for a wired mapping"
 3797                     " is missing"));
 3798 
 3799                 /*
 3800                  * Invalidate the 2MB page mapping and return "failure" if the
 3801                  * mapping was never accessed or the allocation of the new
 3802                  * page table page fails.  If the 2MB page mapping belongs to
 3803                  * the direct map region of the kernel's address space, then
 3804                  * the page allocation request specifies the highest possible
 3805                  * priority (VM_ALLOC_INTERRUPT).  Otherwise, the priority is
 3806                  * normal.  Page table pages are preallocated for every other
 3807                  * part of the kernel address space, so the direct map region
 3808                  * is the only part of the kernel address space that must be
 3809                  * handled here.
 3810                  */
 3811                 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
 3812                     pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va <
 3813                     DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) |
 3814                     VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 3815                         SLIST_INIT(&free);
 3816                         sva = trunc_2mpage(va);
 3817                         pmap_remove_pde(pmap, pde, sva, &free, lockp);
 3818                         if ((oldpde & PG_G) == 0)
 3819                                 pmap_invalidate_pde_page(pmap, sva, oldpde);
 3820                         pmap_free_zero_pages(&free);
 3821                         CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"
 3822                             " in pmap %p", va, pmap);
 3823                         return (FALSE);
 3824                 }
 3825                 if (va < VM_MAXUSER_ADDRESS)
 3826                         pmap_resident_count_inc(pmap, 1);
 3827         }
 3828         mptepa = VM_PAGE_TO_PHYS(mpte);
 3829         firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
 3830         newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
 3831         KASSERT((oldpde & PG_A) != 0,
 3832             ("pmap_demote_pde: oldpde is missing PG_A"));
 3833         KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
 3834             ("pmap_demote_pde: oldpde is missing PG_M"));
 3835         newpte = oldpde & ~PG_PS;
 3836         newpte = pmap_swap_pat(pmap, newpte);
 3837 
 3838         /*
 3839          * If the page table page is new, initialize it.
 3840          */
 3841         if (mpte->wire_count == 1) {
 3842                 mpte->wire_count = NPTEPG;
 3843                 pmap_fill_ptp(firstpte, newpte);
 3844         }
 3845         KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
 3846             ("pmap_demote_pde: firstpte and newpte map different physical"
 3847             " addresses"));
 3848 
 3849         /*
 3850          * If the mapping has changed attributes, update the page table
 3851          * entries.
 3852          */
 3853         if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
 3854                 pmap_fill_ptp(firstpte, newpte);
 3855 
 3856         /*
 3857          * The spare PV entries must be reserved prior to demoting the
 3858          * mapping, that is, prior to changing the PDE.  Otherwise, the state
 3859          * of the PDE and the PV lists will be inconsistent, which can result
 3860          * in reclaim_pv_chunk() attempting to remove a PV entry from the
 3861          * wrong PV list and pmap_pv_demote_pde() failing to find the expected
 3862          * PV entry for the 2MB page mapping that is being demoted.
 3863          */
 3864         if ((oldpde & PG_MANAGED) != 0)
 3865                 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
 3866 
 3867         /*
 3868          * Demote the mapping.  This pmap is locked.  The old PDE has
 3869          * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
 3870          * set.  Thus, there is no danger of a race with another
 3871          * processor changing the setting of PG_A and/or PG_M between
 3872          * the read above and the store below. 
 3873          */
 3874         if (workaround_erratum383)
 3875                 pmap_update_pde(pmap, va, pde, newpde);
 3876         else
 3877                 pde_store(pde, newpde);
 3878 
 3879         /*
 3880          * Invalidate a stale recursive mapping of the page table page.
 3881          */
 3882         if (va >= VM_MAXUSER_ADDRESS)
 3883                 pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 3884 
 3885         /*
 3886          * Demote the PV entry.
 3887          */
 3888         if ((oldpde & PG_MANAGED) != 0)
 3889                 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp);
 3890 
 3891         atomic_add_long(&pmap_pde_demotions, 1);
 3892         CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx"
 3893             " in pmap %p", va, pmap);
 3894         return (TRUE);
 3895 }
 3896 
 3897 /*
 3898  * pmap_remove_kernel_pde: Remove a kernel superpage mapping.
 3899  */
 3900 static void
 3901 pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 3902 {
 3903         pd_entry_t newpde;
 3904         vm_paddr_t mptepa;
 3905         vm_page_t mpte;
 3906 
 3907         KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap));
 3908         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3909         mpte = pmap_remove_pt_page(pmap, va);
 3910         if (mpte == NULL)
 3911                 panic("pmap_remove_kernel_pde: Missing pt page.");
 3912 
 3913         mptepa = VM_PAGE_TO_PHYS(mpte);
 3914         newpde = mptepa | X86_PG_M | X86_PG_A | X86_PG_RW | X86_PG_V;
 3915 
 3916         /*
 3917          * Initialize the page table page.
 3918          */
 3919         pagezero((void *)PHYS_TO_DMAP(mptepa));
 3920 
 3921         /*
 3922          * Demote the mapping.
 3923          */
 3924         if (workaround_erratum383)
 3925                 pmap_update_pde(pmap, va, pde, newpde);
 3926         else
 3927                 pde_store(pde, newpde);
 3928 
 3929         /*
 3930          * Invalidate a stale recursive mapping of the page table page.
 3931          */
 3932         pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 3933 }
 3934 
 3935 /*
 3936  * pmap_remove_pde: do the things to unmap a superpage in a process
 3937  */
 3938 static int
 3939 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 3940     struct spglist *free, struct rwlock **lockp)
 3941 {
 3942         struct md_page *pvh;
 3943         pd_entry_t oldpde;
 3944         vm_offset_t eva, va;
 3945         vm_page_t m, mpte;
 3946         pt_entry_t PG_G, PG_A, PG_M, PG_RW;
 3947 
 3948         PG_G = pmap_global_bit(pmap);
 3949         PG_A = pmap_accessed_bit(pmap);
 3950         PG_M = pmap_modified_bit(pmap);
 3951         PG_RW = pmap_rw_bit(pmap);
 3952 
 3953         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3954         KASSERT((sva & PDRMASK) == 0,
 3955             ("pmap_remove_pde: sva is not 2mpage aligned"));
 3956         oldpde = pte_load_clear(pdq);
 3957         if (oldpde & PG_W)
 3958                 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
 3959         if ((oldpde & PG_G) != 0)
 3960                 pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
 3961         pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
 3962         if (oldpde & PG_MANAGED) {
 3963                 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
 3964                 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
 3965                 pmap_pvh_free(pvh, pmap, sva);
 3966                 eva = sva + NBPDR;
 3967                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 3968                     va < eva; va += PAGE_SIZE, m++) {
 3969                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3970                                 vm_page_dirty(m);
 3971                         if (oldpde & PG_A)
 3972                                 vm_page_aflag_set(m, PGA_REFERENCED);
 3973                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 3974                             TAILQ_EMPTY(&pvh->pv_list))
 3975                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 3976                         pmap_delayed_invl_page(m);
 3977                 }
 3978         }
 3979         if (pmap == kernel_pmap) {
 3980                 pmap_remove_kernel_pde(pmap, pdq, sva);
 3981         } else {
 3982                 mpte = pmap_remove_pt_page(pmap, sva);
 3983                 if (mpte != NULL) {
 3984                         pmap_resident_count_dec(pmap, 1);
 3985                         KASSERT(mpte->wire_count == NPTEPG,
 3986                             ("pmap_remove_pde: pte page wire count error"));
 3987                         mpte->wire_count = 0;
 3988                         pmap_add_delayed_free_list(mpte, free, FALSE);
 3989                 }
 3990         }
 3991         return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free));
 3992 }
 3993 
 3994 /*
 3995  * pmap_remove_pte: do the things to unmap a page in a process
 3996  */
 3997 static int
 3998 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 
 3999     pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp)
 4000 {
 4001         struct md_page *pvh;
 4002         pt_entry_t oldpte, PG_A, PG_M, PG_RW;
 4003         vm_page_t m;
 4004 
 4005         PG_A = pmap_accessed_bit(pmap);
 4006         PG_M = pmap_modified_bit(pmap);
 4007         PG_RW = pmap_rw_bit(pmap);
 4008 
 4009         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4010         oldpte = pte_load_clear(ptq);
 4011         if (oldpte & PG_W)
 4012                 pmap->pm_stats.wired_count -= 1;
 4013         pmap_resident_count_dec(pmap, 1);
 4014         if (oldpte & PG_MANAGED) {
 4015                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 4016                 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 4017                         vm_page_dirty(m);
 4018                 if (oldpte & PG_A)
 4019                         vm_page_aflag_set(m, PGA_REFERENCED);
 4020                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 4021                 pmap_pvh_free(&m->md, pmap, va);
 4022                 if (TAILQ_EMPTY(&m->md.pv_list) &&
 4023                     (m->flags & PG_FICTITIOUS) == 0) {
 4024                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4025                         if (TAILQ_EMPTY(&pvh->pv_list))
 4026                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 4027                 }
 4028                 pmap_delayed_invl_page(m);
 4029         }
 4030         return (pmap_unuse_pt(pmap, va, ptepde, free));
 4031 }
 4032 
 4033 /*
 4034  * Remove a single page from a process address space
 4035  */
 4036 static void
 4037 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
 4038     struct spglist *free)
 4039 {
 4040         struct rwlock *lock;
 4041         pt_entry_t *pte, PG_V;
 4042 
 4043         PG_V = pmap_valid_bit(pmap);
 4044         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4045         if ((*pde & PG_V) == 0)
 4046                 return;
 4047         pte = pmap_pde_to_pte(pde, va);
 4048         if ((*pte & PG_V) == 0)
 4049                 return;
 4050         lock = NULL;
 4051         pmap_remove_pte(pmap, pte, va, *pde, free, &lock);
 4052         if (lock != NULL)
 4053                 rw_wunlock(lock);
 4054         pmap_invalidate_page(pmap, va);
 4055 }
 4056 
 4057 /*
 4058  * Removes the specified range of addresses from the page table page.
 4059  */
 4060 static bool
 4061 pmap_remove_ptes(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
 4062     pd_entry_t *pde, struct spglist *free, struct rwlock **lockp)
 4063 {
 4064         pt_entry_t PG_G, *pte;
 4065         vm_offset_t va;
 4066         bool anyvalid;
 4067 
 4068         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4069         PG_G = pmap_global_bit(pmap);
 4070         anyvalid = false;
 4071         va = eva;
 4072         for (pte = pmap_pde_to_pte(pde, sva); sva != eva; pte++,
 4073             sva += PAGE_SIZE) {
 4074                 if (*pte == 0) {
 4075                         if (va != eva) {
 4076                                 pmap_invalidate_range(pmap, va, sva);
 4077                                 va = eva;
 4078                         }
 4079                         continue;
 4080                 }
 4081                 if ((*pte & PG_G) == 0)
 4082                         anyvalid = true;
 4083                 else if (va == eva)
 4084                         va = sva;
 4085                 if (pmap_remove_pte(pmap, pte, sva, *pde, free, lockp)) {
 4086                         sva += PAGE_SIZE;
 4087                         break;
 4088                 }
 4089         }
 4090         if (va != eva)
 4091                 pmap_invalidate_range(pmap, va, sva);
 4092         return (anyvalid);
 4093 }
 4094 
 4095 /*
 4096  *      Remove the given range of addresses from the specified map.
 4097  *
 4098  *      It is assumed that the start and end are properly
 4099  *      rounded to the page size.
 4100  */
 4101 void
 4102 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 4103 {
 4104         struct rwlock *lock;
 4105         vm_offset_t va_next;
 4106         pml4_entry_t *pml4e;
 4107         pdp_entry_t *pdpe;
 4108         pd_entry_t ptpaddr, *pde;
 4109         pt_entry_t PG_G, PG_V;
 4110         struct spglist free;
 4111         int anyvalid;
 4112 
 4113         PG_G = pmap_global_bit(pmap);
 4114         PG_V = pmap_valid_bit(pmap);
 4115 
 4116         /*
 4117          * Perform an unsynchronized read.  This is, however, safe.
 4118          */
 4119         if (pmap->pm_stats.resident_count == 0)
 4120                 return;
 4121 
 4122         anyvalid = 0;
 4123         SLIST_INIT(&free);
 4124 
 4125         pmap_delayed_invl_started();
 4126         PMAP_LOCK(pmap);
 4127 
 4128         /*
 4129          * special handling of removing one page.  a very
 4130          * common operation and easy to short circuit some
 4131          * code.
 4132          */
 4133         if (sva + PAGE_SIZE == eva) {
 4134                 pde = pmap_pde(pmap, sva);
 4135                 if (pde && (*pde & PG_PS) == 0) {
 4136                         pmap_remove_page(pmap, sva, pde, &free);
 4137                         goto out;
 4138                 }
 4139         }
 4140 
 4141         lock = NULL;
 4142         for (; sva < eva; sva = va_next) {
 4143 
 4144                 if (pmap->pm_stats.resident_count == 0)
 4145                         break;
 4146 
 4147                 pml4e = pmap_pml4e(pmap, sva);
 4148                 if ((*pml4e & PG_V) == 0) {
 4149                         va_next = (sva + NBPML4) & ~PML4MASK;
 4150                         if (va_next < sva)
 4151                                 va_next = eva;
 4152                         continue;
 4153                 }
 4154 
 4155                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 4156                 if ((*pdpe & PG_V) == 0) {
 4157                         va_next = (sva + NBPDP) & ~PDPMASK;
 4158                         if (va_next < sva)
 4159                                 va_next = eva;
 4160                         continue;
 4161                 }
 4162 
 4163                 /*
 4164                  * Calculate index for next page table.
 4165                  */
 4166                 va_next = (sva + NBPDR) & ~PDRMASK;
 4167                 if (va_next < sva)
 4168                         va_next = eva;
 4169 
 4170                 pde = pmap_pdpe_to_pde(pdpe, sva);
 4171                 ptpaddr = *pde;
 4172 
 4173                 /*
 4174                  * Weed out invalid mappings.
 4175                  */
 4176                 if (ptpaddr == 0)
 4177                         continue;
 4178 
 4179                 /*
 4180                  * Check for large page.
 4181                  */
 4182                 if ((ptpaddr & PG_PS) != 0) {
 4183                         /*
 4184                          * Are we removing the entire large page?  If not,
 4185                          * demote the mapping and fall through.
 4186                          */
 4187                         if (sva + NBPDR == va_next && eva >= va_next) {
 4188                                 /*
 4189                                  * The TLB entry for a PG_G mapping is
 4190                                  * invalidated by pmap_remove_pde().
 4191                                  */
 4192                                 if ((ptpaddr & PG_G) == 0)
 4193                                         anyvalid = 1;
 4194                                 pmap_remove_pde(pmap, pde, sva, &free, &lock);
 4195                                 continue;
 4196                         } else if (!pmap_demote_pde_locked(pmap, pde, sva,
 4197                             &lock)) {
 4198                                 /* The large page mapping was destroyed. */
 4199                                 continue;
 4200                         } else
 4201                                 ptpaddr = *pde;
 4202                 }
 4203 
 4204                 /*
 4205                  * Limit our scan to either the end of the va represented
 4206                  * by the current page table page, or to the end of the
 4207                  * range being removed.
 4208                  */
 4209                 if (va_next > eva)
 4210                         va_next = eva;
 4211 
 4212                 if (pmap_remove_ptes(pmap, sva, va_next, pde, &free, &lock))
 4213                         anyvalid = 1;
 4214         }
 4215         if (lock != NULL)
 4216                 rw_wunlock(lock);
 4217 out:
 4218         if (anyvalid)
 4219                 pmap_invalidate_all(pmap);
 4220         PMAP_UNLOCK(pmap);
 4221         pmap_delayed_invl_finished();
 4222         pmap_free_zero_pages(&free);
 4223 }
 4224 
 4225 /*
 4226  *      Routine:        pmap_remove_all
 4227  *      Function:
 4228  *              Removes this physical page from
 4229  *              all physical maps in which it resides.
 4230  *              Reflects back modify bits to the pager.
 4231  *
 4232  *      Notes:
 4233  *              Original versions of this routine were very
 4234  *              inefficient because they iteratively called
 4235  *              pmap_remove (slow...)
 4236  */
 4237 
 4238 void
 4239 pmap_remove_all(vm_page_t m)
 4240 {
 4241         struct md_page *pvh;
 4242         pv_entry_t pv;
 4243         pmap_t pmap;
 4244         struct rwlock *lock;
 4245         pt_entry_t *pte, tpte, PG_A, PG_M, PG_RW;
 4246         pd_entry_t *pde;
 4247         vm_offset_t va;
 4248         struct spglist free;
 4249         int pvh_gen, md_gen;
 4250 
 4251         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4252             ("pmap_remove_all: page %p is not managed", m));
 4253         SLIST_INIT(&free);
 4254         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 4255         pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
 4256             pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4257 retry:
 4258         rw_wlock(lock);
 4259         while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
 4260                 pmap = PV_PMAP(pv);
 4261                 if (!PMAP_TRYLOCK(pmap)) {
 4262                         pvh_gen = pvh->pv_gen;
 4263                         rw_wunlock(lock);
 4264                         PMAP_LOCK(pmap);
 4265                         rw_wlock(lock);
 4266                         if (pvh_gen != pvh->pv_gen) {
 4267                                 rw_wunlock(lock);
 4268                                 PMAP_UNLOCK(pmap);
 4269                                 goto retry;
 4270                         }
 4271                 }
 4272                 va = pv->pv_va;
 4273                 pde = pmap_pde(pmap, va);
 4274                 (void)pmap_demote_pde_locked(pmap, pde, va, &lock);
 4275                 PMAP_UNLOCK(pmap);
 4276         }
 4277         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 4278                 pmap = PV_PMAP(pv);
 4279                 if (!PMAP_TRYLOCK(pmap)) {
 4280                         pvh_gen = pvh->pv_gen;
 4281                         md_gen = m->md.pv_gen;
 4282                         rw_wunlock(lock);
 4283                         PMAP_LOCK(pmap);
 4284                         rw_wlock(lock);
 4285                         if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
 4286                                 rw_wunlock(lock);
 4287                                 PMAP_UNLOCK(pmap);
 4288                                 goto retry;
 4289                         }
 4290                 }
 4291                 PG_A = pmap_accessed_bit(pmap);
 4292                 PG_M = pmap_modified_bit(pmap);
 4293                 PG_RW = pmap_rw_bit(pmap);
 4294                 pmap_resident_count_dec(pmap, 1);
 4295                 pde = pmap_pde(pmap, pv->pv_va);
 4296                 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
 4297                     " a 2mpage in page %p's pv list", m));
 4298                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 4299                 tpte = pte_load_clear(pte);
 4300                 if (tpte & PG_W)
 4301                         pmap->pm_stats.wired_count--;
 4302                 if (tpte & PG_A)
 4303                         vm_page_aflag_set(m, PGA_REFERENCED);
 4304 
 4305                 /*
 4306                  * Update the vm_page_t clean and reference bits.
 4307                  */
 4308                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 4309                         vm_page_dirty(m);
 4310                 pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);
 4311                 pmap_invalidate_page(pmap, pv->pv_va);
 4312                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 4313                 m->md.pv_gen++;
 4314                 free_pv_entry(pmap, pv);
 4315                 PMAP_UNLOCK(pmap);
 4316         }
 4317         vm_page_aflag_clear(m, PGA_WRITEABLE);
 4318         rw_wunlock(lock);
 4319         pmap_delayed_invl_wait(m);
 4320         pmap_free_zero_pages(&free);
 4321 }
 4322 
 4323 /*
 4324  * pmap_protect_pde: do the things to protect a 2mpage in a process
 4325  */
 4326 static boolean_t
 4327 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
 4328 {
 4329         pd_entry_t newpde, oldpde;
 4330         vm_offset_t eva, va;
 4331         vm_page_t m;
 4332         boolean_t anychanged;
 4333         pt_entry_t PG_G, PG_M, PG_RW;
 4334 
 4335         PG_G = pmap_global_bit(pmap);
 4336         PG_M = pmap_modified_bit(pmap);
 4337         PG_RW = pmap_rw_bit(pmap);
 4338 
 4339         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4340         KASSERT((sva & PDRMASK) == 0,
 4341             ("pmap_protect_pde: sva is not 2mpage aligned"));
 4342         anychanged = FALSE;
 4343 retry:
 4344         oldpde = newpde = *pde;
 4345         if ((oldpde & (PG_MANAGED | PG_M | PG_RW)) ==
 4346             (PG_MANAGED | PG_M | PG_RW)) {
 4347                 eva = sva + NBPDR;
 4348                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 4349                     va < eva; va += PAGE_SIZE, m++)
 4350                         vm_page_dirty(m);
 4351         }
 4352         if ((prot & VM_PROT_WRITE) == 0)
 4353                 newpde &= ~(PG_RW | PG_M);
 4354         if ((prot & VM_PROT_EXECUTE) == 0)
 4355                 newpde |= pg_nx;
 4356         if (newpde != oldpde) {
 4357                 /*
 4358                  * As an optimization to future operations on this PDE, clear
 4359                  * PG_PROMOTED.  The impending invalidation will remove any
 4360                  * lingering 4KB page mappings from the TLB.
 4361                  */
 4362                 if (!atomic_cmpset_long(pde, oldpde, newpde & ~PG_PROMOTED))
 4363                         goto retry;
 4364                 if ((oldpde & PG_G) != 0)
 4365                         pmap_invalidate_pde_page(kernel_pmap, sva, oldpde);
 4366                 else
 4367                         anychanged = TRUE;
 4368         }
 4369         return (anychanged);
 4370 }
 4371 
 4372 /*
 4373  *      Set the physical protection on the
 4374  *      specified range of this map as requested.
 4375  */
 4376 void
 4377 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 4378 {
 4379         vm_offset_t va_next;
 4380         pml4_entry_t *pml4e;
 4381         pdp_entry_t *pdpe;
 4382         pd_entry_t ptpaddr, *pde;
 4383         pt_entry_t *pte, PG_G, PG_M, PG_RW, PG_V;
 4384         boolean_t anychanged;
 4385 
 4386         KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
 4387         if (prot == VM_PROT_NONE) {
 4388                 pmap_remove(pmap, sva, eva);
 4389                 return;
 4390         }
 4391 
 4392         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 4393             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 4394                 return;
 4395 
 4396         PG_G = pmap_global_bit(pmap);
 4397         PG_M = pmap_modified_bit(pmap);
 4398         PG_V = pmap_valid_bit(pmap);
 4399         PG_RW = pmap_rw_bit(pmap);
 4400         anychanged = FALSE;
 4401 
 4402         /*
 4403          * Although this function delays and batches the invalidation
 4404          * of stale TLB entries, it does not need to call
 4405          * pmap_delayed_invl_started() and
 4406          * pmap_delayed_invl_finished(), because it does not
 4407          * ordinarily destroy mappings.  Stale TLB entries from
 4408          * protection-only changes need only be invalidated before the
 4409          * pmap lock is released, because protection-only changes do
 4410          * not destroy PV entries.  Even operations that iterate over
 4411          * a physical page's PV list of mappings, like
 4412          * pmap_remove_write(), acquire the pmap lock for each
 4413          * mapping.  Consequently, for protection-only changes, the
 4414          * pmap lock suffices to synchronize both page table and TLB
 4415          * updates.
 4416          *
 4417          * This function only destroys a mapping if pmap_demote_pde()
 4418          * fails.  In that case, stale TLB entries are immediately
 4419          * invalidated.
 4420          */
 4421         
 4422         PMAP_LOCK(pmap);
 4423         for (; sva < eva; sva = va_next) {
 4424 
 4425                 pml4e = pmap_pml4e(pmap, sva);
 4426                 if ((*pml4e & PG_V) == 0) {
 4427                         va_next = (sva + NBPML4) & ~PML4MASK;
 4428                         if (va_next < sva)
 4429                                 va_next = eva;
 4430                         continue;
 4431                 }
 4432 
 4433                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 4434                 if ((*pdpe & PG_V) == 0) {
 4435                         va_next = (sva + NBPDP) & ~PDPMASK;
 4436                         if (va_next < sva)
 4437                                 va_next = eva;
 4438                         continue;
 4439                 }
 4440 
 4441                 va_next = (sva + NBPDR) & ~PDRMASK;
 4442                 if (va_next < sva)
 4443                         va_next = eva;
 4444 
 4445                 pde = pmap_pdpe_to_pde(pdpe, sva);
 4446                 ptpaddr = *pde;
 4447 
 4448                 /*
 4449                  * Weed out invalid mappings.
 4450                  */
 4451                 if (ptpaddr == 0)
 4452                         continue;
 4453 
 4454                 /*
 4455                  * Check for large page.
 4456                  */
 4457                 if ((ptpaddr & PG_PS) != 0) {
 4458                         /*
 4459                          * Are we protecting the entire large page?  If not,
 4460                          * demote the mapping and fall through.
 4461                          */
 4462                         if (sva + NBPDR == va_next && eva >= va_next) {
 4463                                 /*
 4464                                  * The TLB entry for a PG_G mapping is
 4465                                  * invalidated by pmap_protect_pde().
 4466                                  */
 4467                                 if (pmap_protect_pde(pmap, pde, sva, prot))
 4468                                         anychanged = TRUE;
 4469                                 continue;
 4470                         } else if (!pmap_demote_pde(pmap, pde, sva)) {
 4471                                 /*
 4472                                  * The large page mapping was destroyed.
 4473                                  */
 4474                                 continue;
 4475                         }
 4476                 }
 4477 
 4478                 if (va_next > eva)
 4479                         va_next = eva;
 4480 
 4481                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 4482                     sva += PAGE_SIZE) {
 4483                         pt_entry_t obits, pbits;
 4484                         vm_page_t m;
 4485 
 4486 retry:
 4487                         obits = pbits = *pte;
 4488                         if ((pbits & PG_V) == 0)
 4489                                 continue;
 4490 
 4491                         if ((prot & VM_PROT_WRITE) == 0) {
 4492                                 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
 4493                                     (PG_MANAGED | PG_M | PG_RW)) {
 4494                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 4495                                         vm_page_dirty(m);
 4496                                 }
 4497                                 pbits &= ~(PG_RW | PG_M);
 4498                         }
 4499                         if ((prot & VM_PROT_EXECUTE) == 0)
 4500                                 pbits |= pg_nx;
 4501 
 4502                         if (pbits != obits) {
 4503                                 if (!atomic_cmpset_long(pte, obits, pbits))
 4504                                         goto retry;
 4505                                 if (obits & PG_G)
 4506                                         pmap_invalidate_page(pmap, sva);
 4507                                 else
 4508                                         anychanged = TRUE;
 4509                         }
 4510                 }
 4511         }
 4512         if (anychanged)
 4513                 pmap_invalidate_all(pmap);
 4514         PMAP_UNLOCK(pmap);
 4515 }
 4516 
 4517 #if VM_NRESERVLEVEL > 0
 4518 /*
 4519  * Tries to promote the 512, contiguous 4KB page mappings that are within a
 4520  * single page table page (PTP) to a single 2MB page mapping.  For promotion
 4521  * to occur, two conditions must be met: (1) the 4KB page mappings must map
 4522  * aligned, contiguous physical memory and (2) the 4KB page mappings must have
 4523  * identical characteristics. 
 4524  */
 4525 static void
 4526 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
 4527     struct rwlock **lockp)
 4528 {
 4529         pd_entry_t newpde;
 4530         pt_entry_t *firstpte, oldpte, pa, *pte;
 4531         pt_entry_t PG_G, PG_A, PG_M, PG_RW, PG_V;
 4532         vm_page_t mpte;
 4533         int PG_PTE_CACHE;
 4534 
 4535         PG_A = pmap_accessed_bit(pmap);
 4536         PG_G = pmap_global_bit(pmap);
 4537         PG_M = pmap_modified_bit(pmap);
 4538         PG_V = pmap_valid_bit(pmap);
 4539         PG_RW = pmap_rw_bit(pmap);
 4540         PG_PTE_CACHE = pmap_cache_mask(pmap, 0);
 4541 
 4542         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4543 
 4544         /*
 4545          * Examine the first PTE in the specified PTP.  Abort if this PTE is
 4546          * either invalid, unused, or does not map the first 4KB physical page
 4547          * within a 2MB page. 
 4548          */
 4549         firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
 4550 setpde:
 4551         newpde = *firstpte;
 4552         if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
 4553                 atomic_add_long(&pmap_pde_p_failures, 1);
 4554                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 4555                     " in pmap %p", va, pmap);
 4556                 return;
 4557         }
 4558         if ((newpde & (PG_M | PG_RW)) == PG_RW) {
 4559                 /*
 4560                  * When PG_M is already clear, PG_RW can be cleared without
 4561                  * a TLB invalidation.
 4562                  */
 4563                 if (!atomic_cmpset_long(firstpte, newpde, newpde & ~PG_RW))
 4564                         goto setpde;
 4565                 newpde &= ~PG_RW;
 4566         }
 4567 
 4568         /*
 4569          * Examine each of the other PTEs in the specified PTP.  Abort if this
 4570          * PTE maps an unexpected 4KB physical page or does not have identical
 4571          * characteristics to the first PTE.
 4572          */
 4573         pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + NBPDR - PAGE_SIZE;
 4574         for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
 4575 setpte:
 4576                 oldpte = *pte;
 4577                 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
 4578                         atomic_add_long(&pmap_pde_p_failures, 1);
 4579                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 4580                             " in pmap %p", va, pmap);
 4581                         return;
 4582                 }
 4583                 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
 4584                         /*
 4585                          * When PG_M is already clear, PG_RW can be cleared
 4586                          * without a TLB invalidation.
 4587                          */
 4588                         if (!atomic_cmpset_long(pte, oldpte, oldpte & ~PG_RW))
 4589                                 goto setpte;
 4590                         oldpte &= ~PG_RW;
 4591                         CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx"
 4592                             " in pmap %p", (oldpte & PG_FRAME & PDRMASK) |
 4593                             (va & ~PDRMASK), pmap);
 4594                 }
 4595                 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
 4596                         atomic_add_long(&pmap_pde_p_failures, 1);
 4597                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 4598                             " in pmap %p", va, pmap);
 4599                         return;
 4600                 }
 4601                 pa -= PAGE_SIZE;
 4602         }
 4603 
 4604         /*
 4605          * Save the page table page in its current state until the PDE
 4606          * mapping the superpage is demoted by pmap_demote_pde() or
 4607          * destroyed by pmap_remove_pde(). 
 4608          */
 4609         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 4610         KASSERT(mpte >= vm_page_array &&
 4611             mpte < &vm_page_array[vm_page_array_size],
 4612             ("pmap_promote_pde: page table page is out of range"));
 4613         KASSERT(mpte->pindex == pmap_pde_pindex(va),
 4614             ("pmap_promote_pde: page table page's pindex is wrong"));
 4615         if (pmap_insert_pt_page(pmap, mpte)) {
 4616                 atomic_add_long(&pmap_pde_p_failures, 1);
 4617                 CTR2(KTR_PMAP,
 4618                     "pmap_promote_pde: failure for va %#lx in pmap %p", va,
 4619                     pmap);
 4620                 return;
 4621         }
 4622 
 4623         /*
 4624          * Promote the pv entries.
 4625          */
 4626         if ((newpde & PG_MANAGED) != 0)
 4627                 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp);
 4628 
 4629         /*
 4630          * Propagate the PAT index to its proper position.
 4631          */
 4632         newpde = pmap_swap_pat(pmap, newpde);
 4633 
 4634         /*
 4635          * Map the superpage.
 4636          */
 4637         if (workaround_erratum383)
 4638                 pmap_update_pde(pmap, va, pde, PG_PS | newpde);
 4639         else
 4640                 pde_store(pde, PG_PROMOTED | PG_PS | newpde);
 4641 
 4642         atomic_add_long(&pmap_pde_promotions, 1);
 4643         CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
 4644             " in pmap %p", va, pmap);
 4645 }
 4646 #endif /* VM_NRESERVLEVEL > 0 */
 4647 
 4648 /*
 4649  *      Insert the given physical page (p) at
 4650  *      the specified virtual address (v) in the
 4651  *      target physical map with the protection requested.
 4652  *
 4653  *      If specified, the page will be wired down, meaning
 4654  *      that the related pte can not be reclaimed.
 4655  *
 4656  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 4657  *      or lose information.  That is, this routine must actually
 4658  *      insert this page into the given map NOW.
 4659  *
 4660  *      When destroying both a page table and PV entry, this function
 4661  *      performs the TLB invalidation before releasing the PV list
 4662  *      lock, so we do not need pmap_delayed_invl_page() calls here.
 4663  */
 4664 int
 4665 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 4666     u_int flags, int8_t psind)
 4667 {
 4668         struct rwlock *lock;
 4669         pd_entry_t *pde;
 4670         pt_entry_t *pte, PG_G, PG_A, PG_M, PG_RW, PG_V;
 4671         pt_entry_t newpte, origpte;
 4672         pv_entry_t pv;
 4673         vm_paddr_t opa, pa;
 4674         vm_page_t mpte, om;
 4675         int rv;
 4676         boolean_t nosleep;
 4677 
 4678         PG_A = pmap_accessed_bit(pmap);
 4679         PG_G = pmap_global_bit(pmap);
 4680         PG_M = pmap_modified_bit(pmap);
 4681         PG_V = pmap_valid_bit(pmap);
 4682         PG_RW = pmap_rw_bit(pmap);
 4683 
 4684         va = trunc_page(va);
 4685         KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
 4686         KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS,
 4687             ("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)",
 4688             va));
 4689         KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva ||
 4690             va >= kmi.clean_eva,
 4691             ("pmap_enter: managed mapping within the clean submap"));
 4692         if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
 4693                 VM_OBJECT_ASSERT_LOCKED(m->object);
 4694         KASSERT((flags & PMAP_ENTER_RESERVED) == 0,
 4695             ("pmap_enter: flags %u has reserved bits set", flags));
 4696         pa = VM_PAGE_TO_PHYS(m);
 4697         newpte = (pt_entry_t)(pa | PG_A | PG_V);
 4698         if ((flags & VM_PROT_WRITE) != 0)
 4699                 newpte |= PG_M;
 4700         if ((prot & VM_PROT_WRITE) != 0)
 4701                 newpte |= PG_RW;
 4702         KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
 4703             ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't"));
 4704         if ((prot & VM_PROT_EXECUTE) == 0)
 4705                 newpte |= pg_nx;
 4706         if ((flags & PMAP_ENTER_WIRED) != 0)
 4707                 newpte |= PG_W;
 4708         if (va < VM_MAXUSER_ADDRESS)
 4709                 newpte |= PG_U;
 4710         if (pmap == kernel_pmap)
 4711                 newpte |= PG_G;
 4712         newpte |= pmap_cache_bits(pmap, m->md.pat_mode, psind > 0);
 4713 
 4714         /*
 4715          * Set modified bit gratuitously for writeable mappings if
 4716          * the page is unmanaged. We do not want to take a fault
 4717          * to do the dirty bit accounting for these mappings.
 4718          */
 4719         if ((m->oflags & VPO_UNMANAGED) != 0) {
 4720                 if ((newpte & PG_RW) != 0)
 4721                         newpte |= PG_M;
 4722         } else
 4723                 newpte |= PG_MANAGED;
 4724 
 4725         lock = NULL;
 4726         PMAP_LOCK(pmap);
 4727         if (psind == 1) {
 4728                 /* Assert the required virtual and physical alignment. */ 
 4729                 KASSERT((va & PDRMASK) == 0, ("pmap_enter: va unaligned"));
 4730                 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
 4731                 rv = pmap_enter_pde(pmap, va, newpte | PG_PS, flags, m, &lock);
 4732                 goto out;
 4733         }
 4734         mpte = NULL;
 4735 
 4736         /*
 4737          * In the case that a page table page is not
 4738          * resident, we are creating it here.
 4739          */
 4740 retry:
 4741         pde = pmap_pde(pmap, va);
 4742         if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 ||
 4743             pmap_demote_pde_locked(pmap, pde, va, &lock))) {
 4744                 pte = pmap_pde_to_pte(pde, va);
 4745                 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
 4746                         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 4747                         mpte->wire_count++;
 4748                 }
 4749         } else if (va < VM_MAXUSER_ADDRESS) {
 4750                 /*
 4751                  * Here if the pte page isn't mapped, or if it has been
 4752                  * deallocated.
 4753                  */
 4754                 nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0;
 4755                 mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va),
 4756                     nosleep ? NULL : &lock);
 4757                 if (mpte == NULL && nosleep) {
 4758                         rv = KERN_RESOURCE_SHORTAGE;
 4759                         goto out;
 4760                 }
 4761                 goto retry;
 4762         } else
 4763                 panic("pmap_enter: invalid page directory va=%#lx", va);
 4764 
 4765         origpte = *pte;
 4766 
 4767         /*
 4768          * Is the specified virtual address already mapped?
 4769          */
 4770         if ((origpte & PG_V) != 0) {
 4771                 /*
 4772                  * Wiring change, just update stats. We don't worry about
 4773                  * wiring PT pages as they remain resident as long as there
 4774                  * are valid mappings in them. Hence, if a user page is wired,
 4775                  * the PT page will be also.
 4776                  */
 4777                 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
 4778                         pmap->pm_stats.wired_count++;
 4779                 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
 4780                         pmap->pm_stats.wired_count--;
 4781 
 4782                 /*
 4783                  * Remove the extra PT page reference.
 4784                  */
 4785                 if (mpte != NULL) {
 4786                         mpte->wire_count--;
 4787                         KASSERT(mpte->wire_count > 0,
 4788                             ("pmap_enter: missing reference to page table page,"
 4789                              " va: 0x%lx", va));
 4790                 }
 4791 
 4792                 /*
 4793                  * Has the physical page changed?
 4794                  */
 4795                 opa = origpte & PG_FRAME;
 4796                 if (opa == pa) {
 4797                         /*
 4798                          * No, might be a protection or wiring change.
 4799                          */
 4800                         if ((origpte & PG_MANAGED) != 0 &&
 4801                             (newpte & PG_RW) != 0)
 4802                                 vm_page_aflag_set(m, PGA_WRITEABLE);
 4803                         if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
 4804                                 goto unchanged;
 4805                         goto validate;
 4806                 }
 4807         } else {
 4808                 /*
 4809                  * Increment the counters.
 4810                  */
 4811                 if ((newpte & PG_W) != 0)
 4812                         pmap->pm_stats.wired_count++;
 4813                 pmap_resident_count_inc(pmap, 1);
 4814         }
 4815 
 4816         /*
 4817          * Enter on the PV list if part of our managed memory.
 4818          */
 4819         if ((newpte & PG_MANAGED) != 0) {
 4820                 pv = get_pv_entry(pmap, &lock);
 4821                 pv->pv_va = va;
 4822                 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
 4823                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 4824                 m->md.pv_gen++;
 4825                 if ((newpte & PG_RW) != 0)
 4826                         vm_page_aflag_set(m, PGA_WRITEABLE);
 4827         }
 4828 
 4829         /*
 4830          * Update the PTE.
 4831          */
 4832         if ((origpte & PG_V) != 0) {
 4833 validate:
 4834                 origpte = pte_load_store(pte, newpte);
 4835                 opa = origpte & PG_FRAME;
 4836                 if (opa != pa) {
 4837                         if ((origpte & PG_MANAGED) != 0) {
 4838                                 om = PHYS_TO_VM_PAGE(opa);
 4839                                 if ((origpte & (PG_M | PG_RW)) == (PG_M |
 4840                                     PG_RW))
 4841                                         vm_page_dirty(om);
 4842                                 if ((origpte & PG_A) != 0)
 4843                                         vm_page_aflag_set(om, PGA_REFERENCED);
 4844                                 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
 4845                                 pmap_pvh_free(&om->md, pmap, va);
 4846                                 if ((om->aflags & PGA_WRITEABLE) != 0 &&
 4847                                     TAILQ_EMPTY(&om->md.pv_list) &&
 4848                                     ((om->flags & PG_FICTITIOUS) != 0 ||
 4849                                     TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
 4850                                         vm_page_aflag_clear(om, PGA_WRITEABLE);
 4851                         }
 4852                 } else if ((newpte & PG_M) == 0 && (origpte & (PG_M |
 4853                     PG_RW)) == (PG_M | PG_RW)) {
 4854                         if ((origpte & PG_MANAGED) != 0)
 4855                                 vm_page_dirty(m);
 4856 
 4857                         /*
 4858                          * Although the PTE may still have PG_RW set, TLB
 4859                          * invalidation may nonetheless be required because
 4860                          * the PTE no longer has PG_M set.
 4861                          */
 4862                 } else if ((origpte & PG_NX) != 0 || (newpte & PG_NX) == 0) {
 4863                         /*
 4864                          * This PTE change does not require TLB invalidation.
 4865                          */
 4866                         goto unchanged;
 4867                 }
 4868                 if ((origpte & PG_A) != 0)
 4869                         pmap_invalidate_page(pmap, va);
 4870         } else
 4871                 pte_store(pte, newpte);
 4872 
 4873 unchanged:
 4874 
 4875 #if VM_NRESERVLEVEL > 0
 4876         /*
 4877          * If both the page table page and the reservation are fully
 4878          * populated, then attempt promotion.
 4879          */
 4880         if ((mpte == NULL || mpte->wire_count == NPTEPG) &&
 4881             pmap_ps_enabled(pmap) &&
 4882             (m->flags & PG_FICTITIOUS) == 0 &&
 4883             vm_reserv_level_iffullpop(m) == 0)
 4884                 pmap_promote_pde(pmap, pde, va, &lock);
 4885 #endif
 4886 
 4887         rv = KERN_SUCCESS;
 4888 out:
 4889         if (lock != NULL)
 4890                 rw_wunlock(lock);
 4891         PMAP_UNLOCK(pmap);
 4892         return (rv);
 4893 }
 4894 
 4895 /*
 4896  * Tries to create a read- and/or execute-only 2MB page mapping.  Returns true
 4897  * if successful.  Returns false if (1) a page table page cannot be allocated
 4898  * without sleeping, (2) a mapping already exists at the specified virtual
 4899  * address, or (3) a PV entry cannot be allocated without reclaiming another
 4900  * PV entry.
 4901  */
 4902 static bool
 4903 pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 4904     struct rwlock **lockp)
 4905 {
 4906         pd_entry_t newpde;
 4907         pt_entry_t PG_V;
 4908 
 4909         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4910         PG_V = pmap_valid_bit(pmap);
 4911         newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 1) |
 4912             PG_PS | PG_V;
 4913         if ((m->oflags & VPO_UNMANAGED) == 0)
 4914                 newpde |= PG_MANAGED;
 4915         if ((prot & VM_PROT_EXECUTE) == 0)
 4916                 newpde |= pg_nx;
 4917         if (va < VM_MAXUSER_ADDRESS)
 4918                 newpde |= PG_U;
 4919         return (pmap_enter_pde(pmap, va, newpde, PMAP_ENTER_NOSLEEP |
 4920             PMAP_ENTER_NOREPLACE | PMAP_ENTER_NORECLAIM, NULL, lockp) ==
 4921             KERN_SUCCESS);
 4922 }
 4923 
 4924 /*
 4925  * Tries to create the specified 2MB page mapping.  Returns KERN_SUCCESS if
 4926  * the mapping was created, and either KERN_FAILURE or KERN_RESOURCE_SHORTAGE
 4927  * otherwise.  Returns KERN_FAILURE if PMAP_ENTER_NOREPLACE was specified and
 4928  * a mapping already exists at the specified virtual address.  Returns
 4929  * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
 4930  * page allocation failed.  Returns KERN_RESOURCE_SHORTAGE if
 4931  * PMAP_ENTER_NORECLAIM was specified and a PV entry allocation failed.
 4932  *
 4933  * The parameter "m" is only used when creating a managed, writeable mapping.
 4934  */
 4935 static int
 4936 pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t newpde, u_int flags,
 4937     vm_page_t m, struct rwlock **lockp)
 4938 {
 4939         struct spglist free;
 4940         pd_entry_t oldpde, *pde;
 4941         pt_entry_t PG_G, PG_RW, PG_V;
 4942         vm_page_t mt, pdpg;
 4943 
 4944         PG_G = pmap_global_bit(pmap);
 4945         PG_RW = pmap_rw_bit(pmap);
 4946         KASSERT((newpde & (pmap_modified_bit(pmap) | PG_RW)) != PG_RW,
 4947             ("pmap_enter_pde: newpde is missing PG_M"));
 4948         PG_V = pmap_valid_bit(pmap);
 4949         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4950 
 4951         if ((pdpg = pmap_allocpde(pmap, va, (flags & PMAP_ENTER_NOSLEEP) != 0 ?
 4952             NULL : lockp)) == NULL) {
 4953                 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 4954                     " in pmap %p", va, pmap);
 4955                 return (KERN_RESOURCE_SHORTAGE);
 4956         }
 4957         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
 4958         pde = &pde[pmap_pde_index(va)];
 4959         oldpde = *pde;
 4960         if ((oldpde & PG_V) != 0) {
 4961                 KASSERT(pdpg->wire_count > 1,
 4962                     ("pmap_enter_pde: pdpg's wire count is too low"));
 4963                 if ((flags & PMAP_ENTER_NOREPLACE) != 0) {
 4964                         pdpg->wire_count--;
 4965                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 4966                             " in pmap %p", va, pmap);
 4967                         return (KERN_FAILURE);
 4968                 }
 4969                 /* Break the existing mapping(s). */
 4970                 SLIST_INIT(&free);
 4971                 if ((oldpde & PG_PS) != 0) {
 4972                         /*
 4973                          * The reference to the PD page that was acquired by
 4974                          * pmap_allocpde() ensures that it won't be freed.
 4975                          * However, if the PDE resulted from a promotion, then
 4976                          * a reserved PT page could be freed.
 4977                          */
 4978                         (void)pmap_remove_pde(pmap, pde, va, &free, lockp);
 4979                         if ((oldpde & PG_G) == 0)
 4980                                 pmap_invalidate_pde_page(pmap, va, oldpde);
 4981                 } else {
 4982                         pmap_delayed_invl_started();
 4983                         if (pmap_remove_ptes(pmap, va, va + NBPDR, pde, &free,
 4984                             lockp))
 4985                                pmap_invalidate_all(pmap);
 4986                         pmap_delayed_invl_finished();
 4987                 }
 4988                 pmap_free_zero_pages(&free);
 4989                 if (va >= VM_MAXUSER_ADDRESS) {
 4990                         mt = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 4991                         if (pmap_insert_pt_page(pmap, mt)) {
 4992                                 /*
 4993                                  * XXX Currently, this can't happen because
 4994                                  * we do not perform pmap_enter(psind == 1)
 4995                                  * on the kernel pmap.
 4996                                  */
 4997                                 panic("pmap_enter_pde: trie insert failed");
 4998                         }
 4999                 } else
 5000                         KASSERT(*pde == 0, ("pmap_enter_pde: non-zero pde %p",
 5001                             pde));
 5002         }
 5003         if ((newpde & PG_MANAGED) != 0) {
 5004                 /*
 5005                  * Abort this mapping if its PV entry could not be created.
 5006                  */
 5007                 if (!pmap_pv_insert_pde(pmap, va, newpde, flags, lockp)) {
 5008                         SLIST_INIT(&free);
 5009                         if (pmap_unwire_ptp(pmap, va, pdpg, &free)) {
 5010                                 /*
 5011                                  * Although "va" is not mapped, paging-
 5012                                  * structure caches could nonetheless have
 5013                                  * entries that refer to the freed page table
 5014                                  * pages.  Invalidate those entries.
 5015                                  */
 5016                                 pmap_invalidate_page(pmap, va);
 5017                                 pmap_free_zero_pages(&free);
 5018                         }
 5019                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 5020                             " in pmap %p", va, pmap);
 5021                         return (KERN_RESOURCE_SHORTAGE);
 5022                 }
 5023                 if ((newpde & PG_RW) != 0) {
 5024                         for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 5025                                 vm_page_aflag_set(mt, PGA_WRITEABLE);
 5026                 }
 5027         }
 5028 
 5029         /*
 5030          * Increment counters.
 5031          */
 5032         if ((newpde & PG_W) != 0)
 5033                 pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE;
 5034         pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
 5035 
 5036         /*
 5037          * Map the superpage.  (This is not a promoted mapping; there will not
 5038          * be any lingering 4KB page mappings in the TLB.)
 5039          */
 5040         pde_store(pde, newpde);
 5041 
 5042         atomic_add_long(&pmap_pde_mappings, 1);
 5043         CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
 5044             " in pmap %p", va, pmap);
 5045         return (KERN_SUCCESS);
 5046 }
 5047 
 5048 /*
 5049  * Maps a sequence of resident pages belonging to the same object.
 5050  * The sequence begins with the given page m_start.  This page is
 5051  * mapped at the given virtual address start.  Each subsequent page is
 5052  * mapped at a virtual address that is offset from start by the same
 5053  * amount as the page is offset from m_start within the object.  The
 5054  * last page in the sequence is the page with the largest offset from
 5055  * m_start that can be mapped at a virtual address less than the given
 5056  * virtual address end.  Not every virtual page between start and end
 5057  * is mapped; only those for which a resident page exists with the
 5058  * corresponding offset from m_start are mapped.
 5059  */
 5060 void
 5061 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 5062     vm_page_t m_start, vm_prot_t prot)
 5063 {
 5064         struct rwlock *lock;
 5065         vm_offset_t va;
 5066         vm_page_t m, mpte;
 5067         vm_pindex_t diff, psize;
 5068 
 5069         VM_OBJECT_ASSERT_LOCKED(m_start->object);
 5070 
 5071         psize = atop(end - start);
 5072         mpte = NULL;
 5073         m = m_start;
 5074         lock = NULL;
 5075         PMAP_LOCK(pmap);
 5076         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 5077                 va = start + ptoa(diff);
 5078                 if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
 5079                     m->psind == 1 && pmap_ps_enabled(pmap) &&
 5080                     pmap_enter_2mpage(pmap, va, m, prot, &lock))
 5081                         m = &m[NBPDR / PAGE_SIZE - 1];
 5082                 else
 5083                         mpte = pmap_enter_quick_locked(pmap, va, m, prot,
 5084                             mpte, &lock);
 5085                 m = TAILQ_NEXT(m, listq);
 5086         }
 5087         if (lock != NULL)
 5088                 rw_wunlock(lock);
 5089         PMAP_UNLOCK(pmap);
 5090 }
 5091 
 5092 /*
 5093  * this code makes some *MAJOR* assumptions:
 5094  * 1. Current pmap & pmap exists.
 5095  * 2. Not wired.
 5096  * 3. Read access.
 5097  * 4. No page table pages.
 5098  * but is *MUCH* faster than pmap_enter...
 5099  */
 5100 
 5101 void
 5102 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 5103 {
 5104         struct rwlock *lock;
 5105 
 5106         lock = NULL;
 5107         PMAP_LOCK(pmap);
 5108         (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);
 5109         if (lock != NULL)
 5110                 rw_wunlock(lock);
 5111         PMAP_UNLOCK(pmap);
 5112 }
 5113 
 5114 static vm_page_t
 5115 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 5116     vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp)
 5117 {
 5118         struct spglist free;
 5119         pt_entry_t *pte, PG_V;
 5120         vm_paddr_t pa;
 5121 
 5122         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 5123             (m->oflags & VPO_UNMANAGED) != 0,
 5124             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 5125         PG_V = pmap_valid_bit(pmap);
 5126         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 5127 
 5128         /*
 5129          * In the case that a page table page is not
 5130          * resident, we are creating it here.
 5131          */
 5132         if (va < VM_MAXUSER_ADDRESS) {
 5133                 vm_pindex_t ptepindex;
 5134                 pd_entry_t *ptepa;
 5135 
 5136                 /*
 5137                  * Calculate pagetable page index
 5138                  */
 5139                 ptepindex = pmap_pde_pindex(va);
 5140                 if (mpte && (mpte->pindex == ptepindex)) {
 5141                         mpte->wire_count++;
 5142                 } else {
 5143                         /*
 5144                          * Get the page directory entry
 5145                          */
 5146                         ptepa = pmap_pde(pmap, va);
 5147 
 5148                         /*
 5149                          * If the page table page is mapped, we just increment
 5150                          * the hold count, and activate it.  Otherwise, we
 5151                          * attempt to allocate a page table page.  If this
 5152                          * attempt fails, we don't retry.  Instead, we give up.
 5153                          */
 5154                         if (ptepa && (*ptepa & PG_V) != 0) {
 5155                                 if (*ptepa & PG_PS)
 5156                                         return (NULL);
 5157                                 mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
 5158                                 mpte->wire_count++;
 5159                         } else {
 5160                                 /*
 5161                                  * Pass NULL instead of the PV list lock
 5162                                  * pointer, because we don't intend to sleep.
 5163                                  */
 5164                                 mpte = _pmap_allocpte(pmap, ptepindex, NULL);
 5165                                 if (mpte == NULL)
 5166                                         return (mpte);
 5167                         }
 5168                 }
 5169                 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
 5170                 pte = &pte[pmap_pte_index(va)];
 5171         } else {
 5172                 mpte = NULL;
 5173                 pte = vtopte(va);
 5174         }
 5175         if (*pte) {
 5176                 if (mpte != NULL) {
 5177                         mpte->wire_count--;
 5178                         mpte = NULL;
 5179                 }
 5180                 return (mpte);
 5181         }
 5182 
 5183         /*
 5184          * Enter on the PV list if part of our managed memory.
 5185          */
 5186         if ((m->oflags & VPO_UNMANAGED) == 0 &&
 5187             !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
 5188                 if (mpte != NULL) {
 5189                         SLIST_INIT(&free);
 5190                         if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
 5191                                 /*
 5192                                  * Although "va" is not mapped, paging-
 5193                                  * structure caches could nonetheless have
 5194                                  * entries that refer to the freed page table
 5195                                  * pages.  Invalidate those entries.
 5196                                  */
 5197                                 pmap_invalidate_page(pmap, va);
 5198                                 pmap_free_zero_pages(&free);
 5199                         }
 5200                         mpte = NULL;
 5201                 }
 5202                 return (mpte);
 5203         }
 5204 
 5205         /*
 5206          * Increment counters
 5207          */
 5208         pmap_resident_count_inc(pmap, 1);
 5209 
 5210         pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 0);
 5211         if ((prot & VM_PROT_EXECUTE) == 0)
 5212                 pa |= pg_nx;
 5213 
 5214         /*
 5215          * Now validate mapping with RO protection
 5216          */
 5217         if ((m->oflags & VPO_UNMANAGED) != 0)
 5218                 pte_store(pte, pa | PG_V | PG_U);
 5219         else
 5220                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 5221         return (mpte);
 5222 }
 5223 
 5224 /*
 5225  * Make a temporary mapping for a physical address.  This is only intended
 5226  * to be used for panic dumps.
 5227  */
 5228 void *
 5229 pmap_kenter_temporary(vm_paddr_t pa, int i)
 5230 {
 5231         vm_offset_t va;
 5232 
 5233         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 5234         pmap_kenter(va, pa);
 5235         invlpg(va);
 5236         return ((void *)crashdumpmap);
 5237 }
 5238 
 5239 /*
 5240  * This code maps large physical mmap regions into the
 5241  * processor address space.  Note that some shortcuts
 5242  * are taken, but the code works.
 5243  */
 5244 void
 5245 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
 5246     vm_pindex_t pindex, vm_size_t size)
 5247 {
 5248         pd_entry_t *pde;
 5249         pt_entry_t PG_A, PG_M, PG_RW, PG_V;
 5250         vm_paddr_t pa, ptepa;
 5251         vm_page_t p, pdpg;
 5252         int pat_mode;
 5253 
 5254         PG_A = pmap_accessed_bit(pmap);
 5255         PG_M = pmap_modified_bit(pmap);
 5256         PG_V = pmap_valid_bit(pmap);
 5257         PG_RW = pmap_rw_bit(pmap);
 5258 
 5259         VM_OBJECT_ASSERT_WLOCKED(object);
 5260         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 5261             ("pmap_object_init_pt: non-device object"));
 5262         if ((addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
 5263                 if (!pmap_ps_enabled(pmap))
 5264                         return;
 5265                 if (!vm_object_populate(object, pindex, pindex + atop(size)))
 5266                         return;
 5267                 p = vm_page_lookup(object, pindex);
 5268                 KASSERT(p->valid == VM_PAGE_BITS_ALL,
 5269                     ("pmap_object_init_pt: invalid page %p", p));
 5270                 pat_mode = p->md.pat_mode;
 5271 
 5272                 /*
 5273                  * Abort the mapping if the first page is not physically
 5274                  * aligned to a 2MB page boundary.
 5275                  */
 5276                 ptepa = VM_PAGE_TO_PHYS(p);
 5277                 if (ptepa & (NBPDR - 1))
 5278                         return;
 5279 
 5280                 /*
 5281                  * Skip the first page.  Abort the mapping if the rest of
 5282                  * the pages are not physically contiguous or have differing
 5283                  * memory attributes.
 5284                  */
 5285                 p = TAILQ_NEXT(p, listq);
 5286                 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
 5287                     pa += PAGE_SIZE) {
 5288                         KASSERT(p->valid == VM_PAGE_BITS_ALL,
 5289                             ("pmap_object_init_pt: invalid page %p", p));
 5290                         if (pa != VM_PAGE_TO_PHYS(p) ||
 5291                             pat_mode != p->md.pat_mode)
 5292                                 return;
 5293                         p = TAILQ_NEXT(p, listq);
 5294                 }
 5295 
 5296                 /*
 5297                  * Map using 2MB pages.  Since "ptepa" is 2M aligned and
 5298                  * "size" is a multiple of 2M, adding the PAT setting to "pa"
 5299                  * will not affect the termination of this loop.
 5300                  */ 
 5301                 PMAP_LOCK(pmap);
 5302                 for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, 1);
 5303                     pa < ptepa + size; pa += NBPDR) {
 5304                         pdpg = pmap_allocpde(pmap, addr, NULL);
 5305                         if (pdpg == NULL) {
 5306                                 /*
 5307                                  * The creation of mappings below is only an
 5308                                  * optimization.  If a page directory page
 5309                                  * cannot be allocated without blocking,
 5310                                  * continue on to the next mapping rather than
 5311                                  * blocking.
 5312                                  */
 5313                                 addr += NBPDR;
 5314                                 continue;
 5315                         }
 5316                         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
 5317                         pde = &pde[pmap_pde_index(addr)];
 5318                         if ((*pde & PG_V) == 0) {
 5319                                 pde_store(pde, pa | PG_PS | PG_M | PG_A |
 5320                                     PG_U | PG_RW | PG_V);
 5321                                 pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
 5322                                 atomic_add_long(&pmap_pde_mappings, 1);
 5323                         } else {
 5324                                 /* Continue on if the PDE is already valid. */
 5325                                 pdpg->wire_count--;
 5326                                 KASSERT(pdpg->wire_count > 0,
 5327                                     ("pmap_object_init_pt: missing reference "
 5328                                     "to page directory page, va: 0x%lx", addr));
 5329                         }
 5330                         addr += NBPDR;
 5331                 }
 5332                 PMAP_UNLOCK(pmap);
 5333         }
 5334 }
 5335 
 5336 /*
 5337  *      Clear the wired attribute from the mappings for the specified range of
 5338  *      addresses in the given pmap.  Every valid mapping within that range
 5339  *      must have the wired attribute set.  In contrast, invalid mappings
 5340  *      cannot have the wired attribute set, so they are ignored.
 5341  *
 5342  *      The wired attribute of the page table entry is not a hardware
 5343  *      feature, so there is no need to invalidate any TLB entries.
 5344  *      Since pmap_demote_pde() for the wired entry must never fail,
 5345  *      pmap_delayed_invl_started()/finished() calls around the
 5346  *      function are not needed.
 5347  */
 5348 void
 5349 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 5350 {
 5351         vm_offset_t va_next;
 5352         pml4_entry_t *pml4e;
 5353         pdp_entry_t *pdpe;
 5354         pd_entry_t *pde;
 5355         pt_entry_t *pte, PG_V;
 5356 
 5357         PG_V = pmap_valid_bit(pmap);
 5358         PMAP_LOCK(pmap);
 5359         for (; sva < eva; sva = va_next) {
 5360                 pml4e = pmap_pml4e(pmap, sva);
 5361                 if ((*pml4e & PG_V) == 0) {
 5362                         va_next = (sva + NBPML4) & ~PML4MASK;
 5363                         if (va_next < sva)
 5364                                 va_next = eva;
 5365                         continue;
 5366                 }
 5367                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 5368                 if ((*pdpe & PG_V) == 0) {
 5369                         va_next = (sva + NBPDP) & ~PDPMASK;
 5370                         if (va_next < sva)
 5371                                 va_next = eva;
 5372                         continue;
 5373                 }
 5374                 va_next = (sva + NBPDR) & ~PDRMASK;
 5375                 if (va_next < sva)
 5376                         va_next = eva;
 5377                 pde = pmap_pdpe_to_pde(pdpe, sva);
 5378                 if ((*pde & PG_V) == 0)
 5379                         continue;
 5380                 if ((*pde & PG_PS) != 0) {
 5381                         if ((*pde & PG_W) == 0)
 5382                                 panic("pmap_unwire: pde %#jx is missing PG_W",
 5383                                     (uintmax_t)*pde);
 5384 
 5385                         /*
 5386                          * Are we unwiring the entire large page?  If not,
 5387                          * demote the mapping and fall through.
 5388                          */
 5389                         if (sva + NBPDR == va_next && eva >= va_next) {
 5390                                 atomic_clear_long(pde, PG_W);
 5391                                 pmap->pm_stats.wired_count -= NBPDR /
 5392                                     PAGE_SIZE;
 5393                                 continue;
 5394                         } else if (!pmap_demote_pde(pmap, pde, sva))
 5395                                 panic("pmap_unwire: demotion failed");
 5396                 }
 5397                 if (va_next > eva)
 5398                         va_next = eva;
 5399                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 5400                     sva += PAGE_SIZE) {
 5401                         if ((*pte & PG_V) == 0)
 5402                                 continue;
 5403                         if ((*pte & PG_W) == 0)
 5404                                 panic("pmap_unwire: pte %#jx is missing PG_W",
 5405                                     (uintmax_t)*pte);
 5406 
 5407                         /*
 5408                          * PG_W must be cleared atomically.  Although the pmap
 5409                          * lock synchronizes access to PG_W, another processor
 5410                          * could be setting PG_M and/or PG_A concurrently.
 5411                          */
 5412                         atomic_clear_long(pte, PG_W);
 5413                         pmap->pm_stats.wired_count--;
 5414                 }
 5415         }
 5416         PMAP_UNLOCK(pmap);
 5417 }
 5418 
 5419 /*
 5420  *      Copy the range specified by src_addr/len
 5421  *      from the source map to the range dst_addr/len
 5422  *      in the destination map.
 5423  *
 5424  *      This routine is only advisory and need not do anything.
 5425  */
 5426 
 5427 void
 5428 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 5429     vm_offset_t src_addr)
 5430 {
 5431         struct rwlock *lock;
 5432         struct spglist free;
 5433         vm_offset_t addr;
 5434         vm_offset_t end_addr = src_addr + len;
 5435         vm_offset_t va_next;
 5436         vm_page_t dst_pdpg, dstmpte, srcmpte;
 5437         pt_entry_t PG_A, PG_M, PG_V;
 5438 
 5439         if (dst_addr != src_addr)
 5440                 return;
 5441 
 5442         if (dst_pmap->pm_type != src_pmap->pm_type)
 5443                 return;
 5444 
 5445         /*
 5446          * EPT page table entries that require emulation of A/D bits are
 5447          * sensitive to clearing the PG_A bit (aka EPT_PG_READ). Although
 5448          * we clear PG_M (aka EPT_PG_WRITE) concomitantly, the PG_U bit
 5449          * (aka EPT_PG_EXECUTE) could still be set. Since some EPT
 5450          * implementations flag an EPT misconfiguration for exec-only
 5451          * mappings we skip this function entirely for emulated pmaps.
 5452          */
 5453         if (pmap_emulate_ad_bits(dst_pmap))
 5454                 return;
 5455 
 5456         lock = NULL;
 5457         if (dst_pmap < src_pmap) {
 5458                 PMAP_LOCK(dst_pmap);
 5459                 PMAP_LOCK(src_pmap);
 5460         } else {
 5461                 PMAP_LOCK(src_pmap);
 5462                 PMAP_LOCK(dst_pmap);
 5463         }
 5464 
 5465         PG_A = pmap_accessed_bit(dst_pmap);
 5466         PG_M = pmap_modified_bit(dst_pmap);
 5467         PG_V = pmap_valid_bit(dst_pmap);
 5468 
 5469         for (addr = src_addr; addr < end_addr; addr = va_next) {
 5470                 pt_entry_t *src_pte, *dst_pte;
 5471                 pml4_entry_t *pml4e;
 5472                 pdp_entry_t *pdpe;
 5473                 pd_entry_t srcptepaddr, *pde;
 5474 
 5475                 KASSERT(addr < UPT_MIN_ADDRESS,
 5476                     ("pmap_copy: invalid to pmap_copy page tables"));
 5477 
 5478                 pml4e = pmap_pml4e(src_pmap, addr);
 5479                 if ((*pml4e & PG_V) == 0) {
 5480                         va_next = (addr + NBPML4) & ~PML4MASK;
 5481                         if (va_next < addr)
 5482                                 va_next = end_addr;
 5483                         continue;
 5484                 }
 5485 
 5486                 pdpe = pmap_pml4e_to_pdpe(pml4e, addr);
 5487                 if ((*pdpe & PG_V) == 0) {
 5488                         va_next = (addr + NBPDP) & ~PDPMASK;
 5489                         if (va_next < addr)
 5490                                 va_next = end_addr;
 5491                         continue;
 5492                 }
 5493 
 5494                 va_next = (addr + NBPDR) & ~PDRMASK;
 5495                 if (va_next < addr)
 5496                         va_next = end_addr;
 5497 
 5498                 pde = pmap_pdpe_to_pde(pdpe, addr);
 5499                 srcptepaddr = *pde;
 5500                 if (srcptepaddr == 0)
 5501                         continue;
 5502                         
 5503                 if (srcptepaddr & PG_PS) {
 5504                         if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr)
 5505                                 continue;
 5506                         dst_pdpg = pmap_allocpde(dst_pmap, addr, NULL);
 5507                         if (dst_pdpg == NULL)
 5508                                 break;
 5509                         pde = (pd_entry_t *)
 5510                             PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dst_pdpg));
 5511                         pde = &pde[pmap_pde_index(addr)];
 5512                         if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
 5513                             pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr,
 5514                             PMAP_ENTER_NORECLAIM, &lock))) {
 5515                                 *pde = srcptepaddr & ~PG_W;
 5516                                 pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE);
 5517                                 atomic_add_long(&pmap_pde_mappings, 1);
 5518                         } else
 5519                                 dst_pdpg->wire_count--;
 5520                         continue;
 5521                 }
 5522 
 5523                 srcptepaddr &= PG_FRAME;
 5524                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
 5525                 KASSERT(srcmpte->wire_count > 0,
 5526                     ("pmap_copy: source page table page is unused"));
 5527 
 5528                 if (va_next > end_addr)
 5529                         va_next = end_addr;
 5530 
 5531                 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
 5532                 src_pte = &src_pte[pmap_pte_index(addr)];
 5533                 dstmpte = NULL;
 5534                 while (addr < va_next) {
 5535                         pt_entry_t ptetemp;
 5536                         ptetemp = *src_pte;
 5537                         /*
 5538                          * we only virtual copy managed pages
 5539                          */
 5540                         if ((ptetemp & PG_MANAGED) != 0) {
 5541                                 if (dstmpte != NULL &&
 5542                                     dstmpte->pindex == pmap_pde_pindex(addr))
 5543                                         dstmpte->wire_count++;
 5544                                 else if ((dstmpte = pmap_allocpte(dst_pmap,
 5545                                     addr, NULL)) == NULL)
 5546                                         goto out;
 5547                                 dst_pte = (pt_entry_t *)
 5548                                     PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
 5549                                 dst_pte = &dst_pte[pmap_pte_index(addr)];
 5550                                 if (*dst_pte == 0 &&
 5551                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 5552                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
 5553                                     &lock)) {
 5554                                         /*
 5555                                          * Clear the wired, modified, and
 5556                                          * accessed (referenced) bits
 5557                                          * during the copy.
 5558                                          */
 5559                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 5560                                             PG_A);
 5561                                         pmap_resident_count_inc(dst_pmap, 1);
 5562                                 } else {
 5563                                         SLIST_INIT(&free);
 5564                                         if (pmap_unwire_ptp(dst_pmap, addr,
 5565                                             dstmpte, &free)) {
 5566                                                 /*
 5567                                                  * Although "addr" is not
 5568                                                  * mapped, paging-structure
 5569                                                  * caches could nonetheless
 5570                                                  * have entries that refer to
 5571                                                  * the freed page table pages.
 5572                                                  * Invalidate those entries.
 5573                                                  */
 5574                                                 pmap_invalidate_page(dst_pmap,
 5575                                                     addr);
 5576                                                 pmap_free_zero_pages(&free);
 5577                                         }
 5578                                         goto out;
 5579                                 }
 5580                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 5581                                         break;
 5582                         }
 5583                         addr += PAGE_SIZE;
 5584                         src_pte++;
 5585                 }
 5586         }
 5587 out:
 5588         if (lock != NULL)
 5589                 rw_wunlock(lock);
 5590         PMAP_UNLOCK(src_pmap);
 5591         PMAP_UNLOCK(dst_pmap);
 5592 }
 5593 
 5594 /*
 5595  *      pmap_zero_page zeros the specified hardware page by mapping
 5596  *      the page into KVM and using bzero to clear its contents.
 5597  */
 5598 void
 5599 pmap_zero_page(vm_page_t m)
 5600 {
 5601         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 5602 
 5603         pagezero((void *)va);
 5604 }
 5605 
 5606 /*
 5607  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 5608  *      the page into KVM and using bzero to clear its contents.
 5609  *
 5610  *      off and size may not cover an area beyond a single hardware page.
 5611  */
 5612 void
 5613 pmap_zero_page_area(vm_page_t m, int off, int size)
 5614 {
 5615         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 5616 
 5617         if (off == 0 && size == PAGE_SIZE)
 5618                 pagezero((void *)va);
 5619         else
 5620                 bzero((char *)va + off, size);
 5621 }
 5622 
 5623 /*
 5624  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 5625  *      the page into KVM and using bzero to clear its contents.  This
 5626  *      is intended to be called from the vm_pagezero process only and
 5627  *      outside of Giant.
 5628  */
 5629 void
 5630 pmap_zero_page_idle(vm_page_t m)
 5631 {
 5632         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 5633 
 5634         pagezero((void *)va);
 5635 }
 5636 
 5637 /*
 5638  *      pmap_copy_page copies the specified (machine independent)
 5639  *      page by mapping the page into virtual memory and using
 5640  *      bcopy to copy the page, one machine dependent page at a
 5641  *      time.
 5642  */
 5643 void
 5644 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 5645 {
 5646         vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
 5647         vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
 5648 
 5649         pagecopy((void *)src, (void *)dst);
 5650 }
 5651 
 5652 int unmapped_buf_allowed = 1;
 5653 
 5654 void
 5655 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 5656     vm_offset_t b_offset, int xfersize)
 5657 {
 5658         void *a_cp, *b_cp;
 5659         vm_page_t pages[2];
 5660         vm_offset_t vaddr[2], a_pg_offset, b_pg_offset;
 5661         int cnt;
 5662         boolean_t mapped;
 5663 
 5664         while (xfersize > 0) {
 5665                 a_pg_offset = a_offset & PAGE_MASK;
 5666                 pages[0] = ma[a_offset >> PAGE_SHIFT];
 5667                 b_pg_offset = b_offset & PAGE_MASK;
 5668                 pages[1] = mb[b_offset >> PAGE_SHIFT];
 5669                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
 5670                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 5671                 mapped = pmap_map_io_transient(pages, vaddr, 2, FALSE);
 5672                 a_cp = (char *)vaddr[0] + a_pg_offset;
 5673                 b_cp = (char *)vaddr[1] + b_pg_offset;
 5674                 bcopy(a_cp, b_cp, cnt);
 5675                 if (__predict_false(mapped))
 5676                         pmap_unmap_io_transient(pages, vaddr, 2, FALSE);
 5677                 a_offset += cnt;
 5678                 b_offset += cnt;
 5679                 xfersize -= cnt;
 5680         }
 5681 }
 5682 
 5683 /*
 5684  * Returns true if the pmap's pv is one of the first
 5685  * 16 pvs linked to from this page.  This count may
 5686  * be changed upwards or downwards in the future; it
 5687  * is only necessary that true be returned for a small
 5688  * subset of pmaps for proper page aging.
 5689  */
 5690 boolean_t
 5691 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 5692 {
 5693         struct md_page *pvh;
 5694         struct rwlock *lock;
 5695         pv_entry_t pv;
 5696         int loops = 0;
 5697         boolean_t rv;
 5698 
 5699         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5700             ("pmap_page_exists_quick: page %p is not managed", m));
 5701         rv = FALSE;
 5702         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 5703         rw_rlock(lock);
 5704         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5705                 if (PV_PMAP(pv) == pmap) {
 5706                         rv = TRUE;
 5707                         break;
 5708                 }
 5709                 loops++;
 5710                 if (loops >= 16)
 5711                         break;
 5712         }
 5713         if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
 5714                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5715                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 5716                         if (PV_PMAP(pv) == pmap) {
 5717                                 rv = TRUE;
 5718                                 break;
 5719                         }
 5720                         loops++;
 5721                         if (loops >= 16)
 5722                                 break;
 5723                 }
 5724         }
 5725         rw_runlock(lock);
 5726         return (rv);
 5727 }
 5728 
 5729 /*
 5730  *      pmap_page_wired_mappings:
 5731  *
 5732  *      Return the number of managed mappings to the given physical page
 5733  *      that are wired.
 5734  */
 5735 int
 5736 pmap_page_wired_mappings(vm_page_t m)
 5737 {
 5738         struct rwlock *lock;
 5739         struct md_page *pvh;
 5740         pmap_t pmap;
 5741         pt_entry_t *pte;
 5742         pv_entry_t pv;
 5743         int count, md_gen, pvh_gen;
 5744 
 5745         if ((m->oflags & VPO_UNMANAGED) != 0)
 5746                 return (0);
 5747         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 5748         rw_rlock(lock);
 5749 restart:
 5750         count = 0;
 5751         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5752                 pmap = PV_PMAP(pv);
 5753                 if (!PMAP_TRYLOCK(pmap)) {
 5754                         md_gen = m->md.pv_gen;
 5755                         rw_runlock(lock);
 5756                         PMAP_LOCK(pmap);
 5757                         rw_rlock(lock);
 5758                         if (md_gen != m->md.pv_gen) {
 5759                                 PMAP_UNLOCK(pmap);
 5760                                 goto restart;
 5761                         }
 5762                 }
 5763                 pte = pmap_pte(pmap, pv->pv_va);
 5764                 if ((*pte & PG_W) != 0)
 5765                         count++;
 5766                 PMAP_UNLOCK(pmap);
 5767         }
 5768         if ((m->flags & PG_FICTITIOUS) == 0) {
 5769                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5770                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 5771                         pmap = PV_PMAP(pv);
 5772                         if (!PMAP_TRYLOCK(pmap)) {
 5773                                 md_gen = m->md.pv_gen;
 5774                                 pvh_gen = pvh->pv_gen;
 5775                                 rw_runlock(lock);
 5776                                 PMAP_LOCK(pmap);
 5777                                 rw_rlock(lock);
 5778                                 if (md_gen != m->md.pv_gen ||
 5779                                     pvh_gen != pvh->pv_gen) {
 5780                                         PMAP_UNLOCK(pmap);
 5781                                         goto restart;
 5782                                 }
 5783                         }
 5784                         pte = pmap_pde(pmap, pv->pv_va);
 5785                         if ((*pte & PG_W) != 0)
 5786                                 count++;
 5787                         PMAP_UNLOCK(pmap);
 5788                 }
 5789         }
 5790         rw_runlock(lock);
 5791         return (count);
 5792 }
 5793 
 5794 /*
 5795  * Returns TRUE if the given page is mapped individually or as part of
 5796  * a 2mpage.  Otherwise, returns FALSE.
 5797  */
 5798 boolean_t
 5799 pmap_page_is_mapped(vm_page_t m)
 5800 {
 5801         struct rwlock *lock;
 5802         boolean_t rv;
 5803 
 5804         if ((m->oflags & VPO_UNMANAGED) != 0)
 5805                 return (FALSE);
 5806         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 5807         rw_rlock(lock);
 5808         rv = !TAILQ_EMPTY(&m->md.pv_list) ||
 5809             ((m->flags & PG_FICTITIOUS) == 0 &&
 5810             !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
 5811         rw_runlock(lock);
 5812         return (rv);
 5813 }
 5814 
 5815 /*
 5816  * Destroy all managed, non-wired mappings in the given user-space
 5817  * pmap.  This pmap cannot be active on any processor besides the
 5818  * caller.
 5819  *
 5820  * This function cannot be applied to the kernel pmap.  Moreover, it
 5821  * is not intended for general use.  It is only to be used during
 5822  * process termination.  Consequently, it can be implemented in ways
 5823  * that make it faster than pmap_remove().  First, it can more quickly
 5824  * destroy mappings by iterating over the pmap's collection of PV
 5825  * entries, rather than searching the page table.  Second, it doesn't
 5826  * have to test and clear the page table entries atomically, because
 5827  * no processor is currently accessing the user address space.  In
 5828  * particular, a page table entry's dirty bit won't change state once
 5829  * this function starts.
 5830  *
 5831  * Although this function destroys all of the pmap's managed,
 5832  * non-wired mappings, it can delay and batch the invalidation of TLB
 5833  * entries without calling pmap_delayed_invl_started() and
 5834  * pmap_delayed_invl_finished().  Because the pmap is not active on
 5835  * any other processor, none of these TLB entries will ever be used
 5836  * before their eventual invalidation.  Consequently, there is no need
 5837  * for either pmap_remove_all() or pmap_remove_write() to wait for
 5838  * that eventual TLB invalidation.
 5839  */
 5840 void
 5841 pmap_remove_pages(pmap_t pmap)
 5842 {
 5843         pd_entry_t ptepde;
 5844         pt_entry_t *pte, tpte;
 5845         pt_entry_t PG_M, PG_RW, PG_V;
 5846         struct spglist free;
 5847         vm_page_t m, mpte, mt;
 5848         pv_entry_t pv;
 5849         struct md_page *pvh;
 5850         struct pv_chunk *pc, *npc;
 5851         struct rwlock *lock;
 5852         int64_t bit;
 5853         uint64_t inuse, bitmask;
 5854         int allfree, field, freed, idx;
 5855         boolean_t superpage;
 5856         vm_paddr_t pa;
 5857 
 5858         /*
 5859          * Assert that the given pmap is only active on the current
 5860          * CPU.  Unfortunately, we cannot block another CPU from
 5861          * activating the pmap while this function is executing.
 5862          */
 5863         KASSERT(pmap == PCPU_GET(curpmap), ("non-current pmap %p", pmap));
 5864 #ifdef INVARIANTS
 5865         {
 5866                 cpuset_t other_cpus;
 5867 
 5868                 other_cpus = all_cpus;
 5869                 critical_enter();
 5870                 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
 5871                 CPU_AND(&other_cpus, &pmap->pm_active);
 5872                 critical_exit();
 5873                 KASSERT(CPU_EMPTY(&other_cpus), ("pmap active %p", pmap));
 5874         }
 5875 #endif
 5876 
 5877         lock = NULL;
 5878         PG_M = pmap_modified_bit(pmap);
 5879         PG_V = pmap_valid_bit(pmap);
 5880         PG_RW = pmap_rw_bit(pmap);
 5881 
 5882         SLIST_INIT(&free);
 5883         PMAP_LOCK(pmap);
 5884         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 5885                 allfree = 1;
 5886                 freed = 0;
 5887                 for (field = 0; field < _NPCM; field++) {
 5888                         inuse = ~pc->pc_map[field] & pc_freemask[field];
 5889                         while (inuse != 0) {
 5890                                 bit = bsfq(inuse);
 5891                                 bitmask = 1UL << bit;
 5892                                 idx = field * 64 + bit;
 5893                                 pv = &pc->pc_pventry[idx];
 5894                                 inuse &= ~bitmask;
 5895 
 5896                                 pte = pmap_pdpe(pmap, pv->pv_va);
 5897                                 ptepde = *pte;
 5898                                 pte = pmap_pdpe_to_pde(pte, pv->pv_va);
 5899                                 tpte = *pte;
 5900                                 if ((tpte & (PG_PS | PG_V)) == PG_V) {
 5901                                         superpage = FALSE;
 5902                                         ptepde = tpte;
 5903                                         pte = (pt_entry_t *)PHYS_TO_DMAP(tpte &
 5904                                             PG_FRAME);
 5905                                         pte = &pte[pmap_pte_index(pv->pv_va)];
 5906                                         tpte = *pte;
 5907                                 } else {
 5908                                         /*
 5909                                          * Keep track whether 'tpte' is a
 5910                                          * superpage explicitly instead of
 5911                                          * relying on PG_PS being set.
 5912                                          *
 5913                                          * This is because PG_PS is numerically
 5914                                          * identical to PG_PTE_PAT and thus a
 5915                                          * regular page could be mistaken for
 5916                                          * a superpage.
 5917                                          */
 5918                                         superpage = TRUE;
 5919                                 }
 5920 
 5921                                 if ((tpte & PG_V) == 0) {
 5922                                         panic("bad pte va %lx pte %lx",
 5923                                             pv->pv_va, tpte);
 5924                                 }
 5925 
 5926 /*
 5927  * We cannot remove wired pages from a process' mapping at this time
 5928  */
 5929                                 if (tpte & PG_W) {
 5930                                         allfree = 0;
 5931                                         continue;
 5932                                 }
 5933 
 5934                                 if (superpage)
 5935                                         pa = tpte & PG_PS_FRAME;
 5936                                 else
 5937                                         pa = tpte & PG_FRAME;
 5938 
 5939                                 m = PHYS_TO_VM_PAGE(pa);
 5940                                 KASSERT(m->phys_addr == pa,
 5941                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 5942                                     m, (uintmax_t)m->phys_addr,
 5943                                     (uintmax_t)tpte));
 5944 
 5945                                 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
 5946                                     m < &vm_page_array[vm_page_array_size],
 5947                                     ("pmap_remove_pages: bad tpte %#jx",
 5948                                     (uintmax_t)tpte));
 5949 
 5950                                 pte_clear(pte);
 5951 
 5952                                 /*
 5953                                  * Update the vm_page_t clean/reference bits.
 5954                                  */
 5955                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5956                                         if (superpage) {
 5957                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 5958                                                         vm_page_dirty(mt);
 5959                                         } else
 5960                                                 vm_page_dirty(m);
 5961                                 }
 5962 
 5963                                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
 5964 
 5965                                 /* Mark free */
 5966                                 pc->pc_map[field] |= bitmask;
 5967                                 if (superpage) {
 5968                                         pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
 5969                                         pvh = pa_to_pvh(tpte & PG_PS_FRAME);
 5970                                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 5971                                         pvh->pv_gen++;
 5972                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 5973                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 5974                                                         if ((mt->aflags & PGA_WRITEABLE) != 0 &&
 5975                                                             TAILQ_EMPTY(&mt->md.pv_list))
 5976                                                                 vm_page_aflag_clear(mt, PGA_WRITEABLE);
 5977                                         }
 5978                                         mpte = pmap_remove_pt_page(pmap, pv->pv_va);
 5979                                         if (mpte != NULL) {
 5980                                                 pmap_resident_count_dec(pmap, 1);
 5981                                                 KASSERT(mpte->wire_count == NPTEPG,
 5982                                                     ("pmap_remove_pages: pte page wire count error"));
 5983                                                 mpte->wire_count = 0;
 5984                                                 pmap_add_delayed_free_list(mpte, &free, FALSE);
 5985                                         }
 5986                                 } else {
 5987                                         pmap_resident_count_dec(pmap, 1);
 5988                                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 5989                                         m->md.pv_gen++;
 5990                                         if ((m->aflags & PGA_WRITEABLE) != 0 &&
 5991                                             TAILQ_EMPTY(&m->md.pv_list) &&
 5992                                             (m->flags & PG_FICTITIOUS) == 0) {
 5993                                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5994                                                 if (TAILQ_EMPTY(&pvh->pv_list))
 5995                                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 5996                                         }
 5997                                 }
 5998                                 pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
 5999                                 freed++;
 6000                         }
 6001                 }
 6002                 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
 6003                 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
 6004                 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
 6005                 if (allfree) {
 6006                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 6007                         free_pv_chunk(pc);
 6008                 }