The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/pmap.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2003 Peter Wemm
    9  * All rights reserved.
   10  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
   11  * All rights reserved.
   12  *
   13  * This code is derived from software contributed to Berkeley by
   14  * the Systems Programming Group of the University of Utah Computer
   15  * Science Department and William Jolitz of UUNET Technologies Inc.
   16  *
   17  * Redistribution and use in source and binary forms, with or without
   18  * modification, are permitted provided that the following conditions
   19  * are met:
   20  * 1. Redistributions of source code must retain the above copyright
   21  *    notice, this list of conditions and the following disclaimer.
   22  * 2. Redistributions in binary form must reproduce the above copyright
   23  *    notice, this list of conditions and the following disclaimer in the
   24  *    documentation and/or other materials provided with the distribution.
   25  * 3. All advertising materials mentioning features or use of this software
   26  *    must display the following acknowledgement:
   27  *      This product includes software developed by the University of
   28  *      California, Berkeley and its contributors.
   29  * 4. Neither the name of the University nor the names of its contributors
   30  *    may be used to endorse or promote products derived from this software
   31  *    without specific prior written permission.
   32  *
   33  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   34  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   35  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   36  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   37  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   38  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   39  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   40  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   41  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   42  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   43  * SUCH DAMAGE.
   44  *
   45  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   46  */
   47 /*-
   48  * Copyright (c) 2003 Networks Associates Technology, Inc.
   49  * All rights reserved.
   50  *
   51  * This software was developed for the FreeBSD Project by Jake Burkholder,
   52  * Safeport Network Services, and Network Associates Laboratories, the
   53  * Security Research Division of Network Associates, Inc. under
   54  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   55  * CHATS research program.
   56  *
   57  * Redistribution and use in source and binary forms, with or without
   58  * modification, are permitted provided that the following conditions
   59  * are met:
   60  * 1. Redistributions of source code must retain the above copyright
   61  *    notice, this list of conditions and the following disclaimer.
   62  * 2. Redistributions in binary form must reproduce the above copyright
   63  *    notice, this list of conditions and the following disclaimer in the
   64  *    documentation and/or other materials provided with the distribution.
   65  *
   66  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   67  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   68  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   69  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   70  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   71  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   72  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   73  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   74  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   75  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   76  * SUCH DAMAGE.
   77  */
   78 
   79 #include <sys/cdefs.h>
   80 __FBSDID("$FreeBSD: releng/9.2/sys/amd64/amd64/pmap.c 251897 2013-06-18 05:21:40Z scottl $");
   81 
   82 /*
   83  *      Manages physical address maps.
   84  *
   85  *      In addition to hardware address maps, this
   86  *      module is called upon to provide software-use-only
   87  *      maps which may or may not be stored in the same
   88  *      form as hardware maps.  These pseudo-maps are
   89  *      used to store intermediate results from copy
   90  *      operations to and from address spaces.
   91  *
   92  *      Since the information managed by this module is
   93  *      also stored by the logical address mapping module,
   94  *      this module may throw away valid virtual-to-physical
   95  *      mappings at almost any time.  However, invalidations
   96  *      of virtual-to-physical mappings must be done as
   97  *      requested.
   98  *
   99  *      In order to cope with hardware architectures which
  100  *      make virtual-to-physical map invalidates expensive,
  101  *      this module may delay invalidate or reduced protection
  102  *      operations until such time as they are actually
  103  *      necessary.  This module is given full information as
  104  *      to which processors are currently using which maps,
  105  *      and to when physical maps must be made correct.
  106  */
  107 
  108 #include "opt_pmap.h"
  109 #include "opt_vm.h"
  110 
  111 #include <sys/param.h>
  112 #include <sys/bus.h>
  113 #include <sys/systm.h>
  114 #include <sys/kernel.h>
  115 #include <sys/ktr.h>
  116 #include <sys/lock.h>
  117 #include <sys/malloc.h>
  118 #include <sys/mman.h>
  119 #include <sys/mutex.h>
  120 #include <sys/proc.h>
  121 #include <sys/rwlock.h>
  122 #include <sys/sx.h>
  123 #include <sys/vmmeter.h>
  124 #include <sys/sched.h>
  125 #include <sys/sysctl.h>
  126 #ifdef SMP
  127 #include <sys/smp.h>
  128 #else
  129 #include <sys/cpuset.h>
  130 #endif
  131 
  132 #include <vm/vm.h>
  133 #include <vm/vm_param.h>
  134 #include <vm/vm_kern.h>
  135 #include <vm/vm_page.h>
  136 #include <vm/vm_map.h>
  137 #include <vm/vm_object.h>
  138 #include <vm/vm_extern.h>
  139 #include <vm/vm_pageout.h>
  140 #include <vm/vm_pager.h>
  141 #include <vm/vm_reserv.h>
  142 #include <vm/uma.h>
  143 
  144 #include <machine/intr_machdep.h>
  145 #include <machine/apicvar.h>
  146 #include <machine/cpu.h>
  147 #include <machine/cputypes.h>
  148 #include <machine/md_var.h>
  149 #include <machine/pcb.h>
  150 #include <machine/specialreg.h>
  151 #ifdef SMP
  152 #include <machine/smp.h>
  153 #endif
  154 
  155 #if !defined(DIAGNOSTIC)
  156 #ifdef __GNUC_GNU_INLINE__
  157 #define PMAP_INLINE     __attribute__((__gnu_inline__)) inline
  158 #else
  159 #define PMAP_INLINE     extern inline
  160 #endif
  161 #else
  162 #define PMAP_INLINE
  163 #endif
  164 
  165 #ifdef PV_STATS
  166 #define PV_STAT(x)      do { x ; } while (0)
  167 #else
  168 #define PV_STAT(x)      do { } while (0)
  169 #endif
  170 
  171 #define pa_index(pa)    ((pa) >> PDRSHIFT)
  172 #define pa_to_pvh(pa)   (&pv_table[pa_index(pa)])
  173 
  174 #define NPV_LIST_LOCKS  MAXCPU
  175 
  176 #define PHYS_TO_PV_LIST_LOCK(pa)        \
  177                         (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS])
  178 
  179 #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa)  do {    \
  180         struct rwlock **_lockp = (lockp);               \
  181         struct rwlock *_new_lock;                       \
  182                                                         \
  183         _new_lock = PHYS_TO_PV_LIST_LOCK(pa);           \
  184         if (_new_lock != *_lockp) {                     \
  185                 if (*_lockp != NULL)                    \
  186                         rw_wunlock(*_lockp);            \
  187                 *_lockp = _new_lock;                    \
  188                 rw_wlock(*_lockp);                      \
  189         }                                               \
  190 } while (0)
  191 
  192 #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m)        \
  193                         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m))
  194 
  195 #define RELEASE_PV_LIST_LOCK(lockp)             do {    \
  196         struct rwlock **_lockp = (lockp);               \
  197                                                         \
  198         if (*_lockp != NULL) {                          \
  199                 rw_wunlock(*_lockp);                    \
  200                 *_lockp = NULL;                         \
  201         }                                               \
  202 } while (0)
  203 
  204 #define VM_PAGE_TO_PV_LIST_LOCK(m)      \
  205                         PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m))
  206 
  207 struct pmap kernel_pmap_store;
  208 
  209 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  210 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  211 
  212 static int ndmpdp;
  213 static vm_paddr_t dmaplimit;
  214 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
  215 pt_entry_t pg_nx;
  216 
  217 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  218 
  219 static int pat_works = 1;
  220 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1,
  221     "Is page attribute table fully functional?");
  222 
  223 static int pg_ps_enabled = 1;
  224 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN, &pg_ps_enabled, 0,
  225     "Are large page mappings enabled?");
  226 
  227 #define PAT_INDEX_SIZE  8
  228 static int pat_index[PAT_INDEX_SIZE];   /* cache mode to PAT index conversion */
  229 
  230 static u_int64_t        KPTphys;        /* phys addr of kernel level 1 */
  231 static u_int64_t        KPDphys;        /* phys addr of kernel level 2 */
  232 u_int64_t               KPDPphys;       /* phys addr of kernel level 3 */
  233 u_int64_t               KPML4phys;      /* phys addr of kernel level 4 */
  234 
  235 static u_int64_t        DMPDphys;       /* phys addr of direct mapped level 2 */
  236 static u_int64_t        DMPDPphys;      /* phys addr of direct mapped level 3 */
  237 
  238 /*
  239  * Isolate the global pv list lock from data and other locks to prevent false
  240  * sharing within the cache.
  241  */
  242 static struct {
  243         struct rwlock   lock;
  244         char            padding[CACHE_LINE_SIZE - sizeof(struct rwlock)];
  245 } pvh_global __aligned(CACHE_LINE_SIZE);
  246 
  247 #define pvh_global_lock pvh_global.lock
  248 
  249 /*
  250  * Data for the pv entry allocation mechanism
  251  */
  252 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
  253 static struct mtx pv_chunks_mutex;
  254 static struct rwlock pv_list_locks[NPV_LIST_LOCKS];
  255 static struct md_page *pv_table;
  256 
  257 /*
  258  * All those kernel PT submaps that BSD is so fond of
  259  */
  260 pt_entry_t *CMAP1 = 0;
  261 caddr_t CADDR1 = 0;
  262 
  263 /*
  264  * Crashdump maps.
  265  */
  266 static caddr_t crashdumpmap;
  267 
  268 static void     free_pv_chunk(struct pv_chunk *pc);
  269 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  270 static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp);
  271 static int      popcnt_pc_map_elem(uint64_t elem);
  272 static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp);
  273 static void     reserve_pv_entries(pmap_t pmap, int needed,
  274                     struct rwlock **lockp);
  275 static void     pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
  276                     struct rwlock **lockp);
  277 static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
  278                     struct rwlock **lockp);
  279 static void     pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
  280                     struct rwlock **lockp);
  281 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  282 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  283                     vm_offset_t va);
  284 static int      pmap_pvh_wired_mappings(struct md_page *pvh, int count);
  285 
  286 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode);
  287 static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  288 static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde,
  289     vm_offset_t va, struct rwlock **lockp);
  290 static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,
  291     vm_offset_t va);
  292 static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
  293     vm_prot_t prot, struct rwlock **lockp);
  294 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  295     vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp);
  296 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
  297 static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
  298 static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
  299 static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
  300 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
  301 static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
  302 static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
  303 static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
  304     struct rwlock **lockp);
  305 static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
  306     vm_prot_t prot);
  307 static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
  308 static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
  309                 vm_page_t *free, struct rwlock **lockp);
  310 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
  311                 vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free,
  312                 struct rwlock **lockp);
  313 static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
  314 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  315     vm_page_t *free);
  316 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  317     vm_page_t m, struct rwlock **lockp);
  318 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  319     pd_entry_t newpde);
  320 static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
  321 
  322 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex,
  323                 struct rwlock **lockp);
  324 static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va,
  325                 struct rwlock **lockp);
  326 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
  327                 struct rwlock **lockp);
  328 
  329 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m,
  330                 vm_page_t *free);
  331 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
  332 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
  333 
  334 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  335 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  336 
  337 /*
  338  * Move the kernel virtual free pointer to the next
  339  * 2MB.  This is used to help improve performance
  340  * by using a large (2MB) page for much of the kernel
  341  * (.text, .data, .bss)
  342  */
  343 static vm_offset_t
  344 pmap_kmem_choose(vm_offset_t addr)
  345 {
  346         vm_offset_t newaddr = addr;
  347 
  348         newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
  349         return (newaddr);
  350 }
  351 
  352 /********************/
  353 /* Inline functions */
  354 /********************/
  355 
  356 /* Return a non-clipped PD index for a given VA */
  357 static __inline vm_pindex_t
  358 pmap_pde_pindex(vm_offset_t va)
  359 {
  360         return (va >> PDRSHIFT);
  361 }
  362 
  363 
  364 /* Return various clipped indexes for a given VA */
  365 static __inline vm_pindex_t
  366 pmap_pte_index(vm_offset_t va)
  367 {
  368 
  369         return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
  370 }
  371 
  372 static __inline vm_pindex_t
  373 pmap_pde_index(vm_offset_t va)
  374 {
  375 
  376         return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
  377 }
  378 
  379 static __inline vm_pindex_t
  380 pmap_pdpe_index(vm_offset_t va)
  381 {
  382 
  383         return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
  384 }
  385 
  386 static __inline vm_pindex_t
  387 pmap_pml4e_index(vm_offset_t va)
  388 {
  389 
  390         return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
  391 }
  392 
  393 /* Return a pointer to the PML4 slot that corresponds to a VA */
  394 static __inline pml4_entry_t *
  395 pmap_pml4e(pmap_t pmap, vm_offset_t va)
  396 {
  397 
  398         return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
  399 }
  400 
  401 /* Return a pointer to the PDP slot that corresponds to a VA */
  402 static __inline pdp_entry_t *
  403 pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
  404 {
  405         pdp_entry_t *pdpe;
  406 
  407         pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
  408         return (&pdpe[pmap_pdpe_index(va)]);
  409 }
  410 
  411 /* Return a pointer to the PDP slot that corresponds to a VA */
  412 static __inline pdp_entry_t *
  413 pmap_pdpe(pmap_t pmap, vm_offset_t va)
  414 {
  415         pml4_entry_t *pml4e;
  416 
  417         pml4e = pmap_pml4e(pmap, va);
  418         if ((*pml4e & PG_V) == 0)
  419                 return (NULL);
  420         return (pmap_pml4e_to_pdpe(pml4e, va));
  421 }
  422 
  423 /* Return a pointer to the PD slot that corresponds to a VA */
  424 static __inline pd_entry_t *
  425 pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
  426 {
  427         pd_entry_t *pde;
  428 
  429         pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
  430         return (&pde[pmap_pde_index(va)]);
  431 }
  432 
  433 /* Return a pointer to the PD slot that corresponds to a VA */
  434 static __inline pd_entry_t *
  435 pmap_pde(pmap_t pmap, vm_offset_t va)
  436 {
  437         pdp_entry_t *pdpe;
  438 
  439         pdpe = pmap_pdpe(pmap, va);
  440         if (pdpe == NULL || (*pdpe & PG_V) == 0)
  441                 return (NULL);
  442         return (pmap_pdpe_to_pde(pdpe, va));
  443 }
  444 
  445 /* Return a pointer to the PT slot that corresponds to a VA */
  446 static __inline pt_entry_t *
  447 pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
  448 {
  449         pt_entry_t *pte;
  450 
  451         pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
  452         return (&pte[pmap_pte_index(va)]);
  453 }
  454 
  455 /* Return a pointer to the PT slot that corresponds to a VA */
  456 static __inline pt_entry_t *
  457 pmap_pte(pmap_t pmap, vm_offset_t va)
  458 {
  459         pd_entry_t *pde;
  460 
  461         pde = pmap_pde(pmap, va);
  462         if (pde == NULL || (*pde & PG_V) == 0)
  463                 return (NULL);
  464         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  465                 return ((pt_entry_t *)pde);
  466         return (pmap_pde_to_pte(pde, va));
  467 }
  468 
  469 static __inline void
  470 pmap_resident_count_inc(pmap_t pmap, int count)
  471 {
  472 
  473         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
  474         pmap->pm_stats.resident_count += count;
  475 }
  476 
  477 static __inline void
  478 pmap_resident_count_dec(pmap_t pmap, int count)
  479 {
  480 
  481         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
  482         pmap->pm_stats.resident_count -= count;
  483 }
  484 
  485 PMAP_INLINE pt_entry_t *
  486 vtopte(vm_offset_t va)
  487 {
  488         u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  489 
  490         return (PTmap + ((va >> PAGE_SHIFT) & mask));
  491 }
  492 
  493 static __inline pd_entry_t *
  494 vtopde(vm_offset_t va)
  495 {
  496         u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  497 
  498         return (PDmap + ((va >> PDRSHIFT) & mask));
  499 }
  500 
  501 static u_int64_t
  502 allocpages(vm_paddr_t *firstaddr, int n)
  503 {
  504         u_int64_t ret;
  505 
  506         ret = *firstaddr;
  507         bzero((void *)ret, n * PAGE_SIZE);
  508         *firstaddr += n * PAGE_SIZE;
  509         return (ret);
  510 }
  511 
  512 CTASSERT(powerof2(NDMPML4E));
  513 
  514 static void
  515 create_pagetables(vm_paddr_t *firstaddr)
  516 {
  517         int i, j, ndm1g;
  518 
  519         /* Allocate pages */
  520         KPTphys = allocpages(firstaddr, NKPT);
  521         KPML4phys = allocpages(firstaddr, 1);
  522         KPDPphys = allocpages(firstaddr, NKPML4E);
  523         KPDphys = allocpages(firstaddr, NKPDPE);
  524 
  525         ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
  526         if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
  527                 ndmpdp = 4;
  528         DMPDPphys = allocpages(firstaddr, NDMPML4E);
  529         ndm1g = 0;
  530         if ((amd_feature & AMDID_PAGE1GB) != 0)
  531                 ndm1g = ptoa(Maxmem) >> PDPSHIFT;
  532         if (ndm1g < ndmpdp)
  533                 DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g);
  534         dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
  535 
  536         /* Fill in the underlying page table pages */
  537         /* Read-only from zero to physfree */
  538         /* XXX not fully used, underneath 2M pages */
  539         for (i = 0; (i << PAGE_SHIFT) < *firstaddr; i++) {
  540                 ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
  541                 ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V | PG_G;
  542         }
  543 
  544         /* Now map the page tables at their location within PTmap */
  545         for (i = 0; i < NKPT; i++) {
  546                 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
  547                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
  548         }
  549 
  550         /* Map from zero to end of allocations under 2M pages */
  551         /* This replaces some of the KPTphys entries above */
  552         for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
  553                 ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
  554                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  555         }
  556 
  557         /* And connect up the PD to the PDP */
  558         for (i = 0; i < NKPDPE; i++) {
  559                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys +
  560                     (i << PAGE_SHIFT);
  561                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
  562         }
  563 
  564         /*
  565          * Now, set up the direct map region using 2MB and/or 1GB pages.  If
  566          * the end of physical memory is not aligned to a 1GB page boundary,
  567          * then the residual physical memory is mapped with 2MB pages.  Later,
  568          * if pmap_mapdev{_attr}() uses the direct map for non-write-back
  569          * memory, pmap_change_attr() will demote any 2MB or 1GB page mappings
  570          * that are partially used. 
  571          */
  572         for (i = NPDEPG * ndm1g, j = 0; i < NPDEPG * ndmpdp; i++, j++) {
  573                 ((pd_entry_t *)DMPDphys)[j] = (vm_paddr_t)i << PDRSHIFT;
  574                 /* Preset PG_M and PG_A because demotion expects it. */
  575                 ((pd_entry_t *)DMPDphys)[j] |= PG_RW | PG_V | PG_PS | PG_G |
  576                     PG_M | PG_A;
  577         }
  578         for (i = 0; i < ndm1g; i++) {
  579                 ((pdp_entry_t *)DMPDPphys)[i] = (vm_paddr_t)i << PDPSHIFT;
  580                 /* Preset PG_M and PG_A because demotion expects it. */
  581                 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_PS | PG_G |
  582                     PG_M | PG_A;
  583         }
  584         for (j = 0; i < ndmpdp; i++, j++) {
  585                 ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + (j << PAGE_SHIFT);
  586                 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
  587         }
  588 
  589         /* And recursively map PML4 to itself in order to get PTmap */
  590         ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
  591         ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
  592 
  593         /* Connect the Direct Map slot(s) up to the PML4. */
  594         for (i = 0; i < NDMPML4E; i++) {
  595                 ((pdp_entry_t *)KPML4phys)[DMPML4I + i] = DMPDPphys +
  596                     (i << PAGE_SHIFT);
  597                 ((pdp_entry_t *)KPML4phys)[DMPML4I + i] |= PG_RW | PG_V | PG_U;
  598         }
  599 
  600         /* Connect the KVA slot up to the PML4 */
  601         ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
  602         ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
  603 }
  604 
  605 /*
  606  *      Bootstrap the system enough to run with virtual memory.
  607  *
  608  *      On amd64 this is called after mapping has already been enabled
  609  *      and just syncs the pmap module with what has already been done.
  610  *      [We can't call it easily with mapping off since the kernel is not
  611  *      mapped with PA == VA, hence we would have to relocate every address
  612  *      from the linked base (virtual) address "KERNBASE" to the actual
  613  *      (physical) address starting relative to 0]
  614  */
  615 void
  616 pmap_bootstrap(vm_paddr_t *firstaddr)
  617 {
  618         vm_offset_t va;
  619         pt_entry_t *pte, *unused;
  620 
  621         /*
  622          * Create an initial set of page tables to run the kernel in.
  623          */
  624         create_pagetables(firstaddr);
  625 
  626         virtual_avail = (vm_offset_t) KERNBASE + *firstaddr;
  627         virtual_avail = pmap_kmem_choose(virtual_avail);
  628 
  629         virtual_end = VM_MAX_KERNEL_ADDRESS;
  630 
  631 
  632         /* XXX do %cr0 as well */
  633         load_cr4(rcr4() | CR4_PGE | CR4_PSE);
  634         load_cr3(KPML4phys);
  635         if (cpu_stdext_feature & CPUID_STDEXT_SMEP)
  636                 load_cr4(rcr4() | CR4_SMEP);
  637 
  638         /*
  639          * Initialize the kernel pmap (which is statically allocated).
  640          */
  641         PMAP_LOCK_INIT(kernel_pmap);
  642         kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys);
  643         kernel_pmap->pm_root = NULL;
  644         CPU_FILL(&kernel_pmap->pm_active);      /* don't allow deactivation */
  645         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  646 
  647         /*
  648          * Initialize the global pv list lock.
  649          */
  650         rw_init(&pvh_global_lock, "pmap pv global");
  651 
  652         /*
  653          * Reserve some special page table entries/VA space for temporary
  654          * mapping of pages.
  655          */
  656 #define SYSMAP(c, p, v, n)      \
  657         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  658 
  659         va = virtual_avail;
  660         pte = vtopte(va);
  661 
  662         /*
  663          * CMAP1 is only used for the memory test.
  664          */
  665         SYSMAP(caddr_t, CMAP1, CADDR1, 1)
  666 
  667         /*
  668          * Crashdump maps.
  669          */
  670         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  671 
  672         virtual_avail = va;
  673 
  674         /* Initialize the PAT MSR. */
  675         pmap_init_pat();
  676 }
  677 
  678 /*
  679  * Setup the PAT MSR.
  680  */
  681 void
  682 pmap_init_pat(void)
  683 {
  684         int pat_table[PAT_INDEX_SIZE];
  685         uint64_t pat_msr;
  686         u_long cr0, cr4;
  687         int i;
  688 
  689         /* Bail if this CPU doesn't implement PAT. */
  690         if ((cpu_feature & CPUID_PAT) == 0)
  691                 panic("no PAT??");
  692 
  693         /* Set default PAT index table. */
  694         for (i = 0; i < PAT_INDEX_SIZE; i++)
  695                 pat_table[i] = -1;
  696         pat_table[PAT_WRITE_BACK] = 0;
  697         pat_table[PAT_WRITE_THROUGH] = 1;
  698         pat_table[PAT_UNCACHEABLE] = 3;
  699         pat_table[PAT_WRITE_COMBINING] = 3;
  700         pat_table[PAT_WRITE_PROTECTED] = 3;
  701         pat_table[PAT_UNCACHED] = 3;
  702 
  703         /* Initialize default PAT entries. */
  704         pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |
  705             PAT_VALUE(1, PAT_WRITE_THROUGH) |
  706             PAT_VALUE(2, PAT_UNCACHED) |
  707             PAT_VALUE(3, PAT_UNCACHEABLE) |
  708             PAT_VALUE(4, PAT_WRITE_BACK) |
  709             PAT_VALUE(5, PAT_WRITE_THROUGH) |
  710             PAT_VALUE(6, PAT_UNCACHED) |
  711             PAT_VALUE(7, PAT_UNCACHEABLE);
  712 
  713         if (pat_works) {
  714                 /*
  715                  * Leave the indices 0-3 at the default of WB, WT, UC-, and UC.
  716                  * Program 5 and 6 as WP and WC.
  717                  * Leave 4 and 7 as WB and UC.
  718                  */
  719                 pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6));
  720                 pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) |
  721                     PAT_VALUE(6, PAT_WRITE_COMBINING);
  722                 pat_table[PAT_UNCACHED] = 2;
  723                 pat_table[PAT_WRITE_PROTECTED] = 5;
  724                 pat_table[PAT_WRITE_COMBINING] = 6;
  725         } else {
  726                 /*
  727                  * Just replace PAT Index 2 with WC instead of UC-.
  728                  */
  729                 pat_msr &= ~PAT_MASK(2);
  730                 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  731                 pat_table[PAT_WRITE_COMBINING] = 2;
  732         }
  733 
  734         /* Disable PGE. */
  735         cr4 = rcr4();
  736         load_cr4(cr4 & ~CR4_PGE);
  737 
  738         /* Disable caches (CD = 1, NW = 0). */
  739         cr0 = rcr0();
  740         load_cr0((cr0 & ~CR0_NW) | CR0_CD);
  741 
  742         /* Flushes caches and TLBs. */
  743         wbinvd();
  744         invltlb();
  745 
  746         /* Update PAT and index table. */
  747         wrmsr(MSR_PAT, pat_msr);
  748         for (i = 0; i < PAT_INDEX_SIZE; i++)
  749                 pat_index[i] = pat_table[i];
  750 
  751         /* Flush caches and TLBs again. */
  752         wbinvd();
  753         invltlb();
  754 
  755         /* Restore caches and PGE. */
  756         load_cr0(cr0);
  757         load_cr4(cr4);
  758 }
  759 
  760 /*
  761  *      Initialize a vm_page's machine-dependent fields.
  762  */
  763 void
  764 pmap_page_init(vm_page_t m)
  765 {
  766 
  767         TAILQ_INIT(&m->md.pv_list);
  768         m->md.pat_mode = PAT_WRITE_BACK;
  769 }
  770 
  771 /*
  772  *      Initialize the pmap module.
  773  *      Called by vm_init, to initialize any structures that the pmap
  774  *      system needs to map virtual memory.
  775  */
  776 void
  777 pmap_init(void)
  778 {
  779         vm_page_t mpte;
  780         vm_size_t s;
  781         int i, pv_npg;
  782 
  783         /*
  784          * Initialize the vm page array entries for the kernel pmap's
  785          * page table pages.
  786          */ 
  787         for (i = 0; i < NKPT; i++) {
  788                 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
  789                 KASSERT(mpte >= vm_page_array &&
  790                     mpte < &vm_page_array[vm_page_array_size],
  791                     ("pmap_init: page table page is out of range"));
  792                 mpte->pindex = pmap_pde_pindex(KERNBASE) + i;
  793                 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
  794         }
  795 
  796         /*
  797          * If the kernel is running in a virtual machine on an AMD Family 10h
  798          * processor, then it must assume that MCA is enabled by the virtual
  799          * machine monitor.
  800          */
  801         if (vm_guest == VM_GUEST_VM && cpu_vendor_id == CPU_VENDOR_AMD &&
  802             CPUID_TO_FAMILY(cpu_id) == 0x10)
  803                 workaround_erratum383 = 1;
  804 
  805         /*
  806          * Are large page mappings enabled?
  807          */
  808         TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
  809         if (pg_ps_enabled) {
  810                 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
  811                     ("pmap_init: can't assign to pagesizes[1]"));
  812                 pagesizes[1] = NBPDR;
  813         }
  814 
  815         /*
  816          * Initialize the pv chunk list mutex.
  817          */
  818         mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF);
  819 
  820         /*
  821          * Initialize the pool of pv list locks.
  822          */
  823         for (i = 0; i < NPV_LIST_LOCKS; i++)
  824                 rw_init(&pv_list_locks[i], "pmap pv list");
  825 
  826         /*
  827          * Calculate the size of the pv head table for superpages.
  828          */
  829         for (i = 0; phys_avail[i + 1]; i += 2);
  830         pv_npg = round_2mpage(phys_avail[(i - 2) + 1]) / NBPDR;
  831 
  832         /*
  833          * Allocate memory for the pv head table for superpages.
  834          */
  835         s = (vm_size_t)(pv_npg * sizeof(struct md_page));
  836         s = round_page(s);
  837         pv_table = (struct md_page *)kmem_alloc(kernel_map, s);
  838         for (i = 0; i < pv_npg; i++)
  839                 TAILQ_INIT(&pv_table[i].pv_list);
  840 }
  841 
  842 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
  843     "2MB page mapping counters");
  844 
  845 static u_long pmap_pde_demotions;
  846 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
  847     &pmap_pde_demotions, 0, "2MB page demotions");
  848 
  849 static u_long pmap_pde_mappings;
  850 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
  851     &pmap_pde_mappings, 0, "2MB page mappings");
  852 
  853 static u_long pmap_pde_p_failures;
  854 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
  855     &pmap_pde_p_failures, 0, "2MB page promotion failures");
  856 
  857 static u_long pmap_pde_promotions;
  858 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
  859     &pmap_pde_promotions, 0, "2MB page promotions");
  860 
  861 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD, 0,
  862     "1GB page mapping counters");
  863 
  864 static u_long pmap_pdpe_demotions;
  865 SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
  866     &pmap_pdpe_demotions, 0, "1GB page demotions");
  867 
  868 /***************************************************
  869  * Low level helper routines.....
  870  ***************************************************/
  871 
  872 /*
  873  * Determine the appropriate bits to set in a PTE or PDE for a specified
  874  * caching mode.
  875  */
  876 static int
  877 pmap_cache_bits(int mode, boolean_t is_pde)
  878 {
  879         int cache_bits, pat_flag, pat_idx;
  880 
  881         if (mode < 0 || mode >= PAT_INDEX_SIZE || pat_index[mode] < 0)
  882                 panic("Unknown caching mode %d\n", mode);
  883 
  884         /* The PAT bit is different for PTE's and PDE's. */
  885         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
  886 
  887         /* Map the caching mode to a PAT index. */
  888         pat_idx = pat_index[mode];
  889 
  890         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
  891         cache_bits = 0;
  892         if (pat_idx & 0x4)
  893                 cache_bits |= pat_flag;
  894         if (pat_idx & 0x2)
  895                 cache_bits |= PG_NC_PCD;
  896         if (pat_idx & 0x1)
  897                 cache_bits |= PG_NC_PWT;
  898         return (cache_bits);
  899 }
  900 
  901 /*
  902  * After changing the page size for the specified virtual address in the page
  903  * table, flush the corresponding entries from the processor's TLB.  Only the
  904  * calling processor's TLB is affected.
  905  *
  906  * The calling thread must be pinned to a processor.
  907  */
  908 static void
  909 pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
  910 {
  911         u_long cr4;
  912 
  913         if ((newpde & PG_PS) == 0)
  914                 /* Demotion: flush a specific 2MB page mapping. */
  915                 invlpg(va);
  916         else if ((newpde & PG_G) == 0)
  917                 /*
  918                  * Promotion: flush every 4KB page mapping from the TLB
  919                  * because there are too many to flush individually.
  920                  */
  921                 invltlb();
  922         else {
  923                 /*
  924                  * Promotion: flush every 4KB page mapping from the TLB,
  925                  * including any global (PG_G) mappings.
  926                  */
  927                 cr4 = rcr4();
  928                 load_cr4(cr4 & ~CR4_PGE);
  929                 /*
  930                  * Although preemption at this point could be detrimental to
  931                  * performance, it would not lead to an error.  PG_G is simply
  932                  * ignored if CR4.PGE is clear.  Moreover, in case this block
  933                  * is re-entered, the load_cr4() either above or below will
  934                  * modify CR4.PGE flushing the TLB.
  935                  */
  936                 load_cr4(cr4 | CR4_PGE);
  937         }
  938 }
  939 #ifdef SMP
  940 /*
  941  * For SMP, these functions have to use the IPI mechanism for coherence.
  942  *
  943  * N.B.: Before calling any of the following TLB invalidation functions,
  944  * the calling processor must ensure that all stores updating a non-
  945  * kernel page table are globally performed.  Otherwise, another
  946  * processor could cache an old, pre-update entry without being
  947  * invalidated.  This can happen one of two ways: (1) The pmap becomes
  948  * active on another processor after its pm_active field is checked by
  949  * one of the following functions but before a store updating the page
  950  * table is globally performed. (2) The pmap becomes active on another
  951  * processor before its pm_active field is checked but due to
  952  * speculative loads one of the following functions stills reads the
  953  * pmap as inactive on the other processor.
  954  * 
  955  * The kernel page table is exempt because its pm_active field is
  956  * immutable.  The kernel page table is always active on every
  957  * processor.
  958  */
  959 void
  960 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  961 {
  962         cpuset_t other_cpus;
  963         u_int cpuid;
  964 
  965         sched_pin();
  966         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
  967                 invlpg(va);
  968                 smp_invlpg(va);
  969         } else {
  970                 cpuid = PCPU_GET(cpuid);
  971                 other_cpus = all_cpus;
  972                 CPU_CLR(cpuid, &other_cpus);
  973                 if (CPU_ISSET(cpuid, &pmap->pm_active))
  974                         invlpg(va);
  975                 CPU_AND(&other_cpus, &pmap->pm_active);
  976                 if (!CPU_EMPTY(&other_cpus))
  977                         smp_masked_invlpg(other_cpus, va);
  978         }
  979         sched_unpin();
  980 }
  981 
  982 void
  983 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  984 {
  985         cpuset_t other_cpus;
  986         vm_offset_t addr;
  987         u_int cpuid;
  988 
  989         sched_pin();
  990         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
  991                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  992                         invlpg(addr);
  993                 smp_invlpg_range(sva, eva);
  994         } else {
  995                 cpuid = PCPU_GET(cpuid);
  996                 other_cpus = all_cpus;
  997                 CPU_CLR(cpuid, &other_cpus);
  998                 if (CPU_ISSET(cpuid, &pmap->pm_active))
  999                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1000                                 invlpg(addr);
 1001                 CPU_AND(&other_cpus, &pmap->pm_active);
 1002                 if (!CPU_EMPTY(&other_cpus))
 1003                         smp_masked_invlpg_range(other_cpus, sva, eva);
 1004         }
 1005         sched_unpin();
 1006 }
 1007 
 1008 void
 1009 pmap_invalidate_all(pmap_t pmap)
 1010 {
 1011         cpuset_t other_cpus;
 1012         u_int cpuid;
 1013 
 1014         sched_pin();
 1015         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1016                 invltlb();
 1017                 smp_invltlb();
 1018         } else {
 1019                 cpuid = PCPU_GET(cpuid);
 1020                 other_cpus = all_cpus;
 1021                 CPU_CLR(cpuid, &other_cpus);
 1022                 if (CPU_ISSET(cpuid, &pmap->pm_active))
 1023                         invltlb();
 1024                 CPU_AND(&other_cpus, &pmap->pm_active);
 1025                 if (!CPU_EMPTY(&other_cpus))
 1026                         smp_masked_invltlb(other_cpus);
 1027         }
 1028         sched_unpin();
 1029 }
 1030 
 1031 void
 1032 pmap_invalidate_cache(void)
 1033 {
 1034 
 1035         sched_pin();
 1036         wbinvd();
 1037         smp_cache_flush();
 1038         sched_unpin();
 1039 }
 1040 
 1041 struct pde_action {
 1042         cpuset_t invalidate;    /* processors that invalidate their TLB */
 1043         vm_offset_t va;
 1044         pd_entry_t *pde;
 1045         pd_entry_t newpde;
 1046         u_int store;            /* processor that updates the PDE */
 1047 };
 1048 
 1049 static void
 1050 pmap_update_pde_action(void *arg)
 1051 {
 1052         struct pde_action *act = arg;
 1053 
 1054         if (act->store == PCPU_GET(cpuid))
 1055                 pde_store(act->pde, act->newpde);
 1056 }
 1057 
 1058 static void
 1059 pmap_update_pde_teardown(void *arg)
 1060 {
 1061         struct pde_action *act = arg;
 1062 
 1063         if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
 1064                 pmap_update_pde_invalidate(act->va, act->newpde);
 1065 }
 1066 
 1067 /*
 1068  * Change the page size for the specified virtual address in a way that
 1069  * prevents any possibility of the TLB ever having two entries that map the
 1070  * same virtual address using different page sizes.  This is the recommended
 1071  * workaround for Erratum 383 on AMD Family 10h processors.  It prevents a
 1072  * machine check exception for a TLB state that is improperly diagnosed as a
 1073  * hardware error.
 1074  */
 1075 static void
 1076 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1077 {
 1078         struct pde_action act;
 1079         cpuset_t active, other_cpus;
 1080         u_int cpuid;
 1081 
 1082         sched_pin();
 1083         cpuid = PCPU_GET(cpuid);
 1084         other_cpus = all_cpus;
 1085         CPU_CLR(cpuid, &other_cpus);
 1086         if (pmap == kernel_pmap)
 1087                 active = all_cpus;
 1088         else
 1089                 active = pmap->pm_active;
 1090         if (CPU_OVERLAP(&active, &other_cpus)) { 
 1091                 act.store = cpuid;
 1092                 act.invalidate = active;
 1093                 act.va = va;
 1094                 act.pde = pde;
 1095                 act.newpde = newpde;
 1096                 CPU_SET(cpuid, &active);
 1097                 smp_rendezvous_cpus(active,
 1098                     smp_no_rendevous_barrier, pmap_update_pde_action,
 1099                     pmap_update_pde_teardown, &act);
 1100         } else {
 1101                 pde_store(pde, newpde);
 1102                 if (CPU_ISSET(cpuid, &active))
 1103                         pmap_update_pde_invalidate(va, newpde);
 1104         }
 1105         sched_unpin();
 1106 }
 1107 #else /* !SMP */
 1108 /*
 1109  * Normal, non-SMP, invalidation functions.
 1110  * We inline these within pmap.c for speed.
 1111  */
 1112 PMAP_INLINE void
 1113 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 1114 {
 1115 
 1116         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1117                 invlpg(va);
 1118 }
 1119 
 1120 PMAP_INLINE void
 1121 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1122 {
 1123         vm_offset_t addr;
 1124 
 1125         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1126                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1127                         invlpg(addr);
 1128 }
 1129 
 1130 PMAP_INLINE void
 1131 pmap_invalidate_all(pmap_t pmap)
 1132 {
 1133 
 1134         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1135                 invltlb();
 1136 }
 1137 
 1138 PMAP_INLINE void
 1139 pmap_invalidate_cache(void)
 1140 {
 1141 
 1142         wbinvd();
 1143 }
 1144 
 1145 static void
 1146 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1147 {
 1148 
 1149         pde_store(pde, newpde);
 1150         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1151                 pmap_update_pde_invalidate(va, newpde);
 1152 }
 1153 #endif /* !SMP */
 1154 
 1155 #define PMAP_CLFLUSH_THRESHOLD   (2 * 1024 * 1024)
 1156 
 1157 void
 1158 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
 1159 {
 1160 
 1161         KASSERT((sva & PAGE_MASK) == 0,
 1162             ("pmap_invalidate_cache_range: sva not page-aligned"));
 1163         KASSERT((eva & PAGE_MASK) == 0,
 1164             ("pmap_invalidate_cache_range: eva not page-aligned"));
 1165 
 1166         if (cpu_feature & CPUID_SS)
 1167                 ; /* If "Self Snoop" is supported, do nothing. */
 1168         else if ((cpu_feature & CPUID_CLFSH) != 0 &&
 1169             eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 1170 
 1171                 /*
 1172                  * XXX: Some CPUs fault, hang, or trash the local APIC
 1173                  * registers if we use CLFLUSH on the local APIC
 1174                  * range.  The local APIC is always uncached, so we
 1175                  * don't need to flush for that range anyway.
 1176                  */
 1177                 if (pmap_kextract(sva) == lapic_paddr)
 1178                         return;
 1179 
 1180                 /*
 1181                  * Otherwise, do per-cache line flush.  Use the mfence
 1182                  * instruction to insure that previous stores are
 1183                  * included in the write-back.  The processor
 1184                  * propagates flush to other processors in the cache
 1185                  * coherence domain.
 1186                  */
 1187                 mfence();
 1188                 for (; sva < eva; sva += cpu_clflush_line_size)
 1189                         clflush(sva);
 1190                 mfence();
 1191         } else {
 1192 
 1193                 /*
 1194                  * No targeted cache flush methods are supported by CPU,
 1195                  * or the supplied range is bigger than 2MB.
 1196                  * Globally invalidate cache.
 1197                  */
 1198                 pmap_invalidate_cache();
 1199         }
 1200 }
 1201 
 1202 /*
 1203  * Remove the specified set of pages from the data and instruction caches.
 1204  *
 1205  * In contrast to pmap_invalidate_cache_range(), this function does not
 1206  * rely on the CPU's self-snoop feature, because it is intended for use
 1207  * when moving pages into a different cache domain.
 1208  */
 1209 void
 1210 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
 1211 {
 1212         vm_offset_t daddr, eva;
 1213         int i;
 1214 
 1215         if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
 1216             (cpu_feature & CPUID_CLFSH) == 0)
 1217                 pmap_invalidate_cache();
 1218         else {
 1219                 mfence();
 1220                 for (i = 0; i < count; i++) {
 1221                         daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i]));
 1222                         eva = daddr + PAGE_SIZE;
 1223                         for (; daddr < eva; daddr += cpu_clflush_line_size)
 1224                                 clflush(daddr);
 1225                 }
 1226                 mfence();
 1227         }
 1228 }
 1229 
 1230 /*
 1231  * Are we current address space or kernel?
 1232  */
 1233 static __inline int
 1234 pmap_is_current(pmap_t pmap)
 1235 {
 1236         return (pmap == kernel_pmap ||
 1237             (pmap->pm_pml4[PML4PML4I] & PG_FRAME) == (PML4pml4e[0] & PG_FRAME));
 1238 }
 1239 
 1240 /*
 1241  *      Routine:        pmap_extract
 1242  *      Function:
 1243  *              Extract the physical page address associated
 1244  *              with the given map/virtual_address pair.
 1245  */
 1246 vm_paddr_t 
 1247 pmap_extract(pmap_t pmap, vm_offset_t va)
 1248 {
 1249         pdp_entry_t *pdpe;
 1250         pd_entry_t *pde;
 1251         pt_entry_t *pte;
 1252         vm_paddr_t pa;
 1253 
 1254         pa = 0;
 1255         PMAP_LOCK(pmap);
 1256         pdpe = pmap_pdpe(pmap, va);
 1257         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 1258                 if ((*pdpe & PG_PS) != 0)
 1259                         pa = (*pdpe & PG_PS_FRAME) | (va & PDPMASK);
 1260                 else {
 1261                         pde = pmap_pdpe_to_pde(pdpe, va);
 1262                         if ((*pde & PG_V) != 0) {
 1263                                 if ((*pde & PG_PS) != 0) {
 1264                                         pa = (*pde & PG_PS_FRAME) |
 1265                                             (va & PDRMASK);
 1266                                 } else {
 1267                                         pte = pmap_pde_to_pte(pde, va);
 1268                                         pa = (*pte & PG_FRAME) |
 1269                                             (va & PAGE_MASK);
 1270                                 }
 1271                         }
 1272                 }
 1273         }
 1274         PMAP_UNLOCK(pmap);
 1275         return (pa);
 1276 }
 1277 
 1278 /*
 1279  *      Routine:        pmap_extract_and_hold
 1280  *      Function:
 1281  *              Atomically extract and hold the physical page
 1282  *              with the given pmap and virtual address pair
 1283  *              if that mapping permits the given protection.
 1284  */
 1285 vm_page_t
 1286 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 1287 {
 1288         pd_entry_t pde, *pdep;
 1289         pt_entry_t pte;
 1290         vm_paddr_t pa;
 1291         vm_page_t m;
 1292 
 1293         pa = 0;
 1294         m = NULL;
 1295         PMAP_LOCK(pmap);
 1296 retry:
 1297         pdep = pmap_pde(pmap, va);
 1298         if (pdep != NULL && (pde = *pdep)) {
 1299                 if (pde & PG_PS) {
 1300                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
 1301                                 if (vm_page_pa_tryrelock(pmap, (pde &
 1302                                     PG_PS_FRAME) | (va & PDRMASK), &pa))
 1303                                         goto retry;
 1304                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
 1305                                     (va & PDRMASK));
 1306                                 vm_page_hold(m);
 1307                         }
 1308                 } else {
 1309                         pte = *pmap_pde_to_pte(pdep, va);
 1310                         if ((pte & PG_V) &&
 1311                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
 1312                                 if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME,
 1313                                     &pa))
 1314                                         goto retry;
 1315                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
 1316                                 vm_page_hold(m);
 1317                         }
 1318                 }
 1319         }
 1320         PA_UNLOCK_COND(pa);
 1321         PMAP_UNLOCK(pmap);
 1322         return (m);
 1323 }
 1324 
 1325 vm_paddr_t
 1326 pmap_kextract(vm_offset_t va)
 1327 {
 1328         pd_entry_t pde;
 1329         vm_paddr_t pa;
 1330 
 1331         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
 1332                 pa = DMAP_TO_PHYS(va);
 1333         } else {
 1334                 pde = *vtopde(va);
 1335                 if (pde & PG_PS) {
 1336                         pa = (pde & PG_PS_FRAME) | (va & PDRMASK);
 1337                 } else {
 1338                         /*
 1339                          * Beware of a concurrent promotion that changes the
 1340                          * PDE at this point!  For example, vtopte() must not
 1341                          * be used to access the PTE because it would use the
 1342                          * new PDE.  It is, however, safe to use the old PDE
 1343                          * because the page table page is preserved by the
 1344                          * promotion.
 1345                          */
 1346                         pa = *pmap_pde_to_pte(&pde, va);
 1347                         pa = (pa & PG_FRAME) | (va & PAGE_MASK);
 1348                 }
 1349         }
 1350         return (pa);
 1351 }
 1352 
 1353 /***************************************************
 1354  * Low level mapping routines.....
 1355  ***************************************************/
 1356 
 1357 /*
 1358  * Add a wired page to the kva.
 1359  * Note: not SMP coherent.
 1360  */
 1361 PMAP_INLINE void 
 1362 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
 1363 {
 1364         pt_entry_t *pte;
 1365 
 1366         pte = vtopte(va);
 1367         pte_store(pte, pa | PG_RW | PG_V | PG_G);
 1368 }
 1369 
 1370 static __inline void
 1371 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
 1372 {
 1373         pt_entry_t *pte;
 1374 
 1375         pte = vtopte(va);
 1376         pte_store(pte, pa | PG_RW | PG_V | PG_G | pmap_cache_bits(mode, 0));
 1377 }
 1378 
 1379 /*
 1380  * Remove a page from the kernel pagetables.
 1381  * Note: not SMP coherent.
 1382  */
 1383 PMAP_INLINE void
 1384 pmap_kremove(vm_offset_t va)
 1385 {
 1386         pt_entry_t *pte;
 1387 
 1388         pte = vtopte(va);
 1389         pte_clear(pte);
 1390 }
 1391 
 1392 /*
 1393  *      Used to map a range of physical addresses into kernel
 1394  *      virtual address space.
 1395  *
 1396  *      The value passed in '*virt' is a suggested virtual address for
 1397  *      the mapping. Architectures which can support a direct-mapped
 1398  *      physical to virtual region can return the appropriate address
 1399  *      within that region, leaving '*virt' unchanged. Other
 1400  *      architectures should map the pages starting at '*virt' and
 1401  *      update '*virt' with the first usable address after the mapped
 1402  *      region.
 1403  */
 1404 vm_offset_t
 1405 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 1406 {
 1407         return PHYS_TO_DMAP(start);
 1408 }
 1409 
 1410 
 1411 /*
 1412  * Add a list of wired pages to the kva
 1413  * this routine is only used for temporary
 1414  * kernel mappings that do not need to have
 1415  * page modification or references recorded.
 1416  * Note that old mappings are simply written
 1417  * over.  The page *must* be wired.
 1418  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1419  */
 1420 void
 1421 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 1422 {
 1423         pt_entry_t *endpte, oldpte, pa, *pte;
 1424         vm_page_t m;
 1425 
 1426         oldpte = 0;
 1427         pte = vtopte(sva);
 1428         endpte = pte + count;
 1429         while (pte < endpte) {
 1430                 m = *ma++;
 1431                 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0);
 1432                 if ((*pte & (PG_FRAME | PG_PTE_CACHE)) != pa) {
 1433                         oldpte |= *pte;
 1434                         pte_store(pte, pa | PG_G | PG_RW | PG_V);
 1435                 }
 1436                 pte++;
 1437         }
 1438         if (__predict_false((oldpte & PG_V) != 0))
 1439                 pmap_invalidate_range(kernel_pmap, sva, sva + count *
 1440                     PAGE_SIZE);
 1441 }
 1442 
 1443 /*
 1444  * This routine tears out page mappings from the
 1445  * kernel -- it is meant only for temporary mappings.
 1446  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1447  */
 1448 void
 1449 pmap_qremove(vm_offset_t sva, int count)
 1450 {
 1451         vm_offset_t va;
 1452 
 1453         va = sva;
 1454         while (count-- > 0) {
 1455                 KASSERT(va >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", va));
 1456                 pmap_kremove(va);
 1457                 va += PAGE_SIZE;
 1458         }
 1459         pmap_invalidate_range(kernel_pmap, sva, va);
 1460 }
 1461 
 1462 /***************************************************
 1463  * Page table page management routines.....
 1464  ***************************************************/
 1465 static __inline void
 1466 pmap_free_zero_pages(vm_page_t free)
 1467 {
 1468         vm_page_t m;
 1469 
 1470         while (free != NULL) {
 1471                 m = free;
 1472                 free = m->right;
 1473                 /* Preserve the page's PG_ZERO setting. */
 1474                 vm_page_free_toq(m);
 1475         }
 1476 }
 1477 
 1478 /*
 1479  * Schedule the specified unused page table page to be freed.  Specifically,
 1480  * add the page to the specified list of pages that will be released to the
 1481  * physical memory manager after the TLB has been updated.
 1482  */
 1483 static __inline void
 1484 pmap_add_delayed_free_list(vm_page_t m, vm_page_t *free, boolean_t set_PG_ZERO)
 1485 {
 1486 
 1487         if (set_PG_ZERO)
 1488                 m->flags |= PG_ZERO;
 1489         else
 1490                 m->flags &= ~PG_ZERO;
 1491         m->right = *free;
 1492         *free = m;
 1493 }
 1494         
 1495 /*
 1496  * Inserts the specified page table page into the specified pmap's collection
 1497  * of idle page table pages.  Each of a pmap's page table pages is responsible
 1498  * for mapping a distinct range of virtual addresses.  The pmap's collection is
 1499  * ordered by this virtual address range.
 1500  */
 1501 static void
 1502 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
 1503 {
 1504         vm_page_t root;
 1505 
 1506         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1507         root = pmap->pm_root;
 1508         if (root == NULL) {
 1509                 mpte->left = NULL;
 1510                 mpte->right = NULL;
 1511         } else {
 1512                 root = vm_page_splay(mpte->pindex, root);
 1513                 if (mpte->pindex < root->pindex) {
 1514                         mpte->left = root->left;
 1515                         mpte->right = root;
 1516                         root->left = NULL;
 1517                 } else if (mpte->pindex == root->pindex)
 1518                         panic("pmap_insert_pt_page: pindex already inserted");
 1519                 else {
 1520                         mpte->right = root->right;
 1521                         mpte->left = root;
 1522                         root->right = NULL;
 1523                 }
 1524         }
 1525         pmap->pm_root = mpte;
 1526 }
 1527 
 1528 /*
 1529  * Looks for a page table page mapping the specified virtual address in the
 1530  * specified pmap's collection of idle page table pages.  Returns NULL if there
 1531  * is no page table page corresponding to the specified virtual address.
 1532  */
 1533 static vm_page_t
 1534 pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
 1535 {
 1536         vm_page_t mpte;
 1537         vm_pindex_t pindex = pmap_pde_pindex(va);
 1538 
 1539         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1540         if ((mpte = pmap->pm_root) != NULL && mpte->pindex != pindex) {
 1541                 mpte = vm_page_splay(pindex, mpte);
 1542                 if ((pmap->pm_root = mpte)->pindex != pindex)
 1543                         mpte = NULL;
 1544         }
 1545         return (mpte);
 1546 }
 1547 
 1548 /*
 1549  * Removes the specified page table page from the specified pmap's collection
 1550  * of idle page table pages.  The specified page table page must be a member of
 1551  * the pmap's collection.
 1552  */
 1553 static void
 1554 pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
 1555 {
 1556         vm_page_t root;
 1557 
 1558         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1559         if (mpte != pmap->pm_root) {
 1560                 root = vm_page_splay(mpte->pindex, pmap->pm_root);
 1561                 KASSERT(mpte == root,
 1562                     ("pmap_remove_pt_page: mpte %p is missing from pmap %p",
 1563                     mpte, pmap));
 1564         }
 1565         if (mpte->left == NULL)
 1566                 root = mpte->right;
 1567         else {
 1568                 root = vm_page_splay(mpte->pindex, mpte->left);
 1569                 root->right = mpte->right;
 1570         }
 1571         pmap->pm_root = root;
 1572 }
 1573 
 1574 /*
 1575  * Decrements a page table page's wire count, which is used to record the
 1576  * number of valid page table entries within the page.  If the wire count
 1577  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
 1578  * page table page was unmapped and FALSE otherwise.
 1579  */
 1580 static inline boolean_t
 1581 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free)
 1582 {
 1583 
 1584         --m->wire_count;
 1585         if (m->wire_count == 0) {
 1586                 _pmap_unwire_ptp(pmap, va, m, free);
 1587                 return (TRUE);
 1588         } else
 1589                 return (FALSE);
 1590 }
 1591 
 1592 static void
 1593 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free)
 1594 {
 1595 
 1596         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1597         /*
 1598          * unmap the page table page
 1599          */
 1600         if (m->pindex >= (NUPDE + NUPDPE)) {
 1601                 /* PDP page */
 1602                 pml4_entry_t *pml4;
 1603                 pml4 = pmap_pml4e(pmap, va);
 1604                 *pml4 = 0;
 1605         } else if (m->pindex >= NUPDE) {
 1606                 /* PD page */
 1607                 pdp_entry_t *pdp;
 1608                 pdp = pmap_pdpe(pmap, va);
 1609                 *pdp = 0;
 1610         } else {
 1611                 /* PTE page */
 1612                 pd_entry_t *pd;
 1613                 pd = pmap_pde(pmap, va);
 1614                 *pd = 0;
 1615         }
 1616         pmap_resident_count_dec(pmap, 1);
 1617         if (m->pindex < NUPDE) {
 1618                 /* We just released a PT, unhold the matching PD */
 1619                 vm_page_t pdpg;
 1620 
 1621                 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
 1622                 pmap_unwire_ptp(pmap, va, pdpg, free);
 1623         }
 1624         if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 1625                 /* We just released a PD, unhold the matching PDP */
 1626                 vm_page_t pdppg;
 1627 
 1628                 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
 1629                 pmap_unwire_ptp(pmap, va, pdppg, free);
 1630         }
 1631 
 1632         /*
 1633          * This is a release store so that the ordinary store unmapping
 1634          * the page table page is globally performed before TLB shoot-
 1635          * down is begun.
 1636          */
 1637         atomic_subtract_rel_int(&cnt.v_wire_count, 1);
 1638 
 1639         /* 
 1640          * Put page on a list so that it is released after
 1641          * *ALL* TLB shootdown is done
 1642          */
 1643         pmap_add_delayed_free_list(m, free, TRUE);
 1644 }
 1645 
 1646 /*
 1647  * After removing a page table entry, this routine is used to
 1648  * conditionally free the page, and manage the hold/wire counts.
 1649  */
 1650 static int
 1651 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, vm_page_t *free)
 1652 {
 1653         vm_page_t mpte;
 1654 
 1655         if (va >= VM_MAXUSER_ADDRESS)
 1656                 return (0);
 1657         KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 1658         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1659         return (pmap_unwire_ptp(pmap, va, mpte, free));
 1660 }
 1661 
 1662 void
 1663 pmap_pinit0(pmap_t pmap)
 1664 {
 1665 
 1666         PMAP_LOCK_INIT(pmap);
 1667         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
 1668         pmap->pm_root = NULL;
 1669         CPU_ZERO(&pmap->pm_active);
 1670         PCPU_SET(curpmap, pmap);
 1671         TAILQ_INIT(&pmap->pm_pvchunk);
 1672         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1673 }
 1674 
 1675 /*
 1676  * Initialize a preallocated and zeroed pmap structure,
 1677  * such as one in a vmspace structure.
 1678  */
 1679 int
 1680 pmap_pinit(pmap_t pmap)
 1681 {
 1682         vm_page_t pml4pg;
 1683         int i;
 1684 
 1685         PMAP_LOCK_INIT(pmap);
 1686 
 1687         /*
 1688          * allocate the page directory page
 1689          */
 1690         while ((pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 1691             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
 1692                 VM_WAIT;
 1693 
 1694         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
 1695 
 1696         if ((pml4pg->flags & PG_ZERO) == 0)
 1697                 pagezero(pmap->pm_pml4);
 1698 
 1699         /* Wire in kernel global address entries. */
 1700         pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
 1701         for (i = 0; i < NDMPML4E; i++) {
 1702                 pmap->pm_pml4[DMPML4I + i] = (DMPDPphys + (i << PAGE_SHIFT)) |
 1703                     PG_RW | PG_V | PG_U;
 1704         }
 1705 
 1706         /* install self-referential address mapping entry(s) */
 1707         pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
 1708 
 1709         pmap->pm_root = NULL;
 1710         CPU_ZERO(&pmap->pm_active);
 1711         TAILQ_INIT(&pmap->pm_pvchunk);
 1712         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1713 
 1714         return (1);
 1715 }
 1716 
 1717 /*
 1718  * This routine is called if the desired page table page does not exist.
 1719  *
 1720  * If page table page allocation fails, this routine may sleep before
 1721  * returning NULL.  It sleeps only if a lock pointer was given.
 1722  *
 1723  * Note: If a page allocation fails at page table level two or three,
 1724  * one or two pages may be held during the wait, only to be released
 1725  * afterwards.  This conservative approach is easily argued to avoid
 1726  * race conditions.
 1727  */
 1728 static vm_page_t
 1729 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
 1730 {
 1731         vm_page_t m, pdppg, pdpg;
 1732 
 1733         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1734 
 1735         /*
 1736          * Allocate a page table page.
 1737          */
 1738         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1739             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1740                 if (lockp != NULL) {
 1741                         RELEASE_PV_LIST_LOCK(lockp);
 1742                         PMAP_UNLOCK(pmap);
 1743                         rw_runlock(&pvh_global_lock);
 1744                         VM_WAIT;
 1745                         rw_rlock(&pvh_global_lock);
 1746                         PMAP_LOCK(pmap);
 1747                 }
 1748 
 1749                 /*
 1750                  * Indicate the need to retry.  While waiting, the page table
 1751                  * page may have been allocated.
 1752                  */
 1753                 return (NULL);
 1754         }
 1755         if ((m->flags & PG_ZERO) == 0)
 1756                 pmap_zero_page(m);
 1757 
 1758         /*
 1759          * Map the pagetable page into the process address space, if
 1760          * it isn't already there.
 1761          */
 1762 
 1763         if (ptepindex >= (NUPDE + NUPDPE)) {
 1764                 pml4_entry_t *pml4;
 1765                 vm_pindex_t pml4index;
 1766 
 1767                 /* Wire up a new PDPE page */
 1768                 pml4index = ptepindex - (NUPDE + NUPDPE);
 1769                 pml4 = &pmap->pm_pml4[pml4index];
 1770                 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1771 
 1772         } else if (ptepindex >= NUPDE) {
 1773                 vm_pindex_t pml4index;
 1774                 vm_pindex_t pdpindex;
 1775                 pml4_entry_t *pml4;
 1776                 pdp_entry_t *pdp;
 1777 
 1778                 /* Wire up a new PDE page */
 1779                 pdpindex = ptepindex - NUPDE;
 1780                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1781 
 1782                 pml4 = &pmap->pm_pml4[pml4index];
 1783                 if ((*pml4 & PG_V) == 0) {
 1784                         /* Have to allocate a new pdp, recurse */
 1785                         if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
 1786                             lockp) == NULL) {
 1787                                 --m->wire_count;
 1788                                 atomic_subtract_int(&cnt.v_wire_count, 1);
 1789                                 vm_page_free_zero(m);
 1790                                 return (NULL);
 1791                         }
 1792                 } else {
 1793                         /* Add reference to pdp page */
 1794                         pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
 1795                         pdppg->wire_count++;
 1796                 }
 1797                 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1798 
 1799                 /* Now find the pdp page */
 1800                 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1801                 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1802 
 1803         } else {
 1804                 vm_pindex_t pml4index;
 1805                 vm_pindex_t pdpindex;
 1806                 pml4_entry_t *pml4;
 1807                 pdp_entry_t *pdp;
 1808                 pd_entry_t *pd;
 1809 
 1810                 /* Wire up a new PTE page */
 1811                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1812                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1813 
 1814                 /* First, find the pdp and check that its valid. */
 1815                 pml4 = &pmap->pm_pml4[pml4index];
 1816                 if ((*pml4 & PG_V) == 0) {
 1817                         /* Have to allocate a new pd, recurse */
 1818                         if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1819                             lockp) == NULL) {
 1820                                 --m->wire_count;
 1821                                 atomic_subtract_int(&cnt.v_wire_count, 1);
 1822                                 vm_page_free_zero(m);
 1823                                 return (NULL);
 1824                         }
 1825                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1826                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1827                 } else {
 1828                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1829                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1830                         if ((*pdp & PG_V) == 0) {
 1831                                 /* Have to allocate a new pd, recurse */
 1832                                 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1833                                     lockp) == NULL) {
 1834                                         --m->wire_count;
 1835                                         atomic_subtract_int(&cnt.v_wire_count,
 1836                                             1);
 1837                                         vm_page_free_zero(m);
 1838                                         return (NULL);
 1839                                 }
 1840                         } else {
 1841                                 /* Add reference to the pd page */
 1842                                 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
 1843                                 pdpg->wire_count++;
 1844                         }
 1845                 }
 1846                 pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
 1847 
 1848                 /* Now we know where the page directory page is */
 1849                 pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
 1850                 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1851         }
 1852 
 1853         pmap_resident_count_inc(pmap, 1);
 1854 
 1855         return (m);
 1856 }
 1857 
 1858 static vm_page_t
 1859 pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
 1860 {
 1861         vm_pindex_t pdpindex, ptepindex;
 1862         pdp_entry_t *pdpe;
 1863         vm_page_t pdpg;
 1864 
 1865 retry:
 1866         pdpe = pmap_pdpe(pmap, va);
 1867         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 1868                 /* Add a reference to the pd page. */
 1869                 pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
 1870                 pdpg->wire_count++;
 1871         } else {
 1872                 /* Allocate a pd page. */
 1873                 ptepindex = pmap_pde_pindex(va);
 1874                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1875                 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
 1876                 if (pdpg == NULL && lockp != NULL)
 1877                         goto retry;
 1878         }
 1879         return (pdpg);
 1880 }
 1881 
 1882 static vm_page_t
 1883 pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
 1884 {
 1885         vm_pindex_t ptepindex;
 1886         pd_entry_t *pd;
 1887         vm_page_t m;
 1888 
 1889         /*
 1890          * Calculate pagetable page index
 1891          */
 1892         ptepindex = pmap_pde_pindex(va);
 1893 retry:
 1894         /*
 1895          * Get the page directory entry
 1896          */
 1897         pd = pmap_pde(pmap, va);
 1898 
 1899         /*
 1900          * This supports switching from a 2MB page to a
 1901          * normal 4K page.
 1902          */
 1903         if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
 1904                 if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) {
 1905                         /*
 1906                          * Invalidation of the 2MB page mapping may have caused
 1907                          * the deallocation of the underlying PD page.
 1908                          */
 1909                         pd = NULL;
 1910                 }
 1911         }
 1912 
 1913         /*
 1914          * If the page table page is mapped, we just increment the
 1915          * hold count, and activate it.
 1916          */
 1917         if (pd != NULL && (*pd & PG_V) != 0) {
 1918                 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
 1919                 m->wire_count++;
 1920         } else {
 1921                 /*
 1922                  * Here if the pte page isn't mapped, or if it has been
 1923                  * deallocated.
 1924                  */
 1925                 m = _pmap_allocpte(pmap, ptepindex, lockp);
 1926                 if (m == NULL && lockp != NULL)
 1927                         goto retry;
 1928         }
 1929         return (m);
 1930 }
 1931 
 1932 
 1933 /***************************************************
 1934  * Pmap allocation/deallocation routines.
 1935  ***************************************************/
 1936 
 1937 /*
 1938  * Release any resources held by the given physical map.
 1939  * Called when a pmap initialized by pmap_pinit is being released.
 1940  * Should only be called if the map contains no valid mappings.
 1941  */
 1942 void
 1943 pmap_release(pmap_t pmap)
 1944 {
 1945         vm_page_t m;
 1946         int i;
 1947 
 1948         KASSERT(pmap->pm_stats.resident_count == 0,
 1949             ("pmap_release: pmap resident count %ld != 0",
 1950             pmap->pm_stats.resident_count));
 1951         KASSERT(pmap->pm_root == NULL,
 1952             ("pmap_release: pmap has reserved page table page(s)"));
 1953 
 1954         m = PHYS_TO_VM_PAGE(pmap->pm_pml4[PML4PML4I] & PG_FRAME);
 1955 
 1956         pmap->pm_pml4[KPML4I] = 0;      /* KVA */
 1957         for (i = 0; i < NDMPML4E; i++)  /* Direct Map */
 1958                 pmap->pm_pml4[DMPML4I + i] = 0;
 1959         pmap->pm_pml4[PML4PML4I] = 0;   /* Recursive Mapping */
 1960 
 1961         m->wire_count--;
 1962         atomic_subtract_int(&cnt.v_wire_count, 1);
 1963         vm_page_free_zero(m);
 1964         PMAP_LOCK_DESTROY(pmap);
 1965 }
 1966 
 1967 static int
 1968 kvm_size(SYSCTL_HANDLER_ARGS)
 1969 {
 1970         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
 1971 
 1972         return sysctl_handle_long(oidp, &ksize, 0, req);
 1973 }
 1974 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 1975     0, 0, kvm_size, "LU", "Size of KVM");
 1976 
 1977 static int
 1978 kvm_free(SYSCTL_HANDLER_ARGS)
 1979 {
 1980         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 1981 
 1982         return sysctl_handle_long(oidp, &kfree, 0, req);
 1983 }
 1984 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 1985     0, 0, kvm_free, "LU", "Amount of KVM free");
 1986 
 1987 /*
 1988  * grow the number of kernel page table entries, if needed
 1989  */
 1990 void
 1991 pmap_growkernel(vm_offset_t addr)
 1992 {
 1993         vm_paddr_t paddr;
 1994         vm_page_t nkpg;
 1995         pd_entry_t *pde, newpdir;
 1996         pdp_entry_t *pdpe;
 1997 
 1998         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1999 
 2000         /*
 2001          * Return if "addr" is within the range of kernel page table pages
 2002          * that were preallocated during pmap bootstrap.  Moreover, leave
 2003          * "kernel_vm_end" and the kernel page table as they were.
 2004          *
 2005          * The correctness of this action is based on the following
 2006          * argument: vm_map_findspace() allocates contiguous ranges of the
 2007          * kernel virtual address space.  It calls this function if a range
 2008          * ends after "kernel_vm_end".  If the kernel is mapped between
 2009          * "kernel_vm_end" and "addr", then the range cannot begin at
 2010          * "kernel_vm_end".  In fact, its beginning address cannot be less
 2011          * than the kernel.  Thus, there is no immediate need to allocate
 2012          * any new kernel page table pages between "kernel_vm_end" and
 2013          * "KERNBASE".
 2014          */
 2015         if (KERNBASE < addr && addr <= KERNBASE + NKPT * NBPDR)
 2016                 return;
 2017 
 2018         addr = roundup2(addr, NBPDR);
 2019         if (addr - 1 >= kernel_map->max_offset)
 2020                 addr = kernel_map->max_offset;
 2021         while (kernel_vm_end < addr) {
 2022                 pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end);
 2023                 if ((*pdpe & PG_V) == 0) {
 2024                         /* We need a new PDP entry */
 2025                         nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT,
 2026                             VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
 2027                             VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 2028                         if (nkpg == NULL)
 2029                                 panic("pmap_growkernel: no memory to grow kernel");
 2030                         if ((nkpg->flags & PG_ZERO) == 0)
 2031                                 pmap_zero_page(nkpg);
 2032                         paddr = VM_PAGE_TO_PHYS(nkpg);
 2033                         *pdpe = (pdp_entry_t)
 2034                                 (paddr | PG_V | PG_RW | PG_A | PG_M);
 2035                         continue; /* try again */
 2036                 }
 2037                 pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end);
 2038                 if ((*pde & PG_V) != 0) {
 2039                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2040                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 2041                                 kernel_vm_end = kernel_map->max_offset;
 2042                                 break;                       
 2043                         }
 2044                         continue;
 2045                 }
 2046 
 2047                 nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end),
 2048                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 2049                     VM_ALLOC_ZERO);
 2050                 if (nkpg == NULL)
 2051                         panic("pmap_growkernel: no memory to grow kernel");
 2052                 if ((nkpg->flags & PG_ZERO) == 0)
 2053                         pmap_zero_page(nkpg);
 2054                 paddr = VM_PAGE_TO_PHYS(nkpg);
 2055                 newpdir = (pd_entry_t) (paddr | PG_V | PG_RW | PG_A | PG_M);
 2056                 pde_store(pde, newpdir);
 2057 
 2058                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2059                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 2060                         kernel_vm_end = kernel_map->max_offset;
 2061                         break;                       
 2062                 }
 2063         }
 2064 }
 2065 
 2066 
 2067 /***************************************************
 2068  * page management routines.
 2069  ***************************************************/
 2070 
 2071 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 2072 CTASSERT(_NPCM == 3);
 2073 CTASSERT(_NPCPV == 168);
 2074 
 2075 static __inline struct pv_chunk *
 2076 pv_to_chunk(pv_entry_t pv)
 2077 {
 2078 
 2079         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
 2080 }
 2081 
 2082 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 2083 
 2084 #define PC_FREE0        0xfffffffffffffffful
 2085 #define PC_FREE1        0xfffffffffffffffful
 2086 #define PC_FREE2        0x000000fffffffffful
 2087 
 2088 static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
 2089 
 2090 #ifdef PV_STATS
 2091 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 2092 
 2093 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 2094         "Current number of pv entry chunks");
 2095 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 2096         "Current number of pv entry chunks allocated");
 2097 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 2098         "Current number of pv entry chunks frees");
 2099 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 2100         "Number of times tried to get a chunk page but failed.");
 2101 
 2102 static long pv_entry_frees, pv_entry_allocs, pv_entry_count;
 2103 static int pv_entry_spare;
 2104 
 2105 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 2106         "Current number of pv entry frees");
 2107 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 2108         "Current number of pv entry allocs");
 2109 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 2110         "Current number of pv entries");
 2111 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 2112         "Current number of spare pv entries");
 2113 #endif
 2114 
 2115 /*
 2116  * We are in a serious low memory condition.  Resort to
 2117  * drastic measures to free some pages so we can allocate
 2118  * another pv entry chunk.
 2119  *
 2120  * Returns NULL if PV entries were reclaimed from the specified pmap.
 2121  *
 2122  * We do not, however, unmap 2mpages because subsequent accesses will
 2123  * allocate per-page pv entries until repromotion occurs, thereby
 2124  * exacerbating the shortage of free pv entries.
 2125  */
 2126 static vm_page_t
 2127 reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp)
 2128 {
 2129         struct pch new_tail;
 2130         struct pv_chunk *pc;
 2131         struct md_page *pvh;
 2132         pd_entry_t *pde;
 2133         pmap_t pmap;
 2134         pt_entry_t *pte, tpte;
 2135         pv_entry_t pv;
 2136         vm_offset_t va;
 2137         vm_page_t free, m, m_pc;
 2138         uint64_t inuse;
 2139         int bit, field, freed;
 2140         
 2141         rw_assert(&pvh_global_lock, RA_LOCKED);
 2142         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
 2143         KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL"));
 2144         pmap = NULL;
 2145         free = m_pc = NULL;
 2146         TAILQ_INIT(&new_tail);
 2147         mtx_lock(&pv_chunks_mutex);
 2148         while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && free == NULL) {
 2149                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2150                 mtx_unlock(&pv_chunks_mutex);
 2151                 if (pmap != pc->pc_pmap) {
 2152                         if (pmap != NULL) {
 2153                                 pmap_invalidate_all(pmap);
 2154                                 if (pmap != locked_pmap)
 2155                                         PMAP_UNLOCK(pmap);
 2156                         }
 2157                         pmap = pc->pc_pmap;
 2158                         /* Avoid deadlock and lock recursion. */
 2159                         if (pmap > locked_pmap) {
 2160                                 RELEASE_PV_LIST_LOCK(lockp);
 2161                                 PMAP_LOCK(pmap);
 2162                         } else if (pmap != locked_pmap &&
 2163                             !PMAP_TRYLOCK(pmap)) {
 2164                                 pmap = NULL;
 2165                                 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
 2166                                 mtx_lock(&pv_chunks_mutex);
 2167                                 continue;
 2168                         }
 2169                 }
 2170 
 2171                 /*
 2172                  * Destroy every non-wired, 4 KB page mapping in the chunk.
 2173                  */
 2174                 freed = 0;
 2175                 for (field = 0; field < _NPCM; field++) {
 2176                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
 2177                             inuse != 0; inuse &= ~(1UL << bit)) {
 2178                                 bit = bsfq(inuse);
 2179                                 pv = &pc->pc_pventry[field * 64 + bit];
 2180                                 va = pv->pv_va;
 2181                                 pde = pmap_pde(pmap, va);
 2182                                 if ((*pde & PG_PS) != 0)
 2183                                         continue;
 2184                                 pte = pmap_pde_to_pte(pde, va);
 2185                                 if ((*pte & PG_W) != 0)
 2186                                         continue;
 2187                                 tpte = pte_load_clear(pte);
 2188                                 if ((tpte & PG_G) != 0)
 2189                                         pmap_invalidate_page(pmap, va);
 2190                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2191                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2192                                         vm_page_dirty(m);
 2193                                 if ((tpte & PG_A) != 0)
 2194                                         vm_page_aflag_set(m, PGA_REFERENCED);
 2195                                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 2196                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2197                                 if (TAILQ_EMPTY(&m->md.pv_list) &&
 2198                                     (m->flags & PG_FICTITIOUS) == 0) {
 2199                                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2200                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 2201                                                 vm_page_aflag_clear(m,
 2202                                                     PGA_WRITEABLE);
 2203                                         }
 2204                                 }
 2205                                 pc->pc_map[field] |= 1UL << bit;
 2206                                 pmap_unuse_pt(pmap, va, *pde, &free);   
 2207                                 freed++;
 2208                         }
 2209                 }
 2210                 if (freed == 0) {
 2211                         TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
 2212                         mtx_lock(&pv_chunks_mutex);
 2213                         continue;
 2214                 }
 2215                 /* Every freed mapping is for a 4 KB page. */
 2216                 pmap_resident_count_dec(pmap, freed);
 2217                 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
 2218                 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
 2219                 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
 2220                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2221                 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 &&
 2222                     pc->pc_map[2] == PC_FREE2) {
 2223                         PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
 2224                         PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
 2225                         PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
 2226                         /* Entire chunk is free; return it. */
 2227                         m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 2228                         dump_drop_page(m_pc->phys_addr);
 2229                         mtx_lock(&pv_chunks_mutex);
 2230                         break;
 2231                 }
 2232                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2233                 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
 2234                 mtx_lock(&pv_chunks_mutex);
 2235                 /* One freed pv entry in locked_pmap is sufficient. */
 2236                 if (pmap == locked_pmap)
 2237                         break;
 2238         }
 2239         TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
 2240         mtx_unlock(&pv_chunks_mutex);
 2241         if (pmap != NULL) {
 2242                 pmap_invalidate_all(pmap);
 2243                 if (pmap != locked_pmap)
 2244                         PMAP_UNLOCK(pmap);
 2245         }
 2246         if (m_pc == NULL && free != NULL) {
 2247                 m_pc = free;
 2248                 free = m_pc->right;
 2249                 /* Recycle a freed page table page. */
 2250                 m_pc->wire_count = 1;
 2251                 atomic_add_int(&cnt.v_wire_count, 1);
 2252         }
 2253         pmap_free_zero_pages(free);
 2254         return (m_pc);
 2255 }
 2256 
 2257 /*
 2258  * free the pv_entry back to the free list
 2259  */
 2260 static void
 2261 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 2262 {
 2263         struct pv_chunk *pc;
 2264         int idx, field, bit;
 2265 
 2266         rw_assert(&pvh_global_lock, RA_LOCKED);
 2267         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2268         PV_STAT(atomic_add_long(&pv_entry_frees, 1));
 2269         PV_STAT(atomic_add_int(&pv_entry_spare, 1));
 2270         PV_STAT(atomic_subtract_long(&pv_entry_count, 1));
 2271         pc = pv_to_chunk(pv);
 2272         idx = pv - &pc->pc_pventry[0];
 2273         field = idx / 64;
 2274         bit = idx % 64;
 2275         pc->pc_map[field] |= 1ul << bit;
 2276         if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 ||
 2277             pc->pc_map[2] != PC_FREE2) {
 2278                 /* 98% of the time, pc is already at the head of the list. */
 2279                 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
 2280                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2281                         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2282                 }
 2283                 return;
 2284         }
 2285         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2286         free_pv_chunk(pc);
 2287 }
 2288 
 2289 static void
 2290 free_pv_chunk(struct pv_chunk *pc)
 2291 {
 2292         vm_page_t m;
 2293 
 2294         mtx_lock(&pv_chunks_mutex);
 2295         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2296         mtx_unlock(&pv_chunks_mutex);
 2297         PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV));
 2298         PV_STAT(atomic_subtract_int(&pc_chunk_count, 1));
 2299         PV_STAT(atomic_add_int(&pc_chunk_frees, 1));
 2300         /* entire chunk is free, return it */
 2301         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 2302         dump_drop_page(m->phys_addr);
 2303         vm_page_unwire(m, 0);
 2304         vm_page_free(m);
 2305 }
 2306 
 2307 /*
 2308  * Returns a new PV entry, allocating a new PV chunk from the system when
 2309  * needed.  If this PV chunk allocation fails and a PV list lock pointer was
 2310  * given, a PV chunk is reclaimed from an arbitrary pmap.  Otherwise, NULL is
 2311  * returned.
 2312  *
 2313  * The given PV list lock may be released.
 2314  */
 2315 static pv_entry_t
 2316 get_pv_entry(pmap_t pmap, struct rwlock **lockp)
 2317 {
 2318         int bit, field;
 2319         pv_entry_t pv;
 2320         struct pv_chunk *pc;
 2321         vm_page_t m;
 2322 
 2323         rw_assert(&pvh_global_lock, RA_LOCKED);
 2324         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2325         PV_STAT(atomic_add_long(&pv_entry_allocs, 1));
 2326 retry:
 2327         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 2328         if (pc != NULL) {
 2329                 for (field = 0; field < _NPCM; field++) {
 2330                         if (pc->pc_map[field]) {
 2331                                 bit = bsfq(pc->pc_map[field]);
 2332                                 break;
 2333                         }
 2334                 }
 2335                 if (field < _NPCM) {
 2336                         pv = &pc->pc_pventry[field * 64 + bit];
 2337                         pc->pc_map[field] &= ~(1ul << bit);
 2338                         /* If this was the last item, move it to tail */
 2339                         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 &&
 2340                             pc->pc_map[2] == 0) {
 2341                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2342                                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
 2343                                     pc_list);
 2344                         }
 2345                         PV_STAT(atomic_add_long(&pv_entry_count, 1));
 2346                         PV_STAT(atomic_subtract_int(&pv_entry_spare, 1));
 2347                         return (pv);
 2348                 }
 2349         }
 2350         /* No free items, allocate another chunk */
 2351         m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 2352             VM_ALLOC_WIRED);
 2353         if (m == NULL) {
 2354                 if (lockp == NULL) {
 2355                         PV_STAT(pc_chunk_tryfail++);
 2356                         return (NULL);
 2357                 }
 2358                 m = reclaim_pv_chunk(pmap, lockp);
 2359                 if (m == NULL)
 2360                         goto retry;
 2361         }
 2362         PV_STAT(atomic_add_int(&pc_chunk_count, 1));
 2363         PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
 2364         dump_add_page(m->phys_addr);
 2365         pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 2366         pc->pc_pmap = pmap;
 2367         pc->pc_map[0] = PC_FREE0 & ~1ul;        /* preallocated bit 0 */
 2368         pc->pc_map[1] = PC_FREE1;
 2369         pc->pc_map[2] = PC_FREE2;
 2370         mtx_lock(&pv_chunks_mutex);
 2371         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 2372         mtx_unlock(&pv_chunks_mutex);
 2373         pv = &pc->pc_pventry[0];
 2374         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2375         PV_STAT(atomic_add_long(&pv_entry_count, 1));
 2376         PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
 2377         return (pv);
 2378 }
 2379 
 2380 /*
 2381  * Returns the number of one bits within the given PV chunk map element.
 2382  */
 2383 static int
 2384 popcnt_pc_map_elem(uint64_t elem)
 2385 {
 2386         int count;
 2387 
 2388         /*
 2389          * This simple method of counting the one bits performs well because
 2390          * the given element typically contains more zero bits than one bits.
 2391          */
 2392         count = 0;
 2393         for (; elem != 0; elem &= elem - 1)
 2394                 count++;
 2395         return (count);
 2396 }
 2397 
 2398 /*
 2399  * Ensure that the number of spare PV entries in the specified pmap meets or
 2400  * exceeds the given count, "needed".
 2401  *
 2402  * The given PV list lock may be released.
 2403  */
 2404 static void
 2405 reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp)
 2406 {
 2407         struct pch new_tail;
 2408         struct pv_chunk *pc;
 2409         int avail, free;
 2410         vm_page_t m;
 2411 
 2412         rw_assert(&pvh_global_lock, RA_LOCKED);
 2413         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2414         KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL"));
 2415 
 2416         /*
 2417          * Newly allocated PV chunks must be stored in a private list until
 2418          * the required number of PV chunks have been allocated.  Otherwise,
 2419          * reclaim_pv_chunk() could recycle one of these chunks.  In
 2420          * contrast, these chunks must be added to the pmap upon allocation.
 2421          */
 2422         TAILQ_INIT(&new_tail);
 2423 retry:
 2424         avail = 0;
 2425         TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
 2426                 if ((cpu_feature2 & CPUID2_POPCNT) == 0) {
 2427                         free = popcnt_pc_map_elem(pc->pc_map[0]);
 2428                         free += popcnt_pc_map_elem(pc->pc_map[1]);
 2429                         free += popcnt_pc_map_elem(pc->pc_map[2]);
 2430                 } else {
 2431                         free = popcntq(pc->pc_map[0]);
 2432                         free += popcntq(pc->pc_map[1]);
 2433                         free += popcntq(pc->pc_map[2]);
 2434                 }
 2435                 if (free == 0)
 2436                         break;
 2437                 avail += free;
 2438                 if (avail >= needed)
 2439                         break;
 2440         }
 2441         for (; avail < needed; avail += _NPCPV) {
 2442                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 2443                     VM_ALLOC_WIRED);
 2444                 if (m == NULL) {
 2445                         m = reclaim_pv_chunk(pmap, lockp);
 2446                         if (m == NULL)
 2447                                 goto retry;
 2448                 }
 2449                 PV_STAT(atomic_add_int(&pc_chunk_count, 1));
 2450                 PV_STAT(atomic_add_int(&pc_chunk_allocs, 1));
 2451                 dump_add_page(m->phys_addr);
 2452                 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 2453                 pc->pc_pmap = pmap;
 2454                 pc->pc_map[0] = PC_FREE0;
 2455                 pc->pc_map[1] = PC_FREE1;
 2456                 pc->pc_map[2] = PC_FREE2;
 2457                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2458                 TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru);
 2459                 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV));
 2460         }
 2461         if (!TAILQ_EMPTY(&new_tail)) {
 2462                 mtx_lock(&pv_chunks_mutex);
 2463                 TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru);
 2464                 mtx_unlock(&pv_chunks_mutex);
 2465         }
 2466 }
 2467 
 2468 /*
 2469  * First find and then remove the pv entry for the specified pmap and virtual
 2470  * address from the specified pv list.  Returns the pv entry if found and NULL
 2471  * otherwise.  This operation can be performed on pv lists for either 4KB or
 2472  * 2MB page mappings.
 2473  */
 2474 static __inline pv_entry_t
 2475 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2476 {
 2477         pv_entry_t pv;
 2478 
 2479         rw_assert(&pvh_global_lock, RA_LOCKED);
 2480         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 2481                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 2482                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
 2483                         break;
 2484                 }
 2485         }
 2486         return (pv);
 2487 }
 2488 
 2489 /*
 2490  * After demotion from a 2MB page mapping to 512 4KB page mappings,
 2491  * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
 2492  * entries for each of the 4KB page mappings.
 2493  */
 2494 static void
 2495 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
 2496     struct rwlock **lockp)
 2497 {
 2498         struct md_page *pvh;
 2499         struct pv_chunk *pc;
 2500         pv_entry_t pv;
 2501         vm_offset_t va_last;
 2502         vm_page_t m;
 2503         int bit, field;
 2504 
 2505         rw_assert(&pvh_global_lock, RA_LOCKED);
 2506         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2507         KASSERT((pa & PDRMASK) == 0,
 2508             ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
 2509         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
 2510 
 2511         /*
 2512          * Transfer the 2mpage's pv entry for this mapping to the first
 2513          * page's pv list.  Once this transfer begins, the pv list lock
 2514          * must not be released until the last pv entry is reinstantiated.
 2515          */
 2516         pvh = pa_to_pvh(pa);
 2517         va = trunc_2mpage(va);
 2518         pv = pmap_pvh_remove(pvh, pmap, va);
 2519         KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
 2520         m = PHYS_TO_VM_PAGE(pa);
 2521         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2522         /* Instantiate the remaining NPTEPG - 1 pv entries. */
 2523         PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
 2524         va_last = va + NBPDR - PAGE_SIZE;
 2525         for (;;) {
 2526                 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 2527                 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 ||
 2528                     pc->pc_map[2] != 0, ("pmap_pv_demote_pde: missing spare"));
 2529                 for (field = 0; field < _NPCM; field++) {
 2530                         while (pc->pc_map[field]) {
 2531                                 bit = bsfq(pc->pc_map[field]);
 2532                                 pc->pc_map[field] &= ~(1ul << bit);
 2533                                 pv = &pc->pc_pventry[field * 64 + bit];
 2534                                 va += PAGE_SIZE;
 2535                                 pv->pv_va = va;
 2536                                 m++;
 2537                                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 2538                             ("pmap_pv_demote_pde: page %p is not managed", m));
 2539                                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2540                                 if (va == va_last)
 2541                                         goto out;
 2542                         }
 2543                 }
 2544                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2545                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 2546         }
 2547 out:
 2548         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) {
 2549                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2550                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 2551         }
 2552         PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
 2553         PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
 2554 }
 2555 
 2556 /*
 2557  * After promotion from 512 4KB page mappings to a single 2MB page mapping,
 2558  * replace the many pv entries for the 4KB page mappings by a single pv entry
 2559  * for the 2MB page mapping.
 2560  */
 2561 static void
 2562 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
 2563     struct rwlock **lockp)
 2564 {
 2565         struct md_page *pvh;
 2566         pv_entry_t pv;
 2567         vm_offset_t va_last;
 2568         vm_page_t m;
 2569 
 2570         rw_assert(&pvh_global_lock, RA_LOCKED);
 2571         KASSERT((pa & PDRMASK) == 0,
 2572             ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
 2573         CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
 2574 
 2575         /*
 2576          * Transfer the first page's pv entry for this mapping to the 2mpage's
 2577          * pv list.  Aside from avoiding the cost of a call to get_pv_entry(),
 2578          * a transfer avoids the possibility that get_pv_entry() calls
 2579          * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the
 2580          * mappings that is being promoted.
 2581          */
 2582         m = PHYS_TO_VM_PAGE(pa);
 2583         va = trunc_2mpage(va);
 2584         pv = pmap_pvh_remove(&m->md, pmap, va);
 2585         KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
 2586         pvh = pa_to_pvh(pa);
 2587         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
 2588         /* Free the remaining NPTEPG - 1 pv entries. */
 2589         va_last = va + NBPDR - PAGE_SIZE;
 2590         do {
 2591                 m++;
 2592                 va += PAGE_SIZE;
 2593                 pmap_pvh_free(&m->md, pmap, va);
 2594         } while (va < va_last);
 2595 }
 2596 
 2597 /*
 2598  * First find and then destroy the pv entry for the specified pmap and virtual
 2599  * address.  This operation can be performed on pv lists for either 4KB or 2MB
 2600  * page mappings.
 2601  */
 2602 static void
 2603 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2604 {
 2605         pv_entry_t pv;
 2606 
 2607         pv = pmap_pvh_remove(pvh, pmap, va);
 2608         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 2609         free_pv_entry(pmap, pv);
 2610 }
 2611 
 2612 /*
 2613  * Conditionally create the PV entry for a 4KB page mapping if the required
 2614  * memory can be allocated without resorting to reclamation.
 2615  */
 2616 static boolean_t
 2617 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m,
 2618     struct rwlock **lockp)
 2619 {
 2620         pv_entry_t pv;
 2621 
 2622         rw_assert(&pvh_global_lock, RA_LOCKED);
 2623         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2624         /* Pass NULL instead of the lock pointer to disable reclamation. */
 2625         if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
 2626                 pv->pv_va = va;
 2627                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 2628                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2629                 return (TRUE);
 2630         } else
 2631                 return (FALSE);
 2632 }
 2633 
 2634 /*
 2635  * Conditionally create the PV entry for a 2MB page mapping if the required
 2636  * memory can be allocated without resorting to reclamation.
 2637  */
 2638 static boolean_t
 2639 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
 2640     struct rwlock **lockp)
 2641 {
 2642         struct md_page *pvh;
 2643         pv_entry_t pv;
 2644 
 2645         rw_assert(&pvh_global_lock, RA_LOCKED);
 2646         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2647         /* Pass NULL instead of the lock pointer to disable reclamation. */
 2648         if ((pv = get_pv_entry(pmap, NULL)) != NULL) {
 2649                 pv->pv_va = va;
 2650                 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
 2651                 pvh = pa_to_pvh(pa);
 2652                 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
 2653                 return (TRUE);
 2654         } else
 2655                 return (FALSE);
 2656 }
 2657 
 2658 /*
 2659  * Fills a page table page with mappings to consecutive physical pages.
 2660  */
 2661 static void
 2662 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
 2663 {
 2664         pt_entry_t *pte;
 2665 
 2666         for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
 2667                 *pte = newpte;
 2668                 newpte += PAGE_SIZE;
 2669         }
 2670 }
 2671 
 2672 /*
 2673  * Tries to demote a 2MB page mapping.  If demotion fails, the 2MB page
 2674  * mapping is invalidated.
 2675  */
 2676 static boolean_t
 2677 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2678 {
 2679         struct rwlock *lock;
 2680         boolean_t rv;
 2681 
 2682         lock = NULL;
 2683         rv = pmap_demote_pde_locked(pmap, pde, va, &lock);
 2684         if (lock != NULL)
 2685                 rw_wunlock(lock);
 2686         return (rv);
 2687 }
 2688 
 2689 static boolean_t
 2690 pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
 2691     struct rwlock **lockp)
 2692 {
 2693         pd_entry_t newpde, oldpde;
 2694         pt_entry_t *firstpte, newpte;
 2695         vm_paddr_t mptepa;
 2696         vm_page_t free, mpte;
 2697 
 2698         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2699         oldpde = *pde;
 2700         KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
 2701             ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
 2702         mpte = pmap_lookup_pt_page(pmap, va);
 2703         if (mpte != NULL)
 2704                 pmap_remove_pt_page(pmap, mpte);
 2705         else {
 2706                 KASSERT((oldpde & PG_W) == 0,
 2707                     ("pmap_demote_pde: page table page for a wired mapping"
 2708                     " is missing"));
 2709 
 2710                 /*
 2711                  * Invalidate the 2MB page mapping and return "failure" if the
 2712                  * mapping was never accessed or the allocation of the new
 2713                  * page table page fails.  If the 2MB page mapping belongs to
 2714                  * the direct map region of the kernel's address space, then
 2715                  * the page allocation request specifies the highest possible
 2716                  * priority (VM_ALLOC_INTERRUPT).  Otherwise, the priority is
 2717                  * normal.  Page table pages are preallocated for every other
 2718                  * part of the kernel address space, so the direct map region
 2719                  * is the only part of the kernel address space that must be
 2720                  * handled here.
 2721                  */
 2722                 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
 2723                     pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va <
 2724                     DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) |
 2725                     VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 2726                         free = NULL;
 2727                         pmap_remove_pde(pmap, pde, trunc_2mpage(va), &free,
 2728                             lockp);
 2729                         pmap_invalidate_page(pmap, trunc_2mpage(va));
 2730                         pmap_free_zero_pages(free);
 2731                         CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"
 2732                             " in pmap %p", va, pmap);
 2733                         return (FALSE);
 2734                 }
 2735                 if (va < VM_MAXUSER_ADDRESS)
 2736                         pmap_resident_count_inc(pmap, 1);
 2737         }
 2738         mptepa = VM_PAGE_TO_PHYS(mpte);
 2739         firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
 2740         newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
 2741         KASSERT((oldpde & PG_A) != 0,
 2742             ("pmap_demote_pde: oldpde is missing PG_A"));
 2743         KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
 2744             ("pmap_demote_pde: oldpde is missing PG_M"));
 2745         newpte = oldpde & ~PG_PS;
 2746         if ((newpte & PG_PDE_PAT) != 0)
 2747                 newpte ^= PG_PDE_PAT | PG_PTE_PAT;
 2748 
 2749         /*
 2750          * If the page table page is new, initialize it.
 2751          */
 2752         if (mpte->wire_count == 1) {
 2753                 mpte->wire_count = NPTEPG;
 2754                 pmap_fill_ptp(firstpte, newpte);
 2755         }
 2756         KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
 2757             ("pmap_demote_pde: firstpte and newpte map different physical"
 2758             " addresses"));
 2759 
 2760         /*
 2761          * If the mapping has changed attributes, update the page table
 2762          * entries.
 2763          */
 2764         if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
 2765                 pmap_fill_ptp(firstpte, newpte);
 2766 
 2767         /*
 2768          * The spare PV entries must be reserved prior to demoting the
 2769          * mapping, that is, prior to changing the PDE.  Otherwise, the state
 2770          * of the PDE and the PV lists will be inconsistent, which can result
 2771          * in reclaim_pv_chunk() attempting to remove a PV entry from the
 2772          * wrong PV list and pmap_pv_demote_pde() failing to find the expected
 2773          * PV entry for the 2MB page mapping that is being demoted.
 2774          */
 2775         if ((oldpde & PG_MANAGED) != 0)
 2776                 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
 2777 
 2778         /*
 2779          * Demote the mapping.  This pmap is locked.  The old PDE has
 2780          * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
 2781          * set.  Thus, there is no danger of a race with another
 2782          * processor changing the setting of PG_A and/or PG_M between
 2783          * the read above and the store below. 
 2784          */
 2785         if (workaround_erratum383)
 2786                 pmap_update_pde(pmap, va, pde, newpde);
 2787         else
 2788                 pde_store(pde, newpde);
 2789 
 2790         /*
 2791          * Invalidate a stale recursive mapping of the page table page.
 2792          */
 2793         if (va >= VM_MAXUSER_ADDRESS)
 2794                 pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 2795 
 2796         /*
 2797          * Demote the PV entry.
 2798          */
 2799         if ((oldpde & PG_MANAGED) != 0)
 2800                 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp);
 2801 
 2802         atomic_add_long(&pmap_pde_demotions, 1);
 2803         CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx"
 2804             " in pmap %p", va, pmap);
 2805         return (TRUE);
 2806 }
 2807 
 2808 /*
 2809  * pmap_remove_pde: do the things to unmap a superpage in a process
 2810  */
 2811 static int
 2812 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 2813     vm_page_t *free, struct rwlock **lockp)
 2814 {
 2815         struct md_page *pvh;
 2816         pd_entry_t oldpde;
 2817         vm_offset_t eva, va;
 2818         vm_page_t m, mpte;
 2819 
 2820         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2821         KASSERT((sva & PDRMASK) == 0,
 2822             ("pmap_remove_pde: sva is not 2mpage aligned"));
 2823         oldpde = pte_load_clear(pdq);
 2824         if (oldpde & PG_W)
 2825                 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
 2826 
 2827         /*
 2828          * Machines that don't support invlpg, also don't support
 2829          * PG_G.
 2830          */
 2831         if (oldpde & PG_G)
 2832                 pmap_invalidate_page(kernel_pmap, sva);
 2833         pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
 2834         if (oldpde & PG_MANAGED) {
 2835                 CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME);
 2836                 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
 2837                 pmap_pvh_free(pvh, pmap, sva);
 2838                 eva = sva + NBPDR;
 2839                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 2840                     va < eva; va += PAGE_SIZE, m++) {
 2841                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2842                                 vm_page_dirty(m);
 2843                         if (oldpde & PG_A)
 2844                                 vm_page_aflag_set(m, PGA_REFERENCED);
 2845                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 2846                             TAILQ_EMPTY(&pvh->pv_list))
 2847                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 2848                 }
 2849         }
 2850         if (pmap == kernel_pmap) {
 2851                 if (!pmap_demote_pde_locked(pmap, pdq, sva, lockp))
 2852                         panic("pmap_remove_pde: failed demotion");
 2853         } else {
 2854                 mpte = pmap_lookup_pt_page(pmap, sva);
 2855                 if (mpte != NULL) {
 2856                         pmap_remove_pt_page(pmap, mpte);
 2857                         pmap_resident_count_dec(pmap, 1);
 2858                         KASSERT(mpte->wire_count == NPTEPG,
 2859                             ("pmap_remove_pde: pte page wire count error"));
 2860                         mpte->wire_count = 0;
 2861                         pmap_add_delayed_free_list(mpte, free, FALSE);
 2862                         atomic_subtract_int(&cnt.v_wire_count, 1);
 2863                 }
 2864         }
 2865         return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free));
 2866 }
 2867 
 2868 /*
 2869  * pmap_remove_pte: do the things to unmap a page in a process
 2870  */
 2871 static int
 2872 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 
 2873     pd_entry_t ptepde, vm_page_t *free, struct rwlock **lockp)
 2874 {
 2875         struct md_page *pvh;
 2876         pt_entry_t oldpte;
 2877         vm_page_t m;
 2878 
 2879         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2880         oldpte = pte_load_clear(ptq);
 2881         if (oldpte & PG_W)
 2882                 pmap->pm_stats.wired_count -= 1;
 2883         pmap_resident_count_dec(pmap, 1);
 2884         if (oldpte & PG_MANAGED) {
 2885                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 2886                 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2887                         vm_page_dirty(m);
 2888                 if (oldpte & PG_A)
 2889                         vm_page_aflag_set(m, PGA_REFERENCED);
 2890                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m);
 2891                 pmap_pvh_free(&m->md, pmap, va);
 2892                 if (TAILQ_EMPTY(&m->md.pv_list) &&
 2893                     (m->flags & PG_FICTITIOUS) == 0) {
 2894                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2895                         if (TAILQ_EMPTY(&pvh->pv_list))
 2896                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 2897                 }
 2898         }
 2899         return (pmap_unuse_pt(pmap, va, ptepde, free));
 2900 }
 2901 
 2902 /*
 2903  * Remove a single page from a process address space
 2904  */
 2905 static void
 2906 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, vm_page_t *free)
 2907 {
 2908         struct rwlock *lock;
 2909         pt_entry_t *pte;
 2910 
 2911         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2912         if ((*pde & PG_V) == 0)
 2913                 return;
 2914         pte = pmap_pde_to_pte(pde, va);
 2915         if ((*pte & PG_V) == 0)
 2916                 return;
 2917         lock = NULL;
 2918         pmap_remove_pte(pmap, pte, va, *pde, free, &lock);
 2919         if (lock != NULL)
 2920                 rw_wunlock(lock);
 2921         pmap_invalidate_page(pmap, va);
 2922 }
 2923 
 2924 /*
 2925  *      Remove the given range of addresses from the specified map.
 2926  *
 2927  *      It is assumed that the start and end are properly
 2928  *      rounded to the page size.
 2929  */
 2930 void
 2931 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 2932 {
 2933         struct rwlock *lock;
 2934         vm_offset_t va, va_next;
 2935         pml4_entry_t *pml4e;
 2936         pdp_entry_t *pdpe;
 2937         pd_entry_t ptpaddr, *pde;
 2938         pt_entry_t *pte;
 2939         vm_page_t free = NULL;
 2940         int anyvalid;
 2941 
 2942         /*
 2943          * Perform an unsynchronized read.  This is, however, safe.
 2944          */
 2945         if (pmap->pm_stats.resident_count == 0)
 2946                 return;
 2947 
 2948         anyvalid = 0;
 2949 
 2950         rw_rlock(&pvh_global_lock);
 2951         PMAP_LOCK(pmap);
 2952 
 2953         /*
 2954          * special handling of removing one page.  a very
 2955          * common operation and easy to short circuit some
 2956          * code.
 2957          */
 2958         if (sva + PAGE_SIZE == eva) {
 2959                 pde = pmap_pde(pmap, sva);
 2960                 if (pde && (*pde & PG_PS) == 0) {
 2961                         pmap_remove_page(pmap, sva, pde, &free);
 2962                         goto out;
 2963                 }
 2964         }
 2965 
 2966         lock = NULL;
 2967         for (; sva < eva; sva = va_next) {
 2968 
 2969                 if (pmap->pm_stats.resident_count == 0)
 2970                         break;
 2971 
 2972                 pml4e = pmap_pml4e(pmap, sva);
 2973                 if ((*pml4e & PG_V) == 0) {
 2974                         va_next = (sva + NBPML4) & ~PML4MASK;
 2975                         if (va_next < sva)
 2976                                 va_next = eva;
 2977                         continue;
 2978                 }
 2979 
 2980                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 2981                 if ((*pdpe & PG_V) == 0) {
 2982                         va_next = (sva + NBPDP) & ~PDPMASK;
 2983                         if (va_next < sva)
 2984                                 va_next = eva;
 2985                         continue;
 2986                 }
 2987 
 2988                 /*
 2989                  * Calculate index for next page table.
 2990                  */
 2991                 va_next = (sva + NBPDR) & ~PDRMASK;
 2992                 if (va_next < sva)
 2993                         va_next = eva;
 2994 
 2995                 pde = pmap_pdpe_to_pde(pdpe, sva);
 2996                 ptpaddr = *pde;
 2997 
 2998                 /*
 2999                  * Weed out invalid mappings.
 3000                  */
 3001                 if (ptpaddr == 0)
 3002                         continue;
 3003 
 3004                 /*
 3005                  * Check for large page.
 3006                  */
 3007                 if ((ptpaddr & PG_PS) != 0) {
 3008                         /*
 3009                          * Are we removing the entire large page?  If not,
 3010                          * demote the mapping and fall through.
 3011                          */
 3012                         if (sva + NBPDR == va_next && eva >= va_next) {
 3013                                 /*
 3014                                  * The TLB entry for a PG_G mapping is
 3015                                  * invalidated by pmap_remove_pde().
 3016                                  */
 3017                                 if ((ptpaddr & PG_G) == 0)
 3018                                         anyvalid = 1;
 3019                                 pmap_remove_pde(pmap, pde, sva, &free, &lock);
 3020                                 continue;
 3021                         } else if (!pmap_demote_pde_locked(pmap, pde, sva,
 3022                             &lock)) {
 3023                                 /* The large page mapping was destroyed. */
 3024                                 continue;
 3025                         } else
 3026                                 ptpaddr = *pde;
 3027                 }
 3028 
 3029                 /*
 3030                  * Limit our scan to either the end of the va represented
 3031                  * by the current page table page, or to the end of the
 3032                  * range being removed.
 3033                  */
 3034                 if (va_next > eva)
 3035                         va_next = eva;
 3036 
 3037                 va = va_next;
 3038                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 3039                     sva += PAGE_SIZE) {
 3040                         if (*pte == 0) {
 3041                                 if (va != va_next) {
 3042                                         pmap_invalidate_range(pmap, va, sva);
 3043                                         va = va_next;
 3044                                 }
 3045                                 continue;
 3046                         }
 3047                         if ((*pte & PG_G) == 0)
 3048                                 anyvalid = 1;
 3049                         else if (va == va_next)
 3050                                 va = sva;
 3051                         if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free,
 3052                             &lock)) {
 3053                                 sva += PAGE_SIZE;
 3054                                 break;
 3055                         }
 3056                 }
 3057                 if (va != va_next)
 3058                         pmap_invalidate_range(pmap, va, sva);
 3059         }
 3060         if (lock != NULL)
 3061                 rw_wunlock(lock);
 3062 out:
 3063         if (anyvalid)
 3064                 pmap_invalidate_all(pmap);
 3065         rw_runlock(&pvh_global_lock);   
 3066         PMAP_UNLOCK(pmap);
 3067         pmap_free_zero_pages(free);
 3068 }
 3069 
 3070 /*
 3071  *      Routine:        pmap_remove_all
 3072  *      Function:
 3073  *              Removes this physical page from
 3074  *              all physical maps in which it resides.
 3075  *              Reflects back modify bits to the pager.
 3076  *
 3077  *      Notes:
 3078  *              Original versions of this routine were very
 3079  *              inefficient because they iteratively called
 3080  *              pmap_remove (slow...)
 3081  */
 3082 
 3083 void
 3084 pmap_remove_all(vm_page_t m)
 3085 {
 3086         struct md_page *pvh;
 3087         pv_entry_t pv;
 3088         pmap_t pmap;
 3089         pt_entry_t *pte, tpte;
 3090         pd_entry_t *pde;
 3091         vm_offset_t va;
 3092         vm_page_t free;
 3093 
 3094         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3095             ("pmap_remove_all: page %p is not managed", m));
 3096         free = NULL;
 3097         rw_wlock(&pvh_global_lock);
 3098         if ((m->flags & PG_FICTITIOUS) != 0)
 3099                 goto small_mappings;
 3100         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 3101         while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
 3102                 pmap = PV_PMAP(pv);
 3103                 PMAP_LOCK(pmap);
 3104                 va = pv->pv_va;
 3105                 pde = pmap_pde(pmap, va);
 3106                 (void)pmap_demote_pde(pmap, pde, va);
 3107                 PMAP_UNLOCK(pmap);
 3108         }
 3109 small_mappings:
 3110         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 3111                 pmap = PV_PMAP(pv);
 3112                 PMAP_LOCK(pmap);
 3113                 pmap_resident_count_dec(pmap, 1);
 3114                 pde = pmap_pde(pmap, pv->pv_va);
 3115                 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
 3116                     " a 2mpage in page %p's pv list", m));
 3117                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 3118                 tpte = pte_load_clear(pte);
 3119                 if (tpte & PG_W)
 3120                         pmap->pm_stats.wired_count--;
 3121                 if (tpte & PG_A)
 3122                         vm_page_aflag_set(m, PGA_REFERENCED);
 3123 
 3124                 /*
 3125                  * Update the vm_page_t clean and reference bits.
 3126                  */
 3127                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3128                         vm_page_dirty(m);
 3129                 pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);
 3130                 pmap_invalidate_page(pmap, pv->pv_va);
 3131                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 3132                 free_pv_entry(pmap, pv);
 3133                 PMAP_UNLOCK(pmap);
 3134         }
 3135         vm_page_aflag_clear(m, PGA_WRITEABLE);
 3136         rw_wunlock(&pvh_global_lock);
 3137         pmap_free_zero_pages(free);
 3138 }
 3139 
 3140 /*
 3141  * pmap_protect_pde: do the things to protect a 2mpage in a process
 3142  */
 3143 static boolean_t
 3144 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
 3145 {
 3146         pd_entry_t newpde, oldpde;
 3147         vm_offset_t eva, va;
 3148         vm_page_t m;
 3149         boolean_t anychanged;
 3150 
 3151         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3152         KASSERT((sva & PDRMASK) == 0,
 3153             ("pmap_protect_pde: sva is not 2mpage aligned"));
 3154         anychanged = FALSE;
 3155 retry:
 3156         oldpde = newpde = *pde;
 3157         if (oldpde & PG_MANAGED) {
 3158                 eva = sva + NBPDR;
 3159                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 3160                     va < eva; va += PAGE_SIZE, m++)
 3161                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3162                                 vm_page_dirty(m);
 3163         }
 3164         if ((prot & VM_PROT_WRITE) == 0)
 3165                 newpde &= ~(PG_RW | PG_M);
 3166         if ((prot & VM_PROT_EXECUTE) == 0)
 3167                 newpde |= pg_nx;
 3168         if (newpde != oldpde) {
 3169                 if (!atomic_cmpset_long(pde, oldpde, newpde))
 3170                         goto retry;
 3171                 if (oldpde & PG_G)
 3172                         pmap_invalidate_page(pmap, sva);
 3173                 else
 3174                         anychanged = TRUE;
 3175         }
 3176         return (anychanged);
 3177 }
 3178 
 3179 /*
 3180  *      Set the physical protection on the
 3181  *      specified range of this map as requested.
 3182  */
 3183 void
 3184 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 3185 {
 3186         vm_offset_t va_next;
 3187         pml4_entry_t *pml4e;
 3188         pdp_entry_t *pdpe;
 3189         pd_entry_t ptpaddr, *pde;
 3190         pt_entry_t *pte;
 3191         boolean_t anychanged, pv_lists_locked;
 3192 
 3193         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 3194                 pmap_remove(pmap, sva, eva);
 3195                 return;
 3196         }
 3197 
 3198         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 3199             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 3200                 return;
 3201 
 3202         pv_lists_locked = FALSE;
 3203 resume:
 3204         anychanged = FALSE;
 3205 
 3206         PMAP_LOCK(pmap);
 3207         for (; sva < eva; sva = va_next) {
 3208 
 3209                 pml4e = pmap_pml4e(pmap, sva);
 3210                 if ((*pml4e & PG_V) == 0) {
 3211                         va_next = (sva + NBPML4) & ~PML4MASK;
 3212                         if (va_next < sva)
 3213                                 va_next = eva;
 3214                         continue;
 3215                 }
 3216 
 3217                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 3218                 if ((*pdpe & PG_V) == 0) {
 3219                         va_next = (sva + NBPDP) & ~PDPMASK;
 3220                         if (va_next < sva)
 3221                                 va_next = eva;
 3222                         continue;
 3223                 }
 3224 
 3225                 va_next = (sva + NBPDR) & ~PDRMASK;
 3226                 if (va_next < sva)
 3227                         va_next = eva;
 3228 
 3229                 pde = pmap_pdpe_to_pde(pdpe, sva);
 3230                 ptpaddr = *pde;
 3231 
 3232                 /*
 3233                  * Weed out invalid mappings.
 3234                  */
 3235                 if (ptpaddr == 0)
 3236                         continue;
 3237 
 3238                 /*
 3239                  * Check for large page.
 3240                  */
 3241                 if ((ptpaddr & PG_PS) != 0) {
 3242                         /*
 3243                          * Are we protecting the entire large page?  If not,
 3244                          * demote the mapping and fall through.
 3245                          */
 3246                         if (sva + NBPDR == va_next && eva >= va_next) {
 3247                                 /*
 3248                                  * The TLB entry for a PG_G mapping is
 3249                                  * invalidated by pmap_protect_pde().
 3250                                  */
 3251                                 if (pmap_protect_pde(pmap, pde, sva, prot))
 3252                                         anychanged = TRUE;
 3253                                 continue;
 3254                         } else {
 3255                                 if (!pv_lists_locked) {
 3256                                         pv_lists_locked = TRUE;
 3257                                         if (!rw_try_rlock(&pvh_global_lock)) {
 3258                                                 if (anychanged)
 3259                                                         pmap_invalidate_all(
 3260                                                             pmap);
 3261                                                 PMAP_UNLOCK(pmap);
 3262                                                 rw_rlock(&pvh_global_lock);
 3263                                                 goto resume;
 3264                                         }
 3265                                 }
 3266                                 if (!pmap_demote_pde(pmap, pde, sva)) {
 3267                                         /*
 3268                                          * The large page mapping was
 3269                                          * destroyed.
 3270                                          */
 3271                                         continue;
 3272                                 }
 3273                         }
 3274                 }
 3275 
 3276                 if (va_next > eva)
 3277                         va_next = eva;
 3278 
 3279                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 3280                     sva += PAGE_SIZE) {
 3281                         pt_entry_t obits, pbits;
 3282                         vm_page_t m;
 3283 
 3284 retry:
 3285                         obits = pbits = *pte;
 3286                         if ((pbits & PG_V) == 0)
 3287                                 continue;
 3288 
 3289                         if ((prot & VM_PROT_WRITE) == 0) {
 3290                                 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
 3291                                     (PG_MANAGED | PG_M | PG_RW)) {
 3292                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 3293                                         vm_page_dirty(m);
 3294                                 }
 3295                                 pbits &= ~(PG_RW | PG_M);
 3296                         }
 3297                         if ((prot & VM_PROT_EXECUTE) == 0)
 3298                                 pbits |= pg_nx;
 3299 
 3300                         if (pbits != obits) {
 3301                                 if (!atomic_cmpset_long(pte, obits, pbits))
 3302                                         goto retry;
 3303                                 if (obits & PG_G)
 3304                                         pmap_invalidate_page(pmap, sva);
 3305                                 else
 3306                                         anychanged = TRUE;
 3307                         }
 3308                 }
 3309         }
 3310         if (anychanged)
 3311                 pmap_invalidate_all(pmap);
 3312         if (pv_lists_locked)
 3313                 rw_runlock(&pvh_global_lock);
 3314         PMAP_UNLOCK(pmap);
 3315 }
 3316 
 3317 /*
 3318  * Tries to promote the 512, contiguous 4KB page mappings that are within a
 3319  * single page table page (PTP) to a single 2MB page mapping.  For promotion
 3320  * to occur, two conditions must be met: (1) the 4KB page mappings must map
 3321  * aligned, contiguous physical memory and (2) the 4KB page mappings must have
 3322  * identical characteristics. 
 3323  */
 3324 static void
 3325 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
 3326     struct rwlock **lockp)
 3327 {
 3328         pd_entry_t newpde;
 3329         pt_entry_t *firstpte, oldpte, pa, *pte;
 3330         vm_offset_t oldpteva;
 3331         vm_page_t mpte;
 3332 
 3333         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3334 
 3335         /*
 3336          * Examine the first PTE in the specified PTP.  Abort if this PTE is
 3337          * either invalid, unused, or does not map the first 4KB physical page
 3338          * within a 2MB page. 
 3339          */
 3340         firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
 3341 setpde:
 3342         newpde = *firstpte;
 3343         if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
 3344                 atomic_add_long(&pmap_pde_p_failures, 1);
 3345                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 3346                     " in pmap %p", va, pmap);
 3347                 return;
 3348         }
 3349         if ((newpde & (PG_M | PG_RW)) == PG_RW) {
 3350                 /*
 3351                  * When PG_M is already clear, PG_RW can be cleared without
 3352                  * a TLB invalidation.
 3353                  */
 3354                 if (!atomic_cmpset_long(firstpte, newpde, newpde & ~PG_RW))
 3355                         goto setpde;
 3356                 newpde &= ~PG_RW;
 3357         }
 3358 
 3359         /*
 3360          * Examine each of the other PTEs in the specified PTP.  Abort if this
 3361          * PTE maps an unexpected 4KB physical page or does not have identical
 3362          * characteristics to the first PTE.
 3363          */
 3364         pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + NBPDR - PAGE_SIZE;
 3365         for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
 3366 setpte:
 3367                 oldpte = *pte;
 3368                 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
 3369                         atomic_add_long(&pmap_pde_p_failures, 1);
 3370                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 3371                             " in pmap %p", va, pmap);
 3372                         return;
 3373                 }
 3374                 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
 3375                         /*
 3376                          * When PG_M is already clear, PG_RW can be cleared
 3377                          * without a TLB invalidation.
 3378                          */
 3379                         if (!atomic_cmpset_long(pte, oldpte, oldpte & ~PG_RW))
 3380                                 goto setpte;
 3381                         oldpte &= ~PG_RW;
 3382                         oldpteva = (oldpte & PG_FRAME & PDRMASK) |
 3383                             (va & ~PDRMASK);
 3384                         CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx"
 3385                             " in pmap %p", oldpteva, pmap);
 3386                 }
 3387                 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
 3388                         atomic_add_long(&pmap_pde_p_failures, 1);
 3389                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 3390                             " in pmap %p", va, pmap);
 3391                         return;
 3392                 }
 3393                 pa -= PAGE_SIZE;
 3394         }
 3395 
 3396         /*
 3397          * Save the page table page in its current state until the PDE
 3398          * mapping the superpage is demoted by pmap_demote_pde() or
 3399          * destroyed by pmap_remove_pde(). 
 3400          */
 3401         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 3402         KASSERT(mpte >= vm_page_array &&
 3403             mpte < &vm_page_array[vm_page_array_size],
 3404             ("pmap_promote_pde: page table page is out of range"));
 3405         KASSERT(mpte->pindex == pmap_pde_pindex(va),
 3406             ("pmap_promote_pde: page table page's pindex is wrong"));
 3407         pmap_insert_pt_page(pmap, mpte);
 3408 
 3409         /*
 3410          * Promote the pv entries.
 3411          */
 3412         if ((newpde & PG_MANAGED) != 0)
 3413                 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp);
 3414 
 3415         /*
 3416          * Propagate the PAT index to its proper position.
 3417          */
 3418         if ((newpde & PG_PTE_PAT) != 0)
 3419                 newpde ^= PG_PDE_PAT | PG_PTE_PAT;
 3420 
 3421         /*
 3422          * Map the superpage.
 3423          */
 3424         if (workaround_erratum383)
 3425                 pmap_update_pde(pmap, va, pde, PG_PS | newpde);
 3426         else
 3427                 pde_store(pde, PG_PS | newpde);
 3428 
 3429         atomic_add_long(&pmap_pde_promotions, 1);
 3430         CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
 3431             " in pmap %p", va, pmap);
 3432 }
 3433 
 3434 /*
 3435  *      Insert the given physical page (p) at
 3436  *      the specified virtual address (v) in the
 3437  *      target physical map with the protection requested.
 3438  *
 3439  *      If specified, the page will be wired down, meaning
 3440  *      that the related pte can not be reclaimed.
 3441  *
 3442  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 3443  *      or lose information.  That is, this routine must actually
 3444  *      insert this page into the given map NOW.
 3445  */
 3446 void
 3447 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
 3448     vm_prot_t prot, boolean_t wired)
 3449 {
 3450         struct rwlock *lock;
 3451         pd_entry_t *pde;
 3452         pt_entry_t *pte;
 3453         pt_entry_t newpte, origpte;
 3454         pv_entry_t pv;
 3455         vm_paddr_t opa, pa;
 3456         vm_page_t mpte, om;
 3457 
 3458         va = trunc_page(va);
 3459         KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
 3460         KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS,
 3461             ("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)",
 3462             va));
 3463         KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva ||
 3464             va >= kmi.clean_eva,
 3465             ("pmap_enter: managed mapping within the clean submap"));
 3466         KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 ||
 3467             VM_OBJECT_LOCKED(m->object),
 3468             ("pmap_enter: page %p is not busy", m));
 3469         pa = VM_PAGE_TO_PHYS(m);
 3470         newpte = (pt_entry_t)(pa | PG_A | PG_V);
 3471         if ((access & VM_PROT_WRITE) != 0)
 3472                 newpte |= PG_M;
 3473         if ((prot & VM_PROT_WRITE) != 0)
 3474                 newpte |= PG_RW;
 3475         KASSERT((newpte & (PG_M | PG_RW)) != PG_M,
 3476             ("pmap_enter: access includes VM_PROT_WRITE but prot doesn't"));
 3477         if ((prot & VM_PROT_EXECUTE) == 0)
 3478                 newpte |= pg_nx;
 3479         if (wired)
 3480                 newpte |= PG_W;
 3481         if (va < VM_MAXUSER_ADDRESS)
 3482                 newpte |= PG_U;
 3483         if (pmap == kernel_pmap)
 3484                 newpte |= PG_G;
 3485         newpte |= pmap_cache_bits(m->md.pat_mode, 0);
 3486 
 3487         mpte = NULL;
 3488 
 3489         lock = NULL;
 3490         rw_rlock(&pvh_global_lock);
 3491         PMAP_LOCK(pmap);
 3492 
 3493         /*
 3494          * In the case that a page table page is not
 3495          * resident, we are creating it here.
 3496          */
 3497 retry:
 3498         pde = pmap_pde(pmap, va);
 3499         if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 ||
 3500             pmap_demote_pde_locked(pmap, pde, va, &lock))) {
 3501                 pte = pmap_pde_to_pte(pde, va);
 3502                 if (va < VM_MAXUSER_ADDRESS && mpte == NULL) {
 3503                         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 3504                         mpte->wire_count++;
 3505                 }
 3506         } else if (va < VM_MAXUSER_ADDRESS) {
 3507                 /*
 3508                  * Here if the pte page isn't mapped, or if it has been
 3509                  * deallocated.
 3510                  */
 3511                 mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), &lock);
 3512                 goto retry;
 3513         } else
 3514                 panic("pmap_enter: invalid page directory va=%#lx", va);
 3515 
 3516         origpte = *pte;
 3517 
 3518         /*
 3519          * Is the specified virtual address already mapped?
 3520          */
 3521         if ((origpte & PG_V) != 0) {
 3522                 /*
 3523                  * Wiring change, just update stats. We don't worry about
 3524                  * wiring PT pages as they remain resident as long as there
 3525                  * are valid mappings in them. Hence, if a user page is wired,
 3526                  * the PT page will be also.
 3527                  */
 3528                 if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0)
 3529                         pmap->pm_stats.wired_count++;
 3530                 else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0)
 3531                         pmap->pm_stats.wired_count--;
 3532 
 3533                 /*
 3534                  * Remove the extra PT page reference.
 3535                  */
 3536                 if (mpte != NULL) {
 3537                         mpte->wire_count--;
 3538                         KASSERT(mpte->wire_count > 0,
 3539                             ("pmap_enter: missing reference to page table page,"
 3540                              " va: 0x%lx", va));
 3541                 }
 3542 
 3543                 /*
 3544                  * Has the physical page changed?
 3545                  */
 3546                 opa = origpte & PG_FRAME;
 3547                 if (opa == pa) {
 3548                         /*
 3549                          * No, might be a protection or wiring change.
 3550                          */
 3551                         if ((origpte & PG_MANAGED) != 0) {
 3552                                 newpte |= PG_MANAGED;
 3553                                 if ((newpte & PG_RW) != 0)
 3554                                         vm_page_aflag_set(m, PGA_WRITEABLE);
 3555                         }
 3556                         if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0)
 3557                                 goto unchanged;
 3558                         goto validate;
 3559                 }
 3560         } else {
 3561                 /*
 3562                  * Increment the counters.
 3563                  */
 3564                 if ((newpte & PG_W) != 0)
 3565                         pmap->pm_stats.wired_count++;
 3566                 pmap_resident_count_inc(pmap, 1);
 3567         }
 3568 
 3569         /*
 3570          * Enter on the PV list if part of our managed memory.
 3571          */
 3572         if ((m->oflags & VPO_UNMANAGED) == 0) {
 3573                 newpte |= PG_MANAGED;
 3574                 pv = get_pv_entry(pmap, &lock);
 3575                 pv->pv_va = va;
 3576                 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa);
 3577                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 3578                 if ((newpte & PG_RW) != 0)
 3579                         vm_page_aflag_set(m, PGA_WRITEABLE);
 3580         }
 3581 
 3582         /*
 3583          * Update the PTE.
 3584          */
 3585         if ((origpte & PG_V) != 0) {
 3586 validate:
 3587                 origpte = pte_load_store(pte, newpte);
 3588                 opa = origpte & PG_FRAME;
 3589                 if (opa != pa) {
 3590                         if ((origpte & PG_MANAGED) != 0) {
 3591                                 om = PHYS_TO_VM_PAGE(opa);
 3592                                 if ((origpte & (PG_M | PG_RW)) == (PG_M |
 3593                                     PG_RW))
 3594                                         vm_page_dirty(om);
 3595                                 if ((origpte & PG_A) != 0)
 3596                                         vm_page_aflag_set(om, PGA_REFERENCED);
 3597                                 CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa);
 3598                                 pmap_pvh_free(&om->md, pmap, va);
 3599                                 if ((om->aflags & PGA_WRITEABLE) != 0 &&
 3600                                     TAILQ_EMPTY(&om->md.pv_list) &&
 3601                                     ((om->flags & PG_FICTITIOUS) != 0 ||
 3602                                     TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
 3603                                         vm_page_aflag_clear(om, PGA_WRITEABLE);
 3604                         }
 3605                 } else if ((newpte & PG_M) == 0 && (origpte & (PG_M |
 3606                     PG_RW)) == (PG_M | PG_RW)) {
 3607                         if ((origpte & PG_MANAGED) != 0)
 3608                                 vm_page_dirty(m);
 3609 
 3610                         /*
 3611                          * Although the PTE may still have PG_RW set, TLB
 3612                          * invalidation may nonetheless be required because
 3613                          * the PTE no longer has PG_M set.
 3614                          */
 3615                 } else if ((origpte & PG_NX) != 0 || (newpte & PG_NX) == 0) {
 3616                         /*
 3617                          * This PTE change does not require TLB invalidation.
 3618                          */
 3619                         goto unchanged;
 3620                 }
 3621                 if ((origpte & PG_A) != 0)
 3622                         pmap_invalidate_page(pmap, va);
 3623         } else
 3624                 pte_store(pte, newpte);
 3625 
 3626 unchanged:
 3627 
 3628         /*
 3629          * If both the page table page and the reservation are fully
 3630          * populated, then attempt promotion.
 3631          */
 3632         if ((mpte == NULL || mpte->wire_count == NPTEPG) &&
 3633             pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 &&
 3634             vm_reserv_level_iffullpop(m) == 0)
 3635                 pmap_promote_pde(pmap, pde, va, &lock);
 3636 
 3637         if (lock != NULL)
 3638                 rw_wunlock(lock);
 3639         rw_runlock(&pvh_global_lock);
 3640         PMAP_UNLOCK(pmap);
 3641 }
 3642 
 3643 /*
 3644  * Tries to create a 2MB page mapping.  Returns TRUE if successful and FALSE
 3645  * otherwise.  Fails if (1) a page table page cannot be allocated without
 3646  * blocking, (2) a mapping already exists at the specified virtual address, or
 3647  * (3) a pv entry cannot be allocated without reclaiming another pv entry. 
 3648  */
 3649 static boolean_t
 3650 pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 3651     struct rwlock **lockp)
 3652 {
 3653         pd_entry_t *pde, newpde;
 3654         vm_page_t free, mpde;
 3655 
 3656         rw_assert(&pvh_global_lock, RA_LOCKED);
 3657         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3658         if ((mpde = pmap_allocpde(pmap, va, NULL)) == NULL) {
 3659                 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3660                     " in pmap %p", va, pmap);
 3661                 return (FALSE);
 3662         }
 3663         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpde));
 3664         pde = &pde[pmap_pde_index(va)];
 3665         if ((*pde & PG_V) != 0) {
 3666                 KASSERT(mpde->wire_count > 1,
 3667                     ("pmap_enter_pde: mpde's wire count is too low"));
 3668                 mpde->wire_count--;
 3669                 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3670                     " in pmap %p", va, pmap);
 3671                 return (FALSE);
 3672         }
 3673         newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 1) |
 3674             PG_PS | PG_V;
 3675         if ((m->oflags & VPO_UNMANAGED) == 0) {
 3676                 newpde |= PG_MANAGED;
 3677 
 3678                 /*
 3679                  * Abort this mapping if its PV entry could not be created.
 3680                  */
 3681                 if (!pmap_pv_insert_pde(pmap, va, VM_PAGE_TO_PHYS(m),
 3682                     lockp)) {
 3683                         free = NULL;
 3684                         if (pmap_unwire_ptp(pmap, va, mpde, &free)) {
 3685                                 pmap_invalidate_page(pmap, va);
 3686                                 pmap_free_zero_pages(free);
 3687                         }
 3688                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3689                             " in pmap %p", va, pmap);
 3690                         return (FALSE);
 3691                 }
 3692         }
 3693         if ((prot & VM_PROT_EXECUTE) == 0)
 3694                 newpde |= pg_nx;
 3695         if (va < VM_MAXUSER_ADDRESS)
 3696                 newpde |= PG_U;
 3697 
 3698         /*
 3699          * Increment counters.
 3700          */
 3701         pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
 3702 
 3703         /*
 3704          * Map the superpage.
 3705          */
 3706         pde_store(pde, newpde);
 3707 
 3708         atomic_add_long(&pmap_pde_mappings, 1);
 3709         CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
 3710             " in pmap %p", va, pmap);
 3711         return (TRUE);
 3712 }
 3713 
 3714 /*
 3715  * Maps a sequence of resident pages belonging to the same object.
 3716  * The sequence begins with the given page m_start.  This page is
 3717  * mapped at the given virtual address start.  Each subsequent page is
 3718  * mapped at a virtual address that is offset from start by the same
 3719  * amount as the page is offset from m_start within the object.  The
 3720  * last page in the sequence is the page with the largest offset from
 3721  * m_start that can be mapped at a virtual address less than the given
 3722  * virtual address end.  Not every virtual page between start and end
 3723  * is mapped; only those for which a resident page exists with the
 3724  * corresponding offset from m_start are mapped.
 3725  */
 3726 void
 3727 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 3728     vm_page_t m_start, vm_prot_t prot)
 3729 {
 3730         struct rwlock *lock;
 3731         vm_offset_t va;
 3732         vm_page_t m, mpte;
 3733         vm_pindex_t diff, psize;
 3734 
 3735         VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
 3736         psize = atop(end - start);
 3737         mpte = NULL;
 3738         m = m_start;
 3739         lock = NULL;
 3740         rw_rlock(&pvh_global_lock);
 3741         PMAP_LOCK(pmap);
 3742         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 3743                 va = start + ptoa(diff);
 3744                 if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
 3745                     (VM_PAGE_TO_PHYS(m) & PDRMASK) == 0 &&
 3746                     pg_ps_enabled && vm_reserv_level_iffullpop(m) == 0 &&
 3747                     pmap_enter_pde(pmap, va, m, prot, &lock))
 3748                         m = &m[NBPDR / PAGE_SIZE - 1];
 3749                 else
 3750                         mpte = pmap_enter_quick_locked(pmap, va, m, prot,
 3751                             mpte, &lock);
 3752                 m = TAILQ_NEXT(m, listq);
 3753         }
 3754         if (lock != NULL)
 3755                 rw_wunlock(lock);
 3756         rw_runlock(&pvh_global_lock);
 3757         PMAP_UNLOCK(pmap);
 3758 }
 3759 
 3760 /*
 3761  * this code makes some *MAJOR* assumptions:
 3762  * 1. Current pmap & pmap exists.
 3763  * 2. Not wired.
 3764  * 3. Read access.
 3765  * 4. No page table pages.
 3766  * but is *MUCH* faster than pmap_enter...
 3767  */
 3768 
 3769 void
 3770 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 3771 {
 3772         struct rwlock *lock;
 3773 
 3774         lock = NULL;
 3775         rw_rlock(&pvh_global_lock);
 3776         PMAP_LOCK(pmap);
 3777         (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock);
 3778         if (lock != NULL)
 3779                 rw_wunlock(lock);
 3780         rw_runlock(&pvh_global_lock);
 3781         PMAP_UNLOCK(pmap);
 3782 }
 3783 
 3784 static vm_page_t
 3785 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 3786     vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp)
 3787 {
 3788         vm_page_t free;
 3789         pt_entry_t *pte;
 3790         vm_paddr_t pa;
 3791 
 3792         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 3793             (m->oflags & VPO_UNMANAGED) != 0,
 3794             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 3795         rw_assert(&pvh_global_lock, RA_LOCKED);
 3796         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3797 
 3798         /*
 3799          * In the case that a page table page is not
 3800          * resident, we are creating it here.
 3801          */
 3802         if (va < VM_MAXUSER_ADDRESS) {
 3803                 vm_pindex_t ptepindex;
 3804                 pd_entry_t *ptepa;
 3805 
 3806                 /*
 3807                  * Calculate pagetable page index
 3808                  */
 3809                 ptepindex = pmap_pde_pindex(va);
 3810                 if (mpte && (mpte->pindex == ptepindex)) {
 3811                         mpte->wire_count++;
 3812                 } else {
 3813                         /*
 3814                          * Get the page directory entry
 3815                          */
 3816                         ptepa = pmap_pde(pmap, va);
 3817 
 3818                         /*
 3819                          * If the page table page is mapped, we just increment
 3820                          * the hold count, and activate it.  Otherwise, we
 3821                          * attempt to allocate a page table page.  If this
 3822                          * attempt fails, we don't retry.  Instead, we give up.
 3823                          */
 3824                         if (ptepa && (*ptepa & PG_V) != 0) {
 3825                                 if (*ptepa & PG_PS)
 3826                                         return (NULL);
 3827                                 mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
 3828                                 mpte->wire_count++;
 3829                         } else {
 3830                                 /*
 3831                                  * Pass NULL instead of the PV list lock
 3832                                  * pointer, because we don't intend to sleep.
 3833                                  */
 3834                                 mpte = _pmap_allocpte(pmap, ptepindex, NULL);
 3835                                 if (mpte == NULL)
 3836                                         return (mpte);
 3837                         }
 3838                 }
 3839                 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
 3840                 pte = &pte[pmap_pte_index(va)];
 3841         } else {
 3842                 mpte = NULL;
 3843                 pte = vtopte(va);
 3844         }
 3845         if (*pte) {
 3846                 if (mpte != NULL) {
 3847                         mpte->wire_count--;
 3848                         mpte = NULL;
 3849                 }
 3850                 return (mpte);
 3851         }
 3852 
 3853         /*
 3854          * Enter on the PV list if part of our managed memory.
 3855          */
 3856         if ((m->oflags & VPO_UNMANAGED) == 0 &&
 3857             !pmap_try_insert_pv_entry(pmap, va, m, lockp)) {
 3858                 if (mpte != NULL) {
 3859                         free = NULL;
 3860                         if (pmap_unwire_ptp(pmap, va, mpte, &free)) {
 3861                                 pmap_invalidate_page(pmap, va);
 3862                                 pmap_free_zero_pages(free);
 3863                         }
 3864                         mpte = NULL;
 3865                 }
 3866                 return (mpte);
 3867         }
 3868 
 3869         /*
 3870          * Increment counters
 3871          */
 3872         pmap_resident_count_inc(pmap, 1);
 3873 
 3874         pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0);
 3875         if ((prot & VM_PROT_EXECUTE) == 0)
 3876                 pa |= pg_nx;
 3877 
 3878         /*
 3879          * Now validate mapping with RO protection
 3880          */
 3881         if ((m->oflags & VPO_UNMANAGED) != 0)
 3882                 pte_store(pte, pa | PG_V | PG_U);
 3883         else
 3884                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 3885         return (mpte);
 3886 }
 3887 
 3888 /*
 3889  * Make a temporary mapping for a physical address.  This is only intended
 3890  * to be used for panic dumps.
 3891  */
 3892 void *
 3893 pmap_kenter_temporary(vm_paddr_t pa, int i)
 3894 {
 3895         vm_offset_t va;
 3896 
 3897         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 3898         pmap_kenter(va, pa);
 3899         invlpg(va);
 3900         return ((void *)crashdumpmap);
 3901 }
 3902 
 3903 /*
 3904  * This code maps large physical mmap regions into the
 3905  * processor address space.  Note that some shortcuts
 3906  * are taken, but the code works.
 3907  */
 3908 void
 3909 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
 3910     vm_pindex_t pindex, vm_size_t size)
 3911 {
 3912         pd_entry_t *pde;
 3913         vm_paddr_t pa, ptepa;
 3914         vm_page_t p, pdpg;
 3915         int pat_mode;
 3916 
 3917         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 3918         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 3919             ("pmap_object_init_pt: non-device object"));
 3920         if ((addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
 3921                 if (!vm_object_populate(object, pindex, pindex + atop(size)))
 3922                         return;
 3923                 p = vm_page_lookup(object, pindex);
 3924                 KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3925                     ("pmap_object_init_pt: invalid page %p", p));
 3926                 pat_mode = p->md.pat_mode;
 3927 
 3928                 /*
 3929                  * Abort the mapping if the first page is not physically
 3930                  * aligned to a 2MB page boundary.
 3931                  */
 3932                 ptepa = VM_PAGE_TO_PHYS(p);
 3933                 if (ptepa & (NBPDR - 1))
 3934                         return;
 3935 
 3936                 /*
 3937                  * Skip the first page.  Abort the mapping if the rest of
 3938                  * the pages are not physically contiguous or have differing
 3939                  * memory attributes.
 3940                  */
 3941                 p = TAILQ_NEXT(p, listq);
 3942                 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
 3943                     pa += PAGE_SIZE) {
 3944                         KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3945                             ("pmap_object_init_pt: invalid page %p", p));
 3946                         if (pa != VM_PAGE_TO_PHYS(p) ||
 3947                             pat_mode != p->md.pat_mode)
 3948                                 return;
 3949                         p = TAILQ_NEXT(p, listq);
 3950                 }
 3951 
 3952                 /*
 3953                  * Map using 2MB pages.  Since "ptepa" is 2M aligned and
 3954                  * "size" is a multiple of 2M, adding the PAT setting to "pa"
 3955                  * will not affect the termination of this loop.
 3956                  */ 
 3957                 PMAP_LOCK(pmap);
 3958                 for (pa = ptepa | pmap_cache_bits(pat_mode, 1); pa < ptepa +
 3959                     size; pa += NBPDR) {
 3960                         pdpg = pmap_allocpde(pmap, addr, NULL);
 3961                         if (pdpg == NULL) {
 3962                                 /*
 3963                                  * The creation of mappings below is only an
 3964                                  * optimization.  If a page directory page
 3965                                  * cannot be allocated without blocking,
 3966                                  * continue on to the next mapping rather than
 3967                                  * blocking.
 3968                                  */
 3969                                 addr += NBPDR;
 3970                                 continue;
 3971                         }
 3972                         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
 3973                         pde = &pde[pmap_pde_index(addr)];
 3974                         if ((*pde & PG_V) == 0) {
 3975                                 pde_store(pde, pa | PG_PS | PG_M | PG_A |
 3976                                     PG_U | PG_RW | PG_V);
 3977                                 pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE);
 3978                                 atomic_add_long(&pmap_pde_mappings, 1);
 3979                         } else {
 3980                                 /* Continue on if the PDE is already valid. */
 3981                                 pdpg->wire_count--;
 3982                                 KASSERT(pdpg->wire_count > 0,
 3983                                     ("pmap_object_init_pt: missing reference "
 3984                                     "to page directory page, va: 0x%lx", addr));
 3985                         }
 3986                         addr += NBPDR;
 3987                 }
 3988                 PMAP_UNLOCK(pmap);
 3989         }
 3990 }
 3991 
 3992 /*
 3993  *      Routine:        pmap_change_wiring
 3994  *      Function:       Change the wiring attribute for a map/virtual-address
 3995  *                      pair.
 3996  *      In/out conditions:
 3997  *                      The mapping must already exist in the pmap.
 3998  */
 3999 void
 4000 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 4001 {
 4002         pd_entry_t *pde;
 4003         pt_entry_t *pte;
 4004         boolean_t pv_lists_locked;
 4005 
 4006         pv_lists_locked = FALSE;
 4007 
 4008         /*
 4009          * Wiring is not a hardware characteristic so there is no need to
 4010          * invalidate TLB.
 4011          */
 4012 retry:
 4013         PMAP_LOCK(pmap);
 4014         pde = pmap_pde(pmap, va);
 4015         if ((*pde & PG_PS) != 0) {
 4016                 if (!wired != ((*pde & PG_W) == 0)) {
 4017                         if (!pv_lists_locked) {
 4018                                 pv_lists_locked = TRUE;
 4019                                 if (!rw_try_rlock(&pvh_global_lock)) {
 4020                                         PMAP_UNLOCK(pmap);
 4021                                         rw_rlock(&pvh_global_lock);
 4022                                         goto retry;
 4023                                 }
 4024                         }
 4025                         if (!pmap_demote_pde(pmap, pde, va))
 4026                                 panic("pmap_change_wiring: demotion failed");
 4027                 } else
 4028                         goto out;
 4029         }
 4030         pte = pmap_pde_to_pte(pde, va);
 4031         if (wired && (*pte & PG_W) == 0) {
 4032                 pmap->pm_stats.wired_count++;
 4033                 atomic_set_long(pte, PG_W);
 4034         } else if (!wired && (*pte & PG_W) != 0) {
 4035                 pmap->pm_stats.wired_count--;
 4036                 atomic_clear_long(pte, PG_W);
 4037         }
 4038 out:
 4039         if (pv_lists_locked)
 4040                 rw_runlock(&pvh_global_lock);
 4041         PMAP_UNLOCK(pmap);
 4042 }
 4043 
 4044 /*
 4045  *      Copy the range specified by src_addr/len
 4046  *      from the source map to the range dst_addr/len
 4047  *      in the destination map.
 4048  *
 4049  *      This routine is only advisory and need not do anything.
 4050  */
 4051 
 4052 void
 4053 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 4054     vm_offset_t src_addr)
 4055 {
 4056         struct rwlock *lock;
 4057         vm_page_t   free;
 4058         vm_offset_t addr;
 4059         vm_offset_t end_addr = src_addr + len;
 4060         vm_offset_t va_next;
 4061 
 4062         if (dst_addr != src_addr)
 4063                 return;
 4064 
 4065         lock = NULL;
 4066         rw_rlock(&pvh_global_lock);
 4067         if (dst_pmap < src_pmap) {
 4068                 PMAP_LOCK(dst_pmap);
 4069                 PMAP_LOCK(src_pmap);
 4070         } else {
 4071                 PMAP_LOCK(src_pmap);
 4072                 PMAP_LOCK(dst_pmap);
 4073         }
 4074         for (addr = src_addr; addr < end_addr; addr = va_next) {
 4075                 pt_entry_t *src_pte, *dst_pte;
 4076                 vm_page_t dstmpde, dstmpte, srcmpte;
 4077                 pml4_entry_t *pml4e;
 4078                 pdp_entry_t *pdpe;
 4079                 pd_entry_t srcptepaddr, *pde;
 4080 
 4081                 KASSERT(addr < UPT_MIN_ADDRESS,
 4082                     ("pmap_copy: invalid to pmap_copy page tables"));
 4083 
 4084                 pml4e = pmap_pml4e(src_pmap, addr);
 4085                 if ((*pml4e & PG_V) == 0) {
 4086                         va_next = (addr + NBPML4) & ~PML4MASK;
 4087                         if (va_next < addr)
 4088                                 va_next = end_addr;
 4089                         continue;
 4090                 }
 4091 
 4092                 pdpe = pmap_pml4e_to_pdpe(pml4e, addr);
 4093                 if ((*pdpe & PG_V) == 0) {
 4094                         va_next = (addr + NBPDP) & ~PDPMASK;
 4095                         if (va_next < addr)
 4096                                 va_next = end_addr;
 4097                         continue;
 4098                 }
 4099 
 4100                 va_next = (addr + NBPDR) & ~PDRMASK;
 4101                 if (va_next < addr)
 4102                         va_next = end_addr;
 4103 
 4104                 pde = pmap_pdpe_to_pde(pdpe, addr);
 4105                 srcptepaddr = *pde;
 4106                 if (srcptepaddr == 0)
 4107                         continue;
 4108                         
 4109                 if (srcptepaddr & PG_PS) {
 4110                         dstmpde = pmap_allocpde(dst_pmap, addr, NULL);
 4111                         if (dstmpde == NULL)
 4112                                 break;
 4113                         pde = (pd_entry_t *)
 4114                             PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpde));
 4115                         pde = &pde[pmap_pde_index(addr)];
 4116                         if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
 4117                             pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr &
 4118                             PG_PS_FRAME, &lock))) {
 4119                                 *pde = srcptepaddr & ~PG_W;
 4120                                 pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE);
 4121                         } else
 4122                                 dstmpde->wire_count--;
 4123                         continue;
 4124                 }
 4125 
 4126                 srcptepaddr &= PG_FRAME;
 4127                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
 4128                 KASSERT(srcmpte->wire_count > 0,
 4129                     ("pmap_copy: source page table page is unused"));
 4130 
 4131                 if (va_next > end_addr)
 4132                         va_next = end_addr;
 4133 
 4134                 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
 4135                 src_pte = &src_pte[pmap_pte_index(addr)];
 4136                 dstmpte = NULL;
 4137                 while (addr < va_next) {
 4138                         pt_entry_t ptetemp;
 4139                         ptetemp = *src_pte;
 4140                         /*
 4141                          * we only virtual copy managed pages
 4142                          */
 4143                         if ((ptetemp & PG_MANAGED) != 0) {
 4144                                 if (dstmpte != NULL &&
 4145                                     dstmpte->pindex == pmap_pde_pindex(addr))
 4146                                         dstmpte->wire_count++;
 4147                                 else if ((dstmpte = pmap_allocpte(dst_pmap,
 4148                                     addr, NULL)) == NULL)
 4149                                         goto out;
 4150                                 dst_pte = (pt_entry_t *)
 4151                                     PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
 4152                                 dst_pte = &dst_pte[pmap_pte_index(addr)];
 4153                                 if (*dst_pte == 0 &&
 4154                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 4155                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME),
 4156                                     &lock)) {
 4157                                         /*
 4158                                          * Clear the wired, modified, and
 4159                                          * accessed (referenced) bits
 4160                                          * during the copy.
 4161                                          */
 4162                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 4163                                             PG_A);
 4164                                         pmap_resident_count_inc(dst_pmap, 1);
 4165                                 } else {
 4166                                         free = NULL;
 4167                                         if (pmap_unwire_ptp(dst_pmap, addr,
 4168                                             dstmpte, &free)) {
 4169                                                 pmap_invalidate_page(dst_pmap,
 4170                                                     addr);
 4171                                                 pmap_free_zero_pages(free);
 4172                                         }
 4173                                         goto out;
 4174                                 }
 4175                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 4176                                         break;
 4177                         }
 4178                         addr += PAGE_SIZE;
 4179                         src_pte++;
 4180                 }
 4181         }
 4182 out:
 4183         if (lock != NULL)
 4184                 rw_wunlock(lock);
 4185         rw_runlock(&pvh_global_lock);
 4186         PMAP_UNLOCK(src_pmap);
 4187         PMAP_UNLOCK(dst_pmap);
 4188 }       
 4189 
 4190 /*
 4191  *      pmap_zero_page zeros the specified hardware page by mapping 
 4192  *      the page into KVM and using bzero to clear its contents.
 4193  */
 4194 void
 4195 pmap_zero_page(vm_page_t m)
 4196 {
 4197         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 4198 
 4199         pagezero((void *)va);
 4200 }
 4201 
 4202 /*
 4203  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 4204  *      the page into KVM and using bzero to clear its contents.
 4205  *
 4206  *      off and size may not cover an area beyond a single hardware page.
 4207  */
 4208 void
 4209 pmap_zero_page_area(vm_page_t m, int off, int size)
 4210 {
 4211         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 4212 
 4213         if (off == 0 && size == PAGE_SIZE)
 4214                 pagezero((void *)va);
 4215         else
 4216                 bzero((char *)va + off, size);
 4217 }
 4218 
 4219 /*
 4220  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 4221  *      the page into KVM and using bzero to clear its contents.  This
 4222  *      is intended to be called from the vm_pagezero process only and
 4223  *      outside of Giant.
 4224  */
 4225 void
 4226 pmap_zero_page_idle(vm_page_t m)
 4227 {
 4228         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 4229 
 4230         pagezero((void *)va);
 4231 }
 4232 
 4233 /*
 4234  *      pmap_copy_page copies the specified (machine independent)
 4235  *      page by mapping the page into virtual memory and using
 4236  *      bcopy to copy the page, one machine dependent page at a
 4237  *      time.
 4238  */
 4239 void
 4240 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 4241 {
 4242         vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
 4243         vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
 4244 
 4245         pagecopy((void *)src, (void *)dst);
 4246 }
 4247 
 4248 int unmapped_buf_allowed = 1;
 4249 
 4250 void
 4251 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 4252     vm_offset_t b_offset, int xfersize)
 4253 {
 4254         void *a_cp, *b_cp;
 4255         vm_offset_t a_pg_offset, b_pg_offset;
 4256         int cnt;
 4257 
 4258         while (xfersize > 0) {
 4259                 a_pg_offset = a_offset & PAGE_MASK;
 4260                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
 4261                 a_cp = (char *)PHYS_TO_DMAP(ma[a_offset >> PAGE_SHIFT]->
 4262                     phys_addr) + a_pg_offset;
 4263                 b_pg_offset = b_offset & PAGE_MASK;
 4264                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 4265                 b_cp = (char *)PHYS_TO_DMAP(mb[b_offset >> PAGE_SHIFT]->
 4266                     phys_addr) + b_pg_offset;
 4267                 bcopy(a_cp, b_cp, cnt);
 4268                 a_offset += cnt;
 4269                 b_offset += cnt;
 4270                 xfersize -= cnt;
 4271         }
 4272 }
 4273 
 4274 /*
 4275  * Returns true if the pmap's pv is one of the first
 4276  * 16 pvs linked to from this page.  This count may
 4277  * be changed upwards or downwards in the future; it
 4278  * is only necessary that true be returned for a small
 4279  * subset of pmaps for proper page aging.
 4280  */
 4281 boolean_t
 4282 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 4283 {
 4284         struct md_page *pvh;
 4285         struct rwlock *lock;
 4286         pv_entry_t pv;
 4287         int loops = 0;
 4288         boolean_t rv;
 4289 
 4290         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4291             ("pmap_page_exists_quick: page %p is not managed", m));
 4292         rv = FALSE;
 4293         rw_rlock(&pvh_global_lock);
 4294         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 4295         rw_rlock(lock);
 4296         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4297                 if (PV_PMAP(pv) == pmap) {
 4298                         rv = TRUE;
 4299                         break;
 4300                 }
 4301                 loops++;
 4302                 if (loops >= 16)
 4303                         break;
 4304         }
 4305         if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
 4306                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4307                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 4308                         if (PV_PMAP(pv) == pmap) {
 4309                                 rv = TRUE;
 4310                                 break;
 4311                         }
 4312                         loops++;
 4313                         if (loops >= 16)
 4314                                 break;
 4315                 }
 4316         }
 4317         rw_runlock(lock);
 4318         rw_runlock(&pvh_global_lock);
 4319         return (rv);
 4320 }
 4321 
 4322 /*
 4323  *      pmap_page_wired_mappings:
 4324  *
 4325  *      Return the number of managed mappings to the given physical page
 4326  *      that are wired.
 4327  */
 4328 int
 4329 pmap_page_wired_mappings(vm_page_t m)
 4330 {
 4331         int count;
 4332 
 4333         count = 0;
 4334         if ((m->oflags & VPO_UNMANAGED) != 0)
 4335                 return (count);
 4336         rw_wlock(&pvh_global_lock);
 4337         count = pmap_pvh_wired_mappings(&m->md, count);
 4338         if ((m->flags & PG_FICTITIOUS) == 0) {
 4339             count = pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)),
 4340                 count);
 4341         }
 4342         rw_wunlock(&pvh_global_lock);
 4343         return (count);
 4344 }
 4345 
 4346 /*
 4347  *      pmap_pvh_wired_mappings:
 4348  *
 4349  *      Return the updated number "count" of managed mappings that are wired.
 4350  */
 4351 static int
 4352 pmap_pvh_wired_mappings(struct md_page *pvh, int count)
 4353 {
 4354         pmap_t pmap;
 4355         pt_entry_t *pte;
 4356         pv_entry_t pv;
 4357 
 4358         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4359         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 4360                 pmap = PV_PMAP(pv);
 4361                 PMAP_LOCK(pmap);
 4362                 pte = pmap_pte(pmap, pv->pv_va);
 4363                 if ((*pte & PG_W) != 0)
 4364                         count++;
 4365                 PMAP_UNLOCK(pmap);
 4366         }
 4367         return (count);
 4368 }
 4369 
 4370 /*
 4371  * Returns TRUE if the given page is mapped individually or as part of
 4372  * a 2mpage.  Otherwise, returns FALSE.
 4373  */
 4374 boolean_t
 4375 pmap_page_is_mapped(vm_page_t m)
 4376 {
 4377         struct rwlock *lock;
 4378         boolean_t rv;
 4379 
 4380         if ((m->oflags & VPO_UNMANAGED) != 0)
 4381                 return (FALSE);
 4382         rw_rlock(&pvh_global_lock);
 4383         lock = VM_PAGE_TO_PV_LIST_LOCK(m);
 4384         rw_rlock(lock);
 4385         rv = !TAILQ_EMPTY(&m->md.pv_list) ||
 4386             ((m->flags & PG_FICTITIOUS) == 0 &&
 4387             !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
 4388         rw_runlock(lock);
 4389         rw_runlock(&pvh_global_lock);
 4390         return (rv);
 4391 }
 4392 
 4393 /*
 4394  * Remove all pages from specified address space
 4395  * this aids process exit speeds.  Also, this code
 4396  * is special cased for current process only, but
 4397  * can have the more generic (and slightly slower)
 4398  * mode enabled.  This is much faster than pmap_remove
 4399  * in the case of running down an entire address space.
 4400  */
 4401 void
 4402 pmap_remove_pages(pmap_t pmap)
 4403 {
 4404         pd_entry_t ptepde;
 4405         pt_entry_t *pte, tpte;
 4406         vm_page_t free = NULL;
 4407         vm_page_t m, mpte, mt;
 4408         pv_entry_t pv;
 4409         struct md_page *pvh;
 4410         struct pv_chunk *pc, *npc;
 4411         struct rwlock *lock;
 4412         int64_t bit;
 4413         uint64_t inuse, bitmask;
 4414         int allfree, field, freed, idx;
 4415 
 4416         if (pmap != PCPU_GET(curpmap)) {
 4417                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 4418                 return;
 4419         }
 4420         lock = NULL;
 4421         rw_rlock(&pvh_global_lock);
 4422         PMAP_LOCK(pmap);
 4423         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 4424                 allfree = 1;
 4425                 freed = 0;
 4426                 for (field = 0; field < _NPCM; field++) {
 4427                         inuse = ~pc->pc_map[field] & pc_freemask[field];
 4428                         while (inuse != 0) {
 4429                                 bit = bsfq(inuse);
 4430                                 bitmask = 1UL << bit;
 4431                                 idx = field * 64 + bit;
 4432                                 pv = &pc->pc_pventry[idx];
 4433                                 inuse &= ~bitmask;
 4434 
 4435                                 pte = pmap_pdpe(pmap, pv->pv_va);
 4436                                 ptepde = *pte;
 4437                                 pte = pmap_pdpe_to_pde(pte, pv->pv_va);
 4438                                 tpte = *pte;
 4439                                 if ((tpte & (PG_PS | PG_V)) == PG_V) {
 4440                                         ptepde = tpte;
 4441                                         pte = (pt_entry_t *)PHYS_TO_DMAP(tpte &
 4442                                             PG_FRAME);
 4443                                         pte = &pte[pmap_pte_index(pv->pv_va)];
 4444                                         tpte = *pte & ~PG_PTE_PAT;
 4445                                 }
 4446                                 if ((tpte & PG_V) == 0) {
 4447                                         panic("bad pte va %lx pte %lx",
 4448                                             pv->pv_va, tpte);
 4449                                 }
 4450 
 4451 /*
 4452  * We cannot remove wired pages from a process' mapping at this time
 4453  */
 4454                                 if (tpte & PG_W) {
 4455                                         allfree = 0;
 4456                                         continue;
 4457                                 }
 4458 
 4459                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 4460                                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 4461                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 4462                                     m, (uintmax_t)m->phys_addr,
 4463                                     (uintmax_t)tpte));
 4464 
 4465                                 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
 4466                                     m < &vm_page_array[vm_page_array_size],
 4467                                     ("pmap_remove_pages: bad tpte %#jx",
 4468                                     (uintmax_t)tpte));
 4469 
 4470                                 pte_clear(pte);
 4471 
 4472                                 /*
 4473                                  * Update the vm_page_t clean/reference bits.
 4474                                  */
 4475                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4476                                         if ((tpte & PG_PS) != 0) {
 4477                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4478                                                         vm_page_dirty(mt);
 4479                                         } else
 4480                                                 vm_page_dirty(m);
 4481                                 }
 4482 
 4483                                 CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
 4484 
 4485                                 /* Mark free */
 4486                                 pc->pc_map[field] |= bitmask;
 4487                                 if ((tpte & PG_PS) != 0) {
 4488                                         pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE);
 4489                                         pvh = pa_to_pvh(tpte & PG_PS_FRAME);
 4490                                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
 4491                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 4492                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4493                                                         if ((mt->aflags & PGA_WRITEABLE) != 0 &&
 4494                                                             TAILQ_EMPTY(&mt->md.pv_list))
 4495                                                                 vm_page_aflag_clear(mt, PGA_WRITEABLE);
 4496                                         }
 4497                                         mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
 4498                                         if (mpte != NULL) {
 4499                                                 pmap_remove_pt_page(pmap, mpte);
 4500                                                 pmap_resident_count_dec(pmap, 1);
 4501                                                 KASSERT(mpte->wire_count == NPTEPG,
 4502                                                     ("pmap_remove_pages: pte page wire count error"));
 4503                                                 mpte->wire_count = 0;
 4504                                                 pmap_add_delayed_free_list(mpte, &free, FALSE);
 4505                                                 atomic_subtract_int(&cnt.v_wire_count, 1);
 4506                                         }
 4507                                 } else {
 4508                                         pmap_resident_count_dec(pmap, 1);
 4509                                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 4510                                         if ((m->aflags & PGA_WRITEABLE) != 0 &&
 4511                                             TAILQ_EMPTY(&m->md.pv_list) &&
 4512                                             (m->flags & PG_FICTITIOUS) == 0) {
 4513                                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4514                                                 if (TAILQ_EMPTY(&pvh->pv_list))
 4515                                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 4516                                         }
 4517                                 }
 4518                                 pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
 4519                                 freed++;
 4520                         }
 4521                 }
 4522                 PV_STAT(atomic_add_long(&pv_entry_frees, freed));
 4523                 PV_STAT(atomic_add_int(&pv_entry_spare, freed));
 4524                 PV_STAT(atomic_subtract_long(&pv_entry_count, freed));
 4525                 if (allfree) {
 4526                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 4527                         free_pv_chunk(pc);
 4528                 }
 4529         }
 4530         if (lock != NULL)
 4531                 rw_wunlock(lock);
 4532         pmap_invalidate_all(pmap);
 4533         rw_runlock(&pvh_global_lock);
 4534         PMAP_UNLOCK(pmap);
 4535         pmap_free_zero_pages(free);
 4536 }
 4537 
 4538 /*
 4539  *      pmap_is_modified:
 4540  *
 4541  *      Return whether or not the specified physical page was modified
 4542  *      in any physical maps.
 4543  */
 4544 boolean_t
 4545 pmap_is_modified(vm_page_t m)
 4546 {
 4547         boolean_t rv;
 4548 
 4549         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4550             ("pmap_is_modified: page %p is not managed", m));
 4551 
 4552         /*
 4553          * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be
 4554          * concurrently set while the object is locked.  Thus, if PGA_WRITEABLE
 4555          * is clear, no PTEs can have PG_M set.
 4556          */
 4557         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 4558         if ((m->oflags & VPO_BUSY) == 0 &&
 4559             (m->aflags & PGA_WRITEABLE) == 0)
 4560                 return (FALSE);
 4561         rw_wlock(&pvh_global_lock);
 4562         rv = pmap_is_modified_pvh(&m->md) ||
 4563             ((m->flags & PG_FICTITIOUS) == 0 &&
 4564             pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4565         rw_wunlock(&pvh_global_lock);
 4566         return (rv);
 4567 }
 4568 
 4569 /*
 4570  * Returns TRUE if any of the given mappings were used to modify
 4571  * physical memory.  Otherwise, returns FALSE.  Both page and 2mpage
 4572  * mappings are supported.
 4573  */
 4574 static boolean_t
 4575 pmap_is_modified_pvh(struct md_page *pvh)
 4576 {
 4577         pv_entry_t pv;
 4578         pt_entry_t *pte;
 4579         pmap_t pmap;
 4580         boolean_t rv;
 4581 
 4582         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4583         rv = FALSE;
 4584         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 4585                 pmap = PV_PMAP(pv);
 4586                 PMAP_LOCK(pmap);
 4587                 pte = pmap_pte(pmap, pv->pv_va);
 4588                 rv = (*pte & (PG_M | PG_RW)) == (PG_M | PG_RW);
 4589                 PMAP_UNLOCK(pmap);
 4590                 if (rv)
 4591                         break;
 4592         }
 4593         return (rv);
 4594 }
 4595 
 4596 /*
 4597  *      pmap_is_prefaultable:
 4598  *
 4599  *      Return whether or not the specified virtual address is elgible
 4600  *      for prefault.
 4601  */
 4602 boolean_t
 4603 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 4604 {
 4605         pd_entry_t *pde;
 4606         pt_entry_t *pte;
 4607         boolean_t rv;
 4608 
 4609         rv = FALSE;
 4610         PMAP_LOCK(pmap);
 4611         pde = pmap_pde(pmap, addr);
 4612         if (pde != NULL && (*pde & (PG_PS | PG_V)) == PG_V) {
 4613                 pte = pmap_pde_to_pte(pde, addr);
 4614                 rv = (*pte & PG_V) == 0;
 4615         }
 4616         PMAP_UNLOCK(pmap);
 4617         return (rv);
 4618 }
 4619 
 4620 /*
 4621  *      pmap_is_referenced:
 4622  *
 4623  *      Return whether or not the specified physical page was referenced
 4624  *      in any physical maps.
 4625  */
 4626 boolean_t
 4627 pmap_is_referenced(vm_page_t m)
 4628 {
 4629         boolean_t rv;
 4630 
 4631         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4632             ("pmap_is_referenced: page %p is not managed", m));
 4633         rw_wlock(&pvh_global_lock);
 4634         rv = pmap_is_referenced_pvh(&m->md) ||
 4635             ((m->flags & PG_FICTITIOUS) == 0 &&
 4636             pmap_is_referenced_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4637         rw_wunlock(&pvh_global_lock);
 4638         return (rv);
 4639 }
 4640 
 4641 /*
 4642  * Returns TRUE if any of the given mappings were referenced and FALSE
 4643  * otherwise.  Both page and 2mpage mappings are supported.
 4644  */
 4645 static boolean_t
 4646 pmap_is_referenced_pvh(struct md_page *pvh)
 4647 {
 4648         pv_entry_t pv;
 4649         pt_entry_t *pte;
 4650         pmap_t pmap;
 4651         boolean_t rv;
 4652 
 4653         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4654         rv = FALSE;
 4655         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 4656                 pmap = PV_PMAP(pv);
 4657                 PMAP_LOCK(pmap);
 4658                 pte = pmap_pte(pmap, pv->pv_va);
 4659                 rv = (*pte & (PG_A | PG_V)) == (PG_A | PG_V);
 4660                 PMAP_UNLOCK(pmap);
 4661                 if (rv)
 4662                         break;
 4663         }
 4664         return (rv);
 4665 }
 4666 
 4667 /*
 4668  * Clear the write and modified bits in each of the given page's mappings.
 4669  */
 4670 void
 4671 pmap_remove_write(vm_page_t m)
 4672 {
 4673         struct md_page *pvh;
 4674         pmap_t pmap;
 4675         pv_entry_t next_pv, pv;
 4676         pd_entry_t *pde;
 4677         pt_entry_t oldpte, *pte;
 4678         vm_offset_t va;
 4679 
 4680         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4681             ("pmap_remove_write: page %p is not managed", m));
 4682 
 4683         /*
 4684          * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by
 4685          * another thread while the object is locked.  Thus, if PGA_WRITEABLE
 4686          * is clear, no page table entries need updating.
 4687          */
 4688         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 4689         if ((m->oflags & VPO_BUSY) == 0 &&
 4690             (m->aflags & PGA_WRITEABLE) == 0)
 4691                 return;
 4692         rw_wlock(&pvh_global_lock);
 4693         if ((m->flags & PG_FICTITIOUS) != 0)
 4694                 goto small_mappings;
 4695         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4696         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
 4697                 pmap = PV_PMAP(pv);
 4698                 PMAP_LOCK(pmap);
 4699                 va = pv->pv_va;
 4700                 pde = pmap_pde(pmap, va);
 4701                 if ((*pde & PG_RW) != 0)
 4702                         (void)pmap_demote_pde(pmap, pde, va);
 4703                 PMAP_UNLOCK(pmap);
 4704         }
 4705 small_mappings:
 4706         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4707                 pmap = PV_PMAP(pv);
 4708                 PMAP_LOCK(pmap);
 4709                 pde = pmap_pde(pmap, pv->pv_va);
 4710                 KASSERT((*pde & PG_PS) == 0,
 4711                     ("pmap_remove_write: found a 2mpage in page %p's pv list",
 4712                     m));
 4713                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 4714 retry:
 4715                 oldpte = *pte;
 4716                 if (oldpte & PG_RW) {
 4717                         if (!atomic_cmpset_long(pte, oldpte, oldpte &
 4718                             ~(PG_RW | PG_M)))
 4719                                 goto retry;
 4720                         if ((oldpte & PG_M) != 0)
 4721                                 vm_page_dirty(m);
 4722                         pmap_invalidate_page(pmap, pv->pv_va);
 4723                 }
 4724                 PMAP_UNLOCK(pmap);
 4725         }
 4726         vm_page_aflag_clear(m, PGA_WRITEABLE);
 4727         rw_wunlock(&pvh_global_lock);
 4728 }
 4729 
 4730 /*
 4731  *      pmap_ts_referenced:
 4732  *
 4733  *      Return a count of reference bits for a page, clearing those bits.
 4734  *      It is not necessary for every reference bit to be cleared, but it
 4735  *      is necessary that 0 only be returned when there are truly no
 4736  *      reference bits set.
 4737  *
 4738  *      XXX: The exact number of bits to check and clear is a matter that
 4739  *      should be tested and standardized at some point in the future for
 4740  *      optimal aging of shared pages.
 4741  */
 4742 int
 4743 pmap_ts_referenced(vm_page_t m)
 4744 {
 4745         struct md_page *pvh;
 4746         pv_entry_t pv, pvf, pvn;
 4747         pmap_t pmap;
 4748         pd_entry_t oldpde, *pde;
 4749         pt_entry_t *pte;
 4750         vm_offset_t va;
 4751         int rtval = 0;
 4752 
 4753         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4754             ("pmap_ts_referenced: page %p is not managed", m));
 4755         rw_wlock(&pvh_global_lock);
 4756         if ((m->flags & PG_FICTITIOUS) != 0)
 4757                 goto small_mappings;
 4758         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4759         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) {
 4760                 pmap = PV_PMAP(pv);
 4761                 PMAP_LOCK(pmap);
 4762                 va = pv->pv_va;
 4763                 pde = pmap_pde(pmap, va);
 4764                 oldpde = *pde;
 4765                 if ((oldpde & PG_A) != 0) {
 4766                         if (pmap_demote_pde(pmap, pde, va)) {
 4767                                 if ((oldpde & PG_W) == 0) {
 4768                                         /*
 4769                                          * Remove the mapping to a single page
 4770                                          * so that a subsequent access may
 4771                                          * repromote.  Since the underlying
 4772                                          * page table page is fully populated,
 4773                                          * this removal never frees a page
 4774                                          * table page.
 4775                                          */
 4776                                         va += VM_PAGE_TO_PHYS(m) - (oldpde &
 4777                                             PG_PS_FRAME);
 4778                                         pmap_remove_page(pmap, va, pde, NULL);
 4779                                         rtval++;
 4780                                         if (rtval > 4) {
 4781                                                 PMAP_UNLOCK(pmap);
 4782                                                 goto out;
 4783                                         }
 4784                                 }
 4785                         }
 4786                 }
 4787                 PMAP_UNLOCK(pmap);
 4788         }
 4789 small_mappings:
 4790         if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 4791                 pvf = pv;
 4792                 do {
 4793                         pvn = TAILQ_NEXT(pv, pv_list);
 4794                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 4795                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 4796                         pmap = PV_PMAP(pv);
 4797                         PMAP_LOCK(pmap);
 4798                         pde = pmap_pde(pmap, pv->pv_va);
 4799                         KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:"
 4800                             " found a 2mpage in page %p's pv list", m));
 4801                         pte = pmap_pde_to_pte(pde, pv->pv_va);
 4802                         if ((*pte & PG_A) != 0) {
 4803                                 atomic_clear_long(pte, PG_A);
 4804                                 pmap_invalidate_page(pmap, pv->pv_va);
 4805                                 rtval++;
 4806                                 if (rtval > 4)
 4807                                         pvn = NULL;
 4808                         }
 4809                         PMAP_UNLOCK(pmap);
 4810                 } while ((pv = pvn) != NULL && pv != pvf);
 4811         }
 4812 out:
 4813         rw_wunlock(&pvh_global_lock);
 4814         return (rtval);
 4815 }
 4816 
 4817 /*
 4818  *      Clear the modify bits on the specified physical page.
 4819  */
 4820 void
 4821 pmap_clear_modify(vm_page_t m)
 4822 {
 4823         struct md_page *pvh;
 4824         pmap_t pmap;
 4825         pv_entry_t next_pv, pv;
 4826         pd_entry_t oldpde, *pde;
 4827         pt_entry_t oldpte, *pte;
 4828         vm_offset_t va;
 4829 
 4830         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4831             ("pmap_clear_modify: page %p is not managed", m));
 4832         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 4833         KASSERT((m->oflags & VPO_BUSY) == 0,
 4834             ("pmap_clear_modify: page %p is busy", m));
 4835 
 4836         /*
 4837          * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
 4838          * If the object containing the page is locked and the page is not
 4839          * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set.
 4840          */
 4841         if ((m->aflags & PGA_WRITEABLE) == 0)
 4842                 return;
 4843         rw_wlock(&pvh_global_lock);
 4844         if ((m->flags & PG_FICTITIOUS) != 0)
 4845                 goto small_mappings;
 4846         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4847         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
 4848                 pmap = PV_PMAP(pv);
 4849                 PMAP_LOCK(pmap);
 4850                 va = pv->pv_va;
 4851                 pde = pmap_pde(pmap, va);
 4852                 oldpde = *pde;
 4853                 if ((oldpde & PG_RW) != 0) {
 4854                         if (pmap_demote_pde(pmap, pde, va)) {
 4855                                 if ((oldpde & PG_W) == 0) {
 4856                                         /*
 4857                                          * Write protect the mapping to a
 4858                                          * single page so that a subsequent
 4859                                          * write access may repromote.
 4860                                          */
 4861                                         va += VM_PAGE_TO_PHYS(m) - (oldpde &
 4862                                             PG_PS_FRAME);
 4863                                         pte = pmap_pde_to_pte(pde, va);
 4864                                         oldpte = *pte;
 4865                                         if ((oldpte & PG_V) != 0) {
 4866                                                 while (!atomic_cmpset_long(pte,
 4867                                                     oldpte,
 4868                                                     oldpte & ~(PG_M | PG_RW)))
 4869                                                         oldpte = *pte;
 4870                                                 vm_page_dirty(m);
 4871                                                 pmap_invalidate_page(pmap, va);
 4872                                         }
 4873                                 }
 4874                         }
 4875                 }
 4876                 PMAP_UNLOCK(pmap);
 4877         }
 4878 small_mappings:
 4879         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4880                 pmap = PV_PMAP(pv);
 4881                 PMAP_LOCK(pmap);
 4882                 pde = pmap_pde(pmap, pv->pv_va);
 4883                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
 4884                     " a 2mpage in page %p's pv list", m));
 4885                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 4886                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4887                         atomic_clear_long(pte, PG_M);
 4888                         pmap_invalidate_page(pmap, pv->pv_va);
 4889                 }
 4890                 PMAP_UNLOCK(pmap);
 4891         }
 4892         rw_wunlock(&pvh_global_lock);
 4893 }
 4894 
 4895 /*
 4896  *      pmap_clear_reference:
 4897  *
 4898  *      Clear the reference bit on the specified physical page.
 4899  */
 4900 void
 4901 pmap_clear_reference(vm_page_t m)
 4902 {
 4903         struct md_page *pvh;
 4904         pmap_t pmap;
 4905         pv_entry_t next_pv, pv;
 4906         pd_entry_t oldpde, *pde;
 4907         pt_entry_t *pte;
 4908         vm_offset_t va;
 4909 
 4910         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4911             ("pmap_clear_reference: page %p is not managed", m));
 4912         rw_wlock(&pvh_global_lock);
 4913         if ((m->flags & PG_FICTITIOUS) != 0)
 4914                 goto small_mappings;
 4915         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4916         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
 4917                 pmap = PV_PMAP(pv);
 4918                 PMAP_LOCK(pmap);
 4919                 va = pv->pv_va;
 4920                 pde = pmap_pde(pmap, va);
 4921                 oldpde = *pde;
 4922                 if ((oldpde & PG_A) != 0) {
 4923                         if (pmap_demote_pde(pmap, pde, va)) {
 4924                                 /*
 4925                                  * Remove the mapping to a single page so
 4926                                  * that a subsequent access may repromote.
 4927                                  * Since the underlying page table page is
 4928                                  * fully populated, this removal never frees
 4929                                  * a page table page.
 4930                                  */
 4931                                 va += VM_PAGE_TO_PHYS(m) - (oldpde &
 4932                                     PG_PS_FRAME);
 4933                                 pmap_remove_page(pmap, va, pde, NULL);
 4934                         }
 4935                 }
 4936                 PMAP_UNLOCK(pmap);
 4937         }
 4938 small_mappings:
 4939         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4940                 pmap = PV_PMAP(pv);
 4941                 PMAP_LOCK(pmap);
 4942                 pde = pmap_pde(pmap, pv->pv_va);
 4943                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_reference: found"
 4944                     " a 2mpage in page %p's pv list", m));
 4945                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 4946                 if (*pte & PG_A) {
 4947                         atomic_clear_long(pte, PG_A);
 4948                         pmap_invalidate_page(pmap, pv->pv_va);
 4949                 }
 4950                 PMAP_UNLOCK(pmap);
 4951         }
 4952         rw_wunlock(&pvh_global_lock);
 4953 }
 4954 
 4955 /*
 4956  * Miscellaneous support routines follow
 4957  */
 4958 
 4959 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
 4960 static __inline void
 4961 pmap_pte_attr(pt_entry_t *pte, int cache_bits)
 4962 {
 4963         u_int opte, npte;
 4964 
 4965         /*
 4966          * The cache mode bits are all in the low 32-bits of the
 4967          * PTE, so we can just spin on updating the low 32-bits.
 4968          */
 4969         do {
 4970                 opte = *(u_int *)pte;
 4971                 npte = opte & ~PG_PTE_CACHE;
 4972                 npte |= cache_bits;
 4973         } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
 4974 }
 4975 
 4976 /* Adjust the cache mode for a 2MB page mapped via a PDE. */
 4977 static __inline void
 4978 pmap_pde_attr(pd_entry_t *pde, int cache_bits)
 4979 {
 4980         u_int opde, npde;
 4981 
 4982         /*
 4983          * The cache mode bits are all in the low 32-bits of the
 4984          * PDE, so we can just spin on updating the low 32-bits.
 4985          */
 4986         do {
 4987                 opde = *(u_int *)pde;
 4988                 npde = opde & ~PG_PDE_CACHE;
 4989                 npde |= cache_bits;
 4990         } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
 4991 }
 4992 
 4993 /*
 4994  * Map a set of physical memory pages into the kernel virtual
 4995  * address space. Return a pointer to where it is mapped. This
 4996  * routine is intended to be used for mapping device memory,
 4997  * NOT real memory.
 4998  */
 4999 void *
 5000 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 5001 {
 5002         vm_offset_t va, offset;
 5003         vm_size_t tmpsize;
 5004 
 5005         /*
 5006          * If the specified range of physical addresses fits within the direct
 5007          * map window, use the direct map. 
 5008          */
 5009         if (pa < dmaplimit && pa + size < dmaplimit) {
 5010                 va = PHYS_TO_DMAP(pa);
 5011                 if (!pmap_change_attr(va, size, mode))
 5012                         return ((void *)va);
 5013         }
 5014         offset = pa & PAGE_MASK;
 5015         size = roundup(offset + size, PAGE_SIZE);
 5016         va = kmem_alloc_nofault(kernel_map, size);
 5017         if (!va)
 5018                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 5019         pa = trunc_page(pa);
 5020         for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
 5021                 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
 5022         pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
 5023         pmap_invalidate_cache_range(va, va + tmpsize);
 5024         return ((void *)(va + offset));
 5025 }
 5026 
 5027 void *
 5028 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 5029 {
 5030 
 5031         return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
 5032 }
 5033 
 5034 void *
 5035 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
 5036 {
 5037 
 5038         return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
 5039 }
 5040 
 5041 void
 5042 pmap_unmapdev(vm_offset_t va, vm_size_t size)
 5043 {
 5044         vm_offset_t base, offset;
 5045 
 5046         /* If we gave a direct map region in pmap_mapdev, do nothing */
 5047         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
 5048                 return;
 5049         base = trunc_page(va);
 5050         offset = va & PAGE_MASK;
 5051         size = roundup(offset + size, PAGE_SIZE);
 5052         kmem_free(kernel_map, base, size);
 5053 }
 5054 
 5055 /*
 5056  * Tries to demote a 1GB page mapping.
 5057  */
 5058 static boolean_t
 5059 pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va)
 5060 {
 5061         pdp_entry_t newpdpe, oldpdpe;
 5062         pd_entry_t *firstpde, newpde, *pde;
 5063         vm_paddr_t mpdepa;
 5064         vm_page_t mpde;
 5065 
 5066         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 5067         oldpdpe = *pdpe;
 5068         KASSERT((oldpdpe & (PG_PS | PG_V)) == (PG_PS | PG_V),
 5069             ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
 5070         if ((mpde = vm_page_alloc(NULL, va >> PDPSHIFT, VM_ALLOC_INTERRUPT |
 5071             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 5072                 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
 5073                     " in pmap %p", va, pmap);
 5074                 return (FALSE);
 5075         }
 5076         mpdepa = VM_PAGE_TO_PHYS(mpde);
 5077         firstpde = (pd_entry_t *)PHYS_TO_DMAP(mpdepa);
 5078         newpdpe = mpdepa | PG_M | PG_A | (oldpdpe & PG_U) | PG_RW | PG_V;
 5079         KASSERT((oldpdpe & PG_A) != 0,
 5080             ("pmap_demote_pdpe: oldpdpe is missing PG_A"));
 5081         KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW,
 5082             ("pmap_demote_pdpe: oldpdpe is missing PG_M"));
 5083         newpde = oldpdpe;
 5084 
 5085         /*
 5086          * Initialize the page directory page.
 5087          */
 5088         for (pde = firstpde; pde < firstpde + NPDEPG; pde++) {
 5089                 *pde = newpde;
 5090                 newpde += NBPDR;
 5091         }
 5092 
 5093         /*
 5094          * Demote the mapping.
 5095          */
 5096         *pdpe = newpdpe;
 5097 
 5098         /*
 5099          * Invalidate a stale recursive mapping of the page directory page.
 5100          */
 5101         pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va));
 5102 
 5103         pmap_pdpe_demotions++;
 5104         CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
 5105             " in pmap %p", va, pmap);
 5106         return (TRUE);
 5107 }
 5108 
 5109 /*
 5110  * Sets the memory attribute for the specified page.
 5111  */
 5112 void
 5113 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
 5114 {
 5115 
 5116         m->md.pat_mode = ma;
 5117 
 5118         /*
 5119          * If "m" is a normal page, update its direct mapping.  This update
 5120          * can be relied upon to perform any cache operations that are
 5121          * required for data coherence.
 5122          */
 5123         if ((m->flags & PG_FICTITIOUS) == 0 &&
 5124             pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), PAGE_SIZE,
 5125             m->md.pat_mode))
 5126                 panic("memory attribute change on the direct map failed");
 5127 }
 5128 
 5129 /*
 5130  * Changes the specified virtual address range's memory type to that given by
 5131  * the parameter "mode".  The specified virtual address range must be
 5132  * completely contained within either the direct map or the kernel map.  If
 5133  * the virtual address range is contained within the kernel map, then the
 5134  * memory type for each of the corresponding ranges of the direct map is also
 5135  * changed.  (The corresponding ranges of the direct map are those ranges that
 5136  * map the same physical pages as the specified virtual address range.)  These
 5137  * changes to the direct map are necessary because Intel describes the
 5138  * behavior of their processors as "undefined" if two or more mappings to the
 5139  * same physical page have different memory types.
 5140  *
 5141  * Returns zero if the change completed successfully, and either EINVAL or
 5142  * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
 5143  * of the virtual address range was not mapped, and ENOMEM is returned if
 5144  * there was insufficient memory available to complete the change.  In the
 5145  * latter case, the memory type may have been changed on some part of the
 5146  * virtual address range or the direct map.
 5147  */
 5148 int
 5149 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
 5150 {
 5151         int error;
 5152 
 5153         PMAP_LOCK(kernel_pmap);
 5154         error = pmap_change_attr_locked(va, size, mode);
 5155         PMAP_UNLOCK(kernel_pmap);
 5156         return (error);
 5157 }
 5158 
 5159 static int
 5160 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode)
 5161 {
 5162         vm_offset_t base, offset, tmpva;
 5163         vm_paddr_t pa_start, pa_end;
 5164         pdp_entry_t *pdpe;
 5165         pd_entry_t *pde;
 5166         pt_entry_t *pte;
 5167         int cache_bits_pte, cache_bits_pde, error;
 5168         boolean_t changed;
 5169 
 5170         PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
 5171         base = trunc_page(va);
 5172         offset = va & PAGE_MASK;
 5173         size = roundup(offset + size, PAGE_SIZE);
 5174 
 5175         /*
 5176          * Only supported on kernel virtual addresses, including the direct
 5177          * map but excluding the recursive map.
 5178          */
 5179         if (base < DMAP_MIN_ADDRESS)
 5180                 return (EINVAL);
 5181 
 5182         cache_bits_pde = pmap_cache_bits(mode, 1);
 5183         cache_bits_pte = pmap_cache_bits(mode, 0);
 5184         changed = FALSE;
 5185 
 5186         /*
 5187          * Pages that aren't mapped aren't supported.  Also break down 2MB pages
 5188          * into 4KB pages if required.
 5189          */
 5190         for (tmpva = base; tmpva < base + size; ) {
 5191                 pdpe = pmap_pdpe(kernel_pmap, tmpva);
 5192                 if (*pdpe == 0)
 5193                         return (EINVAL);
 5194                 if (*pdpe & PG_PS) {
 5195                         /*
 5196                          * If the current 1GB page already has the required
 5197                          * memory type, then we need not demote this page. Just
 5198                          * increment tmpva to the next 1GB page frame.
 5199                          */
 5200                         if ((*pdpe & PG_PDE_CACHE) == cache_bits_pde) {
 5201                                 tmpva = trunc_1gpage(tmpva) + NBPDP;
 5202                                 continue;
 5203                         }
 5204 
 5205                         /*
 5206                          * If the current offset aligns with a 1GB page frame
 5207                          * and there is at least 1GB left within the range, then
 5208                          * we need not break down this page into 2MB pages.
 5209                          */
 5210                         if ((tmpva & PDPMASK) == 0 &&
 5211                             tmpva + PDPMASK < base + size) {
 5212                                 tmpva += NBPDP;
 5213                                 continue;
 5214                         }
 5215                         if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva))
 5216                                 return (ENOMEM);
 5217                 }
 5218                 pde = pmap_pdpe_to_pde(pdpe, tmpva);
 5219                 if (*pde == 0)
 5220                         return (EINVAL);
 5221                 if (*pde & PG_PS) {
 5222                         /*
 5223                          * If the current 2MB page already has the required
 5224                          * memory type, then we need not demote this page. Just
 5225                          * increment tmpva to the next 2MB page frame.
 5226                          */
 5227                         if ((*pde & PG_PDE_CACHE) == cache_bits_pde) {
 5228                                 tmpva = trunc_2mpage(tmpva) + NBPDR;
 5229                                 continue;
 5230                         }
 5231 
 5232                         /*
 5233                          * If the current offset aligns with a 2MB page frame
 5234                          * and there is at least 2MB left within the range, then
 5235                          * we need not break down this page into 4KB pages.
 5236                          */
 5237                         if ((tmpva & PDRMASK) == 0 &&
 5238                             tmpva + PDRMASK < base + size) {
 5239                                 tmpva += NBPDR;
 5240                                 continue;
 5241                         }
 5242                         if (!pmap_demote_pde(kernel_pmap, pde, tmpva))
 5243                                 return (ENOMEM);
 5244                 }
 5245                 pte = pmap_pde_to_pte(pde, tmpva);
 5246                 if (*pte == 0)
 5247                         return (EINVAL);
 5248                 tmpva += PAGE_SIZE;
 5249         }
 5250         error = 0;
 5251 
 5252         /*
 5253          * Ok, all the pages exist, so run through them updating their
 5254          * cache mode if required.
 5255          */
 5256         pa_start = pa_end = 0;
 5257         for (tmpva = base; tmpva < base + size; ) {
 5258                 pdpe = pmap_pdpe(kernel_pmap, tmpva);
 5259                 if (*pdpe & PG_PS) {
 5260                         if ((*pdpe & PG_PDE_CACHE) != cache_bits_pde) {
 5261                                 pmap_pde_attr(pdpe, cache_bits_pde);
 5262                                 changed = TRUE;
 5263                         }
 5264                         if (tmpva >= VM_MIN_KERNEL_ADDRESS) {
 5265                                 if (pa_start == pa_end) {
 5266                                         /* Start physical address run. */
 5267                                         pa_start = *pdpe & PG_PS_FRAME;
 5268                                         pa_end = pa_start + NBPDP;
 5269                                 } else if (pa_end == (*pdpe & PG_PS_FRAME))
 5270                                         pa_end += NBPDP;
 5271                                 else {
 5272                                         /* Run ended, update direct map. */
 5273                                         error = pmap_change_attr_locked(
 5274                                             PHYS_TO_DMAP(pa_start),
 5275                                             pa_end - pa_start, mode);
 5276                                         if (error != 0)
 5277                                                 break;
 5278                                         /* Start physical address run. */
 5279                                         pa_start = *pdpe & PG_PS_FRAME;
 5280                                         pa_end = pa_start + NBPDP;
 5281                                 }
 5282                         }
 5283                         tmpva = trunc_1gpage(tmpva) + NBPDP;
 5284                         continue;
 5285                 }
 5286                 pde = pmap_pdpe_to_pde(pdpe, tmpva);
 5287                 if (*pde & PG_PS) {
 5288                         if ((*pde & PG_PDE_CACHE) != cache_bits_pde) {
 5289                                 pmap_pde_attr(pde, cache_bits_pde);
 5290                                 changed = TRUE;
 5291                         }
 5292                         if (tmpva >= VM_MIN_KERNEL_ADDRESS) {
 5293                                 if (pa_start == pa_end) {
 5294                                         /* Start physical address run. */
 5295                                         pa_start = *pde & PG_PS_FRAME;
 5296                                         pa_end = pa_start + NBPDR;
 5297                                 } else if (pa_end == (*pde & PG_PS_FRAME))
 5298                                         pa_end += NBPDR;
 5299                                 else {
 5300                                         /* Run ended, update direct map. */
 5301                                         error = pmap_change_attr_locked(
 5302                                             PHYS_TO_DMAP(pa_start),
 5303                                             pa_end - pa_start, mode);
 5304                                         if (error != 0)
 5305                                                 break;
 5306                                         /* Start physical address run. */
 5307                                         pa_start = *pde & PG_PS_FRAME;
 5308                                         pa_end = pa_start + NBPDR;
 5309                                 }
 5310                         }
 5311                         tmpva = trunc_2mpage(tmpva) + NBPDR;
 5312                 } else {
 5313                         pte = pmap_pde_to_pte(pde, tmpva);
 5314                         if ((*pte & PG_PTE_CACHE) != cache_bits_pte) {
 5315                                 pmap_pte_attr(pte, cache_bits_pte);
 5316                                 changed = TRUE;
 5317                         }
 5318                         if (tmpva >= VM_MIN_KERNEL_ADDRESS) {
 5319                                 if (pa_start == pa_end) {
 5320                                         /* Start physical address run. */
 5321                                         pa_start = *pte & PG_FRAME;
 5322                                         pa_end = pa_start + PAGE_SIZE;
 5323                                 } else if (pa_end == (*pte & PG_FRAME))
 5324                                         pa_end += PAGE_SIZE;
 5325                                 else {
 5326                                         /* Run ended, update direct map. */
 5327                                         error = pmap_change_attr_locked(
 5328                                             PHYS_TO_DMAP(pa_start),
 5329                                             pa_end - pa_start, mode);
 5330                                         if (error != 0)
 5331                                                 break;
 5332                                         /* Start physical address run. */
 5333                                         pa_start = *pte & PG_FRAME;
 5334                                         pa_end = pa_start + PAGE_SIZE;
 5335                                 }
 5336                         }
 5337                         tmpva += PAGE_SIZE;
 5338                 }
 5339         }
 5340         if (error == 0 && pa_start != pa_end)
 5341                 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start),
 5342                     pa_end - pa_start, mode);
 5343 
 5344         /*
 5345          * Flush CPU caches if required to make sure any data isn't cached that
 5346          * shouldn't be, etc.
 5347          */
 5348         if (changed) {
 5349                 pmap_invalidate_range(kernel_pmap, base, tmpva);
 5350                 pmap_invalidate_cache_range(base, tmpva);
 5351         }
 5352         return (error);
 5353 }
 5354 
 5355 /*
 5356  * Demotes any mapping within the direct map region that covers more than the
 5357  * specified range of physical addresses.  This range's size must be a power
 5358  * of two and its starting address must be a multiple of its size.  Since the
 5359  * demotion does not change any attributes of the mapping, a TLB invalidation
 5360  * is not mandatory.  The caller may, however, request a TLB invalidation.
 5361  */
 5362 void
 5363 pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, boolean_t invalidate)
 5364 {
 5365         pdp_entry_t *pdpe;
 5366         pd_entry_t *pde;
 5367         vm_offset_t va;
 5368         boolean_t changed;
 5369 
 5370         if (len == 0)
 5371                 return;
 5372         KASSERT(powerof2(len), ("pmap_demote_DMAP: len is not a power of 2"));
 5373         KASSERT((base & (len - 1)) == 0,
 5374             ("pmap_demote_DMAP: base is not a multiple of len"));
 5375         if (len < NBPDP && base < dmaplimit) {
 5376                 va = PHYS_TO_DMAP(base);
 5377                 changed = FALSE;
 5378                 PMAP_LOCK(kernel_pmap);
 5379                 pdpe = pmap_pdpe(kernel_pmap, va);
 5380                 if ((*pdpe & PG_V) == 0)
 5381                         panic("pmap_demote_DMAP: invalid PDPE");
 5382                 if ((*pdpe & PG_PS) != 0) {
 5383                         if (!pmap_demote_pdpe(kernel_pmap, pdpe, va))
 5384                                 panic("pmap_demote_DMAP: PDPE failed");
 5385                         changed = TRUE;
 5386                 }
 5387                 if (len < NBPDR) {
 5388                         pde = pmap_pdpe_to_pde(pdpe, va);
 5389                         if ((*pde & PG_V) == 0)
 5390                                 panic("pmap_demote_DMAP: invalid PDE");
 5391                         if ((*pde & PG_PS) != 0) {
 5392                                 if (!pmap_demote_pde(kernel_pmap, pde, va))
 5393                                         panic("pmap_demote_DMAP: PDE failed");
 5394                                 changed = TRUE;
 5395                         }
 5396                 }
 5397                 if (changed && invalidate)
 5398                         pmap_invalidate_page(kernel_pmap, va);
 5399                 PMAP_UNLOCK(kernel_pmap);
 5400         }
 5401 }
 5402 
 5403 /*
 5404  * perform the pmap work for mincore
 5405  */
 5406 int
 5407 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
 5408 {
 5409         pd_entry_t *pdep;
 5410         pt_entry_t pte;
 5411         vm_paddr_t pa;
 5412         int val;
 5413 
 5414         PMAP_LOCK(pmap);
 5415 retry:
 5416         pdep = pmap_pde(pmap, addr);
 5417         if (pdep != NULL && (*pdep & PG_V)) {
 5418                 if (*pdep & PG_PS) {
 5419                         pte = *pdep;
 5420                         /* Compute the physical address of the 4KB page. */
 5421                         pa = ((*pdep & PG_PS_FRAME) | (addr & PDRMASK)) &
 5422                             PG_FRAME;
 5423                         val = MINCORE_SUPER;
 5424                 } else {
 5425                         pte = *pmap_pde_to_pte(pdep, addr);
 5426                         pa = pte & PG_FRAME;
 5427                         val = 0;
 5428                 }
 5429         } else {
 5430                 pte = 0;
 5431                 pa = 0;
 5432                 val = 0;
 5433         }
 5434         if ((pte & PG_V) != 0) {
 5435                 val |= MINCORE_INCORE;
 5436                 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 5437                         val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
 5438                 if ((pte & PG_A) != 0)
 5439                         val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
 5440         }
 5441         if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
 5442             (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
 5443             (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
 5444                 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */
 5445                 if (vm_page_pa_tryrelock(pmap, pa, locked_pa))
 5446                         goto retry;
 5447         } else
 5448                 PA_UNLOCK_COND(*locked_pa);
 5449         PMAP_UNLOCK(pmap);
 5450         return (val);
 5451 }
 5452 
 5453 void
 5454 pmap_activate(struct thread *td)
 5455 {
 5456         pmap_t  pmap, oldpmap;
 5457         u_int   cpuid;
 5458         u_int64_t  cr3;
 5459 
 5460         critical_enter();
 5461         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 5462         oldpmap = PCPU_GET(curpmap);
 5463         cpuid = PCPU_GET(cpuid);
 5464 #ifdef SMP
 5465         CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
 5466         CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
 5467 #else
 5468         CPU_CLR(cpuid, &oldpmap->pm_active);
 5469         CPU_SET(cpuid, &pmap->pm_active);
 5470 #endif
 5471         cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4);
 5472         td->td_pcb->pcb_cr3 = cr3;
 5473         load_cr3(cr3);
 5474         PCPU_SET(curpmap, pmap);
 5475         critical_exit();
 5476 }
 5477 
 5478 void
 5479 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
 5480 {
 5481 }
 5482 
 5483 /*
 5484  *      Increase the starting virtual address of the given mapping if a
 5485  *      different alignment might result in more superpage mappings.
 5486  */
 5487 void
 5488 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
 5489     vm_offset_t *addr, vm_size_t size)
 5490 {
 5491         vm_offset_t superpage_offset;
 5492 
 5493         if (size < NBPDR)
 5494                 return;
 5495         if (object != NULL && (object->flags & OBJ_COLORED) != 0)
 5496                 offset += ptoa(object->pg_color);
 5497         superpage_offset = offset & PDRMASK;
 5498         if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR ||
 5499             (*addr & PDRMASK) == superpage_offset)
 5500                 return;
 5501         if ((*addr & PDRMASK) < superpage_offset)
 5502                 *addr = (*addr & ~PDRMASK) + superpage_offset;
 5503         else
 5504                 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
 5505 }
 5506 
 5507 #include "opt_ddb.h"
 5508 #ifdef DDB
 5509 #include <ddb/ddb.h>
 5510 
 5511 DB_SHOW_COMMAND(pte, pmap_print_pte)
 5512 {
 5513         pmap_t pmap;
 5514         pml4_entry_t *pml4;
 5515         pdp_entry_t *pdp;
 5516         pd_entry_t *pde;
 5517         pt_entry_t *pte;
 5518         vm_offset_t va;
 5519 
 5520         if (have_addr) {
 5521                 va = (vm_offset_t)addr;
 5522                 pmap = PCPU_GET(curpmap); /* XXX */
 5523         } else {
 5524                 db_printf("show pte addr\n");
 5525                 return;
 5526         }
 5527         pml4 = pmap_pml4e(pmap, va);
 5528         db_printf("VA %#016lx pml4e %#016lx", va, *pml4);
 5529         if ((*pml4 & PG_V) == 0) {
 5530                 db_printf("\n");
 5531                 return;
 5532         }
 5533         pdp = pmap_pml4e_to_pdpe(pml4, va);
 5534         db_printf(" pdpe %#016lx", *pdp);
 5535         if ((*pdp & PG_V) == 0 || (*pdp & PG_PS) != 0) {
 5536                 db_printf("\n");
 5537                 return;
 5538         }
 5539         pde = pmap_pdpe_to_pde(pdp, va);
 5540         db_printf(" pde %#016lx", *pde);
 5541         if ((*pde & PG_V) == 0 || (*pde & PG_PS) != 0) {
 5542                 db_printf("\n");
 5543                 return;
 5544         }
 5545         pte = pmap_pde_to_pte(pde, va);
 5546         db_printf(" pte %#016lx\n", *pte);
 5547 }
 5548 
 5549 DB_SHOW_COMMAND(phys2dmap, pmap_phys2dmap)
 5550 {
 5551         vm_paddr_t a;
 5552 
 5553         if (have_addr) {
 5554                 a = (vm_paddr_t)addr;
 5555                 db_printf("0x%jx\n", (uintmax_t)PHYS_TO_DMAP(a));
 5556         } else {
 5557                 db_printf("show phys2dmap addr\n");
 5558         }
 5559 }
 5560 #endif

Cache object: 8934f74de3ccdc7edad29a491878b94c


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.