The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/pmap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2003 Peter Wemm
    9  * All rights reserved.
   10  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
   11  * All rights reserved.
   12  *
   13  * This code is derived from software contributed to Berkeley by
   14  * the Systems Programming Group of the University of Utah Computer
   15  * Science Department and William Jolitz of UUNET Technologies Inc.
   16  *
   17  * Redistribution and use in source and binary forms, with or without
   18  * modification, are permitted provided that the following conditions
   19  * are met:
   20  * 1. Redistributions of source code must retain the above copyright
   21  *    notice, this list of conditions and the following disclaimer.
   22  * 2. Redistributions in binary form must reproduce the above copyright
   23  *    notice, this list of conditions and the following disclaimer in the
   24  *    documentation and/or other materials provided with the distribution.
   25  * 3. All advertising materials mentioning features or use of this software
   26  *    must display the following acknowledgement:
   27  *      This product includes software developed by the University of
   28  *      California, Berkeley and its contributors.
   29  * 4. Neither the name of the University nor the names of its contributors
   30  *    may be used to endorse or promote products derived from this software
   31  *    without specific prior written permission.
   32  *
   33  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   34  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   35  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   36  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   37  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   38  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   39  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   40  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   41  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   42  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   43  * SUCH DAMAGE.
   44  *
   45  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   46  */
   47 /*-
   48  * Copyright (c) 2003 Networks Associates Technology, Inc.
   49  * All rights reserved.
   50  *
   51  * This software was developed for the FreeBSD Project by Jake Burkholder,
   52  * Safeport Network Services, and Network Associates Laboratories, the
   53  * Security Research Division of Network Associates, Inc. under
   54  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   55  * CHATS research program.
   56  *
   57  * Redistribution and use in source and binary forms, with or without
   58  * modification, are permitted provided that the following conditions
   59  * are met:
   60  * 1. Redistributions of source code must retain the above copyright
   61  *    notice, this list of conditions and the following disclaimer.
   62  * 2. Redistributions in binary form must reproduce the above copyright
   63  *    notice, this list of conditions and the following disclaimer in the
   64  *    documentation and/or other materials provided with the distribution.
   65  *
   66  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   67  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   68  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   69  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   70  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   71  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   72  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   73  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   74  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   75  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   76  * SUCH DAMAGE.
   77  */
   78 
   79 #include <sys/cdefs.h>
   80 __FBSDID("$FreeBSD: releng/8.3/sys/amd64/amd64/pmap.c 228262 2011-12-04 18:55:19Z alc $");
   81 
   82 /*
   83  *      Manages physical address maps.
   84  *
   85  *      In addition to hardware address maps, this
   86  *      module is called upon to provide software-use-only
   87  *      maps which may or may not be stored in the same
   88  *      form as hardware maps.  These pseudo-maps are
   89  *      used to store intermediate results from copy
   90  *      operations to and from address spaces.
   91  *
   92  *      Since the information managed by this module is
   93  *      also stored by the logical address mapping module,
   94  *      this module may throw away valid virtual-to-physical
   95  *      mappings at almost any time.  However, invalidations
   96  *      of virtual-to-physical mappings must be done as
   97  *      requested.
   98  *
   99  *      In order to cope with hardware architectures which
  100  *      make virtual-to-physical map invalidates expensive,
  101  *      this module may delay invalidate or reduced protection
  102  *      operations until such time as they are actually
  103  *      necessary.  This module is given full information as
  104  *      to which processors are currently using which maps,
  105  *      and to when physical maps must be made correct.
  106  */
  107 
  108 #include "opt_pmap.h"
  109 #include "opt_vm.h"
  110 
  111 #include <sys/param.h>
  112 #include <sys/systm.h>
  113 #include <sys/kernel.h>
  114 #include <sys/ktr.h>
  115 #include <sys/lock.h>
  116 #include <sys/malloc.h>
  117 #include <sys/mman.h>
  118 #include <sys/mutex.h>
  119 #include <sys/proc.h>
  120 #include <sys/sx.h>
  121 #include <sys/vmmeter.h>
  122 #include <sys/sched.h>
  123 #include <sys/sysctl.h>
  124 #ifdef SMP
  125 #include <sys/smp.h>
  126 #endif
  127 
  128 #include <vm/vm.h>
  129 #include <vm/vm_param.h>
  130 #include <vm/vm_kern.h>
  131 #include <vm/vm_page.h>
  132 #include <vm/vm_map.h>
  133 #include <vm/vm_object.h>
  134 #include <vm/vm_extern.h>
  135 #include <vm/vm_pageout.h>
  136 #include <vm/vm_pager.h>
  137 #include <vm/vm_reserv.h>
  138 #include <vm/uma.h>
  139 
  140 #include <machine/cpu.h>
  141 #include <machine/cputypes.h>
  142 #include <machine/md_var.h>
  143 #include <machine/pcb.h>
  144 #include <machine/specialreg.h>
  145 #ifdef SMP
  146 #include <machine/smp.h>
  147 #endif
  148 
  149 #ifndef PMAP_SHPGPERPROC
  150 #define PMAP_SHPGPERPROC 200
  151 #endif
  152 
  153 #if !defined(DIAGNOSTIC)
  154 #define PMAP_INLINE     __gnu89_inline
  155 #else
  156 #define PMAP_INLINE
  157 #endif
  158 
  159 #define PV_STATS
  160 #ifdef PV_STATS
  161 #define PV_STAT(x)      do { x ; } while (0)
  162 #else
  163 #define PV_STAT(x)      do { } while (0)
  164 #endif
  165 
  166 #define pa_index(pa)    ((pa) >> PDRSHIFT)
  167 #define pa_to_pvh(pa)   (&pv_table[pa_index(pa)])
  168 
  169 struct pmap kernel_pmap_store;
  170 
  171 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  172 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  173 
  174 static int ndmpdp;
  175 static vm_paddr_t dmaplimit;
  176 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
  177 pt_entry_t pg_nx;
  178 
  179 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  180 
  181 static int pat_works = 1;
  182 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1,
  183     "Is page attribute table fully functional?");
  184 
  185 static int pg_ps_enabled = 1;
  186 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN, &pg_ps_enabled, 0,
  187     "Are large page mappings enabled?");
  188 
  189 #define PAT_INDEX_SIZE  8
  190 static int pat_index[PAT_INDEX_SIZE];   /* cache mode to PAT index conversion */
  191 
  192 static u_int64_t        KPTphys;        /* phys addr of kernel level 1 */
  193 static u_int64_t        KPDphys;        /* phys addr of kernel level 2 */
  194 u_int64_t               KPDPphys;       /* phys addr of kernel level 3 */
  195 u_int64_t               KPML4phys;      /* phys addr of kernel level 4 */
  196 
  197 static u_int64_t        DMPDphys;       /* phys addr of direct mapped level 2 */
  198 static u_int64_t        DMPDPphys;      /* phys addr of direct mapped level 3 */
  199 
  200 /*
  201  * Data for the pv entry allocation mechanism
  202  */
  203 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  204 static struct md_page *pv_table;
  205 static int shpgperproc = PMAP_SHPGPERPROC;
  206 
  207 /*
  208  * All those kernel PT submaps that BSD is so fond of
  209  */
  210 pt_entry_t *CMAP1 = 0;
  211 caddr_t CADDR1 = 0;
  212 
  213 /*
  214  * Crashdump maps.
  215  */
  216 static caddr_t crashdumpmap;
  217 
  218 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  219 static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try);
  220 static void     pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  221 static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  222 static void     pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  223 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  224 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  225                     vm_offset_t va);
  226 static int      pmap_pvh_wired_mappings(struct md_page *pvh, int count);
  227 
  228 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode);
  229 static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  230 static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe,
  231     vm_offset_t va);
  232 static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
  233     vm_prot_t prot);
  234 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  235     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  236 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
  237 static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
  238 static void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
  239 static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
  240 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
  241 static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
  242 static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
  243 static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  244 static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
  245     vm_prot_t prot);
  246 static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
  247 static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
  248                 vm_page_t *free);
  249 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
  250                 vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free);
  251 static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
  252 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  253     vm_page_t *free);
  254 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  255                 vm_offset_t va);
  256 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  257 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  258     vm_page_t m);
  259 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  260     pd_entry_t newpde);
  261 static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
  262 
  263 static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags);
  264 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
  265 
  266 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
  267 static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
  268                 vm_page_t* free);
  269 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
  270 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
  271 
  272 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  273 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  274 
  275 /*
  276  * Move the kernel virtual free pointer to the next
  277  * 2MB.  This is used to help improve performance
  278  * by using a large (2MB) page for much of the kernel
  279  * (.text, .data, .bss)
  280  */
  281 static vm_offset_t
  282 pmap_kmem_choose(vm_offset_t addr)
  283 {
  284         vm_offset_t newaddr = addr;
  285 
  286         newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
  287         return newaddr;
  288 }
  289 
  290 /********************/
  291 /* Inline functions */
  292 /********************/
  293 
  294 /* Return a non-clipped PD index for a given VA */
  295 static __inline vm_pindex_t
  296 pmap_pde_pindex(vm_offset_t va)
  297 {
  298         return va >> PDRSHIFT;
  299 }
  300 
  301 
  302 /* Return various clipped indexes for a given VA */
  303 static __inline vm_pindex_t
  304 pmap_pte_index(vm_offset_t va)
  305 {
  306 
  307         return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
  308 }
  309 
  310 static __inline vm_pindex_t
  311 pmap_pde_index(vm_offset_t va)
  312 {
  313 
  314         return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
  315 }
  316 
  317 static __inline vm_pindex_t
  318 pmap_pdpe_index(vm_offset_t va)
  319 {
  320 
  321         return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
  322 }
  323 
  324 static __inline vm_pindex_t
  325 pmap_pml4e_index(vm_offset_t va)
  326 {
  327 
  328         return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
  329 }
  330 
  331 /* Return a pointer to the PML4 slot that corresponds to a VA */
  332 static __inline pml4_entry_t *
  333 pmap_pml4e(pmap_t pmap, vm_offset_t va)
  334 {
  335 
  336         return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
  337 }
  338 
  339 /* Return a pointer to the PDP slot that corresponds to a VA */
  340 static __inline pdp_entry_t *
  341 pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
  342 {
  343         pdp_entry_t *pdpe;
  344 
  345         pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
  346         return (&pdpe[pmap_pdpe_index(va)]);
  347 }
  348 
  349 /* Return a pointer to the PDP slot that corresponds to a VA */
  350 static __inline pdp_entry_t *
  351 pmap_pdpe(pmap_t pmap, vm_offset_t va)
  352 {
  353         pml4_entry_t *pml4e;
  354 
  355         pml4e = pmap_pml4e(pmap, va);
  356         if ((*pml4e & PG_V) == 0)
  357                 return NULL;
  358         return (pmap_pml4e_to_pdpe(pml4e, va));
  359 }
  360 
  361 /* Return a pointer to the PD slot that corresponds to a VA */
  362 static __inline pd_entry_t *
  363 pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
  364 {
  365         pd_entry_t *pde;
  366 
  367         pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
  368         return (&pde[pmap_pde_index(va)]);
  369 }
  370 
  371 /* Return a pointer to the PD slot that corresponds to a VA */
  372 static __inline pd_entry_t *
  373 pmap_pde(pmap_t pmap, vm_offset_t va)
  374 {
  375         pdp_entry_t *pdpe;
  376 
  377         pdpe = pmap_pdpe(pmap, va);
  378         if (pdpe == NULL || (*pdpe & PG_V) == 0)
  379                  return NULL;
  380         return (pmap_pdpe_to_pde(pdpe, va));
  381 }
  382 
  383 /* Return a pointer to the PT slot that corresponds to a VA */
  384 static __inline pt_entry_t *
  385 pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
  386 {
  387         pt_entry_t *pte;
  388 
  389         pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
  390         return (&pte[pmap_pte_index(va)]);
  391 }
  392 
  393 /* Return a pointer to the PT slot that corresponds to a VA */
  394 static __inline pt_entry_t *
  395 pmap_pte(pmap_t pmap, vm_offset_t va)
  396 {
  397         pd_entry_t *pde;
  398 
  399         pde = pmap_pde(pmap, va);
  400         if (pde == NULL || (*pde & PG_V) == 0)
  401                 return NULL;
  402         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  403                 return ((pt_entry_t *)pde);
  404         return (pmap_pde_to_pte(pde, va));
  405 }
  406 
  407 
  408 PMAP_INLINE pt_entry_t *
  409 vtopte(vm_offset_t va)
  410 {
  411         u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  412 
  413         return (PTmap + ((va >> PAGE_SHIFT) & mask));
  414 }
  415 
  416 static __inline pd_entry_t *
  417 vtopde(vm_offset_t va)
  418 {
  419         u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  420 
  421         return (PDmap + ((va >> PDRSHIFT) & mask));
  422 }
  423 
  424 static u_int64_t
  425 allocpages(vm_paddr_t *firstaddr, int n)
  426 {
  427         u_int64_t ret;
  428 
  429         ret = *firstaddr;
  430         bzero((void *)ret, n * PAGE_SIZE);
  431         *firstaddr += n * PAGE_SIZE;
  432         return (ret);
  433 }
  434 
  435 static void
  436 create_pagetables(vm_paddr_t *firstaddr)
  437 {
  438         int i, j, ndm1g;
  439 
  440         /* Allocate pages */
  441         KPTphys = allocpages(firstaddr, NKPT);
  442         KPML4phys = allocpages(firstaddr, 1);
  443         KPDPphys = allocpages(firstaddr, NKPML4E);
  444         KPDphys = allocpages(firstaddr, NKPDPE);
  445 
  446         ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
  447         if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
  448                 ndmpdp = 4;
  449         DMPDPphys = allocpages(firstaddr, NDMPML4E);
  450         ndm1g = 0;
  451         if ((amd_feature & AMDID_PAGE1GB) != 0)
  452                 ndm1g = ptoa(Maxmem) >> PDPSHIFT;
  453         if (ndm1g < ndmpdp)
  454                 DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g);
  455         dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
  456 
  457         /* Fill in the underlying page table pages */
  458         /* Read-only from zero to physfree */
  459         /* XXX not fully used, underneath 2M pages */
  460         for (i = 0; (i << PAGE_SHIFT) < *firstaddr; i++) {
  461                 ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
  462                 ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V | PG_G;
  463         }
  464 
  465         /* Now map the page tables at their location within PTmap */
  466         for (i = 0; i < NKPT; i++) {
  467                 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
  468                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
  469         }
  470 
  471         /* Map from zero to end of allocations under 2M pages */
  472         /* This replaces some of the KPTphys entries above */
  473         for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
  474                 ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
  475                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  476         }
  477 
  478         /* And connect up the PD to the PDP */
  479         for (i = 0; i < NKPDPE; i++) {
  480                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys +
  481                     (i << PAGE_SHIFT);
  482                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
  483         }
  484 
  485         /*
  486          * Now, set up the direct map region using 2MB and/or 1GB pages.  If
  487          * the end of physical memory is not aligned to a 1GB page boundary,
  488          * then the residual physical memory is mapped with 2MB pages.  Later,
  489          * if pmap_mapdev{_attr}() uses the direct map for non-write-back
  490          * memory, pmap_change_attr() will demote any 2MB or 1GB page mappings
  491          * that are partially used. 
  492          */
  493         for (i = NPDEPG * ndm1g, j = 0; i < NPDEPG * ndmpdp; i++, j++) {
  494                 ((pd_entry_t *)DMPDphys)[j] = (vm_paddr_t)i << PDRSHIFT;
  495                 /* Preset PG_M and PG_A because demotion expects it. */
  496                 ((pd_entry_t *)DMPDphys)[j] |= PG_RW | PG_V | PG_PS | PG_G |
  497                     PG_M | PG_A;
  498         }
  499         for (i = 0; i < ndm1g; i++) {
  500                 ((pdp_entry_t *)DMPDPphys)[i] = (vm_paddr_t)i << PDPSHIFT;
  501                 /* Preset PG_M and PG_A because demotion expects it. */
  502                 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_PS | PG_G |
  503                     PG_M | PG_A;
  504         }
  505         for (j = 0; i < ndmpdp; i++, j++) {
  506                 ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + (j << PAGE_SHIFT);
  507                 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
  508         }
  509 
  510         /* And recursively map PML4 to itself in order to get PTmap */
  511         ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
  512         ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
  513 
  514         /* Connect the Direct Map slot up to the PML4 */
  515         ((pdp_entry_t *)KPML4phys)[DMPML4I] = DMPDPphys;
  516         ((pdp_entry_t *)KPML4phys)[DMPML4I] |= PG_RW | PG_V | PG_U;
  517 
  518         /* Connect the KVA slot up to the PML4 */
  519         ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
  520         ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
  521 }
  522 
  523 /*
  524  *      Bootstrap the system enough to run with virtual memory.
  525  *
  526  *      On amd64 this is called after mapping has already been enabled
  527  *      and just syncs the pmap module with what has already been done.
  528  *      [We can't call it easily with mapping off since the kernel is not
  529  *      mapped with PA == VA, hence we would have to relocate every address
  530  *      from the linked base (virtual) address "KERNBASE" to the actual
  531  *      (physical) address starting relative to 0]
  532  */
  533 void
  534 pmap_bootstrap(vm_paddr_t *firstaddr)
  535 {
  536         vm_offset_t va;
  537         pt_entry_t *pte, *unused;
  538 
  539         /*
  540          * Create an initial set of page tables to run the kernel in.
  541          */
  542         create_pagetables(firstaddr);
  543 
  544         virtual_avail = (vm_offset_t) KERNBASE + *firstaddr;
  545         virtual_avail = pmap_kmem_choose(virtual_avail);
  546 
  547         virtual_end = VM_MAX_KERNEL_ADDRESS;
  548 
  549 
  550         /* XXX do %cr0 as well */
  551         load_cr4(rcr4() | CR4_PGE | CR4_PSE);
  552         load_cr3(KPML4phys);
  553 
  554         /*
  555          * Initialize the kernel pmap (which is statically allocated).
  556          */
  557         PMAP_LOCK_INIT(kernel_pmap);
  558         kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys);
  559         kernel_pmap->pm_root = NULL;
  560         kernel_pmap->pm_active = -1;    /* don't allow deactivation */
  561         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  562 
  563         /*
  564          * Reserve some special page table entries/VA space for temporary
  565          * mapping of pages.
  566          */
  567 #define SYSMAP(c, p, v, n)      \
  568         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  569 
  570         va = virtual_avail;
  571         pte = vtopte(va);
  572 
  573         /*
  574          * CMAP1 is only used for the memory test.
  575          */
  576         SYSMAP(caddr_t, CMAP1, CADDR1, 1)
  577 
  578         /*
  579          * Crashdump maps.
  580          */
  581         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  582 
  583         virtual_avail = va;
  584 
  585         *CMAP1 = 0;
  586 
  587         invltlb();
  588 
  589         /* Initialize the PAT MSR. */
  590         pmap_init_pat();
  591 }
  592 
  593 /*
  594  * Setup the PAT MSR.
  595  */
  596 void
  597 pmap_init_pat(void)
  598 {
  599         int pat_table[PAT_INDEX_SIZE];
  600         uint64_t pat_msr;
  601         u_long cr0, cr4;
  602         int i;
  603 
  604         /* Bail if this CPU doesn't implement PAT. */
  605         if ((cpu_feature & CPUID_PAT) == 0)
  606                 panic("no PAT??");
  607 
  608         /* Set default PAT index table. */
  609         for (i = 0; i < PAT_INDEX_SIZE; i++)
  610                 pat_table[i] = -1;
  611         pat_table[PAT_WRITE_BACK] = 0;
  612         pat_table[PAT_WRITE_THROUGH] = 1;
  613         pat_table[PAT_UNCACHEABLE] = 3;
  614         pat_table[PAT_WRITE_COMBINING] = 3;
  615         pat_table[PAT_WRITE_PROTECTED] = 3;
  616         pat_table[PAT_UNCACHED] = 3;
  617 
  618         /* Initialize default PAT entries. */
  619         pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |
  620             PAT_VALUE(1, PAT_WRITE_THROUGH) |
  621             PAT_VALUE(2, PAT_UNCACHED) |
  622             PAT_VALUE(3, PAT_UNCACHEABLE) |
  623             PAT_VALUE(4, PAT_WRITE_BACK) |
  624             PAT_VALUE(5, PAT_WRITE_THROUGH) |
  625             PAT_VALUE(6, PAT_UNCACHED) |
  626             PAT_VALUE(7, PAT_UNCACHEABLE);
  627 
  628         if (pat_works) {
  629                 /*
  630                  * Leave the indices 0-3 at the default of WB, WT, UC-, and UC.
  631                  * Program 5 and 6 as WP and WC.
  632                  * Leave 4 and 7 as WB and UC.
  633                  */
  634                 pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6));
  635                 pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) |
  636                     PAT_VALUE(6, PAT_WRITE_COMBINING);
  637                 pat_table[PAT_UNCACHED] = 2;
  638                 pat_table[PAT_WRITE_PROTECTED] = 5;
  639                 pat_table[PAT_WRITE_COMBINING] = 6;
  640         } else {
  641                 /*
  642                  * Just replace PAT Index 2 with WC instead of UC-.
  643                  */
  644                 pat_msr &= ~PAT_MASK(2);
  645                 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  646                 pat_table[PAT_WRITE_COMBINING] = 2;
  647         }
  648 
  649         /* Disable PGE. */
  650         cr4 = rcr4();
  651         load_cr4(cr4 & ~CR4_PGE);
  652 
  653         /* Disable caches (CD = 1, NW = 0). */
  654         cr0 = rcr0();
  655         load_cr0((cr0 & ~CR0_NW) | CR0_CD);
  656 
  657         /* Flushes caches and TLBs. */
  658         wbinvd();
  659         invltlb();
  660 
  661         /* Update PAT and index table. */
  662         wrmsr(MSR_PAT, pat_msr);
  663         for (i = 0; i < PAT_INDEX_SIZE; i++)
  664                 pat_index[i] = pat_table[i];
  665 
  666         /* Flush caches and TLBs again. */
  667         wbinvd();
  668         invltlb();
  669 
  670         /* Restore caches and PGE. */
  671         load_cr0(cr0);
  672         load_cr4(cr4);
  673 }
  674 
  675 /*
  676  *      Initialize a vm_page's machine-dependent fields.
  677  */
  678 void
  679 pmap_page_init(vm_page_t m)
  680 {
  681 
  682         TAILQ_INIT(&m->md.pv_list);
  683         m->md.pat_mode = PAT_WRITE_BACK;
  684 }
  685 
  686 /*
  687  *      Initialize the pmap module.
  688  *      Called by vm_init, to initialize any structures that the pmap
  689  *      system needs to map virtual memory.
  690  */
  691 void
  692 pmap_init(void)
  693 {
  694         vm_page_t mpte;
  695         vm_size_t s;
  696         int i, pv_npg;
  697 
  698         /*
  699          * Initialize the vm page array entries for the kernel pmap's
  700          * page table pages.
  701          */ 
  702         for (i = 0; i < NKPT; i++) {
  703                 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
  704                 KASSERT(mpte >= vm_page_array &&
  705                     mpte < &vm_page_array[vm_page_array_size],
  706                     ("pmap_init: page table page is out of range"));
  707                 mpte->pindex = pmap_pde_pindex(KERNBASE) + i;
  708                 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
  709         }
  710 
  711         /*
  712          * Initialize the address space (zone) for the pv entries.  Set a
  713          * high water mark so that the system can recover from excessive
  714          * numbers of pv entries.
  715          */
  716         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  717         pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  718         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  719         pv_entry_high_water = 9 * (pv_entry_max / 10);
  720 
  721         /*
  722          * If the kernel is running in a virtual machine on an AMD Family 10h
  723          * processor, then it must assume that MCA is enabled by the virtual
  724          * machine monitor.
  725          */
  726         if (vm_guest == VM_GUEST_VM && cpu_vendor_id == CPU_VENDOR_AMD &&
  727             CPUID_TO_FAMILY(cpu_id) == 0x10)
  728                 workaround_erratum383 = 1;
  729 
  730         /*
  731          * Are large page mappings enabled?
  732          */
  733         TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
  734         if (pg_ps_enabled) {
  735                 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
  736                     ("pmap_init: can't assign to pagesizes[1]"));
  737                 pagesizes[1] = NBPDR;
  738         }
  739 
  740         /*
  741          * Calculate the size of the pv head table for superpages.
  742          */
  743         for (i = 0; phys_avail[i + 1]; i += 2);
  744         pv_npg = round_2mpage(phys_avail[(i - 2) + 1]) / NBPDR;
  745 
  746         /*
  747          * Allocate memory for the pv head table for superpages.
  748          */
  749         s = (vm_size_t)(pv_npg * sizeof(struct md_page));
  750         s = round_page(s);
  751         pv_table = (struct md_page *)kmem_alloc(kernel_map, s);
  752         for (i = 0; i < pv_npg; i++)
  753                 TAILQ_INIT(&pv_table[i].pv_list);
  754 }
  755 
  756 static int
  757 pmap_pventry_proc(SYSCTL_HANDLER_ARGS)
  758 {
  759         int error;
  760 
  761         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
  762         if (error == 0 && req->newptr) {
  763                 shpgperproc = (pv_entry_max - cnt.v_page_count) / maxproc;
  764                 pv_entry_high_water = 9 * (pv_entry_max / 10);
  765         }
  766         return (error);
  767 }
  768 SYSCTL_PROC(_vm_pmap, OID_AUTO, pv_entry_max, CTLTYPE_INT|CTLFLAG_RW, 
  769     &pv_entry_max, 0, pmap_pventry_proc, "IU", "Max number of PV entries");
  770 
  771 static int
  772 pmap_shpgperproc_proc(SYSCTL_HANDLER_ARGS)
  773 {
  774         int error;
  775 
  776         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
  777         if (error == 0 && req->newptr) {
  778                 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  779                 pv_entry_high_water = 9 * (pv_entry_max / 10);
  780         }
  781         return (error);
  782 }
  783 SYSCTL_PROC(_vm_pmap, OID_AUTO, shpgperproc, CTLTYPE_INT|CTLFLAG_RW, 
  784     &shpgperproc, 0, pmap_shpgperproc_proc, "IU", "Page share factor per proc");
  785 
  786 SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
  787     "2MB page mapping counters");
  788 
  789 static u_long pmap_pde_demotions;
  790 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
  791     &pmap_pde_demotions, 0, "2MB page demotions");
  792 
  793 static u_long pmap_pde_mappings;
  794 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
  795     &pmap_pde_mappings, 0, "2MB page mappings");
  796 
  797 static u_long pmap_pde_p_failures;
  798 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
  799     &pmap_pde_p_failures, 0, "2MB page promotion failures");
  800 
  801 static u_long pmap_pde_promotions;
  802 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
  803     &pmap_pde_promotions, 0, "2MB page promotions");
  804 
  805 SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD, 0,
  806     "1GB page mapping counters");
  807 
  808 static u_long pmap_pdpe_demotions;
  809 SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD,
  810     &pmap_pdpe_demotions, 0, "1GB page demotions");
  811 
  812 
  813 /***************************************************
  814  * Low level helper routines.....
  815  ***************************************************/
  816 
  817 /*
  818  * Determine the appropriate bits to set in a PTE or PDE for a specified
  819  * caching mode.
  820  */
  821 static int
  822 pmap_cache_bits(int mode, boolean_t is_pde)
  823 {
  824         int cache_bits, pat_flag, pat_idx;
  825 
  826         if (mode < 0 || mode >= PAT_INDEX_SIZE || pat_index[mode] < 0)
  827                 panic("Unknown caching mode %d\n", mode);
  828 
  829         /* The PAT bit is different for PTE's and PDE's. */
  830         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
  831 
  832         /* Map the caching mode to a PAT index. */
  833         pat_idx = pat_index[mode];
  834 
  835         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
  836         cache_bits = 0;
  837         if (pat_idx & 0x4)
  838                 cache_bits |= pat_flag;
  839         if (pat_idx & 0x2)
  840                 cache_bits |= PG_NC_PCD;
  841         if (pat_idx & 0x1)
  842                 cache_bits |= PG_NC_PWT;
  843         return (cache_bits);
  844 }
  845 
  846 /*
  847  * After changing the page size for the specified virtual address in the page
  848  * table, flush the corresponding entries from the processor's TLB.  Only the
  849  * calling processor's TLB is affected.
  850  *
  851  * The calling thread must be pinned to a processor.
  852  */
  853 static void
  854 pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
  855 {
  856         u_long cr4;
  857 
  858         if ((newpde & PG_PS) == 0)
  859                 /* Demotion: flush a specific 2MB page mapping. */
  860                 invlpg(va);
  861         else if ((newpde & PG_G) == 0)
  862                 /*
  863                  * Promotion: flush every 4KB page mapping from the TLB
  864                  * because there are too many to flush individually.
  865                  */
  866                 invltlb();
  867         else {
  868                 /*
  869                  * Promotion: flush every 4KB page mapping from the TLB,
  870                  * including any global (PG_G) mappings.
  871                  */
  872                 cr4 = rcr4();
  873                 load_cr4(cr4 & ~CR4_PGE);
  874                 /*
  875                  * Although preemption at this point could be detrimental to
  876                  * performance, it would not lead to an error.  PG_G is simply
  877                  * ignored if CR4.PGE is clear.  Moreover, in case this block
  878                  * is re-entered, the load_cr4() either above or below will
  879                  * modify CR4.PGE flushing the TLB.
  880                  */
  881                 load_cr4(cr4 | CR4_PGE);
  882         }
  883 }
  884 #ifdef SMP
  885 /*
  886  * For SMP, these functions have to use the IPI mechanism for coherence.
  887  *
  888  * N.B.: Before calling any of the following TLB invalidation functions,
  889  * the calling processor must ensure that all stores updating a non-
  890  * kernel page table are globally performed.  Otherwise, another
  891  * processor could cache an old, pre-update entry without being
  892  * invalidated.  This can happen one of two ways: (1) The pmap becomes
  893  * active on another processor after its pm_active field is checked by
  894  * one of the following functions but before a store updating the page
  895  * table is globally performed. (2) The pmap becomes active on another
  896  * processor before its pm_active field is checked but due to
  897  * speculative loads one of the following functions stills reads the
  898  * pmap as inactive on the other processor.
  899  * 
  900  * The kernel page table is exempt because its pm_active field is
  901  * immutable.  The kernel page table is always active on every
  902  * processor.
  903  */
  904 void
  905 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  906 {
  907         cpumask_t cpumask, other_cpus;
  908 
  909         sched_pin();
  910         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  911                 invlpg(va);
  912                 smp_invlpg(va);
  913         } else {
  914                 cpumask = PCPU_GET(cpumask);
  915                 other_cpus = PCPU_GET(other_cpus);
  916                 if (pmap->pm_active & cpumask)
  917                         invlpg(va);
  918                 if (pmap->pm_active & other_cpus)
  919                         smp_masked_invlpg(pmap->pm_active & other_cpus, va);
  920         }
  921         sched_unpin();
  922 }
  923 
  924 void
  925 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  926 {
  927         cpumask_t cpumask, other_cpus;
  928         vm_offset_t addr;
  929 
  930         sched_pin();
  931         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  932                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  933                         invlpg(addr);
  934                 smp_invlpg_range(sva, eva);
  935         } else {
  936                 cpumask = PCPU_GET(cpumask);
  937                 other_cpus = PCPU_GET(other_cpus);
  938                 if (pmap->pm_active & cpumask)
  939                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
  940                                 invlpg(addr);
  941                 if (pmap->pm_active & other_cpus)
  942                         smp_masked_invlpg_range(pmap->pm_active & other_cpus,
  943                             sva, eva);
  944         }
  945         sched_unpin();
  946 }
  947 
  948 void
  949 pmap_invalidate_all(pmap_t pmap)
  950 {
  951         cpumask_t cpumask, other_cpus;
  952 
  953         sched_pin();
  954         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  955                 invltlb();
  956                 smp_invltlb();
  957         } else {
  958                 cpumask = PCPU_GET(cpumask);
  959                 other_cpus = PCPU_GET(other_cpus);
  960                 if (pmap->pm_active & cpumask)
  961                         invltlb();
  962                 if (pmap->pm_active & other_cpus)
  963                         smp_masked_invltlb(pmap->pm_active & other_cpus);
  964         }
  965         sched_unpin();
  966 }
  967 
  968 void
  969 pmap_invalidate_cache(void)
  970 {
  971 
  972         sched_pin();
  973         wbinvd();
  974         smp_cache_flush();
  975         sched_unpin();
  976 }
  977 
  978 struct pde_action {
  979         cpumask_t store;        /* processor that updates the PDE */
  980         cpumask_t invalidate;   /* processors that invalidate their TLB */
  981         vm_offset_t va;
  982         pd_entry_t *pde;
  983         pd_entry_t newpde;
  984 };
  985 
  986 static void
  987 pmap_update_pde_action(void *arg)
  988 {
  989         struct pde_action *act = arg;
  990 
  991         if (act->store == PCPU_GET(cpumask))
  992                 pde_store(act->pde, act->newpde);
  993 }
  994 
  995 static void
  996 pmap_update_pde_teardown(void *arg)
  997 {
  998         struct pde_action *act = arg;
  999 
 1000         if ((act->invalidate & PCPU_GET(cpumask)) != 0)
 1001                 pmap_update_pde_invalidate(act->va, act->newpde);
 1002 }
 1003 
 1004 /*
 1005  * Change the page size for the specified virtual address in a way that
 1006  * prevents any possibility of the TLB ever having two entries that map the
 1007  * same virtual address using different page sizes.  This is the recommended
 1008  * workaround for Erratum 383 on AMD Family 10h processors.  It prevents a
 1009  * machine check exception for a TLB state that is improperly diagnosed as a
 1010  * hardware error.
 1011  */
 1012 static void
 1013 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1014 {
 1015         struct pde_action act;
 1016         cpumask_t active, cpumask;
 1017 
 1018         sched_pin();
 1019         cpumask = PCPU_GET(cpumask);
 1020         if (pmap == kernel_pmap)
 1021                 active = all_cpus;
 1022         else
 1023                 active = pmap->pm_active;
 1024         if ((active & PCPU_GET(other_cpus)) != 0) {
 1025                 act.store = cpumask;
 1026                 act.invalidate = active;
 1027                 act.va = va;
 1028                 act.pde = pde;
 1029                 act.newpde = newpde;
 1030                 smp_rendezvous_cpus(cpumask | active,
 1031                     smp_no_rendevous_barrier, pmap_update_pde_action,
 1032                     pmap_update_pde_teardown, &act);
 1033         } else {
 1034                 pde_store(pde, newpde);
 1035                 if ((active & cpumask) != 0)
 1036                         pmap_update_pde_invalidate(va, newpde);
 1037         }
 1038         sched_unpin();
 1039 }
 1040 #else /* !SMP */
 1041 /*
 1042  * Normal, non-SMP, invalidation functions.
 1043  * We inline these within pmap.c for speed.
 1044  */
 1045 PMAP_INLINE void
 1046 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 1047 {
 1048 
 1049         if (pmap == kernel_pmap || pmap->pm_active)
 1050                 invlpg(va);
 1051 }
 1052 
 1053 PMAP_INLINE void
 1054 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1055 {
 1056         vm_offset_t addr;
 1057 
 1058         if (pmap == kernel_pmap || pmap->pm_active)
 1059                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1060                         invlpg(addr);
 1061 }
 1062 
 1063 PMAP_INLINE void
 1064 pmap_invalidate_all(pmap_t pmap)
 1065 {
 1066 
 1067         if (pmap == kernel_pmap || pmap->pm_active)
 1068                 invltlb();
 1069 }
 1070 
 1071 PMAP_INLINE void
 1072 pmap_invalidate_cache(void)
 1073 {
 1074 
 1075         wbinvd();
 1076 }
 1077 
 1078 static void
 1079 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1080 {
 1081 
 1082         pde_store(pde, newpde);
 1083         if (pmap == kernel_pmap || pmap->pm_active)
 1084                 pmap_update_pde_invalidate(va, newpde);
 1085 }
 1086 #endif /* !SMP */
 1087 
 1088 static void
 1089 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
 1090 {
 1091 
 1092         KASSERT((sva & PAGE_MASK) == 0,
 1093             ("pmap_invalidate_cache_range: sva not page-aligned"));
 1094         KASSERT((eva & PAGE_MASK) == 0,
 1095             ("pmap_invalidate_cache_range: eva not page-aligned"));
 1096 
 1097         if (cpu_feature & CPUID_SS)
 1098                 ; /* If "Self Snoop" is supported, do nothing. */
 1099         else if ((cpu_feature & CPUID_CLFSH) != 0 &&
 1100                  eva - sva < 2 * 1024 * 1024) {
 1101 
 1102                 /*
 1103                  * Otherwise, do per-cache line flush.  Use the mfence
 1104                  * instruction to insure that previous stores are
 1105                  * included in the write-back.  The processor
 1106                  * propagates flush to other processors in the cache
 1107                  * coherence domain.
 1108                  */
 1109                 mfence();
 1110                 for (; sva < eva; sva += cpu_clflush_line_size)
 1111                         clflush(sva);
 1112                 mfence();
 1113         } else {
 1114 
 1115                 /*
 1116                  * No targeted cache flush methods are supported by CPU,
 1117                  * or the supplied range is bigger than 2MB.
 1118                  * Globally invalidate cache.
 1119                  */
 1120                 pmap_invalidate_cache();
 1121         }
 1122 }
 1123 
 1124 /*
 1125  * Are we current address space or kernel?
 1126  */
 1127 static __inline int
 1128 pmap_is_current(pmap_t pmap)
 1129 {
 1130         return (pmap == kernel_pmap ||
 1131             (pmap->pm_pml4[PML4PML4I] & PG_FRAME) == (PML4pml4e[0] & PG_FRAME));
 1132 }
 1133 
 1134 /*
 1135  *      Routine:        pmap_extract
 1136  *      Function:
 1137  *              Extract the physical page address associated
 1138  *              with the given map/virtual_address pair.
 1139  */
 1140 vm_paddr_t 
 1141 pmap_extract(pmap_t pmap, vm_offset_t va)
 1142 {
 1143         pdp_entry_t *pdpe;
 1144         pd_entry_t *pde;
 1145         pt_entry_t *pte;
 1146         vm_paddr_t pa;
 1147 
 1148         pa = 0;
 1149         PMAP_LOCK(pmap);
 1150         pdpe = pmap_pdpe(pmap, va);
 1151         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 1152                 if ((*pdpe & PG_PS) != 0)
 1153                         pa = (*pdpe & PG_PS_FRAME) | (va & PDPMASK);
 1154                 else {
 1155                         pde = pmap_pdpe_to_pde(pdpe, va);
 1156                         if ((*pde & PG_V) != 0) {
 1157                                 if ((*pde & PG_PS) != 0) {
 1158                                         pa = (*pde & PG_PS_FRAME) |
 1159                                             (va & PDRMASK);
 1160                                 } else {
 1161                                         pte = pmap_pde_to_pte(pde, va);
 1162                                         pa = (*pte & PG_FRAME) |
 1163                                             (va & PAGE_MASK);
 1164                                 }
 1165                         }
 1166                 }
 1167         }
 1168         PMAP_UNLOCK(pmap);
 1169         return (pa);
 1170 }
 1171 
 1172 /*
 1173  *      Routine:        pmap_extract_and_hold
 1174  *      Function:
 1175  *              Atomically extract and hold the physical page
 1176  *              with the given pmap and virtual address pair
 1177  *              if that mapping permits the given protection.
 1178  */
 1179 vm_page_t
 1180 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 1181 {
 1182         pd_entry_t pde, *pdep;
 1183         pt_entry_t pte;
 1184         vm_page_t m;
 1185 
 1186         m = NULL;
 1187         vm_page_lock_queues();
 1188         PMAP_LOCK(pmap);
 1189         pdep = pmap_pde(pmap, va);
 1190         if (pdep != NULL && (pde = *pdep)) {
 1191                 if (pde & PG_PS) {
 1192                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
 1193                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
 1194                                     (va & PDRMASK));
 1195                                 vm_page_hold(m);
 1196                         }
 1197                 } else {
 1198                         pte = *pmap_pde_to_pte(pdep, va);
 1199                         if ((pte & PG_V) &&
 1200                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
 1201                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
 1202                                 vm_page_hold(m);
 1203                         }
 1204                 }
 1205         }
 1206         vm_page_unlock_queues();
 1207         PMAP_UNLOCK(pmap);
 1208         return (m);
 1209 }
 1210 
 1211 vm_paddr_t
 1212 pmap_kextract(vm_offset_t va)
 1213 {
 1214         pd_entry_t pde;
 1215         vm_paddr_t pa;
 1216 
 1217         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
 1218                 pa = DMAP_TO_PHYS(va);
 1219         } else {
 1220                 pde = *vtopde(va);
 1221                 if (pde & PG_PS) {
 1222                         pa = (pde & PG_PS_FRAME) | (va & PDRMASK);
 1223                 } else {
 1224                         /*
 1225                          * Beware of a concurrent promotion that changes the
 1226                          * PDE at this point!  For example, vtopte() must not
 1227                          * be used to access the PTE because it would use the
 1228                          * new PDE.  It is, however, safe to use the old PDE
 1229                          * because the page table page is preserved by the
 1230                          * promotion.
 1231                          */
 1232                         pa = *pmap_pde_to_pte(&pde, va);
 1233                         pa = (pa & PG_FRAME) | (va & PAGE_MASK);
 1234                 }
 1235         }
 1236         return pa;
 1237 }
 1238 
 1239 /***************************************************
 1240  * Low level mapping routines.....
 1241  ***************************************************/
 1242 
 1243 /*
 1244  * Add a wired page to the kva.
 1245  * Note: not SMP coherent.
 1246  */
 1247 PMAP_INLINE void 
 1248 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
 1249 {
 1250         pt_entry_t *pte;
 1251 
 1252         pte = vtopte(va);
 1253         pte_store(pte, pa | PG_RW | PG_V | PG_G);
 1254 }
 1255 
 1256 static __inline void
 1257 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
 1258 {
 1259         pt_entry_t *pte;
 1260 
 1261         pte = vtopte(va);
 1262         pte_store(pte, pa | PG_RW | PG_V | PG_G | pmap_cache_bits(mode, 0));
 1263 }
 1264 
 1265 /*
 1266  * Remove a page from the kernel pagetables.
 1267  * Note: not SMP coherent.
 1268  */
 1269 PMAP_INLINE void
 1270 pmap_kremove(vm_offset_t va)
 1271 {
 1272         pt_entry_t *pte;
 1273 
 1274         pte = vtopte(va);
 1275         pte_clear(pte);
 1276 }
 1277 
 1278 /*
 1279  *      Used to map a range of physical addresses into kernel
 1280  *      virtual address space.
 1281  *
 1282  *      The value passed in '*virt' is a suggested virtual address for
 1283  *      the mapping. Architectures which can support a direct-mapped
 1284  *      physical to virtual region can return the appropriate address
 1285  *      within that region, leaving '*virt' unchanged. Other
 1286  *      architectures should map the pages starting at '*virt' and
 1287  *      update '*virt' with the first usable address after the mapped
 1288  *      region.
 1289  */
 1290 vm_offset_t
 1291 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 1292 {
 1293         return PHYS_TO_DMAP(start);
 1294 }
 1295 
 1296 
 1297 /*
 1298  * Add a list of wired pages to the kva
 1299  * this routine is only used for temporary
 1300  * kernel mappings that do not need to have
 1301  * page modification or references recorded.
 1302  * Note that old mappings are simply written
 1303  * over.  The page *must* be wired.
 1304  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1305  */
 1306 void
 1307 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 1308 {
 1309         pt_entry_t *endpte, oldpte, pa, *pte;
 1310         vm_page_t m;
 1311 
 1312         oldpte = 0;
 1313         pte = vtopte(sva);
 1314         endpte = pte + count;
 1315         while (pte < endpte) {
 1316                 m = *ma++;
 1317                 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0);
 1318                 if ((*pte & (PG_FRAME | PG_PTE_CACHE)) != pa) {
 1319                         oldpte |= *pte;
 1320                         pte_store(pte, pa | PG_G | PG_RW | PG_V);
 1321                 }
 1322                 pte++;
 1323         }
 1324         if (__predict_false((oldpte & PG_V) != 0))
 1325                 pmap_invalidate_range(kernel_pmap, sva, sva + count *
 1326                     PAGE_SIZE);
 1327 }
 1328 
 1329 /*
 1330  * This routine tears out page mappings from the
 1331  * kernel -- it is meant only for temporary mappings.
 1332  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1333  */
 1334 void
 1335 pmap_qremove(vm_offset_t sva, int count)
 1336 {
 1337         vm_offset_t va;
 1338 
 1339         va = sva;
 1340         while (count-- > 0) {
 1341                 pmap_kremove(va);
 1342                 va += PAGE_SIZE;
 1343         }
 1344         pmap_invalidate_range(kernel_pmap, sva, va);
 1345 }
 1346 
 1347 /***************************************************
 1348  * Page table page management routines.....
 1349  ***************************************************/
 1350 static __inline void
 1351 pmap_free_zero_pages(vm_page_t free)
 1352 {
 1353         vm_page_t m;
 1354 
 1355         while (free != NULL) {
 1356                 m = free;
 1357                 free = m->right;
 1358                 /* Preserve the page's PG_ZERO setting. */
 1359                 vm_page_free_toq(m);
 1360         }
 1361 }
 1362 
 1363 /*
 1364  * Schedule the specified unused page table page to be freed.  Specifically,
 1365  * add the page to the specified list of pages that will be released to the
 1366  * physical memory manager after the TLB has been updated.
 1367  */
 1368 static __inline void
 1369 pmap_add_delayed_free_list(vm_page_t m, vm_page_t *free, boolean_t set_PG_ZERO)
 1370 {
 1371 
 1372         if (set_PG_ZERO)
 1373                 m->flags |= PG_ZERO;
 1374         else
 1375                 m->flags &= ~PG_ZERO;
 1376         m->right = *free;
 1377         *free = m;
 1378 }
 1379         
 1380 /*
 1381  * Inserts the specified page table page into the specified pmap's collection
 1382  * of idle page table pages.  Each of a pmap's page table pages is responsible
 1383  * for mapping a distinct range of virtual addresses.  The pmap's collection is
 1384  * ordered by this virtual address range.
 1385  */
 1386 static void
 1387 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
 1388 {
 1389         vm_page_t root;
 1390 
 1391         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1392         root = pmap->pm_root;
 1393         if (root == NULL) {
 1394                 mpte->left = NULL;
 1395                 mpte->right = NULL;
 1396         } else {
 1397                 root = vm_page_splay(mpte->pindex, root);
 1398                 if (mpte->pindex < root->pindex) {
 1399                         mpte->left = root->left;
 1400                         mpte->right = root;
 1401                         root->left = NULL;
 1402                 } else if (mpte->pindex == root->pindex)
 1403                         panic("pmap_insert_pt_page: pindex already inserted");
 1404                 else {
 1405                         mpte->right = root->right;
 1406                         mpte->left = root;
 1407                         root->right = NULL;
 1408                 }
 1409         }
 1410         pmap->pm_root = mpte;
 1411 }
 1412 
 1413 /*
 1414  * Looks for a page table page mapping the specified virtual address in the
 1415  * specified pmap's collection of idle page table pages.  Returns NULL if there
 1416  * is no page table page corresponding to the specified virtual address.
 1417  */
 1418 static vm_page_t
 1419 pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
 1420 {
 1421         vm_page_t mpte;
 1422         vm_pindex_t pindex = pmap_pde_pindex(va);
 1423 
 1424         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1425         if ((mpte = pmap->pm_root) != NULL && mpte->pindex != pindex) {
 1426                 mpte = vm_page_splay(pindex, mpte);
 1427                 if ((pmap->pm_root = mpte)->pindex != pindex)
 1428                         mpte = NULL;
 1429         }
 1430         return (mpte);
 1431 }
 1432 
 1433 /*
 1434  * Removes the specified page table page from the specified pmap's collection
 1435  * of idle page table pages.  The specified page table page must be a member of
 1436  * the pmap's collection.
 1437  */
 1438 static void
 1439 pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
 1440 {
 1441         vm_page_t root;
 1442 
 1443         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1444         if (mpte != pmap->pm_root) {
 1445                 root = vm_page_splay(mpte->pindex, pmap->pm_root);
 1446                 KASSERT(mpte == root,
 1447                     ("pmap_remove_pt_page: mpte %p is missing from pmap %p",
 1448                     mpte, pmap));
 1449         }
 1450         if (mpte->left == NULL)
 1451                 root = mpte->right;
 1452         else {
 1453                 root = vm_page_splay(mpte->pindex, mpte->left);
 1454                 root->right = mpte->right;
 1455         }
 1456         pmap->pm_root = root;
 1457 }
 1458 
 1459 /*
 1460  * This routine unholds page table pages, and if the hold count
 1461  * drops to zero, then it decrements the wire count.
 1462  */
 1463 static __inline int
 1464 pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free)
 1465 {
 1466 
 1467         --m->wire_count;
 1468         if (m->wire_count == 0)
 1469                 return _pmap_unwire_pte_hold(pmap, va, m, free);
 1470         else
 1471                 return 0;
 1472 }
 1473 
 1474 static int 
 1475 _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, 
 1476     vm_page_t *free)
 1477 {
 1478 
 1479         /*
 1480          * unmap the page table page
 1481          */
 1482         if (m->pindex >= (NUPDE + NUPDPE)) {
 1483                 /* PDP page */
 1484                 pml4_entry_t *pml4;
 1485                 pml4 = pmap_pml4e(pmap, va);
 1486                 *pml4 = 0;
 1487         } else if (m->pindex >= NUPDE) {
 1488                 /* PD page */
 1489                 pdp_entry_t *pdp;
 1490                 pdp = pmap_pdpe(pmap, va);
 1491                 *pdp = 0;
 1492         } else {
 1493                 /* PTE page */
 1494                 pd_entry_t *pd;
 1495                 pd = pmap_pde(pmap, va);
 1496                 *pd = 0;
 1497         }
 1498         --pmap->pm_stats.resident_count;
 1499         if (m->pindex < NUPDE) {
 1500                 /* We just released a PT, unhold the matching PD */
 1501                 vm_page_t pdpg;
 1502 
 1503                 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
 1504                 pmap_unwire_pte_hold(pmap, va, pdpg, free);
 1505         }
 1506         if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 1507                 /* We just released a PD, unhold the matching PDP */
 1508                 vm_page_t pdppg;
 1509 
 1510                 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
 1511                 pmap_unwire_pte_hold(pmap, va, pdppg, free);
 1512         }
 1513 
 1514         /*
 1515          * This is a release store so that the ordinary store unmapping
 1516          * the page table page is globally performed before TLB shoot-
 1517          * down is begun.
 1518          */
 1519         atomic_subtract_rel_int(&cnt.v_wire_count, 1);
 1520 
 1521         /* 
 1522          * Put page on a list so that it is released after
 1523          * *ALL* TLB shootdown is done
 1524          */
 1525         pmap_add_delayed_free_list(m, free, TRUE);
 1526         
 1527         return 1;
 1528 }
 1529 
 1530 /*
 1531  * After removing a page table entry, this routine is used to
 1532  * conditionally free the page, and manage the hold/wire counts.
 1533  */
 1534 static int
 1535 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, vm_page_t *free)
 1536 {
 1537         vm_page_t mpte;
 1538 
 1539         if (va >= VM_MAXUSER_ADDRESS)
 1540                 return 0;
 1541         KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 1542         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1543         return pmap_unwire_pte_hold(pmap, va, mpte, free);
 1544 }
 1545 
 1546 void
 1547 pmap_pinit0(pmap_t pmap)
 1548 {
 1549 
 1550         PMAP_LOCK_INIT(pmap);
 1551         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
 1552         pmap->pm_root = NULL;
 1553         pmap->pm_active = 0;
 1554         PCPU_SET(curpmap, pmap);
 1555         TAILQ_INIT(&pmap->pm_pvchunk);
 1556         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1557 }
 1558 
 1559 /*
 1560  * Initialize a preallocated and zeroed pmap structure,
 1561  * such as one in a vmspace structure.
 1562  */
 1563 int
 1564 pmap_pinit(pmap_t pmap)
 1565 {
 1566         vm_page_t pml4pg;
 1567         static vm_pindex_t color;
 1568 
 1569         PMAP_LOCK_INIT(pmap);
 1570 
 1571         /*
 1572          * allocate the page directory page
 1573          */
 1574         while ((pml4pg = vm_page_alloc(NULL, color++, VM_ALLOC_NOOBJ |
 1575             VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
 1576                 VM_WAIT;
 1577 
 1578         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
 1579 
 1580         if ((pml4pg->flags & PG_ZERO) == 0)
 1581                 pagezero(pmap->pm_pml4);
 1582 
 1583         /* Wire in kernel global address entries. */
 1584         pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
 1585         pmap->pm_pml4[DMPML4I] = DMPDPphys | PG_RW | PG_V | PG_U;
 1586 
 1587         /* install self-referential address mapping entry(s) */
 1588         pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
 1589 
 1590         pmap->pm_root = NULL;
 1591         pmap->pm_active = 0;
 1592         TAILQ_INIT(&pmap->pm_pvchunk);
 1593         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1594 
 1595         return (1);
 1596 }
 1597 
 1598 /*
 1599  * this routine is called if the page table page is not
 1600  * mapped correctly.
 1601  *
 1602  * Note: If a page allocation fails at page table level two or three,
 1603  * one or two pages may be held during the wait, only to be released
 1604  * afterwards.  This conservative approach is easily argued to avoid
 1605  * race conditions.
 1606  */
 1607 static vm_page_t
 1608 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 1609 {
 1610         vm_page_t m, pdppg, pdpg;
 1611 
 1612         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1613             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1614             ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1615 
 1616         /*
 1617          * Allocate a page table page.
 1618          */
 1619         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1620             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1621                 if (flags & M_WAITOK) {
 1622                         PMAP_UNLOCK(pmap);
 1623                         vm_page_unlock_queues();
 1624                         VM_WAIT;
 1625                         vm_page_lock_queues();
 1626                         PMAP_LOCK(pmap);
 1627                 }
 1628 
 1629                 /*
 1630                  * Indicate the need to retry.  While waiting, the page table
 1631                  * page may have been allocated.
 1632                  */
 1633                 return (NULL);
 1634         }
 1635         if ((m->flags & PG_ZERO) == 0)
 1636                 pmap_zero_page(m);
 1637 
 1638         /*
 1639          * Map the pagetable page into the process address space, if
 1640          * it isn't already there.
 1641          */
 1642 
 1643         if (ptepindex >= (NUPDE + NUPDPE)) {
 1644                 pml4_entry_t *pml4;
 1645                 vm_pindex_t pml4index;
 1646 
 1647                 /* Wire up a new PDPE page */
 1648                 pml4index = ptepindex - (NUPDE + NUPDPE);
 1649                 pml4 = &pmap->pm_pml4[pml4index];
 1650                 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1651 
 1652         } else if (ptepindex >= NUPDE) {
 1653                 vm_pindex_t pml4index;
 1654                 vm_pindex_t pdpindex;
 1655                 pml4_entry_t *pml4;
 1656                 pdp_entry_t *pdp;
 1657 
 1658                 /* Wire up a new PDE page */
 1659                 pdpindex = ptepindex - NUPDE;
 1660                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1661 
 1662                 pml4 = &pmap->pm_pml4[pml4index];
 1663                 if ((*pml4 & PG_V) == 0) {
 1664                         /* Have to allocate a new pdp, recurse */
 1665                         if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
 1666                             flags) == NULL) {
 1667                                 --m->wire_count;
 1668                                 atomic_subtract_int(&cnt.v_wire_count, 1);
 1669                                 vm_page_free_zero(m);
 1670                                 return (NULL);
 1671                         }
 1672                 } else {
 1673                         /* Add reference to pdp page */
 1674                         pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
 1675                         pdppg->wire_count++;
 1676                 }
 1677                 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1678 
 1679                 /* Now find the pdp page */
 1680                 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1681                 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1682 
 1683         } else {
 1684                 vm_pindex_t pml4index;
 1685                 vm_pindex_t pdpindex;
 1686                 pml4_entry_t *pml4;
 1687                 pdp_entry_t *pdp;
 1688                 pd_entry_t *pd;
 1689 
 1690                 /* Wire up a new PTE page */
 1691                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1692                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1693 
 1694                 /* First, find the pdp and check that its valid. */
 1695                 pml4 = &pmap->pm_pml4[pml4index];
 1696                 if ((*pml4 & PG_V) == 0) {
 1697                         /* Have to allocate a new pd, recurse */
 1698                         if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1699                             flags) == NULL) {
 1700                                 --m->wire_count;
 1701                                 atomic_subtract_int(&cnt.v_wire_count, 1);
 1702                                 vm_page_free_zero(m);
 1703                                 return (NULL);
 1704                         }
 1705                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1706                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1707                 } else {
 1708                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1709                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1710                         if ((*pdp & PG_V) == 0) {
 1711                                 /* Have to allocate a new pd, recurse */
 1712                                 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1713                                     flags) == NULL) {
 1714                                         --m->wire_count;
 1715                                         atomic_subtract_int(&cnt.v_wire_count,
 1716                                             1);
 1717                                         vm_page_free_zero(m);
 1718                                         return (NULL);
 1719                                 }
 1720                         } else {
 1721                                 /* Add reference to the pd page */
 1722                                 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
 1723                                 pdpg->wire_count++;
 1724                         }
 1725                 }
 1726                 pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
 1727 
 1728                 /* Now we know where the page directory page is */
 1729                 pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
 1730                 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1731         }
 1732 
 1733         pmap->pm_stats.resident_count++;
 1734 
 1735         return m;
 1736 }
 1737 
 1738 static vm_page_t
 1739 pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags)
 1740 {
 1741         vm_pindex_t pdpindex, ptepindex;
 1742         pdp_entry_t *pdpe;
 1743         vm_page_t pdpg;
 1744 
 1745         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1746             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1747             ("pmap_allocpde: flags is neither M_NOWAIT nor M_WAITOK"));
 1748 retry:
 1749         pdpe = pmap_pdpe(pmap, va);
 1750         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 1751                 /* Add a reference to the pd page. */
 1752                 pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
 1753                 pdpg->wire_count++;
 1754         } else {
 1755                 /* Allocate a pd page. */
 1756                 ptepindex = pmap_pde_pindex(va);
 1757                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1758                 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, flags);
 1759                 if (pdpg == NULL && (flags & M_WAITOK))
 1760                         goto retry;
 1761         }
 1762         return (pdpg);
 1763 }
 1764 
 1765 static vm_page_t
 1766 pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
 1767 {
 1768         vm_pindex_t ptepindex;
 1769         pd_entry_t *pd;
 1770         vm_page_t m;
 1771 
 1772         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1773             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1774             ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1775 
 1776         /*
 1777          * Calculate pagetable page index
 1778          */
 1779         ptepindex = pmap_pde_pindex(va);
 1780 retry:
 1781         /*
 1782          * Get the page directory entry
 1783          */
 1784         pd = pmap_pde(pmap, va);
 1785 
 1786         /*
 1787          * This supports switching from a 2MB page to a
 1788          * normal 4K page.
 1789          */
 1790         if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
 1791                 if (!pmap_demote_pde(pmap, pd, va)) {
 1792                         /*
 1793                          * Invalidation of the 2MB page mapping may have caused
 1794                          * the deallocation of the underlying PD page.
 1795                          */
 1796                         pd = NULL;
 1797                 }
 1798         }
 1799 
 1800         /*
 1801          * If the page table page is mapped, we just increment the
 1802          * hold count, and activate it.
 1803          */
 1804         if (pd != NULL && (*pd & PG_V) != 0) {
 1805                 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
 1806                 m->wire_count++;
 1807         } else {
 1808                 /*
 1809                  * Here if the pte page isn't mapped, or if it has been
 1810                  * deallocated.
 1811                  */
 1812                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1813                 if (m == NULL && (flags & M_WAITOK))
 1814                         goto retry;
 1815         }
 1816         return (m);
 1817 }
 1818 
 1819 
 1820 /***************************************************
 1821  * Pmap allocation/deallocation routines.
 1822  ***************************************************/
 1823 
 1824 /*
 1825  * Release any resources held by the given physical map.
 1826  * Called when a pmap initialized by pmap_pinit is being released.
 1827  * Should only be called if the map contains no valid mappings.
 1828  */
 1829 void
 1830 pmap_release(pmap_t pmap)
 1831 {
 1832         vm_page_t m;
 1833 
 1834         KASSERT(pmap->pm_stats.resident_count == 0,
 1835             ("pmap_release: pmap resident count %ld != 0",
 1836             pmap->pm_stats.resident_count));
 1837         KASSERT(pmap->pm_root == NULL,
 1838             ("pmap_release: pmap has reserved page table page(s)"));
 1839 
 1840         m = PHYS_TO_VM_PAGE(pmap->pm_pml4[PML4PML4I] & PG_FRAME);
 1841 
 1842         pmap->pm_pml4[KPML4I] = 0;      /* KVA */
 1843         pmap->pm_pml4[DMPML4I] = 0;     /* Direct Map */
 1844         pmap->pm_pml4[PML4PML4I] = 0;   /* Recursive Mapping */
 1845 
 1846         m->wire_count--;
 1847         atomic_subtract_int(&cnt.v_wire_count, 1);
 1848         vm_page_free_zero(m);
 1849         PMAP_LOCK_DESTROY(pmap);
 1850 }
 1851 
 1852 static int
 1853 kvm_size(SYSCTL_HANDLER_ARGS)
 1854 {
 1855         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
 1856 
 1857         return sysctl_handle_long(oidp, &ksize, 0, req);
 1858 }
 1859 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 1860     0, 0, kvm_size, "LU", "Size of KVM");
 1861 
 1862 static int
 1863 kvm_free(SYSCTL_HANDLER_ARGS)
 1864 {
 1865         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 1866 
 1867         return sysctl_handle_long(oidp, &kfree, 0, req);
 1868 }
 1869 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 1870     0, 0, kvm_free, "LU", "Amount of KVM free");
 1871 
 1872 /*
 1873  * grow the number of kernel page table entries, if needed
 1874  */
 1875 void
 1876 pmap_growkernel(vm_offset_t addr)
 1877 {
 1878         vm_paddr_t paddr;
 1879         vm_page_t nkpg;
 1880         pd_entry_t *pde, newpdir;
 1881         pdp_entry_t *pdpe;
 1882 
 1883         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1884 
 1885         /*
 1886          * Return if "addr" is within the range of kernel page table pages
 1887          * that were preallocated during pmap bootstrap.  Moreover, leave
 1888          * "kernel_vm_end" and the kernel page table as they were.
 1889          *
 1890          * The correctness of this action is based on the following
 1891          * argument: vm_map_findspace() allocates contiguous ranges of the
 1892          * kernel virtual address space.  It calls this function if a range
 1893          * ends after "kernel_vm_end".  If the kernel is mapped between
 1894          * "kernel_vm_end" and "addr", then the range cannot begin at
 1895          * "kernel_vm_end".  In fact, its beginning address cannot be less
 1896          * than the kernel.  Thus, there is no immediate need to allocate
 1897          * any new kernel page table pages between "kernel_vm_end" and
 1898          * "KERNBASE".
 1899          */
 1900         if (KERNBASE < addr && addr <= KERNBASE + NKPT * NBPDR)
 1901                 return;
 1902 
 1903         addr = roundup2(addr, NBPDR);
 1904         if (addr - 1 >= kernel_map->max_offset)
 1905                 addr = kernel_map->max_offset;
 1906         while (kernel_vm_end < addr) {
 1907                 pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end);
 1908                 if ((*pdpe & PG_V) == 0) {
 1909                         /* We need a new PDP entry */
 1910                         nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT,
 1911                             VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
 1912                             VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 1913                         if (nkpg == NULL)
 1914                                 panic("pmap_growkernel: no memory to grow kernel");
 1915                         if ((nkpg->flags & PG_ZERO) == 0)
 1916                                 pmap_zero_page(nkpg);
 1917                         paddr = VM_PAGE_TO_PHYS(nkpg);
 1918                         *pdpe = (pdp_entry_t)
 1919                                 (paddr | PG_V | PG_RW | PG_A | PG_M);
 1920                         continue; /* try again */
 1921                 }
 1922                 pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end);
 1923                 if ((*pde & PG_V) != 0) {
 1924                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 1925                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1926                                 kernel_vm_end = kernel_map->max_offset;
 1927                                 break;                       
 1928                         }
 1929                         continue;
 1930                 }
 1931 
 1932                 nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end),
 1933                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 1934                     VM_ALLOC_ZERO);
 1935                 if (nkpg == NULL)
 1936                         panic("pmap_growkernel: no memory to grow kernel");
 1937                 if ((nkpg->flags & PG_ZERO) == 0)
 1938                         pmap_zero_page(nkpg);
 1939                 paddr = VM_PAGE_TO_PHYS(nkpg);
 1940                 newpdir = (pd_entry_t) (paddr | PG_V | PG_RW | PG_A | PG_M);
 1941                 pde_store(pde, newpdir);
 1942 
 1943                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 1944                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1945                         kernel_vm_end = kernel_map->max_offset;
 1946                         break;                       
 1947                 }
 1948         }
 1949 }
 1950 
 1951 
 1952 /***************************************************
 1953  * page management routines.
 1954  ***************************************************/
 1955 
 1956 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 1957 CTASSERT(_NPCM == 3);
 1958 CTASSERT(_NPCPV == 168);
 1959 
 1960 static __inline struct pv_chunk *
 1961 pv_to_chunk(pv_entry_t pv)
 1962 {
 1963 
 1964         return (struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK);
 1965 }
 1966 
 1967 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 1968 
 1969 #define PC_FREE0        0xfffffffffffffffful
 1970 #define PC_FREE1        0xfffffffffffffffful
 1971 #define PC_FREE2        0x000000fffffffffful
 1972 
 1973 static uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
 1974 
 1975 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 1976         "Current number of pv entries");
 1977 
 1978 #ifdef PV_STATS
 1979 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 1980 
 1981 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 1982         "Current number of pv entry chunks");
 1983 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 1984         "Current number of pv entry chunks allocated");
 1985 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 1986         "Current number of pv entry chunks frees");
 1987 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 1988         "Number of times tried to get a chunk page but failed.");
 1989 
 1990 static long pv_entry_frees, pv_entry_allocs;
 1991 static int pv_entry_spare;
 1992 
 1993 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 1994         "Current number of pv entry frees");
 1995 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 1996         "Current number of pv entry allocs");
 1997 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 1998         "Current number of spare pv entries");
 1999 
 2000 static int pmap_collect_inactive, pmap_collect_active;
 2001 
 2002 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_collect_inactive, CTLFLAG_RD, &pmap_collect_inactive, 0,
 2003         "Current number times pmap_collect called on inactive queue");
 2004 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_collect_active, CTLFLAG_RD, &pmap_collect_active, 0,
 2005         "Current number times pmap_collect called on active queue");
 2006 #endif
 2007 
 2008 /*
 2009  * We are in a serious low memory condition.  Resort to
 2010  * drastic measures to free some pages so we can allocate
 2011  * another pv entry chunk.  This is normally called to
 2012  * unmap inactive pages, and if necessary, active pages.
 2013  *
 2014  * We do not, however, unmap 2mpages because subsequent accesses will
 2015  * allocate per-page pv entries until repromotion occurs, thereby
 2016  * exacerbating the shortage of free pv entries.
 2017  */
 2018 static void
 2019 pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 2020 {
 2021         struct md_page *pvh;
 2022         pd_entry_t *pde;
 2023         pmap_t pmap;
 2024         pt_entry_t *pte, tpte;
 2025         pv_entry_t next_pv, pv;
 2026         vm_offset_t va;
 2027         vm_page_t m, free;
 2028 
 2029         TAILQ_FOREACH(m, &vpq->pl, pageq) {
 2030                 if ((m->flags & PG_MARKER) != 0 || m->hold_count || m->busy)
 2031                         continue;
 2032                 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) {
 2033                         va = pv->pv_va;
 2034                         pmap = PV_PMAP(pv);
 2035                         /* Avoid deadlock and lock recursion. */
 2036                         if (pmap > locked_pmap)
 2037                                 PMAP_LOCK(pmap);
 2038                         else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
 2039                                 continue;
 2040                         pmap->pm_stats.resident_count--;
 2041                         pde = pmap_pde(pmap, va);
 2042                         KASSERT((*pde & PG_PS) == 0, ("pmap_collect: found"
 2043                             " a 2mpage in page %p's pv list", m));
 2044                         pte = pmap_pde_to_pte(pde, va);
 2045                         tpte = pte_load_clear(pte);
 2046                         KASSERT((tpte & PG_W) == 0,
 2047                             ("pmap_collect: wired pte %#lx", tpte));
 2048                         if (tpte & PG_A)
 2049                                 vm_page_flag_set(m, PG_REFERENCED);
 2050                         if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2051                                 vm_page_dirty(m);
 2052                         free = NULL;
 2053                         pmap_unuse_pt(pmap, va, *pde, &free);
 2054                         pmap_invalidate_page(pmap, va);
 2055                         pmap_free_zero_pages(free);
 2056                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2057                         if (TAILQ_EMPTY(&m->md.pv_list)) {
 2058                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2059                                 if (TAILQ_EMPTY(&pvh->pv_list))
 2060                                         vm_page_flag_clear(m, PG_WRITEABLE);
 2061                         }
 2062                         free_pv_entry(pmap, pv);
 2063                         if (pmap != locked_pmap)
 2064                                 PMAP_UNLOCK(pmap);
 2065                 }
 2066         }
 2067 }
 2068 
 2069 
 2070 /*
 2071  * free the pv_entry back to the free list
 2072  */
 2073 static void
 2074 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 2075 {
 2076         vm_page_t m;
 2077         struct pv_chunk *pc;
 2078         int idx, field, bit;
 2079 
 2080         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2081         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2082         PV_STAT(pv_entry_frees++);
 2083         PV_STAT(pv_entry_spare++);
 2084         pv_entry_count--;
 2085         pc = pv_to_chunk(pv);
 2086         idx = pv - &pc->pc_pventry[0];
 2087         field = idx / 64;
 2088         bit = idx % 64;
 2089         pc->pc_map[field] |= 1ul << bit;
 2090         /* move to head of list */
 2091         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2092         if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 ||
 2093             pc->pc_map[2] != PC_FREE2) {
 2094                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2095                 return;
 2096         }
 2097         PV_STAT(pv_entry_spare -= _NPCPV);
 2098         PV_STAT(pc_chunk_count--);
 2099         PV_STAT(pc_chunk_frees++);
 2100         /* entire chunk is free, return it */
 2101         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 2102         dump_drop_page(m->phys_addr);
 2103         vm_page_unwire(m, 0);
 2104         vm_page_free(m);
 2105 }
 2106 
 2107 /*
 2108  * get a new pv_entry, allocating a block from the system
 2109  * when needed.
 2110  */
 2111 static pv_entry_t
 2112 get_pv_entry(pmap_t pmap, int try)
 2113 {
 2114         static const struct timeval printinterval = { 60, 0 };
 2115         static struct timeval lastprint;
 2116         static vm_pindex_t colour;
 2117         struct vpgqueues *pq;
 2118         int bit, field;
 2119         pv_entry_t pv;
 2120         struct pv_chunk *pc;
 2121         vm_page_t m;
 2122 
 2123         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2124         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2125         PV_STAT(pv_entry_allocs++);
 2126         pv_entry_count++;
 2127         if (pv_entry_count > pv_entry_high_water)
 2128                 if (ratecheck(&lastprint, &printinterval))
 2129                         printf("Approaching the limit on PV entries, consider "
 2130                             "increasing either the vm.pmap.shpgperproc or the "
 2131                             "vm.pmap.pv_entry_max sysctl.\n");
 2132         pq = NULL;
 2133 retry:
 2134         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 2135         if (pc != NULL) {
 2136                 for (field = 0; field < _NPCM; field++) {
 2137                         if (pc->pc_map[field]) {
 2138                                 bit = bsfq(pc->pc_map[field]);
 2139                                 break;
 2140                         }
 2141                 }
 2142                 if (field < _NPCM) {
 2143                         pv = &pc->pc_pventry[field * 64 + bit];
 2144                         pc->pc_map[field] &= ~(1ul << bit);
 2145                         /* If this was the last item, move it to tail */
 2146                         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 &&
 2147                             pc->pc_map[2] == 0) {
 2148                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2149                                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 2150                         }
 2151                         PV_STAT(pv_entry_spare--);
 2152                         return (pv);
 2153                 }
 2154         }
 2155         /* No free items, allocate another chunk */
 2156         m = vm_page_alloc(NULL, colour, (pq == &vm_page_queues[PQ_ACTIVE] ?
 2157             VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL) | VM_ALLOC_NOOBJ |
 2158             VM_ALLOC_WIRED);
 2159         if (m == NULL) {
 2160                 if (try) {
 2161                         pv_entry_count--;
 2162                         PV_STAT(pc_chunk_tryfail++);
 2163                         return (NULL);
 2164                 }
 2165                 /*
 2166                  * Reclaim pv entries: At first, destroy mappings to inactive
 2167                  * pages.  After that, if a pv chunk entry is still needed,
 2168                  * destroy mappings to active pages.
 2169                  */
 2170                 if (pq == NULL) {
 2171                         PV_STAT(pmap_collect_inactive++);
 2172                         pq = &vm_page_queues[PQ_INACTIVE];
 2173                 } else if (pq == &vm_page_queues[PQ_INACTIVE]) {
 2174                         PV_STAT(pmap_collect_active++);
 2175                         pq = &vm_page_queues[PQ_ACTIVE];
 2176                 } else
 2177                         panic("get_pv_entry: increase vm.pmap.shpgperproc");
 2178                 pmap_collect(pmap, pq);
 2179                 goto retry;
 2180         }
 2181         PV_STAT(pc_chunk_count++);
 2182         PV_STAT(pc_chunk_allocs++);
 2183         colour++;
 2184         dump_add_page(m->phys_addr);
 2185         pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 2186         pc->pc_pmap = pmap;
 2187         pc->pc_map[0] = PC_FREE0 & ~1ul;        /* preallocated bit 0 */
 2188         pc->pc_map[1] = PC_FREE1;
 2189         pc->pc_map[2] = PC_FREE2;
 2190         pv = &pc->pc_pventry[0];
 2191         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2192         PV_STAT(pv_entry_spare += _NPCPV - 1);
 2193         return (pv);
 2194 }
 2195 
 2196 /*
 2197  * First find and then remove the pv entry for the specified pmap and virtual
 2198  * address from the specified pv list.  Returns the pv entry if found and NULL
 2199  * otherwise.  This operation can be performed on pv lists for either 4KB or
 2200  * 2MB page mappings.
 2201  */
 2202 static __inline pv_entry_t
 2203 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2204 {
 2205         pv_entry_t pv;
 2206 
 2207         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2208         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 2209                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 2210                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
 2211                         break;
 2212                 }
 2213         }
 2214         return (pv);
 2215 }
 2216 
 2217 /*
 2218  * After demotion from a 2MB page mapping to 512 4KB page mappings,
 2219  * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
 2220  * entries for each of the 4KB page mappings.
 2221  */
 2222 static void
 2223 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2224 {
 2225         struct md_page *pvh;
 2226         pv_entry_t pv;
 2227         vm_offset_t va_last;
 2228         vm_page_t m;
 2229 
 2230         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2231         KASSERT((pa & PDRMASK) == 0,
 2232             ("pmap_pv_demote_pde: pa is not 2mpage aligned"));
 2233 
 2234         /*
 2235          * Transfer the 2mpage's pv entry for this mapping to the first
 2236          * page's pv list.
 2237          */
 2238         pvh = pa_to_pvh(pa);
 2239         va = trunc_2mpage(va);
 2240         pv = pmap_pvh_remove(pvh, pmap, va);
 2241         KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
 2242         m = PHYS_TO_VM_PAGE(pa);
 2243         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2244         /* Instantiate the remaining NPTEPG - 1 pv entries. */
 2245         va_last = va + NBPDR - PAGE_SIZE;
 2246         do {
 2247                 m++;
 2248                 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0,
 2249                     ("pmap_pv_demote_pde: page %p is not managed", m));
 2250                 va += PAGE_SIZE;
 2251                 pmap_insert_entry(pmap, va, m);
 2252         } while (va < va_last);
 2253 }
 2254 
 2255 /*
 2256  * After promotion from 512 4KB page mappings to a single 2MB page mapping,
 2257  * replace the many pv entries for the 4KB page mappings by a single pv entry
 2258  * for the 2MB page mapping.
 2259  */
 2260 static void
 2261 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2262 {
 2263         struct md_page *pvh;
 2264         pv_entry_t pv;
 2265         vm_offset_t va_last;
 2266         vm_page_t m;
 2267 
 2268         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2269         KASSERT((pa & PDRMASK) == 0,
 2270             ("pmap_pv_promote_pde: pa is not 2mpage aligned"));
 2271 
 2272         /*
 2273          * Transfer the first page's pv entry for this mapping to the
 2274          * 2mpage's pv list.  Aside from avoiding the cost of a call
 2275          * to get_pv_entry(), a transfer avoids the possibility that
 2276          * get_pv_entry() calls pmap_collect() and that pmap_collect()
 2277          * removes one of the mappings that is being promoted.
 2278          */
 2279         m = PHYS_TO_VM_PAGE(pa);
 2280         va = trunc_2mpage(va);
 2281         pv = pmap_pvh_remove(&m->md, pmap, va);
 2282         KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
 2283         pvh = pa_to_pvh(pa);
 2284         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
 2285         /* Free the remaining NPTEPG - 1 pv entries. */
 2286         va_last = va + NBPDR - PAGE_SIZE;
 2287         do {
 2288                 m++;
 2289                 va += PAGE_SIZE;
 2290                 pmap_pvh_free(&m->md, pmap, va);
 2291         } while (va < va_last);
 2292 }
 2293 
 2294 /*
 2295  * First find and then destroy the pv entry for the specified pmap and virtual
 2296  * address.  This operation can be performed on pv lists for either 4KB or 2MB
 2297  * page mappings.
 2298  */
 2299 static void
 2300 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2301 {
 2302         pv_entry_t pv;
 2303 
 2304         pv = pmap_pvh_remove(pvh, pmap, va);
 2305         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 2306         free_pv_entry(pmap, pv);
 2307 }
 2308 
 2309 static void
 2310 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 2311 {
 2312         struct md_page *pvh;
 2313 
 2314         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2315         pmap_pvh_free(&m->md, pmap, va);
 2316         if (TAILQ_EMPTY(&m->md.pv_list)) {
 2317                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2318                 if (TAILQ_EMPTY(&pvh->pv_list))
 2319                         vm_page_flag_clear(m, PG_WRITEABLE);
 2320         }
 2321 }
 2322 
 2323 /*
 2324  * Create a pv entry for page at pa for
 2325  * (pmap, va).
 2326  */
 2327 static void
 2328 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2329 {
 2330         pv_entry_t pv;
 2331 
 2332         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2333         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2334         pv = get_pv_entry(pmap, FALSE);
 2335         pv->pv_va = va;
 2336         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2337 }
 2338 
 2339 /*
 2340  * Conditionally create a pv entry.
 2341  */
 2342 static boolean_t
 2343 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2344 {
 2345         pv_entry_t pv;
 2346 
 2347         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2348         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2349         if (pv_entry_count < pv_entry_high_water && 
 2350             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 2351                 pv->pv_va = va;
 2352                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2353                 return (TRUE);
 2354         } else
 2355                 return (FALSE);
 2356 }
 2357 
 2358 /*
 2359  * Create the pv entry for a 2MB page mapping.
 2360  */
 2361 static boolean_t
 2362 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2363 {
 2364         struct md_page *pvh;
 2365         pv_entry_t pv;
 2366 
 2367         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2368         if (pv_entry_count < pv_entry_high_water && 
 2369             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 2370                 pv->pv_va = va;
 2371                 pvh = pa_to_pvh(pa);
 2372                 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_list);
 2373                 return (TRUE);
 2374         } else
 2375                 return (FALSE);
 2376 }
 2377 
 2378 /*
 2379  * Fills a page table page with mappings to consecutive physical pages.
 2380  */
 2381 static void
 2382 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
 2383 {
 2384         pt_entry_t *pte;
 2385 
 2386         for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
 2387                 *pte = newpte;
 2388                 newpte += PAGE_SIZE;
 2389         }
 2390 }
 2391 
 2392 /*
 2393  * Tries to demote a 2MB page mapping.  If demotion fails, the 2MB page
 2394  * mapping is invalidated.
 2395  */
 2396 static boolean_t
 2397 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2398 {
 2399         pd_entry_t newpde, oldpde;
 2400         pt_entry_t *firstpte, newpte;
 2401         vm_paddr_t mptepa;
 2402         vm_page_t free, mpte;
 2403 
 2404         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2405         oldpde = *pde;
 2406         KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
 2407             ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
 2408         mpte = pmap_lookup_pt_page(pmap, va);
 2409         if (mpte != NULL)
 2410                 pmap_remove_pt_page(pmap, mpte);
 2411         else {
 2412                 KASSERT((oldpde & PG_W) == 0,
 2413                     ("pmap_demote_pde: page table page for a wired mapping"
 2414                     " is missing"));
 2415 
 2416                 /*
 2417                  * Invalidate the 2MB page mapping and return "failure" if the
 2418                  * mapping was never accessed or the allocation of the new
 2419                  * page table page fails.  If the 2MB page mapping belongs to
 2420                  * the direct map region of the kernel's address space, then
 2421                  * the page allocation request specifies the highest possible
 2422                  * priority (VM_ALLOC_INTERRUPT).  Otherwise, the priority is
 2423                  * normal.  Page table pages are preallocated for every other
 2424                  * part of the kernel address space, so the direct map region
 2425                  * is the only part of the kernel address space that must be
 2426                  * handled here.
 2427                  */
 2428                 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
 2429                     pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va <
 2430                     DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) |
 2431                     VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 2432                         free = NULL;
 2433                         pmap_remove_pde(pmap, pde, trunc_2mpage(va), &free);
 2434                         pmap_invalidate_page(pmap, trunc_2mpage(va));
 2435                         pmap_free_zero_pages(free);
 2436                         CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx"
 2437                             " in pmap %p", va, pmap);
 2438                         return (FALSE);
 2439                 }
 2440                 if (va < VM_MAXUSER_ADDRESS)
 2441                         pmap->pm_stats.resident_count++;
 2442         }
 2443         mptepa = VM_PAGE_TO_PHYS(mpte);
 2444         firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa);
 2445         newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
 2446         KASSERT((oldpde & PG_A) != 0,
 2447             ("pmap_demote_pde: oldpde is missing PG_A"));
 2448         KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
 2449             ("pmap_demote_pde: oldpde is missing PG_M"));
 2450         newpte = oldpde & ~PG_PS;
 2451         if ((newpte & PG_PDE_PAT) != 0)
 2452                 newpte ^= PG_PDE_PAT | PG_PTE_PAT;
 2453 
 2454         /*
 2455          * If the page table page is new, initialize it.
 2456          */
 2457         if (mpte->wire_count == 1) {
 2458                 mpte->wire_count = NPTEPG;
 2459                 pmap_fill_ptp(firstpte, newpte);
 2460         }
 2461         KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
 2462             ("pmap_demote_pde: firstpte and newpte map different physical"
 2463             " addresses"));
 2464 
 2465         /*
 2466          * If the mapping has changed attributes, update the page table
 2467          * entries.
 2468          */
 2469         if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
 2470                 pmap_fill_ptp(firstpte, newpte);
 2471 
 2472         /*
 2473          * Demote the mapping.  This pmap is locked.  The old PDE has
 2474          * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
 2475          * set.  Thus, there is no danger of a race with another
 2476          * processor changing the setting of PG_A and/or PG_M between
 2477          * the read above and the store below. 
 2478          */
 2479         if (workaround_erratum383)
 2480                 pmap_update_pde(pmap, va, pde, newpde);
 2481         else
 2482                 pde_store(pde, newpde);
 2483 
 2484         /*
 2485          * Invalidate a stale recursive mapping of the page table page.
 2486          */
 2487         if (va >= VM_MAXUSER_ADDRESS)
 2488                 pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 2489 
 2490         /*
 2491          * Demote the pv entry.  This depends on the earlier demotion
 2492          * of the mapping.  Specifically, the (re)creation of a per-
 2493          * page pv entry might trigger the execution of pmap_collect(),
 2494          * which might reclaim a newly (re)created per-page pv entry
 2495          * and destroy the associated mapping.  In order to destroy
 2496          * the mapping, the PDE must have already changed from mapping
 2497          * the 2mpage to referencing the page table page.
 2498          */
 2499         if ((oldpde & PG_MANAGED) != 0)
 2500                 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);
 2501 
 2502         pmap_pde_demotions++;
 2503         CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx"
 2504             " in pmap %p", va, pmap);
 2505         return (TRUE);
 2506 }
 2507 
 2508 /*
 2509  * pmap_remove_pde: do the things to unmap a superpage in a process
 2510  */
 2511 static int
 2512 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 2513     vm_page_t *free)
 2514 {
 2515         struct md_page *pvh;
 2516         pd_entry_t oldpde;
 2517         vm_offset_t eva, va;
 2518         vm_page_t m, mpte;
 2519 
 2520         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2521         KASSERT((sva & PDRMASK) == 0,
 2522             ("pmap_remove_pde: sva is not 2mpage aligned"));
 2523         oldpde = pte_load_clear(pdq);
 2524         if (oldpde & PG_W)
 2525                 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
 2526 
 2527         /*
 2528          * Machines that don't support invlpg, also don't support
 2529          * PG_G.
 2530          */
 2531         if (oldpde & PG_G)
 2532                 pmap_invalidate_page(kernel_pmap, sva);
 2533         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 2534         if (oldpde & PG_MANAGED) {
 2535                 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
 2536                 pmap_pvh_free(pvh, pmap, sva);
 2537                 eva = sva + NBPDR;
 2538                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 2539                     va < eva; va += PAGE_SIZE, m++) {
 2540                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2541                                 vm_page_dirty(m);
 2542                         if (oldpde & PG_A)
 2543                                 vm_page_flag_set(m, PG_REFERENCED);
 2544                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 2545                             TAILQ_EMPTY(&pvh->pv_list))
 2546                                 vm_page_flag_clear(m, PG_WRITEABLE);
 2547                 }
 2548         }
 2549         if (pmap == kernel_pmap) {
 2550                 if (!pmap_demote_pde(pmap, pdq, sva))
 2551                         panic("pmap_remove_pde: failed demotion");
 2552         } else {
 2553                 mpte = pmap_lookup_pt_page(pmap, sva);
 2554                 if (mpte != NULL) {
 2555                         pmap_remove_pt_page(pmap, mpte);
 2556                         pmap->pm_stats.resident_count--;
 2557                         KASSERT(mpte->wire_count == NPTEPG,
 2558                             ("pmap_remove_pde: pte page wire count error"));
 2559                         mpte->wire_count = 0;
 2560                         pmap_add_delayed_free_list(mpte, free, FALSE);
 2561                         atomic_subtract_int(&cnt.v_wire_count, 1);
 2562                 }
 2563         }
 2564         return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free));
 2565 }
 2566 
 2567 /*
 2568  * pmap_remove_pte: do the things to unmap a page in a process
 2569  */
 2570 static int
 2571 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 
 2572     pd_entry_t ptepde, vm_page_t *free)
 2573 {
 2574         pt_entry_t oldpte;
 2575         vm_page_t m;
 2576 
 2577         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2578         oldpte = pte_load_clear(ptq);
 2579         if (oldpte & PG_W)
 2580                 pmap->pm_stats.wired_count -= 1;
 2581         pmap->pm_stats.resident_count -= 1;
 2582         if (oldpte & PG_MANAGED) {
 2583                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 2584                 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2585                         vm_page_dirty(m);
 2586                 if (oldpte & PG_A)
 2587                         vm_page_flag_set(m, PG_REFERENCED);
 2588                 pmap_remove_entry(pmap, m, va);
 2589         }
 2590         return (pmap_unuse_pt(pmap, va, ptepde, free));
 2591 }
 2592 
 2593 /*
 2594  * Remove a single page from a process address space
 2595  */
 2596 static void
 2597 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, vm_page_t *free)
 2598 {
 2599         pt_entry_t *pte;
 2600 
 2601         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2602         if ((*pde & PG_V) == 0)
 2603                 return;
 2604         pte = pmap_pde_to_pte(pde, va);
 2605         if ((*pte & PG_V) == 0)
 2606                 return;
 2607         pmap_remove_pte(pmap, pte, va, *pde, free);
 2608         pmap_invalidate_page(pmap, va);
 2609 }
 2610 
 2611 /*
 2612  *      Remove the given range of addresses from the specified map.
 2613  *
 2614  *      It is assumed that the start and end are properly
 2615  *      rounded to the page size.
 2616  */
 2617 void
 2618 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 2619 {
 2620         vm_offset_t va, va_next;
 2621         pml4_entry_t *pml4e;
 2622         pdp_entry_t *pdpe;
 2623         pd_entry_t ptpaddr, *pde;
 2624         pt_entry_t *pte;
 2625         vm_page_t free = NULL;
 2626         int anyvalid;
 2627 
 2628         /*
 2629          * Perform an unsynchronized read.  This is, however, safe.
 2630          */
 2631         if (pmap->pm_stats.resident_count == 0)
 2632                 return;
 2633 
 2634         anyvalid = 0;
 2635 
 2636         vm_page_lock_queues();
 2637         PMAP_LOCK(pmap);
 2638 
 2639         /*
 2640          * special handling of removing one page.  a very
 2641          * common operation and easy to short circuit some
 2642          * code.
 2643          */
 2644         if (sva + PAGE_SIZE == eva) {
 2645                 pde = pmap_pde(pmap, sva);
 2646                 if (pde && (*pde & PG_PS) == 0) {
 2647                         pmap_remove_page(pmap, sva, pde, &free);
 2648                         goto out;
 2649                 }
 2650         }
 2651 
 2652         for (; sva < eva; sva = va_next) {
 2653 
 2654                 if (pmap->pm_stats.resident_count == 0)
 2655                         break;
 2656 
 2657                 pml4e = pmap_pml4e(pmap, sva);
 2658                 if ((*pml4e & PG_V) == 0) {
 2659                         va_next = (sva + NBPML4) & ~PML4MASK;
 2660                         if (va_next < sva)
 2661                                 va_next = eva;
 2662                         continue;
 2663                 }
 2664 
 2665                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 2666                 if ((*pdpe & PG_V) == 0) {
 2667                         va_next = (sva + NBPDP) & ~PDPMASK;
 2668                         if (va_next < sva)
 2669                                 va_next = eva;
 2670                         continue;
 2671                 }
 2672 
 2673                 /*
 2674                  * Calculate index for next page table.
 2675                  */
 2676                 va_next = (sva + NBPDR) & ~PDRMASK;
 2677                 if (va_next < sva)
 2678                         va_next = eva;
 2679 
 2680                 pde = pmap_pdpe_to_pde(pdpe, sva);
 2681                 ptpaddr = *pde;
 2682 
 2683                 /*
 2684                  * Weed out invalid mappings.
 2685                  */
 2686                 if (ptpaddr == 0)
 2687                         continue;
 2688 
 2689                 /*
 2690                  * Check for large page.
 2691                  */
 2692                 if ((ptpaddr & PG_PS) != 0) {
 2693                         /*
 2694                          * Are we removing the entire large page?  If not,
 2695                          * demote the mapping and fall through.
 2696                          */
 2697                         if (sva + NBPDR == va_next && eva >= va_next) {
 2698                                 /*
 2699                                  * The TLB entry for a PG_G mapping is
 2700                                  * invalidated by pmap_remove_pde().
 2701                                  */
 2702                                 if ((ptpaddr & PG_G) == 0)
 2703                                         anyvalid = 1;
 2704                                 pmap_remove_pde(pmap, pde, sva, &free);
 2705                                 continue;
 2706                         } else if (!pmap_demote_pde(pmap, pde, sva)) {
 2707                                 /* The large page mapping was destroyed. */
 2708                                 continue;
 2709                         } else
 2710                                 ptpaddr = *pde;
 2711                 }
 2712 
 2713                 /*
 2714                  * Limit our scan to either the end of the va represented
 2715                  * by the current page table page, or to the end of the
 2716                  * range being removed.
 2717                  */
 2718                 if (va_next > eva)
 2719                         va_next = eva;
 2720 
 2721                 va = va_next;
 2722                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 2723                     sva += PAGE_SIZE) {
 2724                         if (*pte == 0) {
 2725                                 if (va != va_next) {
 2726                                         pmap_invalidate_range(pmap, va, sva);
 2727                                         va = va_next;
 2728                                 }
 2729                                 continue;
 2730                         }
 2731                         if ((*pte & PG_G) == 0)
 2732                                 anyvalid = 1;
 2733                         else if (va == va_next)
 2734                                 va = sva;
 2735                         if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free)) {
 2736                                 sva += PAGE_SIZE;
 2737                                 break;
 2738                         }
 2739                 }
 2740                 if (va != va_next)
 2741                         pmap_invalidate_range(pmap, va, sva);
 2742         }
 2743 out:
 2744         if (anyvalid)
 2745                 pmap_invalidate_all(pmap);
 2746         vm_page_unlock_queues();        
 2747         PMAP_UNLOCK(pmap);
 2748         pmap_free_zero_pages(free);
 2749 }
 2750 
 2751 /*
 2752  *      Routine:        pmap_remove_all
 2753  *      Function:
 2754  *              Removes this physical page from
 2755  *              all physical maps in which it resides.
 2756  *              Reflects back modify bits to the pager.
 2757  *
 2758  *      Notes:
 2759  *              Original versions of this routine were very
 2760  *              inefficient because they iteratively called
 2761  *              pmap_remove (slow...)
 2762  */
 2763 
 2764 void
 2765 pmap_remove_all(vm_page_t m)
 2766 {
 2767         struct md_page *pvh;
 2768         pv_entry_t pv;
 2769         pmap_t pmap;
 2770         pt_entry_t *pte, tpte;
 2771         pd_entry_t *pde;
 2772         vm_offset_t va;
 2773         vm_page_t free;
 2774 
 2775         KASSERT((m->flags & PG_FICTITIOUS) == 0,
 2776             ("pmap_remove_all: page %p is fictitious", m));
 2777         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2778         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2779         while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
 2780                 va = pv->pv_va;
 2781                 pmap = PV_PMAP(pv);
 2782                 PMAP_LOCK(pmap);
 2783                 pde = pmap_pde(pmap, va);
 2784                 (void)pmap_demote_pde(pmap, pde, va);
 2785                 PMAP_UNLOCK(pmap);
 2786         }
 2787         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 2788                 pmap = PV_PMAP(pv);
 2789                 PMAP_LOCK(pmap);
 2790                 pmap->pm_stats.resident_count--;
 2791                 pde = pmap_pde(pmap, pv->pv_va);
 2792                 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
 2793                     " a 2mpage in page %p's pv list", m));
 2794                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 2795                 tpte = pte_load_clear(pte);
 2796                 if (tpte & PG_W)
 2797                         pmap->pm_stats.wired_count--;
 2798                 if (tpte & PG_A)
 2799                         vm_page_flag_set(m, PG_REFERENCED);
 2800 
 2801                 /*
 2802                  * Update the vm_page_t clean and reference bits.
 2803                  */
 2804                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2805                         vm_page_dirty(m);
 2806                 free = NULL;
 2807                 pmap_unuse_pt(pmap, pv->pv_va, *pde, &free);
 2808                 pmap_invalidate_page(pmap, pv->pv_va);
 2809                 pmap_free_zero_pages(free);
 2810                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2811                 free_pv_entry(pmap, pv);
 2812                 PMAP_UNLOCK(pmap);
 2813         }
 2814         vm_page_flag_clear(m, PG_WRITEABLE);
 2815 }
 2816 
 2817 /*
 2818  * pmap_protect_pde: do the things to protect a 2mpage in a process
 2819  */
 2820 static boolean_t
 2821 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
 2822 {
 2823         pd_entry_t newpde, oldpde;
 2824         vm_offset_t eva, va;
 2825         vm_page_t m;
 2826         boolean_t anychanged;
 2827 
 2828         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2829         KASSERT((sva & PDRMASK) == 0,
 2830             ("pmap_protect_pde: sva is not 2mpage aligned"));
 2831         anychanged = FALSE;
 2832 retry:
 2833         oldpde = newpde = *pde;
 2834         if (oldpde & PG_MANAGED) {
 2835                 eva = sva + NBPDR;
 2836                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 2837                     va < eva; va += PAGE_SIZE, m++) {
 2838                         /*
 2839                          * In contrast to the analogous operation on a 4KB page
 2840                          * mapping, the mapping's PG_A flag is not cleared and
 2841                          * the page's PG_REFERENCED flag is not set.  The
 2842                          * reason is that pmap_demote_pde() expects that a 2MB
 2843                          * page mapping with a stored page table page has PG_A
 2844                          * set.
 2845                          */
 2846                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2847                                 vm_page_dirty(m);
 2848                 }
 2849         }
 2850         if ((prot & VM_PROT_WRITE) == 0)
 2851                 newpde &= ~(PG_RW | PG_M);
 2852         if ((prot & VM_PROT_EXECUTE) == 0)
 2853                 newpde |= pg_nx;
 2854         if (newpde != oldpde) {
 2855                 if (!atomic_cmpset_long(pde, oldpde, newpde))
 2856                         goto retry;
 2857                 if (oldpde & PG_G)
 2858                         pmap_invalidate_page(pmap, sva);
 2859                 else
 2860                         anychanged = TRUE;
 2861         }
 2862         return (anychanged);
 2863 }
 2864 
 2865 /*
 2866  *      Set the physical protection on the
 2867  *      specified range of this map as requested.
 2868  */
 2869 void
 2870 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 2871 {
 2872         vm_offset_t va_next;
 2873         pml4_entry_t *pml4e;
 2874         pdp_entry_t *pdpe;
 2875         pd_entry_t ptpaddr, *pde;
 2876         pt_entry_t *pte;
 2877         int anychanged;
 2878 
 2879         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 2880                 pmap_remove(pmap, sva, eva);
 2881                 return;
 2882         }
 2883 
 2884         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 2885             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 2886                 return;
 2887 
 2888         anychanged = 0;
 2889 
 2890         vm_page_lock_queues();
 2891         PMAP_LOCK(pmap);
 2892         for (; sva < eva; sva = va_next) {
 2893 
 2894                 pml4e = pmap_pml4e(pmap, sva);
 2895                 if ((*pml4e & PG_V) == 0) {
 2896                         va_next = (sva + NBPML4) & ~PML4MASK;
 2897                         if (va_next < sva)
 2898                                 va_next = eva;
 2899                         continue;
 2900                 }
 2901 
 2902                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 2903                 if ((*pdpe & PG_V) == 0) {
 2904                         va_next = (sva + NBPDP) & ~PDPMASK;
 2905                         if (va_next < sva)
 2906                                 va_next = eva;
 2907                         continue;
 2908                 }
 2909 
 2910                 va_next = (sva + NBPDR) & ~PDRMASK;
 2911                 if (va_next < sva)
 2912                         va_next = eva;
 2913 
 2914                 pde = pmap_pdpe_to_pde(pdpe, sva);
 2915                 ptpaddr = *pde;
 2916 
 2917                 /*
 2918                  * Weed out invalid mappings.
 2919                  */
 2920                 if (ptpaddr == 0)
 2921                         continue;
 2922 
 2923                 /*
 2924                  * Check for large page.
 2925                  */
 2926                 if ((ptpaddr & PG_PS) != 0) {
 2927                         /*
 2928                          * Are we protecting the entire large page?  If not,
 2929                          * demote the mapping and fall through.
 2930                          */
 2931                         if (sva + NBPDR == va_next && eva >= va_next) {
 2932                                 /*
 2933                                  * The TLB entry for a PG_G mapping is
 2934                                  * invalidated by pmap_protect_pde().
 2935                                  */
 2936                                 if (pmap_protect_pde(pmap, pde, sva, prot))
 2937                                         anychanged = 1;
 2938                                 continue;
 2939                         } else if (!pmap_demote_pde(pmap, pde, sva)) {
 2940                                 /* The large page mapping was destroyed. */
 2941                                 continue;
 2942                         }
 2943                 }
 2944 
 2945                 if (va_next > eva)
 2946                         va_next = eva;
 2947 
 2948                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 2949                     sva += PAGE_SIZE) {
 2950                         pt_entry_t obits, pbits;
 2951                         vm_page_t m;
 2952 
 2953 retry:
 2954                         obits = pbits = *pte;
 2955                         if ((pbits & PG_V) == 0)
 2956                                 continue;
 2957                         if (pbits & PG_MANAGED) {
 2958                                 m = NULL;
 2959                                 if (pbits & PG_A) {
 2960                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 2961                                         vm_page_flag_set(m, PG_REFERENCED);
 2962                                         pbits &= ~PG_A;
 2963                                 }
 2964                                 if ((pbits & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 2965                                         if (m == NULL)
 2966                                                 m = PHYS_TO_VM_PAGE(pbits &
 2967                                                     PG_FRAME);
 2968                                         vm_page_dirty(m);
 2969                                 }
 2970                         }
 2971 
 2972                         if ((prot & VM_PROT_WRITE) == 0)
 2973                                 pbits &= ~(PG_RW | PG_M);
 2974                         if ((prot & VM_PROT_EXECUTE) == 0)
 2975                                 pbits |= pg_nx;
 2976 
 2977                         if (pbits != obits) {
 2978                                 if (!atomic_cmpset_long(pte, obits, pbits))
 2979                                         goto retry;
 2980                                 if (obits & PG_G)
 2981                                         pmap_invalidate_page(pmap, sva);
 2982                                 else
 2983                                         anychanged = 1;
 2984                         }
 2985                 }
 2986         }
 2987         if (anychanged)
 2988                 pmap_invalidate_all(pmap);
 2989         vm_page_unlock_queues();
 2990         PMAP_UNLOCK(pmap);
 2991 }
 2992 
 2993 /*
 2994  * Tries to promote the 512, contiguous 4KB page mappings that are within a
 2995  * single page table page (PTP) to a single 2MB page mapping.  For promotion
 2996  * to occur, two conditions must be met: (1) the 4KB page mappings must map
 2997  * aligned, contiguous physical memory and (2) the 4KB page mappings must have
 2998  * identical characteristics. 
 2999  */
 3000 static void
 3001 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 3002 {
 3003         pd_entry_t newpde;
 3004         pt_entry_t *firstpte, oldpte, pa, *pte;
 3005         vm_offset_t oldpteva;
 3006         vm_page_t mpte;
 3007 
 3008         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3009 
 3010         /*
 3011          * Examine the first PTE in the specified PTP.  Abort if this PTE is
 3012          * either invalid, unused, or does not map the first 4KB physical page
 3013          * within a 2MB page. 
 3014          */
 3015         firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
 3016 setpde:
 3017         newpde = *firstpte;
 3018         if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
 3019                 pmap_pde_p_failures++;
 3020                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 3021                     " in pmap %p", va, pmap);
 3022                 return;
 3023         }
 3024         if ((newpde & (PG_M | PG_RW)) == PG_RW) {
 3025                 /*
 3026                  * When PG_M is already clear, PG_RW can be cleared without
 3027                  * a TLB invalidation.
 3028                  */
 3029                 if (!atomic_cmpset_long(firstpte, newpde, newpde & ~PG_RW))
 3030                         goto setpde;
 3031                 newpde &= ~PG_RW;
 3032         }
 3033 
 3034         /*
 3035          * Examine each of the other PTEs in the specified PTP.  Abort if this
 3036          * PTE maps an unexpected 4KB physical page or does not have identical
 3037          * characteristics to the first PTE.
 3038          */
 3039         pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + NBPDR - PAGE_SIZE;
 3040         for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
 3041 setpte:
 3042                 oldpte = *pte;
 3043                 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
 3044                         pmap_pde_p_failures++;
 3045                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 3046                             " in pmap %p", va, pmap);
 3047                         return;
 3048                 }
 3049                 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
 3050                         /*
 3051                          * When PG_M is already clear, PG_RW can be cleared
 3052                          * without a TLB invalidation.
 3053                          */
 3054                         if (!atomic_cmpset_long(pte, oldpte, oldpte & ~PG_RW))
 3055                                 goto setpte;
 3056                         oldpte &= ~PG_RW;
 3057                         oldpteva = (oldpte & PG_FRAME & PDRMASK) |
 3058                             (va & ~PDRMASK);
 3059                         CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx"
 3060                             " in pmap %p", oldpteva, pmap);
 3061                 }
 3062                 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
 3063                         pmap_pde_p_failures++;
 3064                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
 3065                             " in pmap %p", va, pmap);
 3066                         return;
 3067                 }
 3068                 pa -= PAGE_SIZE;
 3069         }
 3070 
 3071         /*
 3072          * Save the page table page in its current state until the PDE
 3073          * mapping the superpage is demoted by pmap_demote_pde() or
 3074          * destroyed by pmap_remove_pde(). 
 3075          */
 3076         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 3077         KASSERT(mpte >= vm_page_array &&
 3078             mpte < &vm_page_array[vm_page_array_size],
 3079             ("pmap_promote_pde: page table page is out of range"));
 3080         KASSERT(mpte->pindex == pmap_pde_pindex(va),
 3081             ("pmap_promote_pde: page table page's pindex is wrong"));
 3082         pmap_insert_pt_page(pmap, mpte);
 3083 
 3084         /*
 3085          * Promote the pv entries.
 3086          */
 3087         if ((newpde & PG_MANAGED) != 0)
 3088                 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
 3089 
 3090         /*
 3091          * Propagate the PAT index to its proper position.
 3092          */
 3093         if ((newpde & PG_PTE_PAT) != 0)
 3094                 newpde ^= PG_PDE_PAT | PG_PTE_PAT;
 3095 
 3096         /*
 3097          * Map the superpage.
 3098          */
 3099         if (workaround_erratum383)
 3100                 pmap_update_pde(pmap, va, pde, PG_PS | newpde);
 3101         else
 3102                 pde_store(pde, PG_PS | newpde);
 3103 
 3104         pmap_pde_promotions++;
 3105         CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
 3106             " in pmap %p", va, pmap);
 3107 }
 3108 
 3109 /*
 3110  *      Insert the given physical page (p) at
 3111  *      the specified virtual address (v) in the
 3112  *      target physical map with the protection requested.
 3113  *
 3114  *      If specified, the page will be wired down, meaning
 3115  *      that the related pte can not be reclaimed.
 3116  *
 3117  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 3118  *      or lose information.  That is, this routine must actually
 3119  *      insert this page into the given map NOW.
 3120  */
 3121 void
 3122 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
 3123     vm_prot_t prot, boolean_t wired)
 3124 {
 3125         vm_paddr_t pa;
 3126         pd_entry_t *pde;
 3127         pt_entry_t *pte;
 3128         vm_paddr_t opa;
 3129         pt_entry_t origpte, newpte;
 3130         vm_page_t mpte, om;
 3131         boolean_t invlva;
 3132 
 3133         va = trunc_page(va);
 3134         KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
 3135         KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS,
 3136             ("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", va));
 3137 
 3138         mpte = NULL;
 3139 
 3140         vm_page_lock_queues();
 3141         PMAP_LOCK(pmap);
 3142 
 3143         /*
 3144          * In the case that a page table page is not
 3145          * resident, we are creating it here.
 3146          */
 3147         if (va < VM_MAXUSER_ADDRESS) {
 3148                 mpte = pmap_allocpte(pmap, va, M_WAITOK);
 3149         }
 3150 
 3151         pde = pmap_pde(pmap, va);
 3152         if (pde != NULL && (*pde & PG_V) != 0) {
 3153                 if ((*pde & PG_PS) != 0)
 3154                         panic("pmap_enter: attempted pmap_enter on 2MB page");
 3155                 pte = pmap_pde_to_pte(pde, va);
 3156         } else
 3157                 panic("pmap_enter: invalid page directory va=%#lx", va);
 3158 
 3159         pa = VM_PAGE_TO_PHYS(m);
 3160         om = NULL;
 3161         origpte = *pte;
 3162         opa = origpte & PG_FRAME;
 3163 
 3164         /*
 3165          * Mapping has not changed, must be protection or wiring change.
 3166          */
 3167         if (origpte && (opa == pa)) {
 3168                 /*
 3169                  * Wiring change, just update stats. We don't worry about
 3170                  * wiring PT pages as they remain resident as long as there
 3171                  * are valid mappings in them. Hence, if a user page is wired,
 3172                  * the PT page will be also.
 3173                  */
 3174                 if (wired && ((origpte & PG_W) == 0))
 3175                         pmap->pm_stats.wired_count++;
 3176                 else if (!wired && (origpte & PG_W))
 3177                         pmap->pm_stats.wired_count--;
 3178 
 3179                 /*
 3180                  * Remove extra pte reference
 3181                  */
 3182                 if (mpte)
 3183                         mpte->wire_count--;
 3184 
 3185                 /*
 3186                  * We might be turning off write access to the page,
 3187                  * so we go ahead and sense modify status.
 3188                  */
 3189                 if (origpte & PG_MANAGED) {
 3190                         om = m;
 3191                         pa |= PG_MANAGED;
 3192                 }
 3193                 goto validate;
 3194         } 
 3195         /*
 3196          * Mapping has changed, invalidate old range and fall through to
 3197          * handle validating new mapping.
 3198          */
 3199         if (opa) {
 3200                 if (origpte & PG_W)
 3201                         pmap->pm_stats.wired_count--;
 3202                 if (origpte & PG_MANAGED) {
 3203                         om = PHYS_TO_VM_PAGE(opa);
 3204                         pmap_remove_entry(pmap, om, va);
 3205                 }
 3206                 if (mpte != NULL) {
 3207                         mpte->wire_count--;
 3208                         KASSERT(mpte->wire_count > 0,
 3209                             ("pmap_enter: missing reference to page table page,"
 3210                              " va: 0x%lx", va));
 3211                 }
 3212         } else
 3213                 pmap->pm_stats.resident_count++;
 3214 
 3215         /*
 3216          * Enter on the PV list if part of our managed memory.
 3217          */
 3218         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
 3219                 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
 3220                     ("pmap_enter: managed mapping within the clean submap"));
 3221                 pmap_insert_entry(pmap, va, m);
 3222                 pa |= PG_MANAGED;
 3223         }
 3224 
 3225         /*
 3226          * Increment counters
 3227          */
 3228         if (wired)
 3229                 pmap->pm_stats.wired_count++;
 3230 
 3231 validate:
 3232         /*
 3233          * Now validate mapping with desired protection/wiring.
 3234          */
 3235         newpte = (pt_entry_t)(pa | pmap_cache_bits(m->md.pat_mode, 0) | PG_V);
 3236         if ((prot & VM_PROT_WRITE) != 0) {
 3237                 newpte |= PG_RW;
 3238                 vm_page_flag_set(m, PG_WRITEABLE);
 3239         }
 3240         if ((prot & VM_PROT_EXECUTE) == 0)
 3241                 newpte |= pg_nx;
 3242         if (wired)
 3243                 newpte |= PG_W;
 3244         if (va < VM_MAXUSER_ADDRESS)
 3245                 newpte |= PG_U;
 3246         if (pmap == kernel_pmap)
 3247                 newpte |= PG_G;
 3248 
 3249         /*
 3250          * if the mapping or permission bits are different, we need
 3251          * to update the pte.
 3252          */
 3253         if ((origpte & ~(PG_M|PG_A)) != newpte) {
 3254                 newpte |= PG_A;
 3255                 if ((access & VM_PROT_WRITE) != 0)
 3256                         newpte |= PG_M;
 3257                 if (origpte & PG_V) {
 3258                         invlva = FALSE;
 3259                         origpte = pte_load_store(pte, newpte);
 3260                         if (origpte & PG_A) {
 3261                                 if (origpte & PG_MANAGED)
 3262                                         vm_page_flag_set(om, PG_REFERENCED);
 3263                                 if (opa != VM_PAGE_TO_PHYS(m) || ((origpte &
 3264                                     PG_NX) == 0 && (newpte & PG_NX)))
 3265                                         invlva = TRUE;
 3266                         }
 3267                         if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 3268                                 if ((origpte & PG_MANAGED) != 0)
 3269                                         vm_page_dirty(om);
 3270                                 if ((newpte & PG_RW) == 0)
 3271                                         invlva = TRUE;
 3272                         }
 3273                         if (invlva)
 3274                                 pmap_invalidate_page(pmap, va);
 3275                 } else
 3276                         pte_store(pte, newpte);
 3277         }
 3278 
 3279         /*
 3280          * If both the page table page and the reservation are fully
 3281          * populated, then attempt promotion.
 3282          */
 3283         if ((mpte == NULL || mpte->wire_count == NPTEPG) &&
 3284             pg_ps_enabled && vm_reserv_level_iffullpop(m) == 0)
 3285                 pmap_promote_pde(pmap, pde, va);
 3286 
 3287         vm_page_unlock_queues();
 3288         PMAP_UNLOCK(pmap);
 3289 }
 3290 
 3291 /*
 3292  * Tries to create a 2MB page mapping.  Returns TRUE if successful and FALSE
 3293  * otherwise.  Fails if (1) a page table page cannot be allocated without
 3294  * blocking, (2) a mapping already exists at the specified virtual address, or
 3295  * (3) a pv entry cannot be allocated without reclaiming another pv entry. 
 3296  */
 3297 static boolean_t
 3298 pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 3299 {
 3300         pd_entry_t *pde, newpde;
 3301         vm_page_t free, mpde;
 3302 
 3303         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3304         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3305         if ((mpde = pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) {
 3306                 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3307                     " in pmap %p", va, pmap);
 3308                 return (FALSE);
 3309         }
 3310         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpde));
 3311         pde = &pde[pmap_pde_index(va)];
 3312         if ((*pde & PG_V) != 0) {
 3313                 KASSERT(mpde->wire_count > 1,
 3314                     ("pmap_enter_pde: mpde's wire count is too low"));
 3315                 mpde->wire_count--;
 3316                 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3317                     " in pmap %p", va, pmap);
 3318                 return (FALSE);
 3319         }
 3320         newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 1) |
 3321             PG_PS | PG_V;
 3322         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
 3323                 newpde |= PG_MANAGED;
 3324 
 3325                 /*
 3326                  * Abort this mapping if its PV entry could not be created.
 3327                  */
 3328                 if (!pmap_pv_insert_pde(pmap, va, VM_PAGE_TO_PHYS(m))) {
 3329                         free = NULL;
 3330                         if (pmap_unwire_pte_hold(pmap, va, mpde, &free)) {
 3331                                 pmap_invalidate_page(pmap, va);
 3332                                 pmap_free_zero_pages(free);
 3333                         }
 3334                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3335                             " in pmap %p", va, pmap);
 3336                         return (FALSE);
 3337                 }
 3338         }
 3339         if ((prot & VM_PROT_EXECUTE) == 0)
 3340                 newpde |= pg_nx;
 3341         if (va < VM_MAXUSER_ADDRESS)
 3342                 newpde |= PG_U;
 3343 
 3344         /*
 3345          * Increment counters.
 3346          */
 3347         pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
 3348 
 3349         /*
 3350          * Map the superpage.
 3351          */
 3352         pde_store(pde, newpde);
 3353 
 3354         pmap_pde_mappings++;
 3355         CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
 3356             " in pmap %p", va, pmap);
 3357         return (TRUE);
 3358 }
 3359 
 3360 /*
 3361  * Maps a sequence of resident pages belonging to the same object.
 3362  * The sequence begins with the given page m_start.  This page is
 3363  * mapped at the given virtual address start.  Each subsequent page is
 3364  * mapped at a virtual address that is offset from start by the same
 3365  * amount as the page is offset from m_start within the object.  The
 3366  * last page in the sequence is the page with the largest offset from
 3367  * m_start that can be mapped at a virtual address less than the given
 3368  * virtual address end.  Not every virtual page between start and end
 3369  * is mapped; only those for which a resident page exists with the
 3370  * corresponding offset from m_start are mapped.
 3371  */
 3372 void
 3373 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 3374     vm_page_t m_start, vm_prot_t prot)
 3375 {
 3376         vm_offset_t va;
 3377         vm_page_t m, mpte;
 3378         vm_pindex_t diff, psize;
 3379 
 3380         VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
 3381         psize = atop(end - start);
 3382         mpte = NULL;
 3383         m = m_start;
 3384         PMAP_LOCK(pmap);
 3385         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 3386                 va = start + ptoa(diff);
 3387                 if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
 3388                     (VM_PAGE_TO_PHYS(m) & PDRMASK) == 0 &&
 3389                     pg_ps_enabled && vm_reserv_level_iffullpop(m) == 0 &&
 3390                     pmap_enter_pde(pmap, va, m, prot))
 3391                         m = &m[NBPDR / PAGE_SIZE - 1];
 3392                 else
 3393                         mpte = pmap_enter_quick_locked(pmap, va, m, prot,
 3394                             mpte);
 3395                 m = TAILQ_NEXT(m, listq);
 3396         }
 3397         PMAP_UNLOCK(pmap);
 3398 }
 3399 
 3400 /*
 3401  * this code makes some *MAJOR* assumptions:
 3402  * 1. Current pmap & pmap exists.
 3403  * 2. Not wired.
 3404  * 3. Read access.
 3405  * 4. No page table pages.
 3406  * but is *MUCH* faster than pmap_enter...
 3407  */
 3408 
 3409 void
 3410 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 3411 {
 3412 
 3413         PMAP_LOCK(pmap);
 3414         (void) pmap_enter_quick_locked(pmap, va, m, prot, NULL);
 3415         PMAP_UNLOCK(pmap);
 3416 }
 3417 
 3418 static vm_page_t
 3419 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 3420     vm_prot_t prot, vm_page_t mpte)
 3421 {
 3422         vm_page_t free;
 3423         pt_entry_t *pte;
 3424         vm_paddr_t pa;
 3425 
 3426         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 3427             (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
 3428             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 3429         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3430         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3431 
 3432         /*
 3433          * In the case that a page table page is not
 3434          * resident, we are creating it here.
 3435          */
 3436         if (va < VM_MAXUSER_ADDRESS) {
 3437                 vm_pindex_t ptepindex;
 3438                 pd_entry_t *ptepa;
 3439 
 3440                 /*
 3441                  * Calculate pagetable page index
 3442                  */
 3443                 ptepindex = pmap_pde_pindex(va);
 3444                 if (mpte && (mpte->pindex == ptepindex)) {
 3445                         mpte->wire_count++;
 3446                 } else {
 3447                         /*
 3448                          * Get the page directory entry
 3449                          */
 3450                         ptepa = pmap_pde(pmap, va);
 3451 
 3452                         /*
 3453                          * If the page table page is mapped, we just increment
 3454                          * the hold count, and activate it.
 3455                          */
 3456                         if (ptepa && (*ptepa & PG_V) != 0) {
 3457                                 if (*ptepa & PG_PS)
 3458                                         return (NULL);
 3459                                 mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
 3460                                 mpte->wire_count++;
 3461                         } else {
 3462                                 mpte = _pmap_allocpte(pmap, ptepindex,
 3463                                     M_NOWAIT);
 3464                                 if (mpte == NULL)
 3465                                         return (mpte);
 3466                         }
 3467                 }
 3468                 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte));
 3469                 pte = &pte[pmap_pte_index(va)];
 3470         } else {
 3471                 mpte = NULL;
 3472                 pte = vtopte(va);
 3473         }
 3474         if (*pte) {
 3475                 if (mpte != NULL) {
 3476                         mpte->wire_count--;
 3477                         mpte = NULL;
 3478                 }
 3479                 return (mpte);
 3480         }
 3481 
 3482         /*
 3483          * Enter on the PV list if part of our managed memory.
 3484          */
 3485         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 &&
 3486             !pmap_try_insert_pv_entry(pmap, va, m)) {
 3487                 if (mpte != NULL) {
 3488                         free = NULL;
 3489                         if (pmap_unwire_pte_hold(pmap, va, mpte, &free)) {
 3490                                 pmap_invalidate_page(pmap, va);
 3491                                 pmap_free_zero_pages(free);
 3492                         }
 3493                         mpte = NULL;
 3494                 }
 3495                 return (mpte);
 3496         }
 3497 
 3498         /*
 3499          * Increment counters
 3500          */
 3501         pmap->pm_stats.resident_count++;
 3502 
 3503         pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0);
 3504         if ((prot & VM_PROT_EXECUTE) == 0)
 3505                 pa |= pg_nx;
 3506 
 3507         /*
 3508          * Now validate mapping with RO protection
 3509          */
 3510         if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED))
 3511                 pte_store(pte, pa | PG_V | PG_U);
 3512         else
 3513                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 3514         return mpte;
 3515 }
 3516 
 3517 /*
 3518  * Make a temporary mapping for a physical address.  This is only intended
 3519  * to be used for panic dumps.
 3520  */
 3521 void *
 3522 pmap_kenter_temporary(vm_paddr_t pa, int i)
 3523 {
 3524         vm_offset_t va;
 3525 
 3526         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 3527         pmap_kenter(va, pa);
 3528         invlpg(va);
 3529         return ((void *)crashdumpmap);
 3530 }
 3531 
 3532 /*
 3533  * This code maps large physical mmap regions into the
 3534  * processor address space.  Note that some shortcuts
 3535  * are taken, but the code works.
 3536  */
 3537 void
 3538 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
 3539     vm_pindex_t pindex, vm_size_t size)
 3540 {
 3541         pd_entry_t *pde;
 3542         vm_paddr_t pa, ptepa;
 3543         vm_page_t p, pdpg;
 3544         int pat_mode;
 3545 
 3546         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 3547         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 3548             ("pmap_object_init_pt: non-device object"));
 3549         if ((addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
 3550                 if (!vm_object_populate(object, pindex, pindex + atop(size)))
 3551                         return;
 3552                 p = vm_page_lookup(object, pindex);
 3553                 KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3554                     ("pmap_object_init_pt: invalid page %p", p));
 3555                 pat_mode = p->md.pat_mode;
 3556 
 3557                 /*
 3558                  * Abort the mapping if the first page is not physically
 3559                  * aligned to a 2MB page boundary.
 3560                  */
 3561                 ptepa = VM_PAGE_TO_PHYS(p);
 3562                 if (ptepa & (NBPDR - 1))
 3563                         return;
 3564 
 3565                 /*
 3566                  * Skip the first page.  Abort the mapping if the rest of
 3567                  * the pages are not physically contiguous or have differing
 3568                  * memory attributes.
 3569                  */
 3570                 p = TAILQ_NEXT(p, listq);
 3571                 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
 3572                     pa += PAGE_SIZE) {
 3573                         KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3574                             ("pmap_object_init_pt: invalid page %p", p));
 3575                         if (pa != VM_PAGE_TO_PHYS(p) ||
 3576                             pat_mode != p->md.pat_mode)
 3577                                 return;
 3578                         p = TAILQ_NEXT(p, listq);
 3579                 }
 3580 
 3581                 /*
 3582                  * Map using 2MB pages.  Since "ptepa" is 2M aligned and
 3583                  * "size" is a multiple of 2M, adding the PAT setting to "pa"
 3584                  * will not affect the termination of this loop.
 3585                  */ 
 3586                 PMAP_LOCK(pmap);
 3587                 for (pa = ptepa | pmap_cache_bits(pat_mode, 1); pa < ptepa +
 3588                     size; pa += NBPDR) {
 3589                         pdpg = pmap_allocpde(pmap, addr, M_NOWAIT);
 3590                         if (pdpg == NULL) {
 3591                                 /*
 3592                                  * The creation of mappings below is only an
 3593                                  * optimization.  If a page directory page
 3594                                  * cannot be allocated without blocking,
 3595                                  * continue on to the next mapping rather than
 3596                                  * blocking.
 3597                                  */
 3598                                 addr += NBPDR;
 3599                                 continue;
 3600                         }
 3601                         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
 3602                         pde = &pde[pmap_pde_index(addr)];
 3603                         if ((*pde & PG_V) == 0) {
 3604                                 pde_store(pde, pa | PG_PS | PG_M | PG_A |
 3605                                     PG_U | PG_RW | PG_V);
 3606                                 pmap->pm_stats.resident_count += NBPDR /
 3607                                     PAGE_SIZE;
 3608                                 pmap_pde_mappings++;
 3609                         } else {
 3610                                 /* Continue on if the PDE is already valid. */
 3611                                 pdpg->wire_count--;
 3612                                 KASSERT(pdpg->wire_count > 0,
 3613                                     ("pmap_object_init_pt: missing reference "
 3614                                     "to page directory page, va: 0x%lx", addr));
 3615                         }
 3616                         addr += NBPDR;
 3617                 }
 3618                 PMAP_UNLOCK(pmap);
 3619         }
 3620 }
 3621 
 3622 /*
 3623  *      Routine:        pmap_change_wiring
 3624  *      Function:       Change the wiring attribute for a map/virtual-address
 3625  *                      pair.
 3626  *      In/out conditions:
 3627  *                      The mapping must already exist in the pmap.
 3628  */
 3629 void
 3630 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 3631 {
 3632         pd_entry_t *pde;
 3633         pt_entry_t *pte;
 3634         boolean_t are_queues_locked;
 3635 
 3636         are_queues_locked = FALSE;
 3637 
 3638         /*
 3639          * Wiring is not a hardware characteristic so there is no need to
 3640          * invalidate TLB.
 3641          */
 3642 retry:
 3643         PMAP_LOCK(pmap);
 3644         pde = pmap_pde(pmap, va);
 3645         if ((*pde & PG_PS) != 0) {
 3646                 if (!wired != ((*pde & PG_W) == 0)) {
 3647                         if (!are_queues_locked) {
 3648                                 are_queues_locked = TRUE;
 3649                                 if (!mtx_trylock(&vm_page_queue_mtx)) {
 3650                                         PMAP_UNLOCK(pmap);
 3651                                         vm_page_lock_queues();
 3652                                         goto retry;
 3653                                 }
 3654                         }
 3655                         if (!pmap_demote_pde(pmap, pde, va))
 3656                                 panic("pmap_change_wiring: demotion failed");
 3657                 } else
 3658                         goto out;
 3659         }
 3660         pte = pmap_pde_to_pte(pde, va);
 3661         if (wired && (*pte & PG_W) == 0) {
 3662                 pmap->pm_stats.wired_count++;
 3663                 atomic_set_long(pte, PG_W);
 3664         } else if (!wired && (*pte & PG_W) != 0) {
 3665                 pmap->pm_stats.wired_count--;
 3666                 atomic_clear_long(pte, PG_W);
 3667         }
 3668 out:
 3669         if (are_queues_locked)
 3670                 vm_page_unlock_queues();
 3671         PMAP_UNLOCK(pmap);
 3672 }
 3673 
 3674 
 3675 
 3676 /*
 3677  *      Copy the range specified by src_addr/len
 3678  *      from the source map to the range dst_addr/len
 3679  *      in the destination map.
 3680  *
 3681  *      This routine is only advisory and need not do anything.
 3682  */
 3683 
 3684 void
 3685 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 3686     vm_offset_t src_addr)
 3687 {
 3688         vm_page_t   free;
 3689         vm_offset_t addr;
 3690         vm_offset_t end_addr = src_addr + len;
 3691         vm_offset_t va_next;
 3692 
 3693         if (dst_addr != src_addr)
 3694                 return;
 3695 
 3696         vm_page_lock_queues();
 3697         if (dst_pmap < src_pmap) {
 3698                 PMAP_LOCK(dst_pmap);
 3699                 PMAP_LOCK(src_pmap);
 3700         } else {
 3701                 PMAP_LOCK(src_pmap);
 3702                 PMAP_LOCK(dst_pmap);
 3703         }
 3704         for (addr = src_addr; addr < end_addr; addr = va_next) {
 3705                 pt_entry_t *src_pte, *dst_pte;
 3706                 vm_page_t dstmpde, dstmpte, srcmpte;
 3707                 pml4_entry_t *pml4e;
 3708                 pdp_entry_t *pdpe;
 3709                 pd_entry_t srcptepaddr, *pde;
 3710 
 3711                 KASSERT(addr < UPT_MIN_ADDRESS,
 3712                     ("pmap_copy: invalid to pmap_copy page tables"));
 3713 
 3714                 pml4e = pmap_pml4e(src_pmap, addr);
 3715                 if ((*pml4e & PG_V) == 0) {
 3716                         va_next = (addr + NBPML4) & ~PML4MASK;
 3717                         if (va_next < addr)
 3718                                 va_next = end_addr;
 3719                         continue;
 3720                 }
 3721 
 3722                 pdpe = pmap_pml4e_to_pdpe(pml4e, addr);
 3723                 if ((*pdpe & PG_V) == 0) {
 3724                         va_next = (addr + NBPDP) & ~PDPMASK;
 3725                         if (va_next < addr)
 3726                                 va_next = end_addr;
 3727                         continue;
 3728                 }
 3729 
 3730                 va_next = (addr + NBPDR) & ~PDRMASK;
 3731                 if (va_next < addr)
 3732                         va_next = end_addr;
 3733 
 3734                 pde = pmap_pdpe_to_pde(pdpe, addr);
 3735                 srcptepaddr = *pde;
 3736                 if (srcptepaddr == 0)
 3737                         continue;
 3738                         
 3739                 if (srcptepaddr & PG_PS) {
 3740                         dstmpde = pmap_allocpde(dst_pmap, addr, M_NOWAIT);
 3741                         if (dstmpde == NULL)
 3742                                 break;
 3743                         pde = (pd_entry_t *)
 3744                             PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpde));
 3745                         pde = &pde[pmap_pde_index(addr)];
 3746                         if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 ||
 3747                             pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr &
 3748                             PG_PS_FRAME))) {
 3749                                 *pde = srcptepaddr & ~PG_W;
 3750                                 dst_pmap->pm_stats.resident_count +=
 3751                                     NBPDR / PAGE_SIZE;
 3752                         } else
 3753                                 dstmpde->wire_count--;
 3754                         continue;
 3755                 }
 3756 
 3757                 srcptepaddr &= PG_FRAME;
 3758                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
 3759                 KASSERT(srcmpte->wire_count > 0,
 3760                     ("pmap_copy: source page table page is unused"));
 3761 
 3762                 if (va_next > end_addr)
 3763                         va_next = end_addr;
 3764 
 3765                 src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr);
 3766                 src_pte = &src_pte[pmap_pte_index(addr)];
 3767                 dstmpte = NULL;
 3768                 while (addr < va_next) {
 3769                         pt_entry_t ptetemp;
 3770                         ptetemp = *src_pte;
 3771                         /*
 3772                          * we only virtual copy managed pages
 3773                          */
 3774                         if ((ptetemp & PG_MANAGED) != 0) {
 3775                                 if (dstmpte != NULL &&
 3776                                     dstmpte->pindex == pmap_pde_pindex(addr))
 3777                                         dstmpte->wire_count++;
 3778                                 else if ((dstmpte = pmap_allocpte(dst_pmap,
 3779                                     addr, M_NOWAIT)) == NULL)
 3780                                         goto out;
 3781                                 dst_pte = (pt_entry_t *)
 3782                                     PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
 3783                                 dst_pte = &dst_pte[pmap_pte_index(addr)];
 3784                                 if (*dst_pte == 0 &&
 3785                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 3786                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
 3787                                         /*
 3788                                          * Clear the wired, modified, and
 3789                                          * accessed (referenced) bits
 3790                                          * during the copy.
 3791                                          */
 3792                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 3793                                             PG_A);
 3794                                         dst_pmap->pm_stats.resident_count++;
 3795                                 } else {
 3796                                         free = NULL;
 3797                                         if (pmap_unwire_pte_hold(dst_pmap,
 3798                                             addr, dstmpte, &free)) {
 3799                                                 pmap_invalidate_page(dst_pmap,
 3800                                                     addr);
 3801                                                 pmap_free_zero_pages(free);
 3802                                         }
 3803                                         goto out;
 3804                                 }
 3805                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 3806                                         break;
 3807                         }
 3808                         addr += PAGE_SIZE;
 3809                         src_pte++;
 3810                 }
 3811         }
 3812 out:
 3813         vm_page_unlock_queues();
 3814         PMAP_UNLOCK(src_pmap);
 3815         PMAP_UNLOCK(dst_pmap);
 3816 }       
 3817 
 3818 /*
 3819  *      pmap_zero_page zeros the specified hardware page by mapping 
 3820  *      the page into KVM and using bzero to clear its contents.
 3821  */
 3822 void
 3823 pmap_zero_page(vm_page_t m)
 3824 {
 3825         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 3826 
 3827         pagezero((void *)va);
 3828 }
 3829 
 3830 /*
 3831  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 3832  *      the page into KVM and using bzero to clear its contents.
 3833  *
 3834  *      off and size may not cover an area beyond a single hardware page.
 3835  */
 3836 void
 3837 pmap_zero_page_area(vm_page_t m, int off, int size)
 3838 {
 3839         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 3840 
 3841         if (off == 0 && size == PAGE_SIZE)
 3842                 pagezero((void *)va);
 3843         else
 3844                 bzero((char *)va + off, size);
 3845 }
 3846 
 3847 /*
 3848  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 3849  *      the page into KVM and using bzero to clear its contents.  This
 3850  *      is intended to be called from the vm_pagezero process only and
 3851  *      outside of Giant.
 3852  */
 3853 void
 3854 pmap_zero_page_idle(vm_page_t m)
 3855 {
 3856         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 3857 
 3858         pagezero((void *)va);
 3859 }
 3860 
 3861 /*
 3862  *      pmap_copy_page copies the specified (machine independent)
 3863  *      page by mapping the page into virtual memory and using
 3864  *      bcopy to copy the page, one machine dependent page at a
 3865  *      time.
 3866  */
 3867 void
 3868 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 3869 {
 3870         vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
 3871         vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
 3872 
 3873         pagecopy((void *)src, (void *)dst);
 3874 }
 3875 
 3876 /*
 3877  * Returns true if the pmap's pv is one of the first
 3878  * 16 pvs linked to from this page.  This count may
 3879  * be changed upwards or downwards in the future; it
 3880  * is only necessary that true be returned for a small
 3881  * subset of pmaps for proper page aging.
 3882  */
 3883 boolean_t
 3884 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 3885 {
 3886         struct md_page *pvh;
 3887         pv_entry_t pv;
 3888         int loops = 0;
 3889 
 3890         if (m->flags & PG_FICTITIOUS)
 3891                 return FALSE;
 3892 
 3893         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3894         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3895                 if (PV_PMAP(pv) == pmap) {
 3896                         return TRUE;
 3897                 }
 3898                 loops++;
 3899                 if (loops >= 16)
 3900                         break;
 3901         }
 3902         if (loops < 16) {
 3903                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 3904                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 3905                         if (PV_PMAP(pv) == pmap)
 3906                                 return (TRUE);
 3907                         loops++;
 3908                         if (loops >= 16)
 3909                                 break;
 3910                 }
 3911         }
 3912         return (FALSE);
 3913 }
 3914 
 3915 /*
 3916  *      pmap_page_wired_mappings:
 3917  *
 3918  *      Return the number of managed mappings to the given physical page
 3919  *      that are wired.
 3920  */
 3921 int
 3922 pmap_page_wired_mappings(vm_page_t m)
 3923 {
 3924         int count;
 3925 
 3926         count = 0;
 3927         if ((m->flags & PG_FICTITIOUS) != 0)
 3928                 return (count);
 3929         count = pmap_pvh_wired_mappings(&m->md, count);
 3930         return (pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)), count));
 3931 }
 3932 
 3933 /*
 3934  *      pmap_pvh_wired_mappings:
 3935  *
 3936  *      Return the updated number "count" of managed mappings that are wired.
 3937  */
 3938 static int
 3939 pmap_pvh_wired_mappings(struct md_page *pvh, int count)
 3940 {
 3941         pmap_t pmap;
 3942         pt_entry_t *pte;
 3943         pv_entry_t pv;
 3944 
 3945         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3946         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 3947                 pmap = PV_PMAP(pv);
 3948                 PMAP_LOCK(pmap);
 3949                 pte = pmap_pte(pmap, pv->pv_va);
 3950                 if ((*pte & PG_W) != 0)
 3951                         count++;
 3952                 PMAP_UNLOCK(pmap);
 3953         }
 3954         return (count);
 3955 }
 3956 
 3957 /*
 3958  * Returns TRUE if the given page is mapped individually or as part of
 3959  * a 2mpage.  Otherwise, returns FALSE.
 3960  */
 3961 boolean_t
 3962 pmap_page_is_mapped(vm_page_t m)
 3963 {
 3964         struct md_page *pvh;
 3965 
 3966         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
 3967                 return (FALSE);
 3968         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3969         if (TAILQ_EMPTY(&m->md.pv_list)) {
 3970                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 3971                 return (!TAILQ_EMPTY(&pvh->pv_list));
 3972         } else
 3973                 return (TRUE);
 3974 }
 3975 
 3976 /*
 3977  * Remove all pages from specified address space
 3978  * this aids process exit speeds.  Also, this code
 3979  * is special cased for current process only, but
 3980  * can have the more generic (and slightly slower)
 3981  * mode enabled.  This is much faster than pmap_remove
 3982  * in the case of running down an entire address space.
 3983  */
 3984 void
 3985 pmap_remove_pages(pmap_t pmap)
 3986 {
 3987         pd_entry_t ptepde;
 3988         pt_entry_t *pte, tpte;
 3989         vm_page_t free = NULL;
 3990         vm_page_t m, mpte, mt;
 3991         pv_entry_t pv;
 3992         struct md_page *pvh;
 3993         struct pv_chunk *pc, *npc;
 3994         int field, idx;
 3995         int64_t bit;
 3996         uint64_t inuse, bitmask;
 3997         int allfree;
 3998 
 3999         if (pmap != PCPU_GET(curpmap)) {
 4000                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 4001                 return;
 4002         }
 4003         vm_page_lock_queues();
 4004         PMAP_LOCK(pmap);
 4005         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 4006                 allfree = 1;
 4007                 for (field = 0; field < _NPCM; field++) {
 4008                         inuse = (~(pc->pc_map[field])) & pc_freemask[field];
 4009                         while (inuse != 0) {
 4010                                 bit = bsfq(inuse);
 4011                                 bitmask = 1UL << bit;
 4012                                 idx = field * 64 + bit;
 4013                                 pv = &pc->pc_pventry[idx];
 4014                                 inuse &= ~bitmask;
 4015 
 4016                                 pte = pmap_pdpe(pmap, pv->pv_va);
 4017                                 ptepde = *pte;
 4018                                 pte = pmap_pdpe_to_pde(pte, pv->pv_va);
 4019                                 tpte = *pte;
 4020                                 if ((tpte & (PG_PS | PG_V)) == PG_V) {
 4021                                         ptepde = tpte;
 4022                                         pte = (pt_entry_t *)PHYS_TO_DMAP(tpte &
 4023                                             PG_FRAME);
 4024                                         pte = &pte[pmap_pte_index(pv->pv_va)];
 4025                                         tpte = *pte & ~PG_PTE_PAT;
 4026                                 }
 4027                                 if ((tpte & PG_V) == 0)
 4028                                         panic("bad pte");
 4029 
 4030 /*
 4031  * We cannot remove wired pages from a process' mapping at this time
 4032  */
 4033                                 if (tpte & PG_W) {
 4034                                         allfree = 0;
 4035                                         continue;
 4036                                 }
 4037 
 4038                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 4039                                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 4040                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 4041                                     m, (uintmax_t)m->phys_addr,
 4042                                     (uintmax_t)tpte));
 4043 
 4044                                 KASSERT(m < &vm_page_array[vm_page_array_size],
 4045                                         ("pmap_remove_pages: bad tpte %#jx",
 4046                                         (uintmax_t)tpte));
 4047 
 4048                                 pte_clear(pte);
 4049 
 4050                                 /*
 4051                                  * Update the vm_page_t clean/reference bits.
 4052                                  */
 4053                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4054                                         if ((tpte & PG_PS) != 0) {
 4055                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4056                                                         vm_page_dirty(mt);
 4057                                         } else
 4058                                                 vm_page_dirty(m);
 4059                                 }
 4060 
 4061                                 /* Mark free */
 4062                                 PV_STAT(pv_entry_frees++);
 4063                                 PV_STAT(pv_entry_spare++);
 4064                                 pv_entry_count--;
 4065                                 pc->pc_map[field] |= bitmask;
 4066                                 if ((tpte & PG_PS) != 0) {
 4067                                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 4068                                         pvh = pa_to_pvh(tpte & PG_PS_FRAME);
 4069                                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
 4070                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 4071                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4072                                                         if (TAILQ_EMPTY(&mt->md.pv_list))
 4073                                                                 vm_page_flag_clear(mt, PG_WRITEABLE);
 4074                                         }
 4075                                         mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
 4076                                         if (mpte != NULL) {
 4077                                                 pmap_remove_pt_page(pmap, mpte);
 4078                                                 pmap->pm_stats.resident_count--;
 4079                                                 KASSERT(mpte->wire_count == NPTEPG,
 4080                                                     ("pmap_remove_pages: pte page wire count error"));
 4081                                                 mpte->wire_count = 0;
 4082                                                 pmap_add_delayed_free_list(mpte, &free, FALSE);
 4083                                                 atomic_subtract_int(&cnt.v_wire_count, 1);
 4084                                         }
 4085                                 } else {
 4086                                         pmap->pm_stats.resident_count--;
 4087                                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 4088                                         if (TAILQ_EMPTY(&m->md.pv_list)) {
 4089                                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4090                                                 if (TAILQ_EMPTY(&pvh->pv_list))
 4091                                                         vm_page_flag_clear(m, PG_WRITEABLE);
 4092                                         }
 4093                                 }
 4094                                 pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
 4095                         }
 4096                 }
 4097                 if (allfree) {
 4098                         PV_STAT(pv_entry_spare -= _NPCPV);
 4099                         PV_STAT(pc_chunk_count--);
 4100                         PV_STAT(pc_chunk_frees++);
 4101                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 4102                         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 4103                         dump_drop_page(m->phys_addr);
 4104                         vm_page_unwire(m, 0);
 4105                         vm_page_free(m);
 4106                 }
 4107         }
 4108         pmap_invalidate_all(pmap);
 4109         vm_page_unlock_queues();
 4110         PMAP_UNLOCK(pmap);
 4111         pmap_free_zero_pages(free);
 4112 }
 4113 
 4114 /*
 4115  *      pmap_is_modified:
 4116  *
 4117  *      Return whether or not the specified physical page was modified
 4118  *      in any physical maps.
 4119  */
 4120 boolean_t
 4121 pmap_is_modified(vm_page_t m)
 4122 {
 4123 
 4124         if (m->flags & PG_FICTITIOUS)
 4125                 return (FALSE);
 4126         if (pmap_is_modified_pvh(&m->md))
 4127                 return (TRUE);
 4128         return (pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4129 }
 4130 
 4131 /*
 4132  * Returns TRUE if any of the given mappings were used to modify
 4133  * physical memory.  Otherwise, returns FALSE.  Both page and 2mpage
 4134  * mappings are supported.
 4135  */
 4136 static boolean_t
 4137 pmap_is_modified_pvh(struct md_page *pvh)
 4138 {
 4139         pv_entry_t pv;
 4140         pt_entry_t *pte;
 4141         pmap_t pmap;
 4142         boolean_t rv;
 4143 
 4144         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 4145         rv = FALSE;
 4146         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 4147                 pmap = PV_PMAP(pv);
 4148                 PMAP_LOCK(pmap);
 4149                 pte = pmap_pte(pmap, pv->pv_va);
 4150                 rv = (*pte & (PG_M | PG_RW)) == (PG_M | PG_RW);
 4151                 PMAP_UNLOCK(pmap);
 4152                 if (rv)
 4153                         break;
 4154         }
 4155         return (rv);
 4156 }
 4157 
 4158 /*
 4159  *      pmap_is_prefaultable:
 4160  *
 4161  *      Return whether or not the specified virtual address is elgible
 4162  *      for prefault.
 4163  */
 4164 boolean_t
 4165 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 4166 {
 4167         pd_entry_t *pde;
 4168         pt_entry_t *pte;
 4169         boolean_t rv;
 4170 
 4171         rv = FALSE;
 4172         PMAP_LOCK(pmap);
 4173         pde = pmap_pde(pmap, addr);
 4174         if (pde != NULL && (*pde & (PG_PS | PG_V)) == PG_V) {
 4175                 pte = pmap_pde_to_pte(pde, addr);
 4176                 rv = (*pte & PG_V) == 0;
 4177         }
 4178         PMAP_UNLOCK(pmap);
 4179         return (rv);
 4180 }
 4181 
 4182 /*
 4183  * Clear the write and modified bits in each of the given page's mappings.
 4184  */
 4185 void
 4186 pmap_remove_write(vm_page_t m)
 4187 {
 4188         struct md_page *pvh;
 4189         pmap_t pmap;
 4190         pv_entry_t next_pv, pv;
 4191         pd_entry_t *pde;
 4192         pt_entry_t oldpte, *pte;
 4193         vm_offset_t va;
 4194 
 4195         if ((m->flags & PG_FICTITIOUS) != 0 ||
 4196             (m->flags & PG_WRITEABLE) == 0)
 4197                 return;
 4198         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 4199         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4200         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
 4201                 va = pv->pv_va;
 4202                 pmap = PV_PMAP(pv);
 4203                 PMAP_LOCK(pmap);
 4204                 pde = pmap_pde(pmap, va);
 4205                 if ((*pde & PG_RW) != 0)
 4206                         (void)pmap_demote_pde(pmap, pde, va);
 4207                 PMAP_UNLOCK(pmap);
 4208         }
 4209         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4210                 pmap = PV_PMAP(pv);
 4211                 PMAP_LOCK(pmap);
 4212                 pde = pmap_pde(pmap, pv->pv_va);
 4213                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_write: found"
 4214                     " a 2mpage in page %p's pv list", m));
 4215                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 4216 retry:
 4217                 oldpte = *pte;
 4218                 if (oldpte & PG_RW) {
 4219                         if (!atomic_cmpset_long(pte, oldpte, oldpte &
 4220                             ~(PG_RW | PG_M)))
 4221                                 goto retry;
 4222                         if ((oldpte & PG_M) != 0)
 4223                                 vm_page_dirty(m);
 4224                         pmap_invalidate_page(pmap, pv->pv_va);
 4225                 }
 4226                 PMAP_UNLOCK(pmap);
 4227         }
 4228         vm_page_flag_clear(m, PG_WRITEABLE);
 4229 }
 4230 
 4231 /*
 4232  *      pmap_ts_referenced:
 4233  *
 4234  *      Return a count of reference bits for a page, clearing those bits.
 4235  *      It is not necessary for every reference bit to be cleared, but it
 4236  *      is necessary that 0 only be returned when there are truly no
 4237  *      reference bits set.
 4238  *
 4239  *      XXX: The exact number of bits to check and clear is a matter that
 4240  *      should be tested and standardized at some point in the future for
 4241  *      optimal aging of shared pages.
 4242  */
 4243 int
 4244 pmap_ts_referenced(vm_page_t m)
 4245 {
 4246         struct md_page *pvh;
 4247         pv_entry_t pv, pvf, pvn;
 4248         pmap_t pmap;
 4249         pd_entry_t oldpde, *pde;
 4250         pt_entry_t *pte;
 4251         vm_offset_t va;
 4252         int rtval = 0;
 4253 
 4254         if (m->flags & PG_FICTITIOUS)
 4255                 return (rtval);
 4256         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 4257         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4258         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) {
 4259                 va = pv->pv_va;
 4260                 pmap = PV_PMAP(pv);
 4261                 PMAP_LOCK(pmap);
 4262                 pde = pmap_pde(pmap, va);
 4263                 oldpde = *pde;
 4264                 if ((oldpde & PG_A) != 0) {
 4265                         if (pmap_demote_pde(pmap, pde, va)) {
 4266                                 if ((oldpde & PG_W) == 0) {
 4267                                         /*
 4268                                          * Remove the mapping to a single page
 4269                                          * so that a subsequent access may
 4270                                          * repromote.  Since the underlying
 4271                                          * page table page is fully populated,
 4272                                          * this removal never frees a page
 4273                                          * table page.
 4274                                          */
 4275                                         va += VM_PAGE_TO_PHYS(m) - (oldpde &
 4276                                             PG_PS_FRAME);
 4277                                         pmap_remove_page(pmap, va, pde, NULL);
 4278                                         rtval++;
 4279                                         if (rtval > 4) {
 4280                                                 PMAP_UNLOCK(pmap);
 4281                                                 return (rtval);
 4282                                         }
 4283                                 }
 4284                         }
 4285                 }
 4286                 PMAP_UNLOCK(pmap);
 4287         }
 4288         if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 4289                 pvf = pv;
 4290                 do {
 4291                         pvn = TAILQ_NEXT(pv, pv_list);
 4292                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 4293                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 4294                         pmap = PV_PMAP(pv);
 4295                         PMAP_LOCK(pmap);
 4296                         pde = pmap_pde(pmap, pv->pv_va);
 4297                         KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced:"
 4298                             " found a 2mpage in page %p's pv list", m));
 4299                         pte = pmap_pde_to_pte(pde, pv->pv_va);
 4300                         if ((*pte & PG_A) != 0) {
 4301                                 atomic_clear_long(pte, PG_A);
 4302                                 pmap_invalidate_page(pmap, pv->pv_va);
 4303                                 rtval++;
 4304                                 if (rtval > 4)
 4305                                         pvn = NULL;
 4306                         }
 4307                         PMAP_UNLOCK(pmap);
 4308                 } while ((pv = pvn) != NULL && pv != pvf);
 4309         }
 4310         return (rtval);
 4311 }
 4312 
 4313 /*
 4314  *      Clear the modify bits on the specified physical page.
 4315  */
 4316 void
 4317 pmap_clear_modify(vm_page_t m)
 4318 {
 4319         struct md_page *pvh;
 4320         pmap_t pmap;
 4321         pv_entry_t next_pv, pv;
 4322         pd_entry_t oldpde, *pde;
 4323         pt_entry_t oldpte, *pte;
 4324         vm_offset_t va;
 4325 
 4326         if ((m->flags & PG_FICTITIOUS) != 0)
 4327                 return;
 4328         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 4329         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4330         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
 4331                 va = pv->pv_va;
 4332                 pmap = PV_PMAP(pv);
 4333                 PMAP_LOCK(pmap);
 4334                 pde = pmap_pde(pmap, va);
 4335                 oldpde = *pde;
 4336                 if ((oldpde & PG_RW) != 0) {
 4337                         if (pmap_demote_pde(pmap, pde, va)) {
 4338                                 if ((oldpde & PG_W) == 0) {
 4339                                         /*
 4340                                          * Write protect the mapping to a
 4341                                          * single page so that a subsequent
 4342                                          * write access may repromote.
 4343                                          */
 4344                                         va += VM_PAGE_TO_PHYS(m) - (oldpde &
 4345                                             PG_PS_FRAME);
 4346                                         pte = pmap_pde_to_pte(pde, va);
 4347                                         oldpte = *pte;
 4348                                         if ((oldpte & PG_V) != 0) {
 4349                                                 while (!atomic_cmpset_long(pte,
 4350                                                     oldpte,
 4351                                                     oldpte & ~(PG_M | PG_RW)))
 4352                                                         oldpte = *pte;
 4353                                                 vm_page_dirty(m);
 4354                                                 pmap_invalidate_page(pmap, va);
 4355                                         }
 4356                                 }
 4357                         }
 4358                 }
 4359                 PMAP_UNLOCK(pmap);
 4360         }
 4361         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4362                 pmap = PV_PMAP(pv);
 4363                 PMAP_LOCK(pmap);
 4364                 pde = pmap_pde(pmap, pv->pv_va);
 4365                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
 4366                     " a 2mpage in page %p's pv list", m));
 4367                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 4368                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4369                         atomic_clear_long(pte, PG_M);
 4370                         pmap_invalidate_page(pmap, pv->pv_va);
 4371                 }
 4372                 PMAP_UNLOCK(pmap);
 4373         }
 4374 }
 4375 
 4376 /*
 4377  *      pmap_clear_reference:
 4378  *
 4379  *      Clear the reference bit on the specified physical page.
 4380  */
 4381 void
 4382 pmap_clear_reference(vm_page_t m)
 4383 {
 4384         struct md_page *pvh;
 4385         pmap_t pmap;
 4386         pv_entry_t next_pv, pv;
 4387         pd_entry_t oldpde, *pde;
 4388         pt_entry_t *pte;
 4389         vm_offset_t va;
 4390 
 4391         if ((m->flags & PG_FICTITIOUS) != 0)
 4392                 return;
 4393         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 4394         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4395         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, next_pv) {
 4396                 va = pv->pv_va;
 4397                 pmap = PV_PMAP(pv);
 4398                 PMAP_LOCK(pmap);
 4399                 pde = pmap_pde(pmap, va);
 4400                 oldpde = *pde;
 4401                 if ((oldpde & PG_A) != 0) {
 4402                         if (pmap_demote_pde(pmap, pde, va)) {
 4403                                 /*
 4404                                  * Remove the mapping to a single page so
 4405                                  * that a subsequent access may repromote.
 4406                                  * Since the underlying page table page is
 4407                                  * fully populated, this removal never frees
 4408                                  * a page table page.
 4409                                  */
 4410                                 va += VM_PAGE_TO_PHYS(m) - (oldpde &
 4411                                     PG_PS_FRAME);
 4412                                 pmap_remove_page(pmap, va, pde, NULL);
 4413                         }
 4414                 }
 4415                 PMAP_UNLOCK(pmap);
 4416         }
 4417         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4418                 pmap = PV_PMAP(pv);
 4419                 PMAP_LOCK(pmap);
 4420                 pde = pmap_pde(pmap, pv->pv_va);
 4421                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_reference: found"
 4422                     " a 2mpage in page %p's pv list", m));
 4423                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 4424                 if (*pte & PG_A) {
 4425                         atomic_clear_long(pte, PG_A);
 4426                         pmap_invalidate_page(pmap, pv->pv_va);
 4427                 }
 4428                 PMAP_UNLOCK(pmap);
 4429         }
 4430 }
 4431 
 4432 /*
 4433  * Miscellaneous support routines follow
 4434  */
 4435 
 4436 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
 4437 static __inline void
 4438 pmap_pte_attr(pt_entry_t *pte, int cache_bits)
 4439 {
 4440         u_int opte, npte;
 4441 
 4442         /*
 4443          * The cache mode bits are all in the low 32-bits of the
 4444          * PTE, so we can just spin on updating the low 32-bits.
 4445          */
 4446         do {
 4447                 opte = *(u_int *)pte;
 4448                 npte = opte & ~PG_PTE_CACHE;
 4449                 npte |= cache_bits;
 4450         } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
 4451 }
 4452 
 4453 /* Adjust the cache mode for a 2MB page mapped via a PDE. */
 4454 static __inline void
 4455 pmap_pde_attr(pd_entry_t *pde, int cache_bits)
 4456 {
 4457         u_int opde, npde;
 4458 
 4459         /*
 4460          * The cache mode bits are all in the low 32-bits of the
 4461          * PDE, so we can just spin on updating the low 32-bits.
 4462          */
 4463         do {
 4464                 opde = *(u_int *)pde;
 4465                 npde = opde & ~PG_PDE_CACHE;
 4466                 npde |= cache_bits;
 4467         } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
 4468 }
 4469 
 4470 /*
 4471  * Map a set of physical memory pages into the kernel virtual
 4472  * address space. Return a pointer to where it is mapped. This
 4473  * routine is intended to be used for mapping device memory,
 4474  * NOT real memory.
 4475  */
 4476 void *
 4477 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 4478 {
 4479         vm_offset_t va, offset;
 4480         vm_size_t tmpsize;
 4481 
 4482         /*
 4483          * If the specified range of physical addresses fits within the direct
 4484          * map window, use the direct map. 
 4485          */
 4486         if (pa < dmaplimit && pa + size < dmaplimit) {
 4487                 va = PHYS_TO_DMAP(pa);
 4488                 if (!pmap_change_attr(va, size, mode))
 4489                         return ((void *)va);
 4490         }
 4491         offset = pa & PAGE_MASK;
 4492         size = roundup(offset + size, PAGE_SIZE);
 4493         va = kmem_alloc_nofault(kernel_map, size);
 4494         if (!va)
 4495                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 4496         pa = trunc_page(pa);
 4497         for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
 4498                 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
 4499         pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
 4500         pmap_invalidate_cache_range(va, va + tmpsize);
 4501         return ((void *)(va + offset));
 4502 }
 4503 
 4504 void *
 4505 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 4506 {
 4507 
 4508         return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
 4509 }
 4510 
 4511 void *
 4512 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
 4513 {
 4514 
 4515         return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
 4516 }
 4517 
 4518 void
 4519 pmap_unmapdev(vm_offset_t va, vm_size_t size)
 4520 {
 4521         vm_offset_t base, offset, tmpva;
 4522 
 4523         /* If we gave a direct map region in pmap_mapdev, do nothing */
 4524         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
 4525                 return;
 4526         base = trunc_page(va);
 4527         offset = va & PAGE_MASK;
 4528         size = roundup(offset + size, PAGE_SIZE);
 4529         for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE)
 4530                 pmap_kremove(tmpva);
 4531         pmap_invalidate_range(kernel_pmap, va, tmpva);
 4532         kmem_free(kernel_map, base, size);
 4533 }
 4534 
 4535 /*
 4536  * Tries to demote a 1GB page mapping.
 4537  */
 4538 static boolean_t
 4539 pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va)
 4540 {
 4541         pdp_entry_t newpdpe, oldpdpe;
 4542         pd_entry_t *firstpde, newpde, *pde;
 4543         vm_paddr_t mpdepa;
 4544         vm_page_t mpde;
 4545 
 4546         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 4547         oldpdpe = *pdpe;
 4548         KASSERT((oldpdpe & (PG_PS | PG_V)) == (PG_PS | PG_V),
 4549             ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V"));
 4550         if ((mpde = vm_page_alloc(NULL, va >> PDPSHIFT, VM_ALLOC_INTERRUPT |
 4551             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 4552                 CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx"
 4553                     " in pmap %p", va, pmap);
 4554                 return (FALSE);
 4555         }
 4556         mpdepa = VM_PAGE_TO_PHYS(mpde);
 4557         firstpde = (pd_entry_t *)PHYS_TO_DMAP(mpdepa);
 4558         newpdpe = mpdepa | PG_M | PG_A | (oldpdpe & PG_U) | PG_RW | PG_V;
 4559         KASSERT((oldpdpe & PG_A) != 0,
 4560             ("pmap_demote_pdpe: oldpdpe is missing PG_A"));
 4561         KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW,
 4562             ("pmap_demote_pdpe: oldpdpe is missing PG_M"));
 4563         newpde = oldpdpe;
 4564 
 4565         /*
 4566          * Initialize the page directory page.
 4567          */
 4568         for (pde = firstpde; pde < firstpde + NPDEPG; pde++) {
 4569                 *pde = newpde;
 4570                 newpde += NBPDR;
 4571         }
 4572 
 4573         /*
 4574          * Demote the mapping.
 4575          */
 4576         *pdpe = newpdpe;
 4577 
 4578         /*
 4579          * Invalidate a stale recursive mapping of the page directory page.
 4580          */
 4581         pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va));
 4582 
 4583         pmap_pdpe_demotions++;
 4584         CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx"
 4585             " in pmap %p", va, pmap);
 4586         return (TRUE);
 4587 }
 4588 
 4589 /*
 4590  * Sets the memory attribute for the specified page.
 4591  */
 4592 void
 4593 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
 4594 {
 4595 
 4596         m->md.pat_mode = ma;
 4597 
 4598         /*
 4599          * If "m" is a normal page, update its direct mapping.  This update
 4600          * can be relied upon to perform any cache operations that are
 4601          * required for data coherence.
 4602          */
 4603         if ((m->flags & PG_FICTITIOUS) == 0 &&
 4604             pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), PAGE_SIZE,
 4605             m->md.pat_mode))
 4606                 panic("memory attribute change on the direct map failed");
 4607 }
 4608 
 4609 /*
 4610  * Changes the specified virtual address range's memory type to that given by
 4611  * the parameter "mode".  The specified virtual address range must be
 4612  * completely contained within either the direct map or the kernel map.  If
 4613  * the virtual address range is contained within the kernel map, then the
 4614  * memory type for each of the corresponding ranges of the direct map is also
 4615  * changed.  (The corresponding ranges of the direct map are those ranges that
 4616  * map the same physical pages as the specified virtual address range.)  These
 4617  * changes to the direct map are necessary because Intel describes the
 4618  * behavior of their processors as "undefined" if two or more mappings to the
 4619  * same physical page have different memory types.
 4620  *
 4621  * Returns zero if the change completed successfully, and either EINVAL or
 4622  * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
 4623  * of the virtual address range was not mapped, and ENOMEM is returned if
 4624  * there was insufficient memory available to complete the change.  In the
 4625  * latter case, the memory type may have been changed on some part of the
 4626  * virtual address range or the direct map.
 4627  */
 4628 int
 4629 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
 4630 {
 4631         int error;
 4632 
 4633         PMAP_LOCK(kernel_pmap);
 4634         error = pmap_change_attr_locked(va, size, mode);
 4635         PMAP_UNLOCK(kernel_pmap);
 4636         return (error);
 4637 }
 4638 
 4639 static int
 4640 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode)
 4641 {
 4642         vm_offset_t base, offset, tmpva;
 4643         vm_paddr_t pa_start, pa_end;
 4644         pdp_entry_t *pdpe;
 4645         pd_entry_t *pde;
 4646         pt_entry_t *pte;
 4647         int cache_bits_pte, cache_bits_pde, error;
 4648         boolean_t changed;
 4649 
 4650         PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED);
 4651         base = trunc_page(va);
 4652         offset = va & PAGE_MASK;
 4653         size = roundup(offset + size, PAGE_SIZE);
 4654 
 4655         /*
 4656          * Only supported on kernel virtual addresses, including the direct
 4657          * map but excluding the recursive map.
 4658          */
 4659         if (base < DMAP_MIN_ADDRESS)
 4660                 return (EINVAL);
 4661 
 4662         cache_bits_pde = pmap_cache_bits(mode, 1);
 4663         cache_bits_pte = pmap_cache_bits(mode, 0);
 4664         changed = FALSE;
 4665 
 4666         /*
 4667          * Pages that aren't mapped aren't supported.  Also break down 2MB pages
 4668          * into 4KB pages if required.
 4669          */
 4670         for (tmpva = base; tmpva < base + size; ) {
 4671                 pdpe = pmap_pdpe(kernel_pmap, tmpva);
 4672                 if (*pdpe == 0)
 4673                         return (EINVAL);
 4674                 if (*pdpe & PG_PS) {
 4675                         /*
 4676                          * If the current 1GB page already has the required
 4677                          * memory type, then we need not demote this page. Just
 4678                          * increment tmpva to the next 1GB page frame.
 4679                          */
 4680                         if ((*pdpe & PG_PDE_CACHE) == cache_bits_pde) {
 4681                                 tmpva = trunc_1gpage(tmpva) + NBPDP;
 4682                                 continue;
 4683                         }
 4684 
 4685                         /*
 4686                          * If the current offset aligns with a 1GB page frame
 4687                          * and there is at least 1GB left within the range, then
 4688                          * we need not break down this page into 2MB pages.
 4689                          */
 4690                         if ((tmpva & PDPMASK) == 0 &&
 4691                             tmpva + PDPMASK < base + size) {
 4692                                 tmpva += NBPDP;
 4693                                 continue;
 4694                         }
 4695                         if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva))
 4696                                 return (ENOMEM);
 4697                 }
 4698                 pde = pmap_pdpe_to_pde(pdpe, tmpva);
 4699                 if (*pde == 0)
 4700                         return (EINVAL);
 4701                 if (*pde & PG_PS) {
 4702                         /*
 4703                          * If the current 2MB page already has the required
 4704                          * memory type, then we need not demote this page. Just
 4705                          * increment tmpva to the next 2MB page frame.
 4706                          */
 4707                         if ((*pde & PG_PDE_CACHE) == cache_bits_pde) {
 4708                                 tmpva = trunc_2mpage(tmpva) + NBPDR;
 4709                                 continue;
 4710                         }
 4711 
 4712                         /*
 4713                          * If the current offset aligns with a 2MB page frame
 4714                          * and there is at least 2MB left within the range, then
 4715                          * we need not break down this page into 4KB pages.
 4716                          */
 4717                         if ((tmpva & PDRMASK) == 0 &&
 4718                             tmpva + PDRMASK < base + size) {
 4719                                 tmpva += NBPDR;
 4720                                 continue;
 4721                         }
 4722                         if (!pmap_demote_pde(kernel_pmap, pde, tmpva))
 4723                                 return (ENOMEM);
 4724                 }
 4725                 pte = pmap_pde_to_pte(pde, tmpva);
 4726                 if (*pte == 0)
 4727                         return (EINVAL);
 4728                 tmpva += PAGE_SIZE;
 4729         }
 4730         error = 0;
 4731 
 4732         /*
 4733          * Ok, all the pages exist, so run through them updating their
 4734          * cache mode if required.
 4735          */
 4736         pa_start = pa_end = 0;
 4737         for (tmpva = base; tmpva < base + size; ) {
 4738                 pdpe = pmap_pdpe(kernel_pmap, tmpva);
 4739                 if (*pdpe & PG_PS) {
 4740                         if ((*pdpe & PG_PDE_CACHE) != cache_bits_pde) {
 4741                                 pmap_pde_attr(pdpe, cache_bits_pde);
 4742                                 changed = TRUE;
 4743                         }
 4744                         if (tmpva >= VM_MIN_KERNEL_ADDRESS) {
 4745                                 if (pa_start == pa_end) {
 4746                                         /* Start physical address run. */
 4747                                         pa_start = *pdpe & PG_PS_FRAME;
 4748                                         pa_end = pa_start + NBPDP;
 4749                                 } else if (pa_end == (*pdpe & PG_PS_FRAME))
 4750                                         pa_end += NBPDP;
 4751                                 else {
 4752                                         /* Run ended, update direct map. */
 4753                                         error = pmap_change_attr_locked(
 4754                                             PHYS_TO_DMAP(pa_start),
 4755                                             pa_end - pa_start, mode);
 4756                                         if (error != 0)
 4757                                                 break;
 4758                                         /* Start physical address run. */
 4759                                         pa_start = *pdpe & PG_PS_FRAME;
 4760                                         pa_end = pa_start + NBPDP;
 4761                                 }
 4762                         }
 4763                         tmpva = trunc_1gpage(tmpva) + NBPDP;
 4764                         continue;
 4765                 }
 4766                 pde = pmap_pdpe_to_pde(pdpe, tmpva);
 4767                 if (*pde & PG_PS) {
 4768                         if ((*pde & PG_PDE_CACHE) != cache_bits_pde) {
 4769                                 pmap_pde_attr(pde, cache_bits_pde);
 4770                                 changed = TRUE;
 4771                         }
 4772                         if (tmpva >= VM_MIN_KERNEL_ADDRESS) {
 4773                                 if (pa_start == pa_end) {
 4774                                         /* Start physical address run. */
 4775                                         pa_start = *pde & PG_PS_FRAME;
 4776                                         pa_end = pa_start + NBPDR;
 4777                                 } else if (pa_end == (*pde & PG_PS_FRAME))
 4778                                         pa_end += NBPDR;
 4779                                 else {
 4780                                         /* Run ended, update direct map. */
 4781                                         error = pmap_change_attr_locked(
 4782                                             PHYS_TO_DMAP(pa_start),
 4783                                             pa_end - pa_start, mode);
 4784                                         if (error != 0)
 4785                                                 break;
 4786                                         /* Start physical address run. */
 4787                                         pa_start = *pde & PG_PS_FRAME;
 4788                                         pa_end = pa_start + NBPDR;
 4789                                 }
 4790                         }
 4791                         tmpva = trunc_2mpage(tmpva) + NBPDR;
 4792                 } else {
 4793                         pte = pmap_pde_to_pte(pde, tmpva);
 4794                         if ((*pte & PG_PTE_CACHE) != cache_bits_pte) {
 4795                                 pmap_pte_attr(pte, cache_bits_pte);
 4796                                 changed = TRUE;
 4797                         }
 4798                         if (tmpva >= VM_MIN_KERNEL_ADDRESS) {
 4799                                 if (pa_start == pa_end) {
 4800                                         /* Start physical address run. */
 4801                                         pa_start = *pte & PG_FRAME;
 4802                                         pa_end = pa_start + PAGE_SIZE;
 4803                                 } else if (pa_end == (*pte & PG_FRAME))
 4804                                         pa_end += PAGE_SIZE;
 4805                                 else {
 4806                                         /* Run ended, update direct map. */
 4807                                         error = pmap_change_attr_locked(
 4808                                             PHYS_TO_DMAP(pa_start),
 4809                                             pa_end - pa_start, mode);
 4810                                         if (error != 0)
 4811                                                 break;
 4812                                         /* Start physical address run. */
 4813                                         pa_start = *pte & PG_FRAME;
 4814                                         pa_end = pa_start + PAGE_SIZE;
 4815                                 }
 4816                         }
 4817                         tmpva += PAGE_SIZE;
 4818                 }
 4819         }
 4820         if (error == 0 && pa_start != pa_end)
 4821                 error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start),
 4822                     pa_end - pa_start, mode);
 4823 
 4824         /*
 4825          * Flush CPU caches if required to make sure any data isn't cached that
 4826          * shouldn't be, etc.
 4827          */
 4828         if (changed) {
 4829                 pmap_invalidate_range(kernel_pmap, base, tmpva);
 4830                 pmap_invalidate_cache_range(base, tmpva);
 4831         }
 4832         return (error);
 4833 }
 4834 
 4835 /*
 4836  * Demotes any mapping within the direct map region that covers more than the
 4837  * specified range of physical addresses.  This range's size must be a power
 4838  * of two and its starting address must be a multiple of its size.  Since the
 4839  * demotion does not change any attributes of the mapping, a TLB invalidation
 4840  * is not mandatory.  The caller may, however, request a TLB invalidation.
 4841  */
 4842 void
 4843 pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, boolean_t invalidate)
 4844 {
 4845         pdp_entry_t *pdpe;
 4846         pd_entry_t *pde;
 4847         vm_offset_t va;
 4848         boolean_t changed;
 4849 
 4850         if (len == 0)
 4851                 return;
 4852         KASSERT(powerof2(len), ("pmap_demote_DMAP: len is not a power of 2"));
 4853         KASSERT((base & (len - 1)) == 0,
 4854             ("pmap_demote_DMAP: base is not a multiple of len"));
 4855         if (len < NBPDP && base < dmaplimit) {
 4856                 va = PHYS_TO_DMAP(base);
 4857                 changed = FALSE;
 4858                 PMAP_LOCK(kernel_pmap);
 4859                 pdpe = pmap_pdpe(kernel_pmap, va);
 4860                 if ((*pdpe & PG_V) == 0)
 4861                         panic("pmap_demote_DMAP: invalid PDPE");
 4862                 if ((*pdpe & PG_PS) != 0) {
 4863                         if (!pmap_demote_pdpe(kernel_pmap, pdpe, va))
 4864                                 panic("pmap_demote_DMAP: PDPE failed");
 4865                         changed = TRUE;
 4866                 }
 4867                 if (len < NBPDR) {
 4868                         pde = pmap_pdpe_to_pde(pdpe, va);
 4869                         if ((*pde & PG_V) == 0)
 4870                                 panic("pmap_demote_DMAP: invalid PDE");
 4871                         if ((*pde & PG_PS) != 0) {
 4872                                 if (!pmap_demote_pde(kernel_pmap, pde, va))
 4873                                         panic("pmap_demote_DMAP: PDE failed");
 4874                                 changed = TRUE;
 4875                         }
 4876                 }
 4877                 if (changed && invalidate)
 4878                         pmap_invalidate_page(kernel_pmap, va);
 4879                 PMAP_UNLOCK(kernel_pmap);
 4880         }
 4881 }
 4882 
 4883 /*
 4884  * perform the pmap work for mincore
 4885  */
 4886 int
 4887 pmap_mincore(pmap_t pmap, vm_offset_t addr)
 4888 {
 4889         pd_entry_t *pdep;
 4890         pt_entry_t pte;
 4891         vm_paddr_t pa;
 4892         vm_page_t m;
 4893         int val = 0;
 4894         
 4895         PMAP_LOCK(pmap);
 4896         pdep = pmap_pde(pmap, addr);
 4897         if (pdep != NULL && (*pdep & PG_V)) {
 4898                 if (*pdep & PG_PS) {
 4899                         pte = *pdep;
 4900                         val = MINCORE_SUPER;
 4901                         /* Compute the physical address of the 4KB page. */
 4902                         pa = ((*pdep & PG_PS_FRAME) | (addr & PDRMASK)) &
 4903                             PG_FRAME;
 4904                 } else {
 4905                         pte = *pmap_pde_to_pte(pdep, addr);
 4906                         pa = pte & PG_FRAME;
 4907                 }
 4908         } else {
 4909                 pte = 0;
 4910                 pa = 0;
 4911         }
 4912         PMAP_UNLOCK(pmap);
 4913 
 4914         if (pte != 0) {
 4915                 val |= MINCORE_INCORE;
 4916                 if ((pte & PG_MANAGED) == 0)
 4917                         return val;
 4918 
 4919                 m = PHYS_TO_VM_PAGE(pa);
 4920 
 4921                 /*
 4922                  * Modified by us
 4923                  */
 4924                 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 4925                         val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
 4926                 else {
 4927                         /*
 4928                          * Modified by someone else
 4929                          */
 4930                         vm_page_lock_queues();
 4931                         if (m->dirty || pmap_is_modified(m))
 4932                                 val |= MINCORE_MODIFIED_OTHER;
 4933                         vm_page_unlock_queues();
 4934                 }
 4935                 /*
 4936                  * Referenced by us
 4937                  */
 4938                 if (pte & PG_A)
 4939                         val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
 4940                 else {
 4941                         /*
 4942                          * Referenced by someone else
 4943                          */
 4944                         vm_page_lock_queues();
 4945                         if ((m->flags & PG_REFERENCED) ||
 4946                             pmap_ts_referenced(m)) {
 4947                                 val |= MINCORE_REFERENCED_OTHER;
 4948                                 vm_page_flag_set(m, PG_REFERENCED);
 4949                         }
 4950                         vm_page_unlock_queues();
 4951                 }
 4952         } 
 4953         return val;
 4954 }
 4955 
 4956 void
 4957 pmap_activate(struct thread *td)
 4958 {
 4959         pmap_t  pmap, oldpmap;
 4960         u_int64_t  cr3;
 4961 
 4962         critical_enter();
 4963         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 4964         oldpmap = PCPU_GET(curpmap);
 4965 #ifdef SMP
 4966         atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
 4967         atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
 4968 #else
 4969         oldpmap->pm_active &= ~PCPU_GET(cpumask);
 4970         pmap->pm_active |= PCPU_GET(cpumask);
 4971 #endif
 4972         cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4);
 4973         td->td_pcb->pcb_cr3 = cr3;
 4974         load_cr3(cr3);
 4975         PCPU_SET(curpmap, pmap);
 4976         critical_exit();
 4977 }
 4978 
 4979 void
 4980 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
 4981 {
 4982 }
 4983 
 4984 /*
 4985  *      Increase the starting virtual address of the given mapping if a
 4986  *      different alignment might result in more superpage mappings.
 4987  */
 4988 void
 4989 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
 4990     vm_offset_t *addr, vm_size_t size)
 4991 {
 4992         vm_offset_t superpage_offset;
 4993 
 4994         if (size < NBPDR)
 4995                 return;
 4996         if (object != NULL && (object->flags & OBJ_COLORED) != 0)
 4997                 offset += ptoa(object->pg_color);
 4998         superpage_offset = offset & PDRMASK;
 4999         if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR ||
 5000             (*addr & PDRMASK) == superpage_offset)
 5001                 return;
 5002         if ((*addr & PDRMASK) < superpage_offset)
 5003                 *addr = (*addr & ~PDRMASK) + superpage_offset;
 5004         else
 5005                 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
 5006 }

Cache object: d6b38efdb882af714d28a0facbc64764


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.