The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/pmap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>
    9  * All rights reserved.
   10  *
   11  * This code is derived from software contributed to Berkeley by
   12  * the Systems Programming Group of the University of Utah Computer
   13  * Science Department and William Jolitz of UUNET Technologies Inc.
   14  *
   15  * Redistribution and use in source and binary forms, with or without
   16  * modification, are permitted provided that the following conditions
   17  * are met:
   18  * 1. Redistributions of source code must retain the above copyright
   19  *    notice, this list of conditions and the following disclaimer.
   20  * 2. Redistributions in binary form must reproduce the above copyright
   21  *    notice, this list of conditions and the following disclaimer in the
   22  *    documentation and/or other materials provided with the distribution.
   23  * 3. All advertising materials mentioning features or use of this software
   24  *    must display the following acknowledgement:
   25  *      This product includes software developed by the University of
   26  *      California, Berkeley and its contributors.
   27  * 4. Neither the name of the University nor the names of its contributors
   28  *    may be used to endorse or promote products derived from this software
   29  *    without specific prior written permission.
   30  *
   31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   41  * SUCH DAMAGE.
   42  *
   43  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   44  */
   45 /*-
   46  * Copyright (c) 2003 Networks Associates Technology, Inc.
   47  * All rights reserved.
   48  *
   49  * This software was developed for the FreeBSD Project by Jake Burkholder,
   50  * Safeport Network Services, and Network Associates Laboratories, the
   51  * Security Research Division of Network Associates, Inc. under
   52  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   53  * CHATS research program.
   54  *
   55  * Redistribution and use in source and binary forms, with or without
   56  * modification, are permitted provided that the following conditions
   57  * are met:
   58  * 1. Redistributions of source code must retain the above copyright
   59  *    notice, this list of conditions and the following disclaimer.
   60  * 2. Redistributions in binary form must reproduce the above copyright
   61  *    notice, this list of conditions and the following disclaimer in the
   62  *    documentation and/or other materials provided with the distribution.
   63  *
   64  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   65  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   66  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   67  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   68  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   69  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   70  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   71  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   72  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   73  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   74  * SUCH DAMAGE.
   75  */
   76 
   77 #include <sys/cdefs.h>
   78 __FBSDID("$FreeBSD: releng/10.1/sys/i386/i386/pmap.c 271905 2014-09-20 14:24:48Z kib $");
   79 
   80 /*
   81  *      Manages physical address maps.
   82  *
   83  *      Since the information managed by this module is
   84  *      also stored by the logical address mapping module,
   85  *      this module may throw away valid virtual-to-physical
   86  *      mappings at almost any time.  However, invalidations
   87  *      of virtual-to-physical mappings must be done as
   88  *      requested.
   89  *
   90  *      In order to cope with hardware architectures which
   91  *      make virtual-to-physical map invalidates expensive,
   92  *      this module may delay invalidate or reduced protection
   93  *      operations until such time as they are actually
   94  *      necessary.  This module is given full information as
   95  *      to which processors are currently using which maps,
   96  *      and to when physical maps must be made correct.
   97  */
   98 
   99 #include "opt_apic.h"
  100 #include "opt_cpu.h"
  101 #include "opt_pmap.h"
  102 #include "opt_smp.h"
  103 #include "opt_xbox.h"
  104 
  105 #include <sys/param.h>
  106 #include <sys/systm.h>
  107 #include <sys/kernel.h>
  108 #include <sys/ktr.h>
  109 #include <sys/lock.h>
  110 #include <sys/malloc.h>
  111 #include <sys/mman.h>
  112 #include <sys/msgbuf.h>
  113 #include <sys/mutex.h>
  114 #include <sys/proc.h>
  115 #include <sys/rwlock.h>
  116 #include <sys/sf_buf.h>
  117 #include <sys/sx.h>
  118 #include <sys/vmmeter.h>
  119 #include <sys/sched.h>
  120 #include <sys/sysctl.h>
  121 #ifdef SMP
  122 #include <sys/smp.h>
  123 #else
  124 #include <sys/cpuset.h>
  125 #endif
  126 
  127 #include <vm/vm.h>
  128 #include <vm/vm_param.h>
  129 #include <vm/vm_kern.h>
  130 #include <vm/vm_page.h>
  131 #include <vm/vm_map.h>
  132 #include <vm/vm_object.h>
  133 #include <vm/vm_extern.h>
  134 #include <vm/vm_pageout.h>
  135 #include <vm/vm_pager.h>
  136 #include <vm/vm_radix.h>
  137 #include <vm/vm_reserv.h>
  138 #include <vm/uma.h>
  139 
  140 #ifdef DEV_APIC
  141 #include <sys/bus.h>
  142 #include <machine/intr_machdep.h>
  143 #include <machine/apicvar.h>
  144 #endif
  145 #include <machine/cpu.h>
  146 #include <machine/cputypes.h>
  147 #include <machine/md_var.h>
  148 #include <machine/pcb.h>
  149 #include <machine/specialreg.h>
  150 #ifdef SMP
  151 #include <machine/smp.h>
  152 #endif
  153 
  154 #ifdef XBOX
  155 #include <machine/xbox.h>
  156 #endif
  157 
  158 #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
  159 #define CPU_ENABLE_SSE
  160 #endif
  161 
  162 #ifndef PMAP_SHPGPERPROC
  163 #define PMAP_SHPGPERPROC 200
  164 #endif
  165 
  166 #if !defined(DIAGNOSTIC)
  167 #ifdef __GNUC_GNU_INLINE__
  168 #define PMAP_INLINE     __attribute__((__gnu_inline__)) inline
  169 #else
  170 #define PMAP_INLINE     extern inline
  171 #endif
  172 #else
  173 #define PMAP_INLINE
  174 #endif
  175 
  176 #ifdef PV_STATS
  177 #define PV_STAT(x)      do { x ; } while (0)
  178 #else
  179 #define PV_STAT(x)      do { } while (0)
  180 #endif
  181 
  182 #define pa_index(pa)    ((pa) >> PDRSHIFT)
  183 #define pa_to_pvh(pa)   (&pv_table[pa_index(pa)])
  184 
  185 /*
  186  * Get PDEs and PTEs for user/kernel address space
  187  */
  188 #define pmap_pde(m, v)  (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
  189 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
  190 
  191 #define pmap_pde_v(pte)         ((*(int *)pte & PG_V) != 0)
  192 #define pmap_pte_w(pte)         ((*(int *)pte & PG_W) != 0)
  193 #define pmap_pte_m(pte)         ((*(int *)pte & PG_M) != 0)
  194 #define pmap_pte_u(pte)         ((*(int *)pte & PG_A) != 0)
  195 #define pmap_pte_v(pte)         ((*(int *)pte & PG_V) != 0)
  196 
  197 #define pmap_pte_set_w(pte, v)  ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \
  198     atomic_clear_int((u_int *)(pte), PG_W))
  199 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
  200 
  201 struct pmap kernel_pmap_store;
  202 LIST_HEAD(pmaplist, pmap);
  203 static struct pmaplist allpmaps;
  204 static struct mtx allpmaps_lock;
  205 
  206 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  207 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  208 int pgeflag = 0;                /* PG_G or-in */
  209 int pseflag = 0;                /* PG_PS or-in */
  210 
  211 static int nkpt = NKPT;
  212 vm_offset_t kernel_vm_end = KERNBASE + NKPT * NBPDR;
  213 extern u_int32_t KERNend;
  214 extern u_int32_t KPTphys;
  215 
  216 #ifdef PAE
  217 pt_entry_t pg_nx;
  218 static uma_zone_t pdptzone;
  219 #endif
  220 
  221 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  222 
  223 static int pat_works = 1;
  224 SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1,
  225     "Is page attribute table fully functional?");
  226 
  227 static int pg_ps_enabled = 1;
  228 SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN, &pg_ps_enabled, 0,
  229     "Are large page mappings enabled?");
  230 
  231 #define PAT_INDEX_SIZE  8
  232 static int pat_index[PAT_INDEX_SIZE];   /* cache mode to PAT index conversion */
  233 
  234 static struct rwlock_padalign pvh_global_lock;
  235 
  236 /*
  237  * Data for the pv entry allocation mechanism
  238  */
  239 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
  240 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  241 static struct md_page *pv_table;
  242 static int shpgperproc = PMAP_SHPGPERPROC;
  243 
  244 struct pv_chunk *pv_chunkbase;          /* KVA block for pv_chunks */
  245 int pv_maxchunks;                       /* How many chunks we have KVA for */
  246 vm_offset_t pv_vafree;                  /* freelist stored in the PTE */
  247 
  248 /*
  249  * All those kernel PT submaps that BSD is so fond of
  250  */
  251 struct sysmaps {
  252         struct  mtx lock;
  253         pt_entry_t *CMAP1;
  254         pt_entry_t *CMAP2;
  255         caddr_t CADDR1;
  256         caddr_t CADDR2;
  257 };
  258 static struct sysmaps sysmaps_pcpu[MAXCPU];
  259 pt_entry_t *CMAP3;
  260 static pd_entry_t *KPTD;
  261 caddr_t ptvmmap = 0;
  262 caddr_t CADDR3;
  263 struct msgbuf *msgbufp = 0;
  264 
  265 /*
  266  * Crashdump maps.
  267  */
  268 static caddr_t crashdumpmap;
  269 
  270 static pt_entry_t *PMAP1 = 0, *PMAP2;
  271 static pt_entry_t *PADDR1 = 0, *PADDR2;
  272 #ifdef SMP
  273 static int PMAP1cpu;
  274 static int PMAP1changedcpu;
  275 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 
  276            &PMAP1changedcpu, 0,
  277            "Number of times pmap_pte_quick changed CPU with same PMAP1");
  278 #endif
  279 static int PMAP1changed;
  280 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 
  281            &PMAP1changed, 0,
  282            "Number of times pmap_pte_quick changed PMAP1");
  283 static int PMAP1unchanged;
  284 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 
  285            &PMAP1unchanged, 0,
  286            "Number of times pmap_pte_quick didn't change PMAP1");
  287 static struct mtx PMAP2mutex;
  288 
  289 static void     free_pv_chunk(struct pv_chunk *pc);
  290 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  291 static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
  292 static void     pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  293 static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  294 static void     pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
  295 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  296 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  297                     vm_offset_t va);
  298 static int      pmap_pvh_wired_mappings(struct md_page *pvh, int count);
  299 
  300 static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  301 static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m,
  302     vm_prot_t prot);
  303 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  304     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  305 static void pmap_flush_page(vm_page_t m);
  306 static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
  307 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
  308 static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
  309 static boolean_t pmap_is_referenced_pvh(struct md_page *pvh);
  310 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
  311 static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde);
  312 static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
  313 static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
  314 static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
  315 static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva,
  316     vm_prot_t prot);
  317 static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
  318 static void pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
  319     struct spglist *free);
  320 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
  321     struct spglist *free);
  322 static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte);
  323 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va,
  324     struct spglist *free);
  325 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  326                                         vm_offset_t va);
  327 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  328 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  329     vm_page_t m);
  330 static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  331     pd_entry_t newpde);
  332 static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
  333 
  334 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);
  335 
  336 static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags);
  337 static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free);
  338 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
  339 static void pmap_pte_release(pt_entry_t *pte);
  340 static int pmap_unuse_pt(pmap_t, vm_offset_t, struct spglist *);
  341 #ifdef PAE
  342 static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
  343 #endif
  344 static void pmap_set_pg(void);
  345 
  346 static __inline void pagezero(void *page);
  347 
  348 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  349 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  350 
  351 /*
  352  * If you get an error here, then you set KVA_PAGES wrong! See the
  353  * description of KVA_PAGES in sys/i386/include/pmap.h. It must be
  354  * multiple of 4 for a normal kernel, or a multiple of 8 for a PAE.
  355  */
  356 CTASSERT(KERNBASE % (1 << 24) == 0);
  357 
  358 /*
  359  *      Bootstrap the system enough to run with virtual memory.
  360  *
  361  *      On the i386 this is called after mapping has already been enabled
  362  *      and just syncs the pmap module with what has already been done.
  363  *      [We can't call it easily with mapping off since the kernel is not
  364  *      mapped with PA == VA, hence we would have to relocate every address
  365  *      from the linked base (virtual) address "KERNBASE" to the actual
  366  *      (physical) address starting relative to 0]
  367  */
  368 void
  369 pmap_bootstrap(vm_paddr_t firstaddr)
  370 {
  371         vm_offset_t va;
  372         pt_entry_t *pte, *unused;
  373         struct sysmaps *sysmaps;
  374         int i;
  375 
  376         /*
  377          * Initialize the first available kernel virtual address.  However,
  378          * using "firstaddr" may waste a few pages of the kernel virtual
  379          * address space, because locore may not have mapped every physical
  380          * page that it allocated.  Preferably, locore would provide a first
  381          * unused virtual address in addition to "firstaddr".
  382          */
  383         virtual_avail = (vm_offset_t) KERNBASE + firstaddr;
  384 
  385         virtual_end = VM_MAX_KERNEL_ADDRESS;
  386 
  387         /*
  388          * Initialize the kernel pmap (which is statically allocated).
  389          */
  390         PMAP_LOCK_INIT(kernel_pmap);
  391         kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD);
  392 #ifdef PAE
  393         kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT);
  394 #endif
  395         CPU_FILL(&kernel_pmap->pm_active);      /* don't allow deactivation */
  396         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  397 
  398         /*
  399          * Initialize the global pv list lock.
  400          */
  401         rw_init(&pvh_global_lock, "pmap pv global");
  402 
  403         LIST_INIT(&allpmaps);
  404 
  405         /*
  406          * Request a spin mutex so that changes to allpmaps cannot be
  407          * preempted by smp_rendezvous_cpus().  Otherwise,
  408          * pmap_update_pde_kernel() could access allpmaps while it is
  409          * being changed.
  410          */
  411         mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN);
  412         mtx_lock_spin(&allpmaps_lock);
  413         LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list);
  414         mtx_unlock_spin(&allpmaps_lock);
  415 
  416         /*
  417          * Reserve some special page table entries/VA space for temporary
  418          * mapping of pages.
  419          */
  420 #define SYSMAP(c, p, v, n)      \
  421         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  422 
  423         va = virtual_avail;
  424         pte = vtopte(va);
  425 
  426         /*
  427          * CMAP1/CMAP2 are used for zeroing and copying pages.
  428          * CMAP3 is used for the idle process page zeroing.
  429          */
  430         for (i = 0; i < MAXCPU; i++) {
  431                 sysmaps = &sysmaps_pcpu[i];
  432                 mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF);
  433                 SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1)
  434                 SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1)
  435         }
  436         SYSMAP(caddr_t, CMAP3, CADDR3, 1)
  437 
  438         /*
  439          * Crashdump maps.
  440          */
  441         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  442 
  443         /*
  444          * ptvmmap is used for reading arbitrary physical pages via /dev/mem.
  445          */
  446         SYSMAP(caddr_t, unused, ptvmmap, 1)
  447 
  448         /*
  449          * msgbufp is used to map the system message buffer.
  450          */
  451         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(msgbufsize)))
  452 
  453         /*
  454          * KPTmap is used by pmap_kextract().
  455          *
  456          * KPTmap is first initialized by locore.  However, that initial
  457          * KPTmap can only support NKPT page table pages.  Here, a larger
  458          * KPTmap is created that can support KVA_PAGES page table pages.
  459          */
  460         SYSMAP(pt_entry_t *, KPTD, KPTmap, KVA_PAGES)
  461 
  462         for (i = 0; i < NKPT; i++)
  463                 KPTD[i] = (KPTphys + (i << PAGE_SHIFT)) | pgeflag | PG_RW | PG_V;
  464 
  465         /*
  466          * Adjust the start of the KPTD and KPTmap so that the implementation
  467          * of pmap_kextract() and pmap_growkernel() can be made simpler.
  468          */
  469         KPTD -= KPTDI;
  470         KPTmap -= i386_btop(KPTDI << PDRSHIFT);
  471 
  472         /*
  473          * PADDR1 and PADDR2 are used by pmap_pte_quick() and pmap_pte(),
  474          * respectively.
  475          */
  476         SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1)
  477         SYSMAP(pt_entry_t *, PMAP2, PADDR2, 1)
  478 
  479         mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
  480 
  481         virtual_avail = va;
  482 
  483         /*
  484          * Leave in place an identity mapping (virt == phys) for the low 1 MB
  485          * physical memory region that is used by the ACPI wakeup code.  This
  486          * mapping must not have PG_G set. 
  487          */
  488 #ifdef XBOX
  489         /* FIXME: This is gross, but needed for the XBOX. Since we are in such
  490          * an early stadium, we cannot yet neatly map video memory ... :-(
  491          * Better fixes are very welcome! */
  492         if (!arch_i386_is_xbox)
  493 #endif
  494         for (i = 1; i < NKPT; i++)
  495                 PTD[i] = 0;
  496 
  497         /* Initialize the PAT MSR if present. */
  498         pmap_init_pat();
  499 
  500         /* Turn on PG_G on kernel page(s) */
  501         pmap_set_pg();
  502 }
  503 
  504 /*
  505  * Setup the PAT MSR.
  506  */
  507 void
  508 pmap_init_pat(void)
  509 {
  510         int pat_table[PAT_INDEX_SIZE];
  511         uint64_t pat_msr;
  512         u_long cr0, cr4;
  513         int i;
  514 
  515         /* Set default PAT index table. */
  516         for (i = 0; i < PAT_INDEX_SIZE; i++)
  517                 pat_table[i] = -1;
  518         pat_table[PAT_WRITE_BACK] = 0;
  519         pat_table[PAT_WRITE_THROUGH] = 1;
  520         pat_table[PAT_UNCACHEABLE] = 3;
  521         pat_table[PAT_WRITE_COMBINING] = 3;
  522         pat_table[PAT_WRITE_PROTECTED] = 3;
  523         pat_table[PAT_UNCACHED] = 3;
  524 
  525         /* Bail if this CPU doesn't implement PAT. */
  526         if ((cpu_feature & CPUID_PAT) == 0) {
  527                 for (i = 0; i < PAT_INDEX_SIZE; i++)
  528                         pat_index[i] = pat_table[i];
  529                 pat_works = 0;
  530                 return;
  531         }
  532 
  533         /*
  534          * Due to some Intel errata, we can only safely use the lower 4
  535          * PAT entries.
  536          *
  537          *   Intel Pentium III Processor Specification Update
  538          * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
  539          * or Mode C Paging)
  540          *
  541          *   Intel Pentium IV  Processor Specification Update
  542          * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
  543          */
  544         if (cpu_vendor_id == CPU_VENDOR_INTEL &&
  545             !(CPUID_TO_FAMILY(cpu_id) == 6 && CPUID_TO_MODEL(cpu_id) >= 0xe))
  546                 pat_works = 0;
  547 
  548         /* Initialize default PAT entries. */
  549         pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) |
  550             PAT_VALUE(1, PAT_WRITE_THROUGH) |
  551             PAT_VALUE(2, PAT_UNCACHED) |
  552             PAT_VALUE(3, PAT_UNCACHEABLE) |
  553             PAT_VALUE(4, PAT_WRITE_BACK) |
  554             PAT_VALUE(5, PAT_WRITE_THROUGH) |
  555             PAT_VALUE(6, PAT_UNCACHED) |
  556             PAT_VALUE(7, PAT_UNCACHEABLE);
  557 
  558         if (pat_works) {
  559                 /*
  560                  * Leave the indices 0-3 at the default of WB, WT, UC-, and UC.
  561                  * Program 5 and 6 as WP and WC.
  562                  * Leave 4 and 7 as WB and UC.
  563                  */
  564                 pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6));
  565                 pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) |
  566                     PAT_VALUE(6, PAT_WRITE_COMBINING);
  567                 pat_table[PAT_UNCACHED] = 2;
  568                 pat_table[PAT_WRITE_PROTECTED] = 5;
  569                 pat_table[PAT_WRITE_COMBINING] = 6;
  570         } else {
  571                 /*
  572                  * Just replace PAT Index 2 with WC instead of UC-.
  573                  */
  574                 pat_msr &= ~PAT_MASK(2);
  575                 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  576                 pat_table[PAT_WRITE_COMBINING] = 2;
  577         }
  578 
  579         /* Disable PGE. */
  580         cr4 = rcr4();
  581         load_cr4(cr4 & ~CR4_PGE);
  582 
  583         /* Disable caches (CD = 1, NW = 0). */
  584         cr0 = rcr0();
  585         load_cr0((cr0 & ~CR0_NW) | CR0_CD);
  586 
  587         /* Flushes caches and TLBs. */
  588         wbinvd();
  589         invltlb();
  590 
  591         /* Update PAT and index table. */
  592         wrmsr(MSR_PAT, pat_msr);
  593         for (i = 0; i < PAT_INDEX_SIZE; i++)
  594                 pat_index[i] = pat_table[i];
  595 
  596         /* Flush caches and TLBs again. */
  597         wbinvd();
  598         invltlb();
  599 
  600         /* Restore caches and PGE. */
  601         load_cr0(cr0);
  602         load_cr4(cr4);
  603 }
  604 
  605 /*
  606  * Set PG_G on kernel pages.  Only the BSP calls this when SMP is turned on.
  607  */
  608 static void
  609 pmap_set_pg(void)
  610 {
  611         pt_entry_t *pte;
  612         vm_offset_t va, endva;
  613 
  614         if (pgeflag == 0)
  615                 return;
  616 
  617         endva = KERNBASE + KERNend;
  618 
  619         if (pseflag) {
  620                 va = KERNBASE + KERNLOAD;
  621                 while (va  < endva) {
  622                         pdir_pde(PTD, va) |= pgeflag;
  623                         invltlb();      /* Play it safe, invltlb() every time */
  624                         va += NBPDR;
  625                 }
  626         } else {
  627                 va = (vm_offset_t)btext;
  628                 while (va < endva) {
  629                         pte = vtopte(va);
  630                         if (*pte)
  631                                 *pte |= pgeflag;
  632                         invltlb();      /* Play it safe, invltlb() every time */
  633                         va += PAGE_SIZE;
  634                 }
  635         }
  636 }
  637 
  638 /*
  639  * Initialize a vm_page's machine-dependent fields.
  640  */
  641 void
  642 pmap_page_init(vm_page_t m)
  643 {
  644 
  645         TAILQ_INIT(&m->md.pv_list);
  646         m->md.pat_mode = PAT_WRITE_BACK;
  647 }
  648 
  649 #ifdef PAE
  650 static void *
  651 pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
  652 {
  653 
  654         /* Inform UMA that this allocator uses kernel_map/object. */
  655         *flags = UMA_SLAB_KERNEL;
  656         return ((void *)kmem_alloc_contig(kernel_arena, bytes, wait, 0x0ULL,
  657             0xffffffffULL, 1, 0, VM_MEMATTR_DEFAULT));
  658 }
  659 #endif
  660 
  661 /*
  662  * ABuse the pte nodes for unmapped kva to thread a kva freelist through.
  663  * Requirements:
  664  *  - Must deal with pages in order to ensure that none of the PG_* bits
  665  *    are ever set, PG_V in particular.
  666  *  - Assumes we can write to ptes without pte_store() atomic ops, even
  667  *    on PAE systems.  This should be ok.
  668  *  - Assumes nothing will ever test these addresses for 0 to indicate
  669  *    no mapping instead of correctly checking PG_V.
  670  *  - Assumes a vm_offset_t will fit in a pte (true for i386).
  671  * Because PG_V is never set, there can be no mappings to invalidate.
  672  */
  673 static vm_offset_t
  674 pmap_ptelist_alloc(vm_offset_t *head)
  675 {
  676         pt_entry_t *pte;
  677         vm_offset_t va;
  678 
  679         va = *head;
  680         if (va == 0)
  681                 panic("pmap_ptelist_alloc: exhausted ptelist KVA");
  682         pte = vtopte(va);
  683         *head = *pte;
  684         if (*head & PG_V)
  685                 panic("pmap_ptelist_alloc: va with PG_V set!");
  686         *pte = 0;
  687         return (va);
  688 }
  689 
  690 static void
  691 pmap_ptelist_free(vm_offset_t *head, vm_offset_t va)
  692 {
  693         pt_entry_t *pte;
  694 
  695         if (va & PG_V)
  696                 panic("pmap_ptelist_free: freeing va with PG_V set!");
  697         pte = vtopte(va);
  698         *pte = *head;           /* virtual! PG_V is 0 though */
  699         *head = va;
  700 }
  701 
  702 static void
  703 pmap_ptelist_init(vm_offset_t *head, void *base, int npages)
  704 {
  705         int i;
  706         vm_offset_t va;
  707 
  708         *head = 0;
  709         for (i = npages - 1; i >= 0; i--) {
  710                 va = (vm_offset_t)base + i * PAGE_SIZE;
  711                 pmap_ptelist_free(head, va);
  712         }
  713 }
  714 
  715 
  716 /*
  717  *      Initialize the pmap module.
  718  *      Called by vm_init, to initialize any structures that the pmap
  719  *      system needs to map virtual memory.
  720  */
  721 void
  722 pmap_init(void)
  723 {
  724         vm_page_t mpte;
  725         vm_size_t s;
  726         int i, pv_npg;
  727 
  728         /*
  729          * Initialize the vm page array entries for the kernel pmap's
  730          * page table pages.
  731          */ 
  732         for (i = 0; i < NKPT; i++) {
  733                 mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT));
  734                 KASSERT(mpte >= vm_page_array &&
  735                     mpte < &vm_page_array[vm_page_array_size],
  736                     ("pmap_init: page table page is out of range"));
  737                 mpte->pindex = i + KPTDI;
  738                 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
  739         }
  740 
  741         /*
  742          * Initialize the address space (zone) for the pv entries.  Set a
  743          * high water mark so that the system can recover from excessive
  744          * numbers of pv entries.
  745          */
  746         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  747         pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  748         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  749         pv_entry_max = roundup(pv_entry_max, _NPCPV);
  750         pv_entry_high_water = 9 * (pv_entry_max / 10);
  751 
  752         /*
  753          * If the kernel is running on a virtual machine, then it must assume
  754          * that MCA is enabled by the hypervisor.  Moreover, the kernel must
  755          * be prepared for the hypervisor changing the vendor and family that
  756          * are reported by CPUID.  Consequently, the workaround for AMD Family
  757          * 10h Erratum 383 is enabled if the processor's feature set does not
  758          * include at least one feature that is only supported by older Intel
  759          * or newer AMD processors.
  760          */
  761         if (vm_guest == VM_GUEST_VM && (cpu_feature & CPUID_SS) == 0 &&
  762             (cpu_feature2 & (CPUID2_SSSE3 | CPUID2_SSE41 | CPUID2_AESNI |
  763             CPUID2_AVX | CPUID2_XSAVE)) == 0 && (amd_feature2 & (AMDID2_XOP |
  764             AMDID2_FMA4)) == 0)
  765                 workaround_erratum383 = 1;
  766 
  767         /*
  768          * Are large page mappings supported and enabled?
  769          */
  770         TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled);
  771         if (pseflag == 0)
  772                 pg_ps_enabled = 0;
  773         else if (pg_ps_enabled) {
  774                 KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0,
  775                     ("pmap_init: can't assign to pagesizes[1]"));
  776                 pagesizes[1] = NBPDR;
  777         }
  778 
  779         /*
  780          * Calculate the size of the pv head table for superpages.
  781          */
  782         for (i = 0; phys_avail[i + 1]; i += 2);
  783         pv_npg = round_4mpage(phys_avail[(i - 2) + 1]) / NBPDR;
  784 
  785         /*
  786          * Allocate memory for the pv head table for superpages.
  787          */
  788         s = (vm_size_t)(pv_npg * sizeof(struct md_page));
  789         s = round_page(s);
  790         pv_table = (struct md_page *)kmem_malloc(kernel_arena, s,
  791             M_WAITOK | M_ZERO);
  792         for (i = 0; i < pv_npg; i++)
  793                 TAILQ_INIT(&pv_table[i].pv_list);
  794 
  795         pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc);
  796         pv_chunkbase = (struct pv_chunk *)kva_alloc(PAGE_SIZE * pv_maxchunks);
  797         if (pv_chunkbase == NULL)
  798                 panic("pmap_init: not enough kvm for pv chunks");
  799         pmap_ptelist_init(&pv_vafree, pv_chunkbase, pv_maxchunks);
  800 #ifdef PAE
  801         pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL,
  802             NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1,
  803             UMA_ZONE_VM | UMA_ZONE_NOFREE);
  804         uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf);
  805 #endif
  806 }
  807 
  808 
  809 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0,
  810         "Max number of PV entries");
  811 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0,
  812         "Page share factor per proc");
  813 
  814 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
  815     "2/4MB page mapping counters");
  816 
  817 static u_long pmap_pde_demotions;
  818 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD,
  819     &pmap_pde_demotions, 0, "2/4MB page demotions");
  820 
  821 static u_long pmap_pde_mappings;
  822 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
  823     &pmap_pde_mappings, 0, "2/4MB page mappings");
  824 
  825 static u_long pmap_pde_p_failures;
  826 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD,
  827     &pmap_pde_p_failures, 0, "2/4MB page promotion failures");
  828 
  829 static u_long pmap_pde_promotions;
  830 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD,
  831     &pmap_pde_promotions, 0, "2/4MB page promotions");
  832 
  833 /***************************************************
  834  * Low level helper routines.....
  835  ***************************************************/
  836 
  837 /*
  838  * Determine the appropriate bits to set in a PTE or PDE for a specified
  839  * caching mode.
  840  */
  841 int
  842 pmap_cache_bits(int mode, boolean_t is_pde)
  843 {
  844         int cache_bits, pat_flag, pat_idx;
  845 
  846         if (mode < 0 || mode >= PAT_INDEX_SIZE || pat_index[mode] < 0)
  847                 panic("Unknown caching mode %d\n", mode);
  848 
  849         /* The PAT bit is different for PTE's and PDE's. */
  850         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
  851 
  852         /* Map the caching mode to a PAT index. */
  853         pat_idx = pat_index[mode];
  854 
  855         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
  856         cache_bits = 0;
  857         if (pat_idx & 0x4)
  858                 cache_bits |= pat_flag;
  859         if (pat_idx & 0x2)
  860                 cache_bits |= PG_NC_PCD;
  861         if (pat_idx & 0x1)
  862                 cache_bits |= PG_NC_PWT;
  863         return (cache_bits);
  864 }
  865 
  866 /*
  867  * The caller is responsible for maintaining TLB consistency.
  868  */
  869 static void
  870 pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde)
  871 {
  872         pd_entry_t *pde;
  873         pmap_t pmap;
  874         boolean_t PTD_updated;
  875 
  876         PTD_updated = FALSE;
  877         mtx_lock_spin(&allpmaps_lock);
  878         LIST_FOREACH(pmap, &allpmaps, pm_list) {
  879                 if ((pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] &
  880                     PG_FRAME))
  881                         PTD_updated = TRUE;
  882                 pde = pmap_pde(pmap, va);
  883                 pde_store(pde, newpde);
  884         }
  885         mtx_unlock_spin(&allpmaps_lock);
  886         KASSERT(PTD_updated,
  887             ("pmap_kenter_pde: current page table is not in allpmaps"));
  888 }
  889 
  890 /*
  891  * After changing the page size for the specified virtual address in the page
  892  * table, flush the corresponding entries from the processor's TLB.  Only the
  893  * calling processor's TLB is affected.
  894  *
  895  * The calling thread must be pinned to a processor.
  896  */
  897 static void
  898 pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde)
  899 {
  900         u_long cr4;
  901 
  902         if ((newpde & PG_PS) == 0)
  903                 /* Demotion: flush a specific 2MB page mapping. */
  904                 invlpg(va);
  905         else if ((newpde & PG_G) == 0)
  906                 /*
  907                  * Promotion: flush every 4KB page mapping from the TLB
  908                  * because there are too many to flush individually.
  909                  */
  910                 invltlb();
  911         else {
  912                 /*
  913                  * Promotion: flush every 4KB page mapping from the TLB,
  914                  * including any global (PG_G) mappings.
  915                  */
  916                 cr4 = rcr4();
  917                 load_cr4(cr4 & ~CR4_PGE);
  918                 /*
  919                  * Although preemption at this point could be detrimental to
  920                  * performance, it would not lead to an error.  PG_G is simply
  921                  * ignored if CR4.PGE is clear.  Moreover, in case this block
  922                  * is re-entered, the load_cr4() either above or below will
  923                  * modify CR4.PGE flushing the TLB.
  924                  */
  925                 load_cr4(cr4 | CR4_PGE);
  926         }
  927 }
  928 #ifdef SMP
  929 /*
  930  * For SMP, these functions have to use the IPI mechanism for coherence.
  931  *
  932  * N.B.: Before calling any of the following TLB invalidation functions,
  933  * the calling processor must ensure that all stores updating a non-
  934  * kernel page table are globally performed.  Otherwise, another
  935  * processor could cache an old, pre-update entry without being
  936  * invalidated.  This can happen one of two ways: (1) The pmap becomes
  937  * active on another processor after its pm_active field is checked by
  938  * one of the following functions but before a store updating the page
  939  * table is globally performed. (2) The pmap becomes active on another
  940  * processor before its pm_active field is checked but due to
  941  * speculative loads one of the following functions stills reads the
  942  * pmap as inactive on the other processor.
  943  * 
  944  * The kernel page table is exempt because its pm_active field is
  945  * immutable.  The kernel page table is always active on every
  946  * processor.
  947  */
  948 void
  949 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  950 {
  951         cpuset_t other_cpus;
  952         u_int cpuid;
  953 
  954         sched_pin();
  955         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
  956                 invlpg(va);
  957                 smp_invlpg(va);
  958         } else {
  959                 cpuid = PCPU_GET(cpuid);
  960                 other_cpus = all_cpus;
  961                 CPU_CLR(cpuid, &other_cpus);
  962                 if (CPU_ISSET(cpuid, &pmap->pm_active))
  963                         invlpg(va);
  964                 CPU_AND(&other_cpus, &pmap->pm_active);
  965                 if (!CPU_EMPTY(&other_cpus))
  966                         smp_masked_invlpg(other_cpus, va);
  967         }
  968         sched_unpin();
  969 }
  970 
  971 void
  972 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  973 {
  974         cpuset_t other_cpus;
  975         vm_offset_t addr;
  976         u_int cpuid;
  977 
  978         sched_pin();
  979         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
  980                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  981                         invlpg(addr);
  982                 smp_invlpg_range(sva, eva);
  983         } else {
  984                 cpuid = PCPU_GET(cpuid);
  985                 other_cpus = all_cpus;
  986                 CPU_CLR(cpuid, &other_cpus);
  987                 if (CPU_ISSET(cpuid, &pmap->pm_active))
  988                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
  989                                 invlpg(addr);
  990                 CPU_AND(&other_cpus, &pmap->pm_active);
  991                 if (!CPU_EMPTY(&other_cpus))
  992                         smp_masked_invlpg_range(other_cpus, sva, eva);
  993         }
  994         sched_unpin();
  995 }
  996 
  997 void
  998 pmap_invalidate_all(pmap_t pmap)
  999 {
 1000         cpuset_t other_cpus;
 1001         u_int cpuid;
 1002 
 1003         sched_pin();
 1004         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
 1005                 invltlb();
 1006                 smp_invltlb();
 1007         } else {
 1008                 cpuid = PCPU_GET(cpuid);
 1009                 other_cpus = all_cpus;
 1010                 CPU_CLR(cpuid, &other_cpus);
 1011                 if (CPU_ISSET(cpuid, &pmap->pm_active))
 1012                         invltlb();
 1013                 CPU_AND(&other_cpus, &pmap->pm_active);
 1014                 if (!CPU_EMPTY(&other_cpus))
 1015                         smp_masked_invltlb(other_cpus);
 1016         }
 1017         sched_unpin();
 1018 }
 1019 
 1020 void
 1021 pmap_invalidate_cache(void)
 1022 {
 1023 
 1024         sched_pin();
 1025         wbinvd();
 1026         smp_cache_flush();
 1027         sched_unpin();
 1028 }
 1029 
 1030 struct pde_action {
 1031         cpuset_t invalidate;    /* processors that invalidate their TLB */
 1032         vm_offset_t va;
 1033         pd_entry_t *pde;
 1034         pd_entry_t newpde;
 1035         u_int store;            /* processor that updates the PDE */
 1036 };
 1037 
 1038 static void
 1039 pmap_update_pde_kernel(void *arg)
 1040 {
 1041         struct pde_action *act = arg;
 1042         pd_entry_t *pde;
 1043         pmap_t pmap;
 1044 
 1045         if (act->store == PCPU_GET(cpuid)) {
 1046 
 1047                 /*
 1048                  * Elsewhere, this operation requires allpmaps_lock for
 1049                  * synchronization.  Here, it does not because it is being
 1050                  * performed in the context of an all_cpus rendezvous.
 1051                  */
 1052                 LIST_FOREACH(pmap, &allpmaps, pm_list) {
 1053                         pde = pmap_pde(pmap, act->va);
 1054                         pde_store(pde, act->newpde);
 1055                 }
 1056         }
 1057 }
 1058 
 1059 static void
 1060 pmap_update_pde_user(void *arg)
 1061 {
 1062         struct pde_action *act = arg;
 1063 
 1064         if (act->store == PCPU_GET(cpuid))
 1065                 pde_store(act->pde, act->newpde);
 1066 }
 1067 
 1068 static void
 1069 pmap_update_pde_teardown(void *arg)
 1070 {
 1071         struct pde_action *act = arg;
 1072 
 1073         if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
 1074                 pmap_update_pde_invalidate(act->va, act->newpde);
 1075 }
 1076 
 1077 /*
 1078  * Change the page size for the specified virtual address in a way that
 1079  * prevents any possibility of the TLB ever having two entries that map the
 1080  * same virtual address using different page sizes.  This is the recommended
 1081  * workaround for Erratum 383 on AMD Family 10h processors.  It prevents a
 1082  * machine check exception for a TLB state that is improperly diagnosed as a
 1083  * hardware error.
 1084  */
 1085 static void
 1086 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1087 {
 1088         struct pde_action act;
 1089         cpuset_t active, other_cpus;
 1090         u_int cpuid;
 1091 
 1092         sched_pin();
 1093         cpuid = PCPU_GET(cpuid);
 1094         other_cpus = all_cpus;
 1095         CPU_CLR(cpuid, &other_cpus);
 1096         if (pmap == kernel_pmap)
 1097                 active = all_cpus;
 1098         else
 1099                 active = pmap->pm_active;
 1100         if (CPU_OVERLAP(&active, &other_cpus)) {
 1101                 act.store = cpuid;
 1102                 act.invalidate = active;
 1103                 act.va = va;
 1104                 act.pde = pde;
 1105                 act.newpde = newpde;
 1106                 CPU_SET(cpuid, &active);
 1107                 smp_rendezvous_cpus(active,
 1108                     smp_no_rendevous_barrier, pmap == kernel_pmap ?
 1109                     pmap_update_pde_kernel : pmap_update_pde_user,
 1110                     pmap_update_pde_teardown, &act);
 1111         } else {
 1112                 if (pmap == kernel_pmap)
 1113                         pmap_kenter_pde(va, newpde);
 1114                 else
 1115                         pde_store(pde, newpde);
 1116                 if (CPU_ISSET(cpuid, &active))
 1117                         pmap_update_pde_invalidate(va, newpde);
 1118         }
 1119         sched_unpin();
 1120 }
 1121 #else /* !SMP */
 1122 /*
 1123  * Normal, non-SMP, 486+ invalidation functions.
 1124  * We inline these within pmap.c for speed.
 1125  */
 1126 PMAP_INLINE void
 1127 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 1128 {
 1129 
 1130         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1131                 invlpg(va);
 1132 }
 1133 
 1134 PMAP_INLINE void
 1135 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1136 {
 1137         vm_offset_t addr;
 1138 
 1139         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1140                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
 1141                         invlpg(addr);
 1142 }
 1143 
 1144 PMAP_INLINE void
 1145 pmap_invalidate_all(pmap_t pmap)
 1146 {
 1147 
 1148         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1149                 invltlb();
 1150 }
 1151 
 1152 PMAP_INLINE void
 1153 pmap_invalidate_cache(void)
 1154 {
 1155 
 1156         wbinvd();
 1157 }
 1158 
 1159 static void
 1160 pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
 1161 {
 1162 
 1163         if (pmap == kernel_pmap)
 1164                 pmap_kenter_pde(va, newpde);
 1165         else
 1166                 pde_store(pde, newpde);
 1167         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
 1168                 pmap_update_pde_invalidate(va, newpde);
 1169 }
 1170 #endif /* !SMP */
 1171 
 1172 #define PMAP_CLFLUSH_THRESHOLD  (2 * 1024 * 1024)
 1173 
 1174 void
 1175 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
 1176 {
 1177 
 1178         KASSERT((sva & PAGE_MASK) == 0,
 1179             ("pmap_invalidate_cache_range: sva not page-aligned"));
 1180         KASSERT((eva & PAGE_MASK) == 0,
 1181             ("pmap_invalidate_cache_range: eva not page-aligned"));
 1182 
 1183         if (cpu_feature & CPUID_SS)
 1184                 ; /* If "Self Snoop" is supported, do nothing. */
 1185         else if ((cpu_feature & CPUID_CLFSH) != 0 &&
 1186             eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 1187 
 1188 #ifdef DEV_APIC
 1189                 /*
 1190                  * XXX: Some CPUs fault, hang, or trash the local APIC
 1191                  * registers if we use CLFLUSH on the local APIC
 1192                  * range.  The local APIC is always uncached, so we
 1193                  * don't need to flush for that range anyway.
 1194                  */
 1195                 if (pmap_kextract(sva) == lapic_paddr)
 1196                         return;
 1197 #endif
 1198                 /*
 1199                  * Otherwise, do per-cache line flush.  Use the mfence
 1200                  * instruction to insure that previous stores are
 1201                  * included in the write-back.  The processor
 1202                  * propagates flush to other processors in the cache
 1203                  * coherence domain.
 1204                  */
 1205                 mfence();
 1206                 for (; sva < eva; sva += cpu_clflush_line_size)
 1207                         clflush(sva);
 1208                 mfence();
 1209         } else {
 1210 
 1211                 /*
 1212                  * No targeted cache flush methods are supported by CPU,
 1213                  * or the supplied range is bigger than 2MB.
 1214                  * Globally invalidate cache.
 1215                  */
 1216                 pmap_invalidate_cache();
 1217         }
 1218 }
 1219 
 1220 void
 1221 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
 1222 {
 1223         int i;
 1224 
 1225         if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
 1226             (cpu_feature & CPUID_CLFSH) == 0) {
 1227                 pmap_invalidate_cache();
 1228         } else {
 1229                 for (i = 0; i < count; i++)
 1230                         pmap_flush_page(pages[i]);
 1231         }
 1232 }
 1233 
 1234 /*
 1235  * Are we current address space or kernel?  N.B. We return FALSE when
 1236  * a pmap's page table is in use because a kernel thread is borrowing
 1237  * it.  The borrowed page table can change spontaneously, making any
 1238  * dependence on its continued use subject to a race condition.
 1239  */
 1240 static __inline int
 1241 pmap_is_current(pmap_t pmap)
 1242 {
 1243 
 1244         return (pmap == kernel_pmap ||
 1245             (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) &&
 1246             (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME)));
 1247 }
 1248 
 1249 /*
 1250  * If the given pmap is not the current or kernel pmap, the returned pte must
 1251  * be released by passing it to pmap_pte_release().
 1252  */
 1253 pt_entry_t *
 1254 pmap_pte(pmap_t pmap, vm_offset_t va)
 1255 {
 1256         pd_entry_t newpf;
 1257         pd_entry_t *pde;
 1258 
 1259         pde = pmap_pde(pmap, va);
 1260         if (*pde & PG_PS)
 1261                 return (pde);
 1262         if (*pde != 0) {
 1263                 /* are we current address space or kernel? */
 1264                 if (pmap_is_current(pmap))
 1265                         return (vtopte(va));
 1266                 mtx_lock(&PMAP2mutex);
 1267                 newpf = *pde & PG_FRAME;
 1268                 if ((*PMAP2 & PG_FRAME) != newpf) {
 1269                         *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1270                         pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
 1271                 }
 1272                 return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
 1273         }
 1274         return (NULL);
 1275 }
 1276 
 1277 /*
 1278  * Releases a pte that was obtained from pmap_pte().  Be prepared for the pte
 1279  * being NULL.
 1280  */
 1281 static __inline void
 1282 pmap_pte_release(pt_entry_t *pte)
 1283 {
 1284 
 1285         if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2)
 1286                 mtx_unlock(&PMAP2mutex);
 1287 }
 1288 
 1289 /*
 1290  * NB:  The sequence of updating a page table followed by accesses to the
 1291  * corresponding pages is subject to the situation described in the "AMD64
 1292  * Architecture Programmer's Manual Volume 2: System Programming" rev. 3.23,
 1293  * "7.3.1 Special Coherency Considerations".  Therefore, issuing the INVLPG
 1294  * right after modifying the PTE bits is crucial.
 1295  */
 1296 static __inline void
 1297 invlcaddr(void *caddr)
 1298 {
 1299 
 1300         invlpg((u_int)caddr);
 1301 }
 1302 
 1303 /*
 1304  * Super fast pmap_pte routine best used when scanning
 1305  * the pv lists.  This eliminates many coarse-grained
 1306  * invltlb calls.  Note that many of the pv list
 1307  * scans are across different pmaps.  It is very wasteful
 1308  * to do an entire invltlb for checking a single mapping.
 1309  *
 1310  * If the given pmap is not the current pmap, pvh_global_lock
 1311  * must be held and curthread pinned to a CPU.
 1312  */
 1313 static pt_entry_t *
 1314 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
 1315 {
 1316         pd_entry_t newpf;
 1317         pd_entry_t *pde;
 1318 
 1319         pde = pmap_pde(pmap, va);
 1320         if (*pde & PG_PS)
 1321                 return (pde);
 1322         if (*pde != 0) {
 1323                 /* are we current address space or kernel? */
 1324                 if (pmap_is_current(pmap))
 1325                         return (vtopte(va));
 1326                 rw_assert(&pvh_global_lock, RA_WLOCKED);
 1327                 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 1328                 newpf = *pde & PG_FRAME;
 1329                 if ((*PMAP1 & PG_FRAME) != newpf) {
 1330                         *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M;
 1331 #ifdef SMP
 1332                         PMAP1cpu = PCPU_GET(cpuid);
 1333 #endif
 1334                         invlcaddr(PADDR1);
 1335                         PMAP1changed++;
 1336                 } else
 1337 #ifdef SMP
 1338                 if (PMAP1cpu != PCPU_GET(cpuid)) {
 1339                         PMAP1cpu = PCPU_GET(cpuid);
 1340                         invlcaddr(PADDR1);
 1341                         PMAP1changedcpu++;
 1342                 } else
 1343 #endif
 1344                         PMAP1unchanged++;
 1345                 return (PADDR1 + (i386_btop(va) & (NPTEPG - 1)));
 1346         }
 1347         return (0);
 1348 }
 1349 
 1350 /*
 1351  *      Routine:        pmap_extract
 1352  *      Function:
 1353  *              Extract the physical page address associated
 1354  *              with the given map/virtual_address pair.
 1355  */
 1356 vm_paddr_t 
 1357 pmap_extract(pmap_t pmap, vm_offset_t va)
 1358 {
 1359         vm_paddr_t rtval;
 1360         pt_entry_t *pte;
 1361         pd_entry_t pde;
 1362 
 1363         rtval = 0;
 1364         PMAP_LOCK(pmap);
 1365         pde = pmap->pm_pdir[va >> PDRSHIFT];
 1366         if (pde != 0) {
 1367                 if ((pde & PG_PS) != 0)
 1368                         rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
 1369                 else {
 1370                         pte = pmap_pte(pmap, va);
 1371                         rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
 1372                         pmap_pte_release(pte);
 1373                 }
 1374         }
 1375         PMAP_UNLOCK(pmap);
 1376         return (rtval);
 1377 }
 1378 
 1379 /*
 1380  *      Routine:        pmap_extract_and_hold
 1381  *      Function:
 1382  *              Atomically extract and hold the physical page
 1383  *              with the given pmap and virtual address pair
 1384  *              if that mapping permits the given protection.
 1385  */
 1386 vm_page_t
 1387 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 1388 {
 1389         pd_entry_t pde;
 1390         pt_entry_t pte, *ptep;
 1391         vm_page_t m;
 1392         vm_paddr_t pa;
 1393 
 1394         pa = 0;
 1395         m = NULL;
 1396         PMAP_LOCK(pmap);
 1397 retry:
 1398         pde = *pmap_pde(pmap, va);
 1399         if (pde != 0) {
 1400                 if (pde & PG_PS) {
 1401                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
 1402                                 if (vm_page_pa_tryrelock(pmap, (pde &
 1403                                     PG_PS_FRAME) | (va & PDRMASK), &pa))
 1404                                         goto retry;
 1405                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
 1406                                     (va & PDRMASK));
 1407                                 vm_page_hold(m);
 1408                         }
 1409                 } else {
 1410                         ptep = pmap_pte(pmap, va);
 1411                         pte = *ptep;
 1412                         pmap_pte_release(ptep);
 1413                         if (pte != 0 &&
 1414                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
 1415                                 if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME,
 1416                                     &pa))
 1417                                         goto retry;
 1418                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
 1419                                 vm_page_hold(m);
 1420                         }
 1421                 }
 1422         }
 1423         PA_UNLOCK_COND(pa);
 1424         PMAP_UNLOCK(pmap);
 1425         return (m);
 1426 }
 1427 
 1428 /***************************************************
 1429  * Low level mapping routines.....
 1430  ***************************************************/
 1431 
 1432 /*
 1433  * Add a wired page to the kva.
 1434  * Note: not SMP coherent.
 1435  *
 1436  * This function may be used before pmap_bootstrap() is called.
 1437  */
 1438 PMAP_INLINE void 
 1439 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
 1440 {
 1441         pt_entry_t *pte;
 1442 
 1443         pte = vtopte(va);
 1444         pte_store(pte, pa | PG_RW | PG_V | pgeflag);
 1445 }
 1446 
 1447 static __inline void
 1448 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
 1449 {
 1450         pt_entry_t *pte;
 1451 
 1452         pte = vtopte(va);
 1453         pte_store(pte, pa | PG_RW | PG_V | pgeflag | pmap_cache_bits(mode, 0));
 1454 }
 1455 
 1456 /*
 1457  * Remove a page from the kernel pagetables.
 1458  * Note: not SMP coherent.
 1459  *
 1460  * This function may be used before pmap_bootstrap() is called.
 1461  */
 1462 PMAP_INLINE void
 1463 pmap_kremove(vm_offset_t va)
 1464 {
 1465         pt_entry_t *pte;
 1466 
 1467         pte = vtopte(va);
 1468         pte_clear(pte);
 1469 }
 1470 
 1471 /*
 1472  *      Used to map a range of physical addresses into kernel
 1473  *      virtual address space.
 1474  *
 1475  *      The value passed in '*virt' is a suggested virtual address for
 1476  *      the mapping. Architectures which can support a direct-mapped
 1477  *      physical to virtual region can return the appropriate address
 1478  *      within that region, leaving '*virt' unchanged. Other
 1479  *      architectures should map the pages starting at '*virt' and
 1480  *      update '*virt' with the first usable address after the mapped
 1481  *      region.
 1482  */
 1483 vm_offset_t
 1484 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 1485 {
 1486         vm_offset_t va, sva;
 1487         vm_paddr_t superpage_offset;
 1488         pd_entry_t newpde;
 1489 
 1490         va = *virt;
 1491         /*
 1492          * Does the physical address range's size and alignment permit at
 1493          * least one superpage mapping to be created?
 1494          */ 
 1495         superpage_offset = start & PDRMASK;
 1496         if ((end - start) - ((NBPDR - superpage_offset) & PDRMASK) >= NBPDR) {
 1497                 /*
 1498                  * Increase the starting virtual address so that its alignment
 1499                  * does not preclude the use of superpage mappings.
 1500                  */
 1501                 if ((va & PDRMASK) < superpage_offset)
 1502                         va = (va & ~PDRMASK) + superpage_offset;
 1503                 else if ((va & PDRMASK) > superpage_offset)
 1504                         va = ((va + PDRMASK) & ~PDRMASK) + superpage_offset;
 1505         }
 1506         sva = va;
 1507         while (start < end) {
 1508                 if ((start & PDRMASK) == 0 && end - start >= NBPDR &&
 1509                     pseflag) {
 1510                         KASSERT((va & PDRMASK) == 0,
 1511                             ("pmap_map: misaligned va %#x", va));
 1512                         newpde = start | PG_PS | pgeflag | PG_RW | PG_V;
 1513                         pmap_kenter_pde(va, newpde);
 1514                         va += NBPDR;
 1515                         start += NBPDR;
 1516                 } else {
 1517                         pmap_kenter(va, start);
 1518                         va += PAGE_SIZE;
 1519                         start += PAGE_SIZE;
 1520                 }
 1521         }
 1522         pmap_invalidate_range(kernel_pmap, sva, va);
 1523         *virt = va;
 1524         return (sva);
 1525 }
 1526 
 1527 
 1528 /*
 1529  * Add a list of wired pages to the kva
 1530  * this routine is only used for temporary
 1531  * kernel mappings that do not need to have
 1532  * page modification or references recorded.
 1533  * Note that old mappings are simply written
 1534  * over.  The page *must* be wired.
 1535  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1536  */
 1537 void
 1538 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 1539 {
 1540         pt_entry_t *endpte, oldpte, pa, *pte;
 1541         vm_page_t m;
 1542 
 1543         oldpte = 0;
 1544         pte = vtopte(sva);
 1545         endpte = pte + count;
 1546         while (pte < endpte) {
 1547                 m = *ma++;
 1548                 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0);
 1549                 if ((*pte & (PG_FRAME | PG_PTE_CACHE)) != pa) {
 1550                         oldpte |= *pte;
 1551                         pte_store(pte, pa | pgeflag | PG_RW | PG_V);
 1552                 }
 1553                 pte++;
 1554         }
 1555         if (__predict_false((oldpte & PG_V) != 0))
 1556                 pmap_invalidate_range(kernel_pmap, sva, sva + count *
 1557                     PAGE_SIZE);
 1558 }
 1559 
 1560 /*
 1561  * This routine tears out page mappings from the
 1562  * kernel -- it is meant only for temporary mappings.
 1563  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1564  */
 1565 void
 1566 pmap_qremove(vm_offset_t sva, int count)
 1567 {
 1568         vm_offset_t va;
 1569 
 1570         va = sva;
 1571         while (count-- > 0) {
 1572                 pmap_kremove(va);
 1573                 va += PAGE_SIZE;
 1574         }
 1575         pmap_invalidate_range(kernel_pmap, sva, va);
 1576 }
 1577 
 1578 /***************************************************
 1579  * Page table page management routines.....
 1580  ***************************************************/
 1581 static __inline void
 1582 pmap_free_zero_pages(struct spglist *free)
 1583 {
 1584         vm_page_t m;
 1585 
 1586         while ((m = SLIST_FIRST(free)) != NULL) {
 1587                 SLIST_REMOVE_HEAD(free, plinks.s.ss);
 1588                 /* Preserve the page's PG_ZERO setting. */
 1589                 vm_page_free_toq(m);
 1590         }
 1591 }
 1592 
 1593 /*
 1594  * Schedule the specified unused page table page to be freed.  Specifically,
 1595  * add the page to the specified list of pages that will be released to the
 1596  * physical memory manager after the TLB has been updated.
 1597  */
 1598 static __inline void
 1599 pmap_add_delayed_free_list(vm_page_t m, struct spglist *free,
 1600     boolean_t set_PG_ZERO)
 1601 {
 1602 
 1603         if (set_PG_ZERO)
 1604                 m->flags |= PG_ZERO;
 1605         else
 1606                 m->flags &= ~PG_ZERO;
 1607         SLIST_INSERT_HEAD(free, m, plinks.s.ss);
 1608 }
 1609 
 1610 /*
 1611  * Inserts the specified page table page into the specified pmap's collection
 1612  * of idle page table pages.  Each of a pmap's page table pages is responsible
 1613  * for mapping a distinct range of virtual addresses.  The pmap's collection is
 1614  * ordered by this virtual address range.
 1615  */
 1616 static __inline int
 1617 pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte)
 1618 {
 1619 
 1620         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1621         return (vm_radix_insert(&pmap->pm_root, mpte));
 1622 }
 1623 
 1624 /*
 1625  * Looks for a page table page mapping the specified virtual address in the
 1626  * specified pmap's collection of idle page table pages.  Returns NULL if there
 1627  * is no page table page corresponding to the specified virtual address.
 1628  */
 1629 static __inline vm_page_t
 1630 pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va)
 1631 {
 1632 
 1633         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1634         return (vm_radix_lookup(&pmap->pm_root, va >> PDRSHIFT));
 1635 }
 1636 
 1637 /*
 1638  * Removes the specified page table page from the specified pmap's collection
 1639  * of idle page table pages.  The specified page table page must be a member of
 1640  * the pmap's collection.
 1641  */
 1642 static __inline void
 1643 pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte)
 1644 {
 1645 
 1646         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1647         vm_radix_remove(&pmap->pm_root, mpte->pindex);
 1648 }
 1649 
 1650 /*
 1651  * Decrements a page table page's wire count, which is used to record the
 1652  * number of valid page table entries within the page.  If the wire count
 1653  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
 1654  * page table page was unmapped and FALSE otherwise.
 1655  */
 1656 static inline boolean_t
 1657 pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
 1658 {
 1659 
 1660         --m->wire_count;
 1661         if (m->wire_count == 0) {
 1662                 _pmap_unwire_ptp(pmap, m, free);
 1663                 return (TRUE);
 1664         } else
 1665                 return (FALSE);
 1666 }
 1667 
 1668 static void
 1669 _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, struct spglist *free)
 1670 {
 1671         vm_offset_t pteva;
 1672 
 1673         /*
 1674          * unmap the page table page
 1675          */
 1676         pmap->pm_pdir[m->pindex] = 0;
 1677         --pmap->pm_stats.resident_count;
 1678 
 1679         /*
 1680          * This is a release store so that the ordinary store unmapping
 1681          * the page table page is globally performed before TLB shoot-
 1682          * down is begun.
 1683          */
 1684         atomic_subtract_rel_int(&cnt.v_wire_count, 1);
 1685 
 1686         /*
 1687          * Do an invltlb to make the invalidated mapping
 1688          * take effect immediately.
 1689          */
 1690         pteva = VM_MAXUSER_ADDRESS + i386_ptob(m->pindex);
 1691         pmap_invalidate_page(pmap, pteva);
 1692 
 1693         /* 
 1694          * Put page on a list so that it is released after
 1695          * *ALL* TLB shootdown is done
 1696          */
 1697         pmap_add_delayed_free_list(m, free, TRUE);
 1698 }
 1699 
 1700 /*
 1701  * After removing a page table entry, this routine is used to
 1702  * conditionally free the page, and manage the hold/wire counts.
 1703  */
 1704 static int
 1705 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, struct spglist *free)
 1706 {
 1707         pd_entry_t ptepde;
 1708         vm_page_t mpte;
 1709 
 1710         if (va >= VM_MAXUSER_ADDRESS)
 1711                 return (0);
 1712         ptepde = *pmap_pde(pmap, va);
 1713         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1714         return (pmap_unwire_ptp(pmap, mpte, free));
 1715 }
 1716 
 1717 /*
 1718  * Initialize the pmap for the swapper process.
 1719  */
 1720 void
 1721 pmap_pinit0(pmap_t pmap)
 1722 {
 1723 
 1724         PMAP_LOCK_INIT(pmap);
 1725         /*
 1726          * Since the page table directory is shared with the kernel pmap,
 1727          * which is already included in the list "allpmaps", this pmap does
 1728          * not need to be inserted into that list.
 1729          */
 1730         pmap->pm_pdir = (pd_entry_t *)(KERNBASE + (vm_offset_t)IdlePTD);
 1731 #ifdef PAE
 1732         pmap->pm_pdpt = (pdpt_entry_t *)(KERNBASE + (vm_offset_t)IdlePDPT);
 1733 #endif
 1734         pmap->pm_root.rt_root = 0;
 1735         CPU_ZERO(&pmap->pm_active);
 1736         PCPU_SET(curpmap, pmap);
 1737         TAILQ_INIT(&pmap->pm_pvchunk);
 1738         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1739 }
 1740 
 1741 /*
 1742  * Initialize a preallocated and zeroed pmap structure,
 1743  * such as one in a vmspace structure.
 1744  */
 1745 int
 1746 pmap_pinit(pmap_t pmap)
 1747 {
 1748         vm_page_t m, ptdpg[NPGPTD];
 1749         vm_paddr_t pa;
 1750         int i;
 1751 
 1752         /*
 1753          * No need to allocate page table space yet but we do need a valid
 1754          * page directory table.
 1755          */
 1756         if (pmap->pm_pdir == NULL) {
 1757                 pmap->pm_pdir = (pd_entry_t *)kva_alloc(NBPTD);
 1758                 if (pmap->pm_pdir == NULL)
 1759                         return (0);
 1760 #ifdef PAE
 1761                 pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
 1762                 KASSERT(((vm_offset_t)pmap->pm_pdpt &
 1763                     ((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0,
 1764                     ("pmap_pinit: pdpt misaligned"));
 1765                 KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
 1766                     ("pmap_pinit: pdpt above 4g"));
 1767 #endif
 1768                 pmap->pm_root.rt_root = 0;
 1769         }
 1770         KASSERT(vm_radix_is_empty(&pmap->pm_root),
 1771             ("pmap_pinit: pmap has reserved page table page(s)"));
 1772 
 1773         /*
 1774          * allocate the page directory page(s)
 1775          */
 1776         for (i = 0; i < NPGPTD;) {
 1777                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 1778                     VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 1779                 if (m == NULL)
 1780                         VM_WAIT;
 1781                 else {
 1782                         ptdpg[i++] = m;
 1783                 }
 1784         }
 1785 
 1786         pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD);
 1787 
 1788         for (i = 0; i < NPGPTD; i++)
 1789                 if ((ptdpg[i]->flags & PG_ZERO) == 0)
 1790                         pagezero(pmap->pm_pdir + (i * NPDEPG));
 1791 
 1792         mtx_lock_spin(&allpmaps_lock);
 1793         LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);
 1794         /* Copy the kernel page table directory entries. */
 1795         bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t));
 1796         mtx_unlock_spin(&allpmaps_lock);
 1797 
 1798         /* install self-referential address mapping entry(s) */
 1799         for (i = 0; i < NPGPTD; i++) {
 1800                 pa = VM_PAGE_TO_PHYS(ptdpg[i]);
 1801                 pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M;
 1802 #ifdef PAE
 1803                 pmap->pm_pdpt[i] = pa | PG_V;
 1804 #endif
 1805         }
 1806 
 1807         CPU_ZERO(&pmap->pm_active);
 1808         TAILQ_INIT(&pmap->pm_pvchunk);
 1809         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1810 
 1811         return (1);
 1812 }
 1813 
 1814 /*
 1815  * this routine is called if the page table page is not
 1816  * mapped correctly.
 1817  */
 1818 static vm_page_t
 1819 _pmap_allocpte(pmap_t pmap, u_int ptepindex, u_int flags)
 1820 {
 1821         vm_paddr_t ptepa;
 1822         vm_page_t m;
 1823 
 1824         /*
 1825          * Allocate a page table page.
 1826          */
 1827         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1828             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1829                 if ((flags & PMAP_ENTER_NOSLEEP) == 0) {
 1830                         PMAP_UNLOCK(pmap);
 1831                         rw_wunlock(&pvh_global_lock);
 1832                         VM_WAIT;
 1833                         rw_wlock(&pvh_global_lock);
 1834                         PMAP_LOCK(pmap);
 1835                 }
 1836 
 1837                 /*
 1838                  * Indicate the need to retry.  While waiting, the page table
 1839                  * page may have been allocated.
 1840                  */
 1841                 return (NULL);
 1842         }
 1843         if ((m->flags & PG_ZERO) == 0)
 1844                 pmap_zero_page(m);
 1845 
 1846         /*
 1847          * Map the pagetable page into the process address space, if
 1848          * it isn't already there.
 1849          */
 1850 
 1851         pmap->pm_stats.resident_count++;
 1852 
 1853         ptepa = VM_PAGE_TO_PHYS(m);
 1854         pmap->pm_pdir[ptepindex] =
 1855                 (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
 1856 
 1857         return (m);
 1858 }
 1859 
 1860 static vm_page_t
 1861 pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)
 1862 {
 1863         u_int ptepindex;
 1864         pd_entry_t ptepa;
 1865         vm_page_t m;
 1866 
 1867         /*
 1868          * Calculate pagetable page index
 1869          */
 1870         ptepindex = va >> PDRSHIFT;
 1871 retry:
 1872         /*
 1873          * Get the page directory entry
 1874          */
 1875         ptepa = pmap->pm_pdir[ptepindex];
 1876 
 1877         /*
 1878          * This supports switching from a 4MB page to a
 1879          * normal 4K page.
 1880          */
 1881         if (ptepa & PG_PS) {
 1882                 (void)pmap_demote_pde(pmap, &pmap->pm_pdir[ptepindex], va);
 1883                 ptepa = pmap->pm_pdir[ptepindex];
 1884         }
 1885 
 1886         /*
 1887          * If the page table page is mapped, we just increment the
 1888          * hold count, and activate it.
 1889          */
 1890         if (ptepa) {
 1891                 m = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 1892                 m->wire_count++;
 1893         } else {
 1894                 /*
 1895                  * Here if the pte page isn't mapped, or if it has
 1896                  * been deallocated. 
 1897                  */
 1898                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1899                 if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0)
 1900                         goto retry;
 1901         }
 1902         return (m);
 1903 }
 1904 
 1905 
 1906 /***************************************************
 1907 * Pmap allocation/deallocation routines.
 1908  ***************************************************/
 1909 
 1910 #ifdef SMP
 1911 /*
 1912  * Deal with a SMP shootdown of other users of the pmap that we are
 1913  * trying to dispose of.  This can be a bit hairy.
 1914  */
 1915 static cpuset_t *lazymask;
 1916 static u_int lazyptd;
 1917 static volatile u_int lazywait;
 1918 
 1919 void pmap_lazyfix_action(void);
 1920 
 1921 void
 1922 pmap_lazyfix_action(void)
 1923 {
 1924 
 1925 #ifdef COUNT_IPIS
 1926         (*ipi_lazypmap_counts[PCPU_GET(cpuid)])++;
 1927 #endif
 1928         if (rcr3() == lazyptd)
 1929                 load_cr3(curpcb->pcb_cr3);
 1930         CPU_CLR_ATOMIC(PCPU_GET(cpuid), lazymask);
 1931         atomic_store_rel_int(&lazywait, 1);
 1932 }
 1933 
 1934 static void
 1935 pmap_lazyfix_self(u_int cpuid)
 1936 {
 1937 
 1938         if (rcr3() == lazyptd)
 1939                 load_cr3(curpcb->pcb_cr3);
 1940         CPU_CLR_ATOMIC(cpuid, lazymask);
 1941 }
 1942 
 1943 
 1944 static void
 1945 pmap_lazyfix(pmap_t pmap)
 1946 {
 1947         cpuset_t mymask, mask;
 1948         u_int cpuid, spins;
 1949         int lsb;
 1950 
 1951         mask = pmap->pm_active;
 1952         while (!CPU_EMPTY(&mask)) {
 1953                 spins = 50000000;
 1954 
 1955                 /* Find least significant set bit. */
 1956                 lsb = CPU_FFS(&mask);
 1957                 MPASS(lsb != 0);
 1958                 lsb--;
 1959                 CPU_SETOF(lsb, &mask);
 1960                 mtx_lock_spin(&smp_ipi_mtx);
 1961 #ifdef PAE
 1962                 lazyptd = vtophys(pmap->pm_pdpt);
 1963 #else
 1964                 lazyptd = vtophys(pmap->pm_pdir);
 1965 #endif
 1966                 cpuid = PCPU_GET(cpuid);
 1967 
 1968                 /* Use a cpuset just for having an easy check. */
 1969                 CPU_SETOF(cpuid, &mymask);
 1970                 if (!CPU_CMP(&mask, &mymask)) {
 1971                         lazymask = &pmap->pm_active;
 1972                         pmap_lazyfix_self(cpuid);
 1973                 } else {
 1974                         atomic_store_rel_int((u_int *)&lazymask,
 1975                             (u_int)&pmap->pm_active);
 1976                         atomic_store_rel_int(&lazywait, 0);
 1977                         ipi_selected(mask, IPI_LAZYPMAP);
 1978                         while (lazywait == 0) {
 1979                                 ia32_pause();
 1980                                 if (--spins == 0)
 1981                                         break;
 1982                         }
 1983                 }
 1984                 mtx_unlock_spin(&smp_ipi_mtx);
 1985                 if (spins == 0)
 1986                         printf("pmap_lazyfix: spun for 50000000\n");
 1987                 mask = pmap->pm_active;
 1988         }
 1989 }
 1990 
 1991 #else   /* SMP */
 1992 
 1993 /*
 1994  * Cleaning up on uniprocessor is easy.  For various reasons, we're
 1995  * unlikely to have to even execute this code, including the fact
 1996  * that the cleanup is deferred until the parent does a wait(2), which
 1997  * means that another userland process has run.
 1998  */
 1999 static void
 2000 pmap_lazyfix(pmap_t pmap)
 2001 {
 2002         u_int cr3;
 2003 
 2004         cr3 = vtophys(pmap->pm_pdir);
 2005         if (cr3 == rcr3()) {
 2006                 load_cr3(curpcb->pcb_cr3);
 2007                 CPU_CLR(PCPU_GET(cpuid), &pmap->pm_active);
 2008         }
 2009 }
 2010 #endif  /* SMP */
 2011 
 2012 /*
 2013  * Release any resources held by the given physical map.
 2014  * Called when a pmap initialized by pmap_pinit is being released.
 2015  * Should only be called if the map contains no valid mappings.
 2016  */
 2017 void
 2018 pmap_release(pmap_t pmap)
 2019 {
 2020         vm_page_t m, ptdpg[NPGPTD];
 2021         int i;
 2022 
 2023         KASSERT(pmap->pm_stats.resident_count == 0,
 2024             ("pmap_release: pmap resident count %ld != 0",
 2025             pmap->pm_stats.resident_count));
 2026         KASSERT(vm_radix_is_empty(&pmap->pm_root),
 2027             ("pmap_release: pmap has reserved page table page(s)"));
 2028 
 2029         pmap_lazyfix(pmap);
 2030         mtx_lock_spin(&allpmaps_lock);
 2031         LIST_REMOVE(pmap, pm_list);
 2032         mtx_unlock_spin(&allpmaps_lock);
 2033 
 2034         for (i = 0; i < NPGPTD; i++)
 2035                 ptdpg[i] = PHYS_TO_VM_PAGE(pmap->pm_pdir[PTDPTDI + i] &
 2036                     PG_FRAME);
 2037 
 2038         bzero(pmap->pm_pdir + PTDPTDI, (nkpt + NPGPTD) *
 2039             sizeof(*pmap->pm_pdir));
 2040 
 2041         pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
 2042 
 2043         for (i = 0; i < NPGPTD; i++) {
 2044                 m = ptdpg[i];
 2045 #ifdef PAE
 2046                 KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
 2047                     ("pmap_release: got wrong ptd page"));
 2048 #endif
 2049                 m->wire_count--;
 2050                 atomic_subtract_int(&cnt.v_wire_count, 1);
 2051                 vm_page_free_zero(m);
 2052         }
 2053 }
 2054 
 2055 static int
 2056 kvm_size(SYSCTL_HANDLER_ARGS)
 2057 {
 2058         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
 2059 
 2060         return (sysctl_handle_long(oidp, &ksize, 0, req));
 2061 }
 2062 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 2063     0, 0, kvm_size, "IU", "Size of KVM");
 2064 
 2065 static int
 2066 kvm_free(SYSCTL_HANDLER_ARGS)
 2067 {
 2068         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 2069 
 2070         return (sysctl_handle_long(oidp, &kfree, 0, req));
 2071 }
 2072 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 2073     0, 0, kvm_free, "IU", "Amount of KVM free");
 2074 
 2075 /*
 2076  * grow the number of kernel page table entries, if needed
 2077  */
 2078 void
 2079 pmap_growkernel(vm_offset_t addr)
 2080 {
 2081         vm_paddr_t ptppaddr;
 2082         vm_page_t nkpg;
 2083         pd_entry_t newpdir;
 2084 
 2085         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 2086         addr = roundup2(addr, NBPDR);
 2087         if (addr - 1 >= kernel_map->max_offset)
 2088                 addr = kernel_map->max_offset;
 2089         while (kernel_vm_end < addr) {
 2090                 if (pdir_pde(PTD, kernel_vm_end)) {
 2091                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2092                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 2093                                 kernel_vm_end = kernel_map->max_offset;
 2094                                 break;
 2095                         }
 2096                         continue;
 2097                 }
 2098 
 2099                 nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDRSHIFT,
 2100                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 2101                     VM_ALLOC_ZERO);
 2102                 if (nkpg == NULL)
 2103                         panic("pmap_growkernel: no memory to grow kernel");
 2104 
 2105                 nkpt++;
 2106 
 2107                 if ((nkpg->flags & PG_ZERO) == 0)
 2108                         pmap_zero_page(nkpg);
 2109                 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
 2110                 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
 2111                 pdir_pde(KPTD, kernel_vm_end) = pgeflag | newpdir;
 2112 
 2113                 pmap_kenter_pde(kernel_vm_end, newpdir);
 2114                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 2115                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 2116                         kernel_vm_end = kernel_map->max_offset;
 2117                         break;
 2118                 }
 2119         }
 2120 }
 2121 
 2122 
 2123 /***************************************************
 2124  * page management routines.
 2125  ***************************************************/
 2126 
 2127 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 2128 CTASSERT(_NPCM == 11);
 2129 CTASSERT(_NPCPV == 336);
 2130 
 2131 static __inline struct pv_chunk *
 2132 pv_to_chunk(pv_entry_t pv)
 2133 {
 2134 
 2135         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
 2136 }
 2137 
 2138 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 2139 
 2140 #define PC_FREE0_9      0xfffffffful    /* Free values for index 0 through 9 */
 2141 #define PC_FREE10       0x0000fffful    /* Free values for index 10 */
 2142 
 2143 static const uint32_t pc_freemask[_NPCM] = {
 2144         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2145         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2146         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 2147         PC_FREE0_9, PC_FREE10
 2148 };
 2149 
 2150 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 2151         "Current number of pv entries");
 2152 
 2153 #ifdef PV_STATS
 2154 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 2155 
 2156 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 2157         "Current number of pv entry chunks");
 2158 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 2159         "Current number of pv entry chunks allocated");
 2160 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 2161         "Current number of pv entry chunks frees");
 2162 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 2163         "Number of times tried to get a chunk page but failed.");
 2164 
 2165 static long pv_entry_frees, pv_entry_allocs;
 2166 static int pv_entry_spare;
 2167 
 2168 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 2169         "Current number of pv entry frees");
 2170 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 2171         "Current number of pv entry allocs");
 2172 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 2173         "Current number of spare pv entries");
 2174 #endif
 2175 
 2176 /*
 2177  * We are in a serious low memory condition.  Resort to
 2178  * drastic measures to free some pages so we can allocate
 2179  * another pv entry chunk.
 2180  */
 2181 static vm_page_t
 2182 pmap_pv_reclaim(pmap_t locked_pmap)
 2183 {
 2184         struct pch newtail;
 2185         struct pv_chunk *pc;
 2186         struct md_page *pvh;
 2187         pd_entry_t *pde;
 2188         pmap_t pmap;
 2189         pt_entry_t *pte, tpte;
 2190         pv_entry_t pv;
 2191         vm_offset_t va;
 2192         vm_page_t m, m_pc;
 2193         struct spglist free;
 2194         uint32_t inuse;
 2195         int bit, field, freed;
 2196 
 2197         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
 2198         pmap = NULL;
 2199         m_pc = NULL;
 2200         SLIST_INIT(&free);
 2201         TAILQ_INIT(&newtail);
 2202         while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 ||
 2203             SLIST_EMPTY(&free))) {
 2204                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2205                 if (pmap != pc->pc_pmap) {
 2206                         if (pmap != NULL) {
 2207                                 pmap_invalidate_all(pmap);
 2208                                 if (pmap != locked_pmap)
 2209                                         PMAP_UNLOCK(pmap);
 2210                         }
 2211                         pmap = pc->pc_pmap;
 2212                         /* Avoid deadlock and lock recursion. */
 2213                         if (pmap > locked_pmap)
 2214                                 PMAP_LOCK(pmap);
 2215                         else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
 2216                                 pmap = NULL;
 2217                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2218                                 continue;
 2219                         }
 2220                 }
 2221 
 2222                 /*
 2223                  * Destroy every non-wired, 4 KB page mapping in the chunk.
 2224                  */
 2225                 freed = 0;
 2226                 for (field = 0; field < _NPCM; field++) {
 2227                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
 2228                             inuse != 0; inuse &= ~(1UL << bit)) {
 2229                                 bit = bsfl(inuse);
 2230                                 pv = &pc->pc_pventry[field * 32 + bit];
 2231                                 va = pv->pv_va;
 2232                                 pde = pmap_pde(pmap, va);
 2233                                 if ((*pde & PG_PS) != 0)
 2234                                         continue;
 2235                                 pte = pmap_pte(pmap, va);
 2236                                 tpte = *pte;
 2237                                 if ((tpte & PG_W) == 0)
 2238                                         tpte = pte_load_clear(pte);
 2239                                 pmap_pte_release(pte);
 2240                                 if ((tpte & PG_W) != 0)
 2241                                         continue;
 2242                                 KASSERT(tpte != 0,
 2243                                     ("pmap_pv_reclaim: pmap %p va %x zero pte",
 2244                                     pmap, va));
 2245                                 if ((tpte & PG_G) != 0)
 2246                                         pmap_invalidate_page(pmap, va);
 2247                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2248                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2249                                         vm_page_dirty(m);
 2250                                 if ((tpte & PG_A) != 0)
 2251                                         vm_page_aflag_set(m, PGA_REFERENCED);
 2252                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 2253                                 if (TAILQ_EMPTY(&m->md.pv_list) &&
 2254                                     (m->flags & PG_FICTITIOUS) == 0) {
 2255                                         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2256                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 2257                                                 vm_page_aflag_clear(m,
 2258                                                     PGA_WRITEABLE);
 2259                                         }
 2260                                 }
 2261                                 pc->pc_map[field] |= 1UL << bit;
 2262                                 pmap_unuse_pt(pmap, va, &free);
 2263                                 freed++;
 2264                         }
 2265                 }
 2266                 if (freed == 0) {
 2267                         TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2268                         continue;
 2269                 }
 2270                 /* Every freed mapping is for a 4 KB page. */
 2271                 pmap->pm_stats.resident_count -= freed;
 2272                 PV_STAT(pv_entry_frees += freed);
 2273                 PV_STAT(pv_entry_spare += freed);
 2274                 pv_entry_count -= freed;
 2275                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2276                 for (field = 0; field < _NPCM; field++)
 2277                         if (pc->pc_map[field] != pc_freemask[field]) {
 2278                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2279                                     pc_list);
 2280                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2281 
 2282                                 /*
 2283                                  * One freed pv entry in locked_pmap is
 2284                                  * sufficient.
 2285                                  */
 2286                                 if (pmap == locked_pmap)
 2287                                         goto out;
 2288                                 break;
 2289                         }
 2290                 if (field == _NPCM) {
 2291                         PV_STAT(pv_entry_spare -= _NPCPV);
 2292                         PV_STAT(pc_chunk_count--);
 2293                         PV_STAT(pc_chunk_frees++);
 2294                         /* Entire chunk is free; return it. */
 2295                         m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2296                         pmap_qremove((vm_offset_t)pc, 1);
 2297                         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2298                         break;
 2299                 }
 2300         }
 2301 out:
 2302         TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
 2303         if (pmap != NULL) {
 2304                 pmap_invalidate_all(pmap);
 2305                 if (pmap != locked_pmap)
 2306                         PMAP_UNLOCK(pmap);
 2307         }
 2308         if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) {
 2309                 m_pc = SLIST_FIRST(&free);
 2310                 SLIST_REMOVE_HEAD(&free, plinks.s.ss);
 2311                 /* Recycle a freed page table page. */
 2312                 m_pc->wire_count = 1;
 2313                 atomic_add_int(&cnt.v_wire_count, 1);
 2314         }
 2315         pmap_free_zero_pages(&free);
 2316         return (m_pc);
 2317 }
 2318 
 2319 /*
 2320  * free the pv_entry back to the free list
 2321  */
 2322 static void
 2323 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 2324 {
 2325         struct pv_chunk *pc;
 2326         int idx, field, bit;
 2327 
 2328         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2329         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2330         PV_STAT(pv_entry_frees++);
 2331         PV_STAT(pv_entry_spare++);
 2332         pv_entry_count--;
 2333         pc = pv_to_chunk(pv);
 2334         idx = pv - &pc->pc_pventry[0];
 2335         field = idx / 32;
 2336         bit = idx % 32;
 2337         pc->pc_map[field] |= 1ul << bit;
 2338         for (idx = 0; idx < _NPCM; idx++)
 2339                 if (pc->pc_map[idx] != pc_freemask[idx]) {
 2340                         /*
 2341                          * 98% of the time, pc is already at the head of the
 2342                          * list.  If it isn't already, move it to the head.
 2343                          */
 2344                         if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
 2345                             pc)) {
 2346                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2347                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2348                                     pc_list);
 2349                         }
 2350                         return;
 2351                 }
 2352         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2353         free_pv_chunk(pc);
 2354 }
 2355 
 2356 static void
 2357 free_pv_chunk(struct pv_chunk *pc)
 2358 {
 2359         vm_page_t m;
 2360 
 2361         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2362         PV_STAT(pv_entry_spare -= _NPCPV);
 2363         PV_STAT(pc_chunk_count--);
 2364         PV_STAT(pc_chunk_frees++);
 2365         /* entire chunk is free, return it */
 2366         m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2367         pmap_qremove((vm_offset_t)pc, 1);
 2368         vm_page_unwire(m, 0);
 2369         vm_page_free(m);
 2370         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2371 }
 2372 
 2373 /*
 2374  * get a new pv_entry, allocating a block from the system
 2375  * when needed.
 2376  */
 2377 static pv_entry_t
 2378 get_pv_entry(pmap_t pmap, boolean_t try)
 2379 {
 2380         static const struct timeval printinterval = { 60, 0 };
 2381         static struct timeval lastprint;
 2382         int bit, field;
 2383         pv_entry_t pv;
 2384         struct pv_chunk *pc;
 2385         vm_page_t m;
 2386 
 2387         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2388         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2389         PV_STAT(pv_entry_allocs++);
 2390         pv_entry_count++;
 2391         if (pv_entry_count > pv_entry_high_water)
 2392                 if (ratecheck(&lastprint, &printinterval))
 2393                         printf("Approaching the limit on PV entries, consider "
 2394                             "increasing either the vm.pmap.shpgperproc or the "
 2395                             "vm.pmap.pv_entry_max tunable.\n");
 2396 retry:
 2397         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 2398         if (pc != NULL) {
 2399                 for (field = 0; field < _NPCM; field++) {
 2400                         if (pc->pc_map[field]) {
 2401                                 bit = bsfl(pc->pc_map[field]);
 2402                                 break;
 2403                         }
 2404                 }
 2405                 if (field < _NPCM) {
 2406                         pv = &pc->pc_pventry[field * 32 + bit];
 2407                         pc->pc_map[field] &= ~(1ul << bit);
 2408                         /* If this was the last item, move it to tail */
 2409                         for (field = 0; field < _NPCM; field++)
 2410                                 if (pc->pc_map[field] != 0) {
 2411                                         PV_STAT(pv_entry_spare--);
 2412                                         return (pv);    /* not full, return */
 2413                                 }
 2414                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2415                         TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 2416                         PV_STAT(pv_entry_spare--);
 2417                         return (pv);
 2418                 }
 2419         }
 2420         /*
 2421          * Access to the ptelist "pv_vafree" is synchronized by the pvh
 2422          * global lock.  If "pv_vafree" is currently non-empty, it will
 2423          * remain non-empty until pmap_ptelist_alloc() completes.
 2424          */
 2425         if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 2426             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 2427                 if (try) {
 2428                         pv_entry_count--;
 2429                         PV_STAT(pc_chunk_tryfail++);
 2430                         return (NULL);
 2431                 }
 2432                 m = pmap_pv_reclaim(pmap);
 2433                 if (m == NULL)
 2434                         goto retry;
 2435         }
 2436         PV_STAT(pc_chunk_count++);
 2437         PV_STAT(pc_chunk_allocs++);
 2438         pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree);
 2439         pmap_qenter((vm_offset_t)pc, &m, 1);
 2440         pc->pc_pmap = pmap;
 2441         pc->pc_map[0] = pc_freemask[0] & ~1ul;  /* preallocated bit 0 */
 2442         for (field = 1; field < _NPCM; field++)
 2443                 pc->pc_map[field] = pc_freemask[field];
 2444         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 2445         pv = &pc->pc_pventry[0];
 2446         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2447         PV_STAT(pv_entry_spare += _NPCPV - 1);
 2448         return (pv);
 2449 }
 2450 
 2451 static __inline pv_entry_t
 2452 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2453 {
 2454         pv_entry_t pv;
 2455 
 2456         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2457         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 2458                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 2459                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 2460                         break;
 2461                 }
 2462         }
 2463         return (pv);
 2464 }
 2465 
 2466 static void
 2467 pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2468 {
 2469         struct md_page *pvh;
 2470         pv_entry_t pv;
 2471         vm_offset_t va_last;
 2472         vm_page_t m;
 2473 
 2474         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2475         KASSERT((pa & PDRMASK) == 0,
 2476             ("pmap_pv_demote_pde: pa is not 4mpage aligned"));
 2477 
 2478         /*
 2479          * Transfer the 4mpage's pv entry for this mapping to the first
 2480          * page's pv list.
 2481          */
 2482         pvh = pa_to_pvh(pa);
 2483         va = trunc_4mpage(va);
 2484         pv = pmap_pvh_remove(pvh, pmap, va);
 2485         KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found"));
 2486         m = PHYS_TO_VM_PAGE(pa);
 2487         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2488         /* Instantiate the remaining NPTEPG - 1 pv entries. */
 2489         va_last = va + NBPDR - PAGE_SIZE;
 2490         do {
 2491                 m++;
 2492                 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 2493                     ("pmap_pv_demote_pde: page %p is not managed", m));
 2494                 va += PAGE_SIZE;
 2495                 pmap_insert_entry(pmap, va, m);
 2496         } while (va < va_last);
 2497 }
 2498 
 2499 static void
 2500 pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2501 {
 2502         struct md_page *pvh;
 2503         pv_entry_t pv;
 2504         vm_offset_t va_last;
 2505         vm_page_t m;
 2506 
 2507         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2508         KASSERT((pa & PDRMASK) == 0,
 2509             ("pmap_pv_promote_pde: pa is not 4mpage aligned"));
 2510 
 2511         /*
 2512          * Transfer the first page's pv entry for this mapping to the
 2513          * 4mpage's pv list.  Aside from avoiding the cost of a call
 2514          * to get_pv_entry(), a transfer avoids the possibility that
 2515          * get_pv_entry() calls pmap_collect() and that pmap_collect()
 2516          * removes one of the mappings that is being promoted.
 2517          */
 2518         m = PHYS_TO_VM_PAGE(pa);
 2519         va = trunc_4mpage(va);
 2520         pv = pmap_pvh_remove(&m->md, pmap, va);
 2521         KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found"));
 2522         pvh = pa_to_pvh(pa);
 2523         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 2524         /* Free the remaining NPTEPG - 1 pv entries. */
 2525         va_last = va + NBPDR - PAGE_SIZE;
 2526         do {
 2527                 m++;
 2528                 va += PAGE_SIZE;
 2529                 pmap_pvh_free(&m->md, pmap, va);
 2530         } while (va < va_last);
 2531 }
 2532 
 2533 static void
 2534 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2535 {
 2536         pv_entry_t pv;
 2537 
 2538         pv = pmap_pvh_remove(pvh, pmap, va);
 2539         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 2540         free_pv_entry(pmap, pv);
 2541 }
 2542 
 2543 static void
 2544 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 2545 {
 2546         struct md_page *pvh;
 2547 
 2548         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2549         pmap_pvh_free(&m->md, pmap, va);
 2550         if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) {
 2551                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 2552                 if (TAILQ_EMPTY(&pvh->pv_list))
 2553                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 2554         }
 2555 }
 2556 
 2557 /*
 2558  * Create a pv entry for page at pa for
 2559  * (pmap, va).
 2560  */
 2561 static void
 2562 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2563 {
 2564         pv_entry_t pv;
 2565 
 2566         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2567         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2568         pv = get_pv_entry(pmap, FALSE);
 2569         pv->pv_va = va;
 2570         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2571 }
 2572 
 2573 /*
 2574  * Conditionally create a pv entry.
 2575  */
 2576 static boolean_t
 2577 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2578 {
 2579         pv_entry_t pv;
 2580 
 2581         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2582         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2583         if (pv_entry_count < pv_entry_high_water && 
 2584             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 2585                 pv->pv_va = va;
 2586                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 2587                 return (TRUE);
 2588         } else
 2589                 return (FALSE);
 2590 }
 2591 
 2592 /*
 2593  * Create the pv entries for each of the pages within a superpage.
 2594  */
 2595 static boolean_t
 2596 pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
 2597 {
 2598         struct md_page *pvh;
 2599         pv_entry_t pv;
 2600 
 2601         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2602         if (pv_entry_count < pv_entry_high_water && 
 2603             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 2604                 pv->pv_va = va;
 2605                 pvh = pa_to_pvh(pa);
 2606                 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 2607                 return (TRUE);
 2608         } else
 2609                 return (FALSE);
 2610 }
 2611 
 2612 /*
 2613  * Fills a page table page with mappings to consecutive physical pages.
 2614  */
 2615 static void
 2616 pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte)
 2617 {
 2618         pt_entry_t *pte;
 2619 
 2620         for (pte = firstpte; pte < firstpte + NPTEPG; pte++) {
 2621                 *pte = newpte;  
 2622                 newpte += PAGE_SIZE;
 2623         }
 2624 }
 2625 
 2626 /*
 2627  * Tries to demote a 2- or 4MB page mapping.  If demotion fails, the
 2628  * 2- or 4MB page mapping is invalidated.
 2629  */
 2630 static boolean_t
 2631 pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2632 {
 2633         pd_entry_t newpde, oldpde;
 2634         pt_entry_t *firstpte, newpte;
 2635         vm_paddr_t mptepa;
 2636         vm_page_t mpte;
 2637         struct spglist free;
 2638 
 2639         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2640         oldpde = *pde;
 2641         KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V),
 2642             ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V"));
 2643         if ((oldpde & PG_A) != 0 && (mpte = pmap_lookup_pt_page(pmap, va)) !=
 2644             NULL)
 2645                 pmap_remove_pt_page(pmap, mpte);
 2646         else {
 2647                 KASSERT((oldpde & PG_W) == 0,
 2648                     ("pmap_demote_pde: page table page for a wired mapping"
 2649                     " is missing"));
 2650 
 2651                 /*
 2652                  * Invalidate the 2- or 4MB page mapping and return
 2653                  * "failure" if the mapping was never accessed or the
 2654                  * allocation of the new page table page fails.
 2655                  */
 2656                 if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL,
 2657                     va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL |
 2658                     VM_ALLOC_WIRED)) == NULL) {
 2659                         SLIST_INIT(&free);
 2660                         pmap_remove_pde(pmap, pde, trunc_4mpage(va), &free);
 2661                         pmap_invalidate_page(pmap, trunc_4mpage(va));
 2662                         pmap_free_zero_pages(&free);
 2663                         CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x"
 2664                             " in pmap %p", va, pmap);
 2665                         return (FALSE);
 2666                 }
 2667                 if (va < VM_MAXUSER_ADDRESS)
 2668                         pmap->pm_stats.resident_count++;
 2669         }
 2670         mptepa = VM_PAGE_TO_PHYS(mpte);
 2671 
 2672         /*
 2673          * If the page mapping is in the kernel's address space, then the
 2674          * KPTmap can provide access to the page table page.  Otherwise,
 2675          * temporarily map the page table page (mpte) into the kernel's
 2676          * address space at either PADDR1 or PADDR2. 
 2677          */
 2678         if (va >= KERNBASE)
 2679                 firstpte = &KPTmap[i386_btop(trunc_4mpage(va))];
 2680         else if (curthread->td_pinned > 0 && rw_wowned(&pvh_global_lock)) {
 2681                 if ((*PMAP1 & PG_FRAME) != mptepa) {
 2682                         *PMAP1 = mptepa | PG_RW | PG_V | PG_A | PG_M;
 2683 #ifdef SMP
 2684                         PMAP1cpu = PCPU_GET(cpuid);
 2685 #endif
 2686                         invlcaddr(PADDR1);
 2687                         PMAP1changed++;
 2688                 } else
 2689 #ifdef SMP
 2690                 if (PMAP1cpu != PCPU_GET(cpuid)) {
 2691                         PMAP1cpu = PCPU_GET(cpuid);
 2692                         invlcaddr(PADDR1);
 2693                         PMAP1changedcpu++;
 2694                 } else
 2695 #endif
 2696                         PMAP1unchanged++;
 2697                 firstpte = PADDR1;
 2698         } else {
 2699                 mtx_lock(&PMAP2mutex);
 2700                 if ((*PMAP2 & PG_FRAME) != mptepa) {
 2701                         *PMAP2 = mptepa | PG_RW | PG_V | PG_A | PG_M;
 2702                         pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
 2703                 }
 2704                 firstpte = PADDR2;
 2705         }
 2706         newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V;
 2707         KASSERT((oldpde & PG_A) != 0,
 2708             ("pmap_demote_pde: oldpde is missing PG_A"));
 2709         KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW,
 2710             ("pmap_demote_pde: oldpde is missing PG_M"));
 2711         newpte = oldpde & ~PG_PS;
 2712         if ((newpte & PG_PDE_PAT) != 0)
 2713                 newpte ^= PG_PDE_PAT | PG_PTE_PAT;
 2714 
 2715         /*
 2716          * If the page table page is new, initialize it.
 2717          */
 2718         if (mpte->wire_count == 1) {
 2719                 mpte->wire_count = NPTEPG;
 2720                 pmap_fill_ptp(firstpte, newpte);
 2721         }
 2722         KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME),
 2723             ("pmap_demote_pde: firstpte and newpte map different physical"
 2724             " addresses"));
 2725 
 2726         /*
 2727          * If the mapping has changed attributes, update the page table
 2728          * entries.
 2729          */ 
 2730         if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE))
 2731                 pmap_fill_ptp(firstpte, newpte);
 2732         
 2733         /*
 2734          * Demote the mapping.  This pmap is locked.  The old PDE has
 2735          * PG_A set.  If the old PDE has PG_RW set, it also has PG_M
 2736          * set.  Thus, there is no danger of a race with another
 2737          * processor changing the setting of PG_A and/or PG_M between
 2738          * the read above and the store below. 
 2739          */
 2740         if (workaround_erratum383)
 2741                 pmap_update_pde(pmap, va, pde, newpde);
 2742         else if (pmap == kernel_pmap)
 2743                 pmap_kenter_pde(va, newpde);
 2744         else
 2745                 pde_store(pde, newpde); 
 2746         if (firstpte == PADDR2)
 2747                 mtx_unlock(&PMAP2mutex);
 2748 
 2749         /*
 2750          * Invalidate the recursive mapping of the page table page.
 2751          */
 2752         pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 2753 
 2754         /*
 2755          * Demote the pv entry.  This depends on the earlier demotion
 2756          * of the mapping.  Specifically, the (re)creation of a per-
 2757          * page pv entry might trigger the execution of pmap_collect(),
 2758          * which might reclaim a newly (re)created per-page pv entry
 2759          * and destroy the associated mapping.  In order to destroy
 2760          * the mapping, the PDE must have already changed from mapping
 2761          * the 2mpage to referencing the page table page.
 2762          */
 2763         if ((oldpde & PG_MANAGED) != 0)
 2764                 pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME);
 2765 
 2766         pmap_pde_demotions++;
 2767         CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#x"
 2768             " in pmap %p", va, pmap);
 2769         return (TRUE);
 2770 }
 2771 
 2772 /*
 2773  * Removes a 2- or 4MB page mapping from the kernel pmap.
 2774  */
 2775 static void
 2776 pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 2777 {
 2778         pd_entry_t newpde;
 2779         vm_paddr_t mptepa;
 2780         vm_page_t mpte;
 2781 
 2782         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2783         mpte = pmap_lookup_pt_page(pmap, va);
 2784         if (mpte == NULL)
 2785                 panic("pmap_remove_kernel_pde: Missing pt page.");
 2786 
 2787         pmap_remove_pt_page(pmap, mpte);
 2788         mptepa = VM_PAGE_TO_PHYS(mpte);
 2789         newpde = mptepa | PG_M | PG_A | PG_RW | PG_V;
 2790 
 2791         /*
 2792          * Initialize the page table page.
 2793          */
 2794         pagezero((void *)&KPTmap[i386_btop(trunc_4mpage(va))]);
 2795 
 2796         /*
 2797          * Remove the mapping.
 2798          */
 2799         if (workaround_erratum383)
 2800                 pmap_update_pde(pmap, va, pde, newpde);
 2801         else 
 2802                 pmap_kenter_pde(va, newpde);
 2803 
 2804         /*
 2805          * Invalidate the recursive mapping of the page table page.
 2806          */
 2807         pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va));
 2808 }
 2809 
 2810 /*
 2811  * pmap_remove_pde: do the things to unmap a superpage in a process
 2812  */
 2813 static void
 2814 pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 2815     struct spglist *free)
 2816 {
 2817         struct md_page *pvh;
 2818         pd_entry_t oldpde;
 2819         vm_offset_t eva, va;
 2820         vm_page_t m, mpte;
 2821 
 2822         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2823         KASSERT((sva & PDRMASK) == 0,
 2824             ("pmap_remove_pde: sva is not 4mpage aligned"));
 2825         oldpde = pte_load_clear(pdq);
 2826         if (oldpde & PG_W)
 2827                 pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE;
 2828 
 2829         /*
 2830          * Machines that don't support invlpg, also don't support
 2831          * PG_G.
 2832          */
 2833         if (oldpde & PG_G)
 2834                 pmap_invalidate_page(kernel_pmap, sva);
 2835         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 2836         if (oldpde & PG_MANAGED) {
 2837                 pvh = pa_to_pvh(oldpde & PG_PS_FRAME);
 2838                 pmap_pvh_free(pvh, pmap, sva);
 2839                 eva = sva + NBPDR;
 2840                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 2841                     va < eva; va += PAGE_SIZE, m++) {
 2842                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2843                                 vm_page_dirty(m);
 2844                         if (oldpde & PG_A)
 2845                                 vm_page_aflag_set(m, PGA_REFERENCED);
 2846                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 2847                             TAILQ_EMPTY(&pvh->pv_list))
 2848                                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 2849                 }
 2850         }
 2851         if (pmap == kernel_pmap) {
 2852                 pmap_remove_kernel_pde(pmap, pdq, sva);
 2853         } else {
 2854                 mpte = pmap_lookup_pt_page(pmap, sva);
 2855                 if (mpte != NULL) {
 2856                         pmap_remove_pt_page(pmap, mpte);
 2857                         pmap->pm_stats.resident_count--;
 2858                         KASSERT(mpte->wire_count == NPTEPG,
 2859                             ("pmap_remove_pde: pte page wire count error"));
 2860                         mpte->wire_count = 0;
 2861                         pmap_add_delayed_free_list(mpte, free, FALSE);
 2862                         atomic_subtract_int(&cnt.v_wire_count, 1);
 2863                 }
 2864         }
 2865 }
 2866 
 2867 /*
 2868  * pmap_remove_pte: do the things to unmap a page in a process
 2869  */
 2870 static int
 2871 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
 2872     struct spglist *free)
 2873 {
 2874         pt_entry_t oldpte;
 2875         vm_page_t m;
 2876 
 2877         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2878         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2879         oldpte = pte_load_clear(ptq);
 2880         KASSERT(oldpte != 0,
 2881             ("pmap_remove_pte: pmap %p va %x zero pte", pmap, va));
 2882         if (oldpte & PG_W)
 2883                 pmap->pm_stats.wired_count -= 1;
 2884         /*
 2885          * Machines that don't support invlpg, also don't support
 2886          * PG_G.
 2887          */
 2888         if (oldpte & PG_G)
 2889                 pmap_invalidate_page(kernel_pmap, va);
 2890         pmap->pm_stats.resident_count -= 1;
 2891         if (oldpte & PG_MANAGED) {
 2892                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 2893                 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2894                         vm_page_dirty(m);
 2895                 if (oldpte & PG_A)
 2896                         vm_page_aflag_set(m, PGA_REFERENCED);
 2897                 pmap_remove_entry(pmap, m, va);
 2898         }
 2899         return (pmap_unuse_pt(pmap, va, free));
 2900 }
 2901 
 2902 /*
 2903  * Remove a single page from a process address space
 2904  */
 2905 static void
 2906 pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free)
 2907 {
 2908         pt_entry_t *pte;
 2909 
 2910         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2911         KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 2912         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2913         if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
 2914                 return;
 2915         pmap_remove_pte(pmap, pte, va, free);
 2916         pmap_invalidate_page(pmap, va);
 2917 }
 2918 
 2919 /*
 2920  *      Remove the given range of addresses from the specified map.
 2921  *
 2922  *      It is assumed that the start and end are properly
 2923  *      rounded to the page size.
 2924  */
 2925 void
 2926 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 2927 {
 2928         vm_offset_t pdnxt;
 2929         pd_entry_t ptpaddr;
 2930         pt_entry_t *pte;
 2931         struct spglist free;
 2932         int anyvalid;
 2933 
 2934         /*
 2935          * Perform an unsynchronized read.  This is, however, safe.
 2936          */
 2937         if (pmap->pm_stats.resident_count == 0)
 2938                 return;
 2939 
 2940         anyvalid = 0;
 2941         SLIST_INIT(&free);
 2942 
 2943         rw_wlock(&pvh_global_lock);
 2944         sched_pin();
 2945         PMAP_LOCK(pmap);
 2946 
 2947         /*
 2948          * special handling of removing one page.  a very
 2949          * common operation and easy to short circuit some
 2950          * code.
 2951          */
 2952         if ((sva + PAGE_SIZE == eva) && 
 2953             ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
 2954                 pmap_remove_page(pmap, sva, &free);
 2955                 goto out;
 2956         }
 2957 
 2958         for (; sva < eva; sva = pdnxt) {
 2959                 u_int pdirindex;
 2960 
 2961                 /*
 2962                  * Calculate index for next page table.
 2963                  */
 2964                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 2965                 if (pdnxt < sva)
 2966                         pdnxt = eva;
 2967                 if (pmap->pm_stats.resident_count == 0)
 2968                         break;
 2969 
 2970                 pdirindex = sva >> PDRSHIFT;
 2971                 ptpaddr = pmap->pm_pdir[pdirindex];
 2972 
 2973                 /*
 2974                  * Weed out invalid mappings. Note: we assume that the page
 2975                  * directory table is always allocated, and in kernel virtual.
 2976                  */
 2977                 if (ptpaddr == 0)
 2978                         continue;
 2979 
 2980                 /*
 2981                  * Check for large page.
 2982                  */
 2983                 if ((ptpaddr & PG_PS) != 0) {
 2984                         /*
 2985                          * Are we removing the entire large page?  If not,
 2986                          * demote the mapping and fall through.
 2987                          */
 2988                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 2989                                 /*
 2990                                  * The TLB entry for a PG_G mapping is
 2991                                  * invalidated by pmap_remove_pde().
 2992                                  */
 2993                                 if ((ptpaddr & PG_G) == 0)
 2994                                         anyvalid = 1;
 2995                                 pmap_remove_pde(pmap,
 2996                                     &pmap->pm_pdir[pdirindex], sva, &free);
 2997                                 continue;
 2998                         } else if (!pmap_demote_pde(pmap,
 2999                             &pmap->pm_pdir[pdirindex], sva)) {
 3000                                 /* The large page mapping was destroyed. */
 3001                                 continue;
 3002                         }
 3003                 }
 3004 
 3005                 /*
 3006                  * Limit our scan to either the end of the va represented
 3007                  * by the current page table page, or to the end of the
 3008                  * range being removed.
 3009                  */
 3010                 if (pdnxt > eva)
 3011                         pdnxt = eva;
 3012 
 3013                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 3014                     sva += PAGE_SIZE) {
 3015                         if (*pte == 0)
 3016                                 continue;
 3017 
 3018                         /*
 3019                          * The TLB entry for a PG_G mapping is invalidated
 3020                          * by pmap_remove_pte().
 3021                          */
 3022                         if ((*pte & PG_G) == 0)
 3023                                 anyvalid = 1;
 3024                         if (pmap_remove_pte(pmap, pte, sva, &free))
 3025                                 break;
 3026                 }
 3027         }
 3028 out:
 3029         sched_unpin();
 3030         if (anyvalid)
 3031                 pmap_invalidate_all(pmap);
 3032         rw_wunlock(&pvh_global_lock);
 3033         PMAP_UNLOCK(pmap);
 3034         pmap_free_zero_pages(&free);
 3035 }
 3036 
 3037 /*
 3038  *      Routine:        pmap_remove_all
 3039  *      Function:
 3040  *              Removes this physical page from
 3041  *              all physical maps in which it resides.
 3042  *              Reflects back modify bits to the pager.
 3043  *
 3044  *      Notes:
 3045  *              Original versions of this routine were very
 3046  *              inefficient because they iteratively called
 3047  *              pmap_remove (slow...)
 3048  */
 3049 
 3050 void
 3051 pmap_remove_all(vm_page_t m)
 3052 {
 3053         struct md_page *pvh;
 3054         pv_entry_t pv;
 3055         pmap_t pmap;
 3056         pt_entry_t *pte, tpte;
 3057         pd_entry_t *pde;
 3058         vm_offset_t va;
 3059         struct spglist free;
 3060 
 3061         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3062             ("pmap_remove_all: page %p is not managed", m));
 3063         SLIST_INIT(&free);
 3064         rw_wlock(&pvh_global_lock);
 3065         sched_pin();
 3066         if ((m->flags & PG_FICTITIOUS) != 0)
 3067                 goto small_mappings;
 3068         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 3069         while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
 3070                 va = pv->pv_va;
 3071                 pmap = PV_PMAP(pv);
 3072                 PMAP_LOCK(pmap);
 3073                 pde = pmap_pde(pmap, va);
 3074                 (void)pmap_demote_pde(pmap, pde, va);
 3075                 PMAP_UNLOCK(pmap);
 3076         }
 3077 small_mappings:
 3078         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 3079                 pmap = PV_PMAP(pv);
 3080                 PMAP_LOCK(pmap);
 3081                 pmap->pm_stats.resident_count--;
 3082                 pde = pmap_pde(pmap, pv->pv_va);
 3083                 KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found"
 3084                     " a 4mpage in page %p's pv list", m));
 3085                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3086                 tpte = pte_load_clear(pte);
 3087                 KASSERT(tpte != 0, ("pmap_remove_all: pmap %p va %x zero pte",
 3088                     pmap, pv->pv_va));
 3089                 if (tpte & PG_W)
 3090                         pmap->pm_stats.wired_count--;
 3091                 if (tpte & PG_A)
 3092                         vm_page_aflag_set(m, PGA_REFERENCED);
 3093 
 3094                 /*
 3095                  * Update the vm_page_t clean and reference bits.
 3096                  */
 3097                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3098                         vm_page_dirty(m);
 3099                 pmap_unuse_pt(pmap, pv->pv_va, &free);
 3100                 pmap_invalidate_page(pmap, pv->pv_va);
 3101                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 3102                 free_pv_entry(pmap, pv);
 3103                 PMAP_UNLOCK(pmap);
 3104         }
 3105         vm_page_aflag_clear(m, PGA_WRITEABLE);
 3106         sched_unpin();
 3107         rw_wunlock(&pvh_global_lock);
 3108         pmap_free_zero_pages(&free);
 3109 }
 3110 
 3111 /*
 3112  * pmap_protect_pde: do the things to protect a 4mpage in a process
 3113  */
 3114 static boolean_t
 3115 pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot)
 3116 {
 3117         pd_entry_t newpde, oldpde;
 3118         vm_offset_t eva, va;
 3119         vm_page_t m;
 3120         boolean_t anychanged;
 3121 
 3122         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3123         KASSERT((sva & PDRMASK) == 0,
 3124             ("pmap_protect_pde: sva is not 4mpage aligned"));
 3125         anychanged = FALSE;
 3126 retry:
 3127         oldpde = newpde = *pde;
 3128         if (oldpde & PG_MANAGED) {
 3129                 eva = sva + NBPDR;
 3130                 for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME);
 3131                     va < eva; va += PAGE_SIZE, m++)
 3132                         if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW))
 3133                                 vm_page_dirty(m);
 3134         }
 3135         if ((prot & VM_PROT_WRITE) == 0)
 3136                 newpde &= ~(PG_RW | PG_M);
 3137 #ifdef PAE
 3138         if ((prot & VM_PROT_EXECUTE) == 0)
 3139                 newpde |= pg_nx;
 3140 #endif
 3141         if (newpde != oldpde) {
 3142                 if (!pde_cmpset(pde, oldpde, newpde))
 3143                         goto retry;
 3144                 if (oldpde & PG_G)
 3145                         pmap_invalidate_page(pmap, sva);
 3146                 else
 3147                         anychanged = TRUE;
 3148         }
 3149         return (anychanged);
 3150 }
 3151 
 3152 /*
 3153  *      Set the physical protection on the
 3154  *      specified range of this map as requested.
 3155  */
 3156 void
 3157 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 3158 {
 3159         vm_offset_t pdnxt;
 3160         pd_entry_t ptpaddr;
 3161         pt_entry_t *pte;
 3162         boolean_t anychanged, pv_lists_locked;
 3163 
 3164         KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot));
 3165         if (prot == VM_PROT_NONE) {
 3166                 pmap_remove(pmap, sva, eva);
 3167                 return;
 3168         }
 3169 
 3170 #ifdef PAE
 3171         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 3172             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 3173                 return;
 3174 #else
 3175         if (prot & VM_PROT_WRITE)
 3176                 return;
 3177 #endif
 3178 
 3179         if (pmap_is_current(pmap))
 3180                 pv_lists_locked = FALSE;
 3181         else {
 3182                 pv_lists_locked = TRUE;
 3183 resume:
 3184                 rw_wlock(&pvh_global_lock);
 3185                 sched_pin();
 3186         }
 3187         anychanged = FALSE;
 3188 
 3189         PMAP_LOCK(pmap);
 3190         for (; sva < eva; sva = pdnxt) {
 3191                 pt_entry_t obits, pbits;
 3192                 u_int pdirindex;
 3193 
 3194                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 3195                 if (pdnxt < sva)
 3196                         pdnxt = eva;
 3197 
 3198                 pdirindex = sva >> PDRSHIFT;
 3199                 ptpaddr = pmap->pm_pdir[pdirindex];
 3200 
 3201                 /*
 3202                  * Weed out invalid mappings. Note: we assume that the page
 3203                  * directory table is always allocated, and in kernel virtual.
 3204                  */
 3205                 if (ptpaddr == 0)
 3206                         continue;
 3207 
 3208                 /*
 3209                  * Check for large page.
 3210                  */
 3211                 if ((ptpaddr & PG_PS) != 0) {
 3212                         /*
 3213                          * Are we protecting the entire large page?  If not,
 3214                          * demote the mapping and fall through.
 3215                          */
 3216                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 3217                                 /*
 3218                                  * The TLB entry for a PG_G mapping is
 3219                                  * invalidated by pmap_protect_pde().
 3220                                  */
 3221                                 if (pmap_protect_pde(pmap,
 3222                                     &pmap->pm_pdir[pdirindex], sva, prot))
 3223                                         anychanged = TRUE;
 3224                                 continue;
 3225                         } else {
 3226                                 if (!pv_lists_locked) {
 3227                                         pv_lists_locked = TRUE;
 3228                                         if (!rw_try_wlock(&pvh_global_lock)) {
 3229                                                 if (anychanged)
 3230                                                         pmap_invalidate_all(
 3231                                                             pmap);
 3232                                                 PMAP_UNLOCK(pmap);
 3233                                                 goto resume;
 3234                                         }
 3235                                         sched_pin();
 3236                                 }
 3237                                 if (!pmap_demote_pde(pmap,
 3238                                     &pmap->pm_pdir[pdirindex], sva)) {
 3239                                         /*
 3240                                          * The large page mapping was
 3241                                          * destroyed.
 3242                                          */
 3243                                         continue;
 3244                                 }
 3245                         }
 3246                 }
 3247 
 3248                 if (pdnxt > eva)
 3249                         pdnxt = eva;
 3250 
 3251                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 3252                     sva += PAGE_SIZE) {
 3253                         vm_page_t m;
 3254 
 3255 retry:
 3256                         /*
 3257                          * Regardless of whether a pte is 32 or 64 bits in
 3258                          * size, PG_RW, PG_A, and PG_M are among the least
 3259                          * significant 32 bits.
 3260                          */
 3261                         obits = pbits = *pte;
 3262                         if ((pbits & PG_V) == 0)
 3263                                 continue;
 3264 
 3265                         if ((prot & VM_PROT_WRITE) == 0) {
 3266                                 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
 3267                                     (PG_MANAGED | PG_M | PG_RW)) {
 3268                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 3269                                         vm_page_dirty(m);
 3270                                 }
 3271                                 pbits &= ~(PG_RW | PG_M);
 3272                         }
 3273 #ifdef PAE
 3274                         if ((prot & VM_PROT_EXECUTE) == 0)
 3275                                 pbits |= pg_nx;
 3276 #endif
 3277 
 3278                         if (pbits != obits) {
 3279 #ifdef PAE
 3280                                 if (!atomic_cmpset_64(pte, obits, pbits))
 3281                                         goto retry;
 3282 #else
 3283                                 if (!atomic_cmpset_int((u_int *)pte, obits,
 3284                                     pbits))
 3285                                         goto retry;
 3286 #endif
 3287                                 if (obits & PG_G)
 3288                                         pmap_invalidate_page(pmap, sva);
 3289                                 else
 3290                                         anychanged = TRUE;
 3291                         }
 3292                 }
 3293         }
 3294         if (anychanged)
 3295                 pmap_invalidate_all(pmap);
 3296         if (pv_lists_locked) {
 3297                 sched_unpin();
 3298                 rw_wunlock(&pvh_global_lock);
 3299         }
 3300         PMAP_UNLOCK(pmap);
 3301 }
 3302 
 3303 /*
 3304  * Tries to promote the 512 or 1024, contiguous 4KB page mappings that are
 3305  * within a single page table page (PTP) to a single 2- or 4MB page mapping.
 3306  * For promotion to occur, two conditions must be met: (1) the 4KB page
 3307  * mappings must map aligned, contiguous physical memory and (2) the 4KB page
 3308  * mappings must have identical characteristics.
 3309  *
 3310  * Managed (PG_MANAGED) mappings within the kernel address space are not
 3311  * promoted.  The reason is that kernel PDEs are replicated in each pmap but
 3312  * pmap_clear_ptes() and pmap_ts_referenced() only read the PDE from the kernel
 3313  * pmap.
 3314  */
 3315 static void
 3316 pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
 3317 {
 3318         pd_entry_t newpde;
 3319         pt_entry_t *firstpte, oldpte, pa, *pte;
 3320         vm_offset_t oldpteva;
 3321         vm_page_t mpte;
 3322 
 3323         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3324 
 3325         /*
 3326          * Examine the first PTE in the specified PTP.  Abort if this PTE is
 3327          * either invalid, unused, or does not map the first 4KB physical page
 3328          * within a 2- or 4MB page.
 3329          */
 3330         firstpte = pmap_pte_quick(pmap, trunc_4mpage(va));
 3331 setpde:
 3332         newpde = *firstpte;
 3333         if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
 3334                 pmap_pde_p_failures++;
 3335                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3336                     " in pmap %p", va, pmap);
 3337                 return;
 3338         }
 3339         if ((*firstpte & PG_MANAGED) != 0 && pmap == kernel_pmap) {
 3340                 pmap_pde_p_failures++;
 3341                 CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3342                     " in pmap %p", va, pmap);
 3343                 return;
 3344         }
 3345         if ((newpde & (PG_M | PG_RW)) == PG_RW) {
 3346                 /*
 3347                  * When PG_M is already clear, PG_RW can be cleared without
 3348                  * a TLB invalidation.
 3349                  */
 3350                 if (!atomic_cmpset_int((u_int *)firstpte, newpde, newpde &
 3351                     ~PG_RW))  
 3352                         goto setpde;
 3353                 newpde &= ~PG_RW;
 3354         }
 3355 
 3356         /* 
 3357          * Examine each of the other PTEs in the specified PTP.  Abort if this
 3358          * PTE maps an unexpected 4KB physical page or does not have identical
 3359          * characteristics to the first PTE.
 3360          */
 3361         pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + NBPDR - PAGE_SIZE;
 3362         for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
 3363 setpte:
 3364                 oldpte = *pte;
 3365                 if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
 3366                         pmap_pde_p_failures++;
 3367                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3368                             " in pmap %p", va, pmap);
 3369                         return;
 3370                 }
 3371                 if ((oldpte & (PG_M | PG_RW)) == PG_RW) {
 3372                         /*
 3373                          * When PG_M is already clear, PG_RW can be cleared
 3374                          * without a TLB invalidation.
 3375                          */
 3376                         if (!atomic_cmpset_int((u_int *)pte, oldpte,
 3377                             oldpte & ~PG_RW))
 3378                                 goto setpte;
 3379                         oldpte &= ~PG_RW;
 3380                         oldpteva = (oldpte & PG_FRAME & PDRMASK) |
 3381                             (va & ~PDRMASK);
 3382                         CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#x"
 3383                             " in pmap %p", oldpteva, pmap);
 3384                 }
 3385                 if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
 3386                         pmap_pde_p_failures++;
 3387                         CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#x"
 3388                             " in pmap %p", va, pmap);
 3389                         return;
 3390                 }
 3391                 pa -= PAGE_SIZE;
 3392         }
 3393 
 3394         /*
 3395          * Save the page table page in its current state until the PDE
 3396          * mapping the superpage is demoted by pmap_demote_pde() or
 3397          * destroyed by pmap_remove_pde(). 
 3398          */
 3399         mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME);
 3400         KASSERT(mpte >= vm_page_array &&
 3401             mpte < &vm_page_array[vm_page_array_size],
 3402             ("pmap_promote_pde: page table page is out of range"));
 3403         KASSERT(mpte->pindex == va >> PDRSHIFT,
 3404             ("pmap_promote_pde: page table page's pindex is wrong"));
 3405         if (pmap_insert_pt_page(pmap, mpte)) {
 3406                 pmap_pde_p_failures++;
 3407                 CTR2(KTR_PMAP,
 3408                     "pmap_promote_pde: failure for va %#x in pmap %p", va,
 3409                     pmap);
 3410                 return;
 3411         }
 3412 
 3413         /*
 3414          * Promote the pv entries.
 3415          */
 3416         if ((newpde & PG_MANAGED) != 0)
 3417                 pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
 3418 
 3419         /*
 3420          * Propagate the PAT index to its proper position.
 3421          */
 3422         if ((newpde & PG_PTE_PAT) != 0)
 3423                 newpde ^= PG_PDE_PAT | PG_PTE_PAT;
 3424 
 3425         /*
 3426          * Map the superpage.
 3427          */
 3428         if (workaround_erratum383)
 3429                 pmap_update_pde(pmap, va, pde, PG_PS | newpde);
 3430         else if (pmap == kernel_pmap)
 3431                 pmap_kenter_pde(va, PG_PS | newpde);
 3432         else
 3433                 pde_store(pde, PG_PS | newpde);
 3434 
 3435         pmap_pde_promotions++;
 3436         CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x"
 3437             " in pmap %p", va, pmap);
 3438 }
 3439 
 3440 /*
 3441  *      Insert the given physical page (p) at
 3442  *      the specified virtual address (v) in the
 3443  *      target physical map with the protection requested.
 3444  *
 3445  *      If specified, the page will be wired down, meaning
 3446  *      that the related pte can not be reclaimed.
 3447  *
 3448  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 3449  *      or lose information.  That is, this routine must actually
 3450  *      insert this page into the given map NOW.
 3451  */
 3452 int
 3453 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 3454     u_int flags, int8_t psind)
 3455 {
 3456         pd_entry_t *pde;
 3457         pt_entry_t *pte;
 3458         pt_entry_t newpte, origpte;
 3459         pv_entry_t pv;
 3460         vm_paddr_t opa, pa;
 3461         vm_page_t mpte, om;
 3462         boolean_t invlva, wired;
 3463 
 3464         va = trunc_page(va);
 3465         mpte = NULL;
 3466         wired = (flags & PMAP_ENTER_WIRED) != 0;
 3467 
 3468         KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
 3469         KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS,
 3470             ("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)",
 3471             va));
 3472         if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
 3473                 VM_OBJECT_ASSERT_LOCKED(m->object);
 3474 
 3475         rw_wlock(&pvh_global_lock);
 3476         PMAP_LOCK(pmap);
 3477         sched_pin();
 3478 
 3479         /*
 3480          * In the case that a page table page is not
 3481          * resident, we are creating it here.
 3482          */
 3483         if (va < VM_MAXUSER_ADDRESS) {
 3484                 mpte = pmap_allocpte(pmap, va, flags);
 3485                 if (mpte == NULL) {
 3486                         KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0,
 3487                             ("pmap_allocpte failed with sleep allowed"));
 3488                         sched_unpin();
 3489                         rw_wunlock(&pvh_global_lock);
 3490                         PMAP_UNLOCK(pmap);
 3491                         return (KERN_RESOURCE_SHORTAGE);
 3492                 }
 3493         }
 3494 
 3495         pde = pmap_pde(pmap, va);
 3496         if ((*pde & PG_PS) != 0)
 3497                 panic("pmap_enter: attempted pmap_enter on 4MB page");
 3498         pte = pmap_pte_quick(pmap, va);
 3499 
 3500         /*
 3501          * Page Directory table entry not valid, we need a new PT page
 3502          */
 3503         if (pte == NULL) {
 3504                 panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x",
 3505                         (uintmax_t)pmap->pm_pdir[PTDPTDI], va);
 3506         }
 3507 
 3508         pa = VM_PAGE_TO_PHYS(m);
 3509         om = NULL;
 3510         origpte = *pte;
 3511         opa = origpte & PG_FRAME;
 3512 
 3513         /*
 3514          * Mapping has not changed, must be protection or wiring change.
 3515          */
 3516         if (origpte && (opa == pa)) {
 3517                 /*
 3518                  * Wiring change, just update stats. We don't worry about
 3519                  * wiring PT pages as they remain resident as long as there
 3520                  * are valid mappings in them. Hence, if a user page is wired,
 3521                  * the PT page will be also.
 3522                  */
 3523                 if (wired && ((origpte & PG_W) == 0))
 3524                         pmap->pm_stats.wired_count++;
 3525                 else if (!wired && (origpte & PG_W))
 3526                         pmap->pm_stats.wired_count--;
 3527 
 3528                 /*
 3529                  * Remove extra pte reference
 3530                  */
 3531                 if (mpte)
 3532                         mpte->wire_count--;
 3533 
 3534                 if (origpte & PG_MANAGED) {
 3535                         om = m;
 3536                         pa |= PG_MANAGED;
 3537                 }
 3538                 goto validate;
 3539         } 
 3540 
 3541         pv = NULL;
 3542 
 3543         /*
 3544          * Mapping has changed, invalidate old range and fall through to
 3545          * handle validating new mapping.
 3546          */
 3547         if (opa) {
 3548                 if (origpte & PG_W)
 3549                         pmap->pm_stats.wired_count--;
 3550                 if (origpte & PG_MANAGED) {
 3551                         om = PHYS_TO_VM_PAGE(opa);
 3552                         pv = pmap_pvh_remove(&om->md, pmap, va);
 3553                 }
 3554                 if (mpte != NULL) {
 3555                         mpte->wire_count--;
 3556                         KASSERT(mpte->wire_count > 0,
 3557                             ("pmap_enter: missing reference to page table page,"
 3558                              " va: 0x%x", va));
 3559                 }
 3560         } else
 3561                 pmap->pm_stats.resident_count++;
 3562 
 3563         /*
 3564          * Enter on the PV list if part of our managed memory.
 3565          */
 3566         if ((m->oflags & VPO_UNMANAGED) == 0) {
 3567                 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
 3568                     ("pmap_enter: managed mapping within the clean submap"));
 3569                 if (pv == NULL)
 3570                         pv = get_pv_entry(pmap, FALSE);
 3571                 pv->pv_va = va;
 3572                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 3573                 pa |= PG_MANAGED;
 3574         } else if (pv != NULL)
 3575                 free_pv_entry(pmap, pv);
 3576 
 3577         /*
 3578          * Increment counters
 3579          */
 3580         if (wired)
 3581                 pmap->pm_stats.wired_count++;
 3582 
 3583 validate:
 3584         /*
 3585          * Now validate mapping with desired protection/wiring.
 3586          */
 3587         newpte = (pt_entry_t)(pa | pmap_cache_bits(m->md.pat_mode, 0) | PG_V);
 3588         if ((prot & VM_PROT_WRITE) != 0) {
 3589                 newpte |= PG_RW;
 3590                 if ((newpte & PG_MANAGED) != 0)
 3591                         vm_page_aflag_set(m, PGA_WRITEABLE);
 3592         }
 3593 #ifdef PAE
 3594         if ((prot & VM_PROT_EXECUTE) == 0)
 3595                 newpte |= pg_nx;
 3596 #endif
 3597         if (wired)
 3598                 newpte |= PG_W;
 3599         if (va < VM_MAXUSER_ADDRESS)
 3600                 newpte |= PG_U;
 3601         if (pmap == kernel_pmap)
 3602                 newpte |= pgeflag;
 3603 
 3604         /*
 3605          * if the mapping or permission bits are different, we need
 3606          * to update the pte.
 3607          */
 3608         if ((origpte & ~(PG_M|PG_A)) != newpte) {
 3609                 newpte |= PG_A;
 3610                 if ((flags & VM_PROT_WRITE) != 0)
 3611                         newpte |= PG_M;
 3612                 if (origpte & PG_V) {
 3613                         invlva = FALSE;
 3614                         origpte = pte_load_store(pte, newpte);
 3615                         if (origpte & PG_A) {
 3616                                 if (origpte & PG_MANAGED)
 3617                                         vm_page_aflag_set(om, PGA_REFERENCED);
 3618                                 if (opa != VM_PAGE_TO_PHYS(m))
 3619                                         invlva = TRUE;
 3620 #ifdef PAE
 3621                                 if ((origpte & PG_NX) == 0 &&
 3622                                     (newpte & PG_NX) != 0)
 3623                                         invlva = TRUE;
 3624 #endif
 3625                         }
 3626                         if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 3627                                 if ((origpte & PG_MANAGED) != 0)
 3628                                         vm_page_dirty(om);
 3629                                 if ((prot & VM_PROT_WRITE) == 0)
 3630                                         invlva = TRUE;
 3631                         }
 3632                         if ((origpte & PG_MANAGED) != 0 &&
 3633                             TAILQ_EMPTY(&om->md.pv_list) &&
 3634                             ((om->flags & PG_FICTITIOUS) != 0 ||
 3635                             TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
 3636                                 vm_page_aflag_clear(om, PGA_WRITEABLE);
 3637                         if (invlva)
 3638                                 pmap_invalidate_page(pmap, va);
 3639                 } else
 3640                         pte_store(pte, newpte);
 3641         }
 3642 
 3643         /*
 3644          * If both the page table page and the reservation are fully
 3645          * populated, then attempt promotion.
 3646          */
 3647         if ((mpte == NULL || mpte->wire_count == NPTEPG) &&
 3648             pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 &&
 3649             vm_reserv_level_iffullpop(m) == 0)
 3650                 pmap_promote_pde(pmap, pde, va);
 3651 
 3652         sched_unpin();
 3653         rw_wunlock(&pvh_global_lock);
 3654         PMAP_UNLOCK(pmap);
 3655         return (KERN_SUCCESS);
 3656 }
 3657 
 3658 /*
 3659  * Tries to create a 2- or 4MB page mapping.  Returns TRUE if successful and
 3660  * FALSE otherwise.  Fails if (1) a page table page cannot be allocated without
 3661  * blocking, (2) a mapping already exists at the specified virtual address, or
 3662  * (3) a pv entry cannot be allocated without reclaiming another pv entry. 
 3663  */
 3664 static boolean_t
 3665 pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 3666 {
 3667         pd_entry_t *pde, newpde;
 3668 
 3669         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3670         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3671         pde = pmap_pde(pmap, va);
 3672         if (*pde != 0) {
 3673                 CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3674                     " in pmap %p", va, pmap);
 3675                 return (FALSE);
 3676         }
 3677         newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 1) |
 3678             PG_PS | PG_V;
 3679         if ((m->oflags & VPO_UNMANAGED) == 0) {
 3680                 newpde |= PG_MANAGED;
 3681 
 3682                 /*
 3683                  * Abort this mapping if its PV entry could not be created.
 3684                  */
 3685                 if (!pmap_pv_insert_pde(pmap, va, VM_PAGE_TO_PHYS(m))) {
 3686                         CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
 3687                             " in pmap %p", va, pmap);
 3688                         return (FALSE);
 3689                 }
 3690         }
 3691 #ifdef PAE
 3692         if ((prot & VM_PROT_EXECUTE) == 0)
 3693                 newpde |= pg_nx;
 3694 #endif
 3695         if (va < VM_MAXUSER_ADDRESS)
 3696                 newpde |= PG_U;
 3697 
 3698         /*
 3699          * Increment counters.
 3700          */
 3701         pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE;
 3702 
 3703         /*
 3704          * Map the superpage.
 3705          */
 3706         pde_store(pde, newpde);
 3707 
 3708         pmap_pde_mappings++;
 3709         CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx"
 3710             " in pmap %p", va, pmap);
 3711         return (TRUE);
 3712 }
 3713 
 3714 /*
 3715  * Maps a sequence of resident pages belonging to the same object.
 3716  * The sequence begins with the given page m_start.  This page is
 3717  * mapped at the given virtual address start.  Each subsequent page is
 3718  * mapped at a virtual address that is offset from start by the same
 3719  * amount as the page is offset from m_start within the object.  The
 3720  * last page in the sequence is the page with the largest offset from
 3721  * m_start that can be mapped at a virtual address less than the given
 3722  * virtual address end.  Not every virtual page between start and end
 3723  * is mapped; only those for which a resident page exists with the
 3724  * corresponding offset from m_start are mapped.
 3725  */
 3726 void
 3727 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 3728     vm_page_t m_start, vm_prot_t prot)
 3729 {
 3730         vm_offset_t va;
 3731         vm_page_t m, mpte;
 3732         vm_pindex_t diff, psize;
 3733 
 3734         VM_OBJECT_ASSERT_LOCKED(m_start->object);
 3735 
 3736         psize = atop(end - start);
 3737         mpte = NULL;
 3738         m = m_start;
 3739         rw_wlock(&pvh_global_lock);
 3740         PMAP_LOCK(pmap);
 3741         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 3742                 va = start + ptoa(diff);
 3743                 if ((va & PDRMASK) == 0 && va + NBPDR <= end &&
 3744                     m->psind == 1 && pg_ps_enabled &&
 3745                     pmap_enter_pde(pmap, va, m, prot))
 3746                         m = &m[NBPDR / PAGE_SIZE - 1];
 3747                 else
 3748                         mpte = pmap_enter_quick_locked(pmap, va, m, prot,
 3749                             mpte);
 3750                 m = TAILQ_NEXT(m, listq);
 3751         }
 3752         rw_wunlock(&pvh_global_lock);
 3753         PMAP_UNLOCK(pmap);
 3754 }
 3755 
 3756 /*
 3757  * this code makes some *MAJOR* assumptions:
 3758  * 1. Current pmap & pmap exists.
 3759  * 2. Not wired.
 3760  * 3. Read access.
 3761  * 4. No page table pages.
 3762  * but is *MUCH* faster than pmap_enter...
 3763  */
 3764 
 3765 void
 3766 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 3767 {
 3768 
 3769         rw_wlock(&pvh_global_lock);
 3770         PMAP_LOCK(pmap);
 3771         (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);
 3772         rw_wunlock(&pvh_global_lock);
 3773         PMAP_UNLOCK(pmap);
 3774 }
 3775 
 3776 static vm_page_t
 3777 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 3778     vm_prot_t prot, vm_page_t mpte)
 3779 {
 3780         pt_entry_t *pte;
 3781         vm_paddr_t pa;
 3782         struct spglist free;
 3783 
 3784         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 3785             (m->oflags & VPO_UNMANAGED) != 0,
 3786             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 3787         rw_assert(&pvh_global_lock, RA_WLOCKED);
 3788         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 3789 
 3790         /*
 3791          * In the case that a page table page is not
 3792          * resident, we are creating it here.
 3793          */
 3794         if (va < VM_MAXUSER_ADDRESS) {
 3795                 u_int ptepindex;
 3796                 pd_entry_t ptepa;
 3797 
 3798                 /*
 3799                  * Calculate pagetable page index
 3800                  */
 3801                 ptepindex = va >> PDRSHIFT;
 3802                 if (mpte && (mpte->pindex == ptepindex)) {
 3803                         mpte->wire_count++;
 3804                 } else {
 3805                         /*
 3806                          * Get the page directory entry
 3807                          */
 3808                         ptepa = pmap->pm_pdir[ptepindex];
 3809 
 3810                         /*
 3811                          * If the page table page is mapped, we just increment
 3812                          * the hold count, and activate it.
 3813                          */
 3814                         if (ptepa) {
 3815                                 if (ptepa & PG_PS)
 3816                                         return (NULL);
 3817                                 mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 3818                                 mpte->wire_count++;
 3819                         } else {
 3820                                 mpte = _pmap_allocpte(pmap, ptepindex,
 3821                                     PMAP_ENTER_NOSLEEP);
 3822                                 if (mpte == NULL)
 3823                                         return (mpte);
 3824                         }
 3825                 }
 3826         } else {
 3827                 mpte = NULL;
 3828         }
 3829 
 3830         /*
 3831          * This call to vtopte makes the assumption that we are
 3832          * entering the page into the current pmap.  In order to support
 3833          * quick entry into any pmap, one would likely use pmap_pte_quick.
 3834          * But that isn't as quick as vtopte.
 3835          */
 3836         pte = vtopte(va);
 3837         if (*pte) {
 3838                 if (mpte != NULL) {
 3839                         mpte->wire_count--;
 3840                         mpte = NULL;
 3841                 }
 3842                 return (mpte);
 3843         }
 3844 
 3845         /*
 3846          * Enter on the PV list if part of our managed memory.
 3847          */
 3848         if ((m->oflags & VPO_UNMANAGED) == 0 &&
 3849             !pmap_try_insert_pv_entry(pmap, va, m)) {
 3850                 if (mpte != NULL) {
 3851                         SLIST_INIT(&free);
 3852                         if (pmap_unwire_ptp(pmap, mpte, &free)) {
 3853                                 pmap_invalidate_page(pmap, va);
 3854                                 pmap_free_zero_pages(&free);
 3855                         }
 3856                         
 3857                         mpte = NULL;
 3858                 }
 3859                 return (mpte);
 3860         }
 3861 
 3862         /*
 3863          * Increment counters
 3864          */
 3865         pmap->pm_stats.resident_count++;
 3866 
 3867         pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.pat_mode, 0);
 3868 #ifdef PAE
 3869         if ((prot & VM_PROT_EXECUTE) == 0)
 3870                 pa |= pg_nx;
 3871 #endif
 3872 
 3873         /*
 3874          * Now validate mapping with RO protection
 3875          */
 3876         if ((m->oflags & VPO_UNMANAGED) != 0)
 3877                 pte_store(pte, pa | PG_V | PG_U);
 3878         else
 3879                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 3880         return (mpte);
 3881 }
 3882 
 3883 /*
 3884  * Make a temporary mapping for a physical address.  This is only intended
 3885  * to be used for panic dumps.
 3886  */
 3887 void *
 3888 pmap_kenter_temporary(vm_paddr_t pa, int i)
 3889 {
 3890         vm_offset_t va;
 3891 
 3892         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 3893         pmap_kenter(va, pa);
 3894         invlpg(va);
 3895         return ((void *)crashdumpmap);
 3896 }
 3897 
 3898 /*
 3899  * This code maps large physical mmap regions into the
 3900  * processor address space.  Note that some shortcuts
 3901  * are taken, but the code works.
 3902  */
 3903 void
 3904 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
 3905     vm_pindex_t pindex, vm_size_t size)
 3906 {
 3907         pd_entry_t *pde;
 3908         vm_paddr_t pa, ptepa;
 3909         vm_page_t p;
 3910         int pat_mode;
 3911 
 3912         VM_OBJECT_ASSERT_WLOCKED(object);
 3913         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 3914             ("pmap_object_init_pt: non-device object"));
 3915         if (pseflag && 
 3916             (addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
 3917                 if (!vm_object_populate(object, pindex, pindex + atop(size)))
 3918                         return;
 3919                 p = vm_page_lookup(object, pindex);
 3920                 KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3921                     ("pmap_object_init_pt: invalid page %p", p));
 3922                 pat_mode = p->md.pat_mode;
 3923 
 3924                 /*
 3925                  * Abort the mapping if the first page is not physically
 3926                  * aligned to a 2/4MB page boundary.
 3927                  */
 3928                 ptepa = VM_PAGE_TO_PHYS(p);
 3929                 if (ptepa & (NBPDR - 1))
 3930                         return;
 3931 
 3932                 /*
 3933                  * Skip the first page.  Abort the mapping if the rest of
 3934                  * the pages are not physically contiguous or have differing
 3935                  * memory attributes.
 3936                  */
 3937                 p = TAILQ_NEXT(p, listq);
 3938                 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
 3939                     pa += PAGE_SIZE) {
 3940                         KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3941                             ("pmap_object_init_pt: invalid page %p", p));
 3942                         if (pa != VM_PAGE_TO_PHYS(p) ||
 3943                             pat_mode != p->md.pat_mode)
 3944                                 return;
 3945                         p = TAILQ_NEXT(p, listq);
 3946                 }
 3947 
 3948                 /*
 3949                  * Map using 2/4MB pages.  Since "ptepa" is 2/4M aligned and
 3950                  * "size" is a multiple of 2/4M, adding the PAT setting to
 3951                  * "pa" will not affect the termination of this loop.
 3952                  */
 3953                 PMAP_LOCK(pmap);
 3954                 for (pa = ptepa | pmap_cache_bits(pat_mode, 1); pa < ptepa +
 3955                     size; pa += NBPDR) {
 3956                         pde = pmap_pde(pmap, addr);
 3957                         if (*pde == 0) {
 3958                                 pde_store(pde, pa | PG_PS | PG_M | PG_A |
 3959                                     PG_U | PG_RW | PG_V);
 3960                                 pmap->pm_stats.resident_count += NBPDR /
 3961                                     PAGE_SIZE;
 3962                                 pmap_pde_mappings++;
 3963                         }
 3964                         /* Else continue on if the PDE is already valid. */
 3965                         addr += NBPDR;
 3966                 }
 3967                 PMAP_UNLOCK(pmap);
 3968         }
 3969 }
 3970 
 3971 /*
 3972  *      Clear the wired attribute from the mappings for the specified range of
 3973  *      addresses in the given pmap.  Every valid mapping within that range
 3974  *      must have the wired attribute set.  In contrast, invalid mappings
 3975  *      cannot have the wired attribute set, so they are ignored.
 3976  *
 3977  *      The wired attribute of the page table entry is not a hardware feature,
 3978  *      so there is no need to invalidate any TLB entries.
 3979  */
 3980 void
 3981 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 3982 {
 3983         vm_offset_t pdnxt;
 3984         pd_entry_t *pde;
 3985         pt_entry_t *pte;
 3986         boolean_t pv_lists_locked;
 3987 
 3988         if (pmap_is_current(pmap))
 3989                 pv_lists_locked = FALSE;
 3990         else {
 3991                 pv_lists_locked = TRUE;
 3992 resume:
 3993                 rw_wlock(&pvh_global_lock);
 3994                 sched_pin();
 3995         }
 3996         PMAP_LOCK(pmap);
 3997         for (; sva < eva; sva = pdnxt) {
 3998                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 3999                 if (pdnxt < sva)
 4000                         pdnxt = eva;
 4001                 pde = pmap_pde(pmap, sva);
 4002                 if ((*pde & PG_V) == 0)
 4003                         continue;
 4004                 if ((*pde & PG_PS) != 0) {
 4005                         if ((*pde & PG_W) == 0)
 4006                                 panic("pmap_unwire: pde %#jx is missing PG_W",
 4007                                     (uintmax_t)*pde);
 4008 
 4009                         /*
 4010                          * Are we unwiring the entire large page?  If not,
 4011                          * demote the mapping and fall through.
 4012                          */
 4013                         if (sva + NBPDR == pdnxt && eva >= pdnxt) {
 4014                                 /*
 4015                                  * Regardless of whether a pde (or pte) is 32
 4016                                  * or 64 bits in size, PG_W is among the least
 4017                                  * significant 32 bits.
 4018                                  */
 4019                                 atomic_clear_int((u_int *)pde, PG_W);
 4020                                 pmap->pm_stats.wired_count -= NBPDR /
 4021                                     PAGE_SIZE;
 4022                                 continue;
 4023                         } else {
 4024                                 if (!pv_lists_locked) {
 4025                                         pv_lists_locked = TRUE;
 4026                                         if (!rw_try_wlock(&pvh_global_lock)) {
 4027                                                 PMAP_UNLOCK(pmap);
 4028                                                 /* Repeat sva. */
 4029                                                 goto resume;
 4030                                         }
 4031                                         sched_pin();
 4032                                 }
 4033                                 if (!pmap_demote_pde(pmap, pde, sva))
 4034                                         panic("pmap_unwire: demotion failed");
 4035                         }
 4036                 }
 4037                 if (pdnxt > eva)
 4038                         pdnxt = eva;
 4039                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 4040                     sva += PAGE_SIZE) {
 4041                         if ((*pte & PG_V) == 0)
 4042                                 continue;
 4043                         if ((*pte & PG_W) == 0)
 4044                                 panic("pmap_unwire: pte %#jx is missing PG_W",
 4045                                     (uintmax_t)*pte);
 4046 
 4047                         /*
 4048                          * PG_W must be cleared atomically.  Although the pmap
 4049                          * lock synchronizes access to PG_W, another processor
 4050                          * could be setting PG_M and/or PG_A concurrently.
 4051                          *
 4052                          * PG_W is among the least significant 32 bits.
 4053                          */
 4054                         atomic_clear_int((u_int *)pte, PG_W);
 4055                         pmap->pm_stats.wired_count--;
 4056                 }
 4057         }
 4058         if (pv_lists_locked) {
 4059                 sched_unpin();
 4060                 rw_wunlock(&pvh_global_lock);
 4061         }
 4062         PMAP_UNLOCK(pmap);
 4063 }
 4064 
 4065 
 4066 /*
 4067  *      Copy the range specified by src_addr/len
 4068  *      from the source map to the range dst_addr/len
 4069  *      in the destination map.
 4070  *
 4071  *      This routine is only advisory and need not do anything.
 4072  */
 4073 
 4074 void
 4075 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 4076     vm_offset_t src_addr)
 4077 {
 4078         struct spglist free;
 4079         vm_offset_t addr;
 4080         vm_offset_t end_addr = src_addr + len;
 4081         vm_offset_t pdnxt;
 4082 
 4083         if (dst_addr != src_addr)
 4084                 return;
 4085 
 4086         if (!pmap_is_current(src_pmap))
 4087                 return;
 4088 
 4089         rw_wlock(&pvh_global_lock);
 4090         if (dst_pmap < src_pmap) {
 4091                 PMAP_LOCK(dst_pmap);
 4092                 PMAP_LOCK(src_pmap);
 4093         } else {
 4094                 PMAP_LOCK(src_pmap);
 4095                 PMAP_LOCK(dst_pmap);
 4096         }
 4097         sched_pin();
 4098         for (addr = src_addr; addr < end_addr; addr = pdnxt) {
 4099                 pt_entry_t *src_pte, *dst_pte;
 4100                 vm_page_t dstmpte, srcmpte;
 4101                 pd_entry_t srcptepaddr;
 4102                 u_int ptepindex;
 4103 
 4104                 KASSERT(addr < UPT_MIN_ADDRESS,
 4105                     ("pmap_copy: invalid to pmap_copy page tables"));
 4106 
 4107                 pdnxt = (addr + NBPDR) & ~PDRMASK;
 4108                 if (pdnxt < addr)
 4109                         pdnxt = end_addr;
 4110                 ptepindex = addr >> PDRSHIFT;
 4111 
 4112                 srcptepaddr = src_pmap->pm_pdir[ptepindex];
 4113                 if (srcptepaddr == 0)
 4114                         continue;
 4115                         
 4116                 if (srcptepaddr & PG_PS) {
 4117                         if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr)
 4118                                 continue;
 4119                         if (dst_pmap->pm_pdir[ptepindex] == 0 &&
 4120                             ((srcptepaddr & PG_MANAGED) == 0 ||
 4121                             pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr &
 4122                             PG_PS_FRAME))) {
 4123                                 dst_pmap->pm_pdir[ptepindex] = srcptepaddr &
 4124                                     ~PG_W;
 4125                                 dst_pmap->pm_stats.resident_count +=
 4126                                     NBPDR / PAGE_SIZE;
 4127                         }
 4128                         continue;
 4129                 }
 4130 
 4131                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 4132                 KASSERT(srcmpte->wire_count > 0,
 4133                     ("pmap_copy: source page table page is unused"));
 4134 
 4135                 if (pdnxt > end_addr)
 4136                         pdnxt = end_addr;
 4137 
 4138                 src_pte = vtopte(addr);
 4139                 while (addr < pdnxt) {
 4140                         pt_entry_t ptetemp;
 4141                         ptetemp = *src_pte;
 4142                         /*
 4143                          * we only virtual copy managed pages
 4144                          */
 4145                         if ((ptetemp & PG_MANAGED) != 0) {
 4146                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 4147                                     PMAP_ENTER_NOSLEEP);
 4148                                 if (dstmpte == NULL)
 4149                                         goto out;
 4150                                 dst_pte = pmap_pte_quick(dst_pmap, addr);
 4151                                 if (*dst_pte == 0 &&
 4152                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 4153                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
 4154                                         /*
 4155                                          * Clear the wired, modified, and
 4156                                          * accessed (referenced) bits
 4157                                          * during the copy.
 4158                                          */
 4159                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 4160                                             PG_A);
 4161                                         dst_pmap->pm_stats.resident_count++;
 4162                                 } else {
 4163                                         SLIST_INIT(&free);
 4164                                         if (pmap_unwire_ptp(dst_pmap, dstmpte,
 4165                                             &free)) {
 4166                                                 pmap_invalidate_page(dst_pmap,
 4167                                                     addr);
 4168                                                 pmap_free_zero_pages(&free);
 4169                                         }
 4170                                         goto out;
 4171                                 }
 4172                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 4173                                         break;
 4174                         }
 4175                         addr += PAGE_SIZE;
 4176                         src_pte++;
 4177                 }
 4178         }
 4179 out:
 4180         sched_unpin();
 4181         rw_wunlock(&pvh_global_lock);
 4182         PMAP_UNLOCK(src_pmap);
 4183         PMAP_UNLOCK(dst_pmap);
 4184 }       
 4185 
 4186 static __inline void
 4187 pagezero(void *page)
 4188 {
 4189 #if defined(I686_CPU)
 4190         if (cpu_class == CPUCLASS_686) {
 4191 #if defined(CPU_ENABLE_SSE)
 4192                 if (cpu_feature & CPUID_SSE2)
 4193                         sse2_pagezero(page);
 4194                 else
 4195 #endif
 4196                         i686_pagezero(page);
 4197         } else
 4198 #endif
 4199                 bzero(page, PAGE_SIZE);
 4200 }
 4201 
 4202 /*
 4203  *      pmap_zero_page zeros the specified hardware page by mapping 
 4204  *      the page into KVM and using bzero to clear its contents.
 4205  */
 4206 void
 4207 pmap_zero_page(vm_page_t m)
 4208 {
 4209         struct sysmaps *sysmaps;
 4210 
 4211         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 4212         mtx_lock(&sysmaps->lock);
 4213         if (*sysmaps->CMAP2)
 4214                 panic("pmap_zero_page: CMAP2 busy");
 4215         sched_pin();
 4216         *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 4217             pmap_cache_bits(m->md.pat_mode, 0);
 4218         invlcaddr(sysmaps->CADDR2);
 4219         pagezero(sysmaps->CADDR2);
 4220         *sysmaps->CMAP2 = 0;
 4221         sched_unpin();
 4222         mtx_unlock(&sysmaps->lock);
 4223 }
 4224 
 4225 /*
 4226  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 4227  *      the page into KVM and using bzero to clear its contents.
 4228  *
 4229  *      off and size may not cover an area beyond a single hardware page.
 4230  */
 4231 void
 4232 pmap_zero_page_area(vm_page_t m, int off, int size)
 4233 {
 4234         struct sysmaps *sysmaps;
 4235 
 4236         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 4237         mtx_lock(&sysmaps->lock);
 4238         if (*sysmaps->CMAP2)
 4239                 panic("pmap_zero_page_area: CMAP2 busy");
 4240         sched_pin();
 4241         *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 4242             pmap_cache_bits(m->md.pat_mode, 0);
 4243         invlcaddr(sysmaps->CADDR2);
 4244         if (off == 0 && size == PAGE_SIZE) 
 4245                 pagezero(sysmaps->CADDR2);
 4246         else
 4247                 bzero((char *)sysmaps->CADDR2 + off, size);
 4248         *sysmaps->CMAP2 = 0;
 4249         sched_unpin();
 4250         mtx_unlock(&sysmaps->lock);
 4251 }
 4252 
 4253 /*
 4254  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 4255  *      the page into KVM and using bzero to clear its contents.  This
 4256  *      is intended to be called from the vm_pagezero process only and
 4257  *      outside of Giant.
 4258  */
 4259 void
 4260 pmap_zero_page_idle(vm_page_t m)
 4261 {
 4262 
 4263         if (*CMAP3)
 4264                 panic("pmap_zero_page_idle: CMAP3 busy");
 4265         sched_pin();
 4266         *CMAP3 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M |
 4267             pmap_cache_bits(m->md.pat_mode, 0);
 4268         invlcaddr(CADDR3);
 4269         pagezero(CADDR3);
 4270         *CMAP3 = 0;
 4271         sched_unpin();
 4272 }
 4273 
 4274 /*
 4275  *      pmap_copy_page copies the specified (machine independent)
 4276  *      page by mapping the page into virtual memory and using
 4277  *      bcopy to copy the page, one machine dependent page at a
 4278  *      time.
 4279  */
 4280 void
 4281 pmap_copy_page(vm_page_t src, vm_page_t dst)
 4282 {
 4283         struct sysmaps *sysmaps;
 4284 
 4285         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 4286         mtx_lock(&sysmaps->lock);
 4287         if (*sysmaps->CMAP1)
 4288                 panic("pmap_copy_page: CMAP1 busy");
 4289         if (*sysmaps->CMAP2)
 4290                 panic("pmap_copy_page: CMAP2 busy");
 4291         sched_pin();
 4292         *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A |
 4293             pmap_cache_bits(src->md.pat_mode, 0);
 4294         invlcaddr(sysmaps->CADDR1);
 4295         *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M |
 4296             pmap_cache_bits(dst->md.pat_mode, 0);
 4297         invlcaddr(sysmaps->CADDR2);
 4298         bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
 4299         *sysmaps->CMAP1 = 0;
 4300         *sysmaps->CMAP2 = 0;
 4301         sched_unpin();
 4302         mtx_unlock(&sysmaps->lock);
 4303 }
 4304 
 4305 int unmapped_buf_allowed = 1;
 4306 
 4307 void
 4308 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 4309     vm_offset_t b_offset, int xfersize)
 4310 {
 4311         struct sysmaps *sysmaps;
 4312         vm_page_t a_pg, b_pg;
 4313         char *a_cp, *b_cp;
 4314         vm_offset_t a_pg_offset, b_pg_offset;
 4315         int cnt;
 4316 
 4317         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 4318         mtx_lock(&sysmaps->lock);
 4319         if (*sysmaps->CMAP1 != 0)
 4320                 panic("pmap_copy_pages: CMAP1 busy");
 4321         if (*sysmaps->CMAP2 != 0)
 4322                 panic("pmap_copy_pages: CMAP2 busy");
 4323         sched_pin();
 4324         while (xfersize > 0) {
 4325                 a_pg = ma[a_offset >> PAGE_SHIFT];
 4326                 a_pg_offset = a_offset & PAGE_MASK;
 4327                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
 4328                 b_pg = mb[b_offset >> PAGE_SHIFT];
 4329                 b_pg_offset = b_offset & PAGE_MASK;
 4330                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 4331                 *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(a_pg) | PG_A |
 4332                     pmap_cache_bits(a_pg->md.pat_mode, 0);
 4333                 invlcaddr(sysmaps->CADDR1);
 4334                 *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(b_pg) | PG_A |
 4335                     PG_M | pmap_cache_bits(b_pg->md.pat_mode, 0);
 4336                 invlcaddr(sysmaps->CADDR2);
 4337                 a_cp = sysmaps->CADDR1 + a_pg_offset;
 4338                 b_cp = sysmaps->CADDR2 + b_pg_offset;
 4339                 bcopy(a_cp, b_cp, cnt);
 4340                 a_offset += cnt;
 4341                 b_offset += cnt;
 4342                 xfersize -= cnt;
 4343         }
 4344         *sysmaps->CMAP1 = 0;
 4345         *sysmaps->CMAP2 = 0;
 4346         sched_unpin();
 4347         mtx_unlock(&sysmaps->lock);
 4348 }
 4349 
 4350 /*
 4351  * Returns true if the pmap's pv is one of the first
 4352  * 16 pvs linked to from this page.  This count may
 4353  * be changed upwards or downwards in the future; it
 4354  * is only necessary that true be returned for a small
 4355  * subset of pmaps for proper page aging.
 4356  */
 4357 boolean_t
 4358 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 4359 {
 4360         struct md_page *pvh;
 4361         pv_entry_t pv;
 4362         int loops = 0;
 4363         boolean_t rv;
 4364 
 4365         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4366             ("pmap_page_exists_quick: page %p is not managed", m));
 4367         rv = FALSE;
 4368         rw_wlock(&pvh_global_lock);
 4369         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 4370                 if (PV_PMAP(pv) == pmap) {
 4371                         rv = TRUE;
 4372                         break;
 4373                 }
 4374                 loops++;
 4375                 if (loops >= 16)
 4376                         break;
 4377         }
 4378         if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
 4379                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4380                 TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4381                         if (PV_PMAP(pv) == pmap) {
 4382                                 rv = TRUE;
 4383                                 break;
 4384                         }
 4385                         loops++;
 4386                         if (loops >= 16)
 4387                                 break;
 4388                 }
 4389         }
 4390         rw_wunlock(&pvh_global_lock);
 4391         return (rv);
 4392 }
 4393 
 4394 /*
 4395  *      pmap_page_wired_mappings:
 4396  *
 4397  *      Return the number of managed mappings to the given physical page
 4398  *      that are wired.
 4399  */
 4400 int
 4401 pmap_page_wired_mappings(vm_page_t m)
 4402 {
 4403         int count;
 4404 
 4405         count = 0;
 4406         if ((m->oflags & VPO_UNMANAGED) != 0)
 4407                 return (count);
 4408         rw_wlock(&pvh_global_lock);
 4409         count = pmap_pvh_wired_mappings(&m->md, count);
 4410         if ((m->flags & PG_FICTITIOUS) == 0) {
 4411             count = pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)),
 4412                 count);
 4413         }
 4414         rw_wunlock(&pvh_global_lock);
 4415         return (count);
 4416 }
 4417 
 4418 /*
 4419  *      pmap_pvh_wired_mappings:
 4420  *
 4421  *      Return the updated number "count" of managed mappings that are wired.
 4422  */
 4423 static int
 4424 pmap_pvh_wired_mappings(struct md_page *pvh, int count)
 4425 {
 4426         pmap_t pmap;
 4427         pt_entry_t *pte;
 4428         pv_entry_t pv;
 4429 
 4430         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4431         sched_pin();
 4432         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4433                 pmap = PV_PMAP(pv);
 4434                 PMAP_LOCK(pmap);
 4435                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4436                 if ((*pte & PG_W) != 0)
 4437                         count++;
 4438                 PMAP_UNLOCK(pmap);
 4439         }
 4440         sched_unpin();
 4441         return (count);
 4442 }
 4443 
 4444 /*
 4445  * Returns TRUE if the given page is mapped individually or as part of
 4446  * a 4mpage.  Otherwise, returns FALSE.
 4447  */
 4448 boolean_t
 4449 pmap_page_is_mapped(vm_page_t m)
 4450 {
 4451         boolean_t rv;
 4452 
 4453         if ((m->oflags & VPO_UNMANAGED) != 0)
 4454                 return (FALSE);
 4455         rw_wlock(&pvh_global_lock);
 4456         rv = !TAILQ_EMPTY(&m->md.pv_list) ||
 4457             ((m->flags & PG_FICTITIOUS) == 0 &&
 4458             !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
 4459         rw_wunlock(&pvh_global_lock);
 4460         return (rv);
 4461 }
 4462 
 4463 /*
 4464  * Remove all pages from specified address space
 4465  * this aids process exit speeds.  Also, this code
 4466  * is special cased for current process only, but
 4467  * can have the more generic (and slightly slower)
 4468  * mode enabled.  This is much faster than pmap_remove
 4469  * in the case of running down an entire address space.
 4470  */
 4471 void
 4472 pmap_remove_pages(pmap_t pmap)
 4473 {
 4474         pt_entry_t *pte, tpte;
 4475         vm_page_t m, mpte, mt;
 4476         pv_entry_t pv;
 4477         struct md_page *pvh;
 4478         struct pv_chunk *pc, *npc;
 4479         struct spglist free;
 4480         int field, idx;
 4481         int32_t bit;
 4482         uint32_t inuse, bitmask;
 4483         int allfree;
 4484 
 4485         if (pmap != PCPU_GET(curpmap)) {
 4486                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 4487                 return;
 4488         }
 4489         SLIST_INIT(&free);
 4490         rw_wlock(&pvh_global_lock);
 4491         PMAP_LOCK(pmap);
 4492         sched_pin();
 4493         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 4494                 KASSERT(pc->pc_pmap == pmap, ("Wrong pmap %p %p", pmap,
 4495                     pc->pc_pmap));
 4496                 allfree = 1;
 4497                 for (field = 0; field < _NPCM; field++) {
 4498                         inuse = ~pc->pc_map[field] & pc_freemask[field];
 4499                         while (inuse != 0) {
 4500                                 bit = bsfl(inuse);
 4501                                 bitmask = 1UL << bit;
 4502                                 idx = field * 32 + bit;
 4503                                 pv = &pc->pc_pventry[idx];
 4504                                 inuse &= ~bitmask;
 4505 
 4506                                 pte = pmap_pde(pmap, pv->pv_va);
 4507                                 tpte = *pte;
 4508                                 if ((tpte & PG_PS) == 0) {
 4509                                         pte = vtopte(pv->pv_va);
 4510                                         tpte = *pte & ~PG_PTE_PAT;
 4511                                 }
 4512 
 4513                                 if (tpte == 0) {
 4514                                         printf(
 4515                                             "TPTE at %p  IS ZERO @ VA %08x\n",
 4516                                             pte, pv->pv_va);
 4517                                         panic("bad pte");
 4518                                 }
 4519 
 4520 /*
 4521  * We cannot remove wired pages from a process' mapping at this time
 4522  */
 4523                                 if (tpte & PG_W) {
 4524                                         allfree = 0;
 4525                                         continue;
 4526                                 }
 4527 
 4528                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 4529                                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 4530                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 4531                                     m, (uintmax_t)m->phys_addr,
 4532                                     (uintmax_t)tpte));
 4533 
 4534                                 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
 4535                                     m < &vm_page_array[vm_page_array_size],
 4536                                     ("pmap_remove_pages: bad tpte %#jx",
 4537                                     (uintmax_t)tpte));
 4538 
 4539                                 pte_clear(pte);
 4540 
 4541                                 /*
 4542                                  * Update the vm_page_t clean/reference bits.
 4543                                  */
 4544                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4545                                         if ((tpte & PG_PS) != 0) {
 4546                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4547                                                         vm_page_dirty(mt);
 4548                                         } else
 4549                                                 vm_page_dirty(m);
 4550                                 }
 4551 
 4552                                 /* Mark free */
 4553                                 PV_STAT(pv_entry_frees++);
 4554                                 PV_STAT(pv_entry_spare++);
 4555                                 pv_entry_count--;
 4556                                 pc->pc_map[field] |= bitmask;
 4557                                 if ((tpte & PG_PS) != 0) {
 4558                                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 4559                                         pvh = pa_to_pvh(tpte & PG_PS_FRAME);
 4560                                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 4561                                         if (TAILQ_EMPTY(&pvh->pv_list)) {
 4562                                                 for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++)
 4563                                                         if (TAILQ_EMPTY(&mt->md.pv_list))
 4564                                                                 vm_page_aflag_clear(mt, PGA_WRITEABLE);
 4565                                         }
 4566                                         mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
 4567                                         if (mpte != NULL) {
 4568                                                 pmap_remove_pt_page(pmap, mpte);
 4569                                                 pmap->pm_stats.resident_count--;
 4570                                                 KASSERT(mpte->wire_count == NPTEPG,
 4571                                                     ("pmap_remove_pages: pte page wire count error"));
 4572                                                 mpte->wire_count = 0;
 4573                                                 pmap_add_delayed_free_list(mpte, &free, FALSE);
 4574                                                 atomic_subtract_int(&cnt.v_wire_count, 1);
 4575                                         }
 4576                                 } else {
 4577                                         pmap->pm_stats.resident_count--;
 4578                                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 4579                                         if (TAILQ_EMPTY(&m->md.pv_list) &&
 4580                                             (m->flags & PG_FICTITIOUS) == 0) {
 4581                                                 pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4582                                                 if (TAILQ_EMPTY(&pvh->pv_list))
 4583                                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 4584                                         }
 4585                                         pmap_unuse_pt(pmap, pv->pv_va, &free);
 4586                                 }
 4587                         }
 4588                 }
 4589                 if (allfree) {
 4590                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 4591                         free_pv_chunk(pc);
 4592                 }
 4593         }
 4594         sched_unpin();
 4595         pmap_invalidate_all(pmap);
 4596         rw_wunlock(&pvh_global_lock);
 4597         PMAP_UNLOCK(pmap);
 4598         pmap_free_zero_pages(&free);
 4599 }
 4600 
 4601 /*
 4602  *      pmap_is_modified:
 4603  *
 4604  *      Return whether or not the specified physical page was modified
 4605  *      in any physical maps.
 4606  */
 4607 boolean_t
 4608 pmap_is_modified(vm_page_t m)
 4609 {
 4610         boolean_t rv;
 4611 
 4612         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4613             ("pmap_is_modified: page %p is not managed", m));
 4614 
 4615         /*
 4616          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
 4617          * concurrently set while the object is locked.  Thus, if PGA_WRITEABLE
 4618          * is clear, no PTEs can have PG_M set.
 4619          */
 4620         VM_OBJECT_ASSERT_WLOCKED(m->object);
 4621         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
 4622                 return (FALSE);
 4623         rw_wlock(&pvh_global_lock);
 4624         rv = pmap_is_modified_pvh(&m->md) ||
 4625             ((m->flags & PG_FICTITIOUS) == 0 &&
 4626             pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4627         rw_wunlock(&pvh_global_lock);
 4628         return (rv);
 4629 }
 4630 
 4631 /*
 4632  * Returns TRUE if any of the given mappings were used to modify
 4633  * physical memory.  Otherwise, returns FALSE.  Both page and 2mpage
 4634  * mappings are supported.
 4635  */
 4636 static boolean_t
 4637 pmap_is_modified_pvh(struct md_page *pvh)
 4638 {
 4639         pv_entry_t pv;
 4640         pt_entry_t *pte;
 4641         pmap_t pmap;
 4642         boolean_t rv;
 4643 
 4644         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4645         rv = FALSE;
 4646         sched_pin();
 4647         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4648                 pmap = PV_PMAP(pv);
 4649                 PMAP_LOCK(pmap);
 4650                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4651                 rv = (*pte & (PG_M | PG_RW)) == (PG_M | PG_RW);
 4652                 PMAP_UNLOCK(pmap);
 4653                 if (rv)
 4654                         break;
 4655         }
 4656         sched_unpin();
 4657         return (rv);
 4658 }
 4659 
 4660 /*
 4661  *      pmap_is_prefaultable:
 4662  *
 4663  *      Return whether or not the specified virtual address is elgible
 4664  *      for prefault.
 4665  */
 4666 boolean_t
 4667 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 4668 {
 4669         pd_entry_t *pde;
 4670         pt_entry_t *pte;
 4671         boolean_t rv;
 4672 
 4673         rv = FALSE;
 4674         PMAP_LOCK(pmap);
 4675         pde = pmap_pde(pmap, addr);
 4676         if (*pde != 0 && (*pde & PG_PS) == 0) {
 4677                 pte = vtopte(addr);
 4678                 rv = *pte == 0;
 4679         }
 4680         PMAP_UNLOCK(pmap);
 4681         return (rv);
 4682 }
 4683 
 4684 /*
 4685  *      pmap_is_referenced:
 4686  *
 4687  *      Return whether or not the specified physical page was referenced
 4688  *      in any physical maps.
 4689  */
 4690 boolean_t
 4691 pmap_is_referenced(vm_page_t m)
 4692 {
 4693         boolean_t rv;
 4694 
 4695         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4696             ("pmap_is_referenced: page %p is not managed", m));
 4697         rw_wlock(&pvh_global_lock);
 4698         rv = pmap_is_referenced_pvh(&m->md) ||
 4699             ((m->flags & PG_FICTITIOUS) == 0 &&
 4700             pmap_is_referenced_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m))));
 4701         rw_wunlock(&pvh_global_lock);
 4702         return (rv);
 4703 }
 4704 
 4705 /*
 4706  * Returns TRUE if any of the given mappings were referenced and FALSE
 4707  * otherwise.  Both page and 4mpage mappings are supported.
 4708  */
 4709 static boolean_t
 4710 pmap_is_referenced_pvh(struct md_page *pvh)
 4711 {
 4712         pv_entry_t pv;
 4713         pt_entry_t *pte;
 4714         pmap_t pmap;
 4715         boolean_t rv;
 4716 
 4717         rw_assert(&pvh_global_lock, RA_WLOCKED);
 4718         rv = FALSE;
 4719         sched_pin();
 4720         TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) {
 4721                 pmap = PV_PMAP(pv);
 4722                 PMAP_LOCK(pmap);
 4723                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4724                 rv = (*pte & (PG_A | PG_V)) == (PG_A | PG_V);
 4725                 PMAP_UNLOCK(pmap);
 4726                 if (rv)
 4727                         break;
 4728         }
 4729         sched_unpin();
 4730         return (rv);
 4731 }
 4732 
 4733 /*
 4734  * Clear the write and modified bits in each of the given page's mappings.
 4735  */
 4736 void
 4737 pmap_remove_write(vm_page_t m)
 4738 {
 4739         struct md_page *pvh;
 4740         pv_entry_t next_pv, pv;
 4741         pmap_t pmap;
 4742         pd_entry_t *pde;
 4743         pt_entry_t oldpte, *pte;
 4744         vm_offset_t va;
 4745 
 4746         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4747             ("pmap_remove_write: page %p is not managed", m));
 4748 
 4749         /*
 4750          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
 4751          * set by another thread while the object is locked.  Thus,
 4752          * if PGA_WRITEABLE is clear, no page table entries need updating.
 4753          */
 4754         VM_OBJECT_ASSERT_WLOCKED(m->object);
 4755         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
 4756                 return;
 4757         rw_wlock(&pvh_global_lock);
 4758         sched_pin();
 4759         if ((m->flags & PG_FICTITIOUS) != 0)
 4760                 goto small_mappings;
 4761         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 4762         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
 4763                 va = pv->pv_va;
 4764                 pmap = PV_PMAP(pv);
 4765                 PMAP_LOCK(pmap);
 4766                 pde = pmap_pde(pmap, va);
 4767                 if ((*pde & PG_RW) != 0)
 4768                         (void)pmap_demote_pde(pmap, pde, va);
 4769                 PMAP_UNLOCK(pmap);
 4770         }
 4771 small_mappings:
 4772         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 4773                 pmap = PV_PMAP(pv);
 4774                 PMAP_LOCK(pmap);
 4775                 pde = pmap_pde(pmap, pv->pv_va);
 4776                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_write: found"
 4777                     " a 4mpage in page %p's pv list", m));
 4778                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4779 retry:
 4780                 oldpte = *pte;
 4781                 if ((oldpte & PG_RW) != 0) {
 4782                         /*
 4783                          * Regardless of whether a pte is 32 or 64 bits
 4784                          * in size, PG_RW and PG_M are among the least
 4785                          * significant 32 bits.
 4786                          */
 4787                         if (!atomic_cmpset_int((u_int *)pte, oldpte,
 4788                             oldpte & ~(PG_RW | PG_M)))
 4789                                 goto retry;
 4790                         if ((oldpte & PG_M) != 0)
 4791                                 vm_page_dirty(m);
 4792                         pmap_invalidate_page(pmap, pv->pv_va);
 4793                 }
 4794                 PMAP_UNLOCK(pmap);
 4795         }
 4796         vm_page_aflag_clear(m, PGA_WRITEABLE);
 4797         sched_unpin();
 4798         rw_wunlock(&pvh_global_lock);
 4799 }
 4800 
 4801 #define PMAP_TS_REFERENCED_MAX  5
 4802 
 4803 /*
 4804  *      pmap_ts_referenced:
 4805  *
 4806  *      Return a count of reference bits for a page, clearing those bits.
 4807  *      It is not necessary for every reference bit to be cleared, but it
 4808  *      is necessary that 0 only be returned when there are truly no
 4809  *      reference bits set.
 4810  *
 4811  *      XXX: The exact number of bits to check and clear is a matter that
 4812  *      should be tested and standardized at some point in the future for
 4813  *      optimal aging of shared pages.
 4814  */
 4815 int
 4816 pmap_ts_referenced(vm_page_t m)
 4817 {
 4818         struct md_page *pvh;
 4819         pv_entry_t pv, pvf;
 4820         pmap_t pmap;
 4821         pd_entry_t *pde;
 4822         pt_entry_t *pte;
 4823         vm_paddr_t pa;
 4824         int rtval = 0;
 4825 
 4826         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 4827             ("pmap_ts_referenced: page %p is not managed", m));
 4828         pa = VM_PAGE_TO_PHYS(m);
 4829         pvh = pa_to_pvh(pa);
 4830         rw_wlock(&pvh_global_lock);
 4831         sched_pin();
 4832         if ((m->flags & PG_FICTITIOUS) != 0 ||
 4833             (pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
 4834                 goto small_mappings;
 4835         pv = pvf;
 4836         do {
 4837                 pmap = PV_PMAP(pv);
 4838                 PMAP_LOCK(pmap);
 4839                 pde = pmap_pde(pmap, pv->pv_va);
 4840                 if ((*pde & PG_A) != 0) {
 4841                         /*
 4842                          * Since this reference bit is shared by either 1024
 4843                          * or 512 4KB pages, it should not be cleared every
 4844                          * time it is tested.  Apply a simple "hash" function
 4845                          * on the physical page number, the virtual superpage
 4846                          * number, and the pmap address to select one 4KB page
 4847                          * out of the 1024 or 512 on which testing the
 4848                          * reference bit will result in clearing that bit.
 4849                          * This function is designed to avoid the selection of
 4850                          * the same 4KB page for every 2- or 4MB page mapping.
 4851                          *
 4852                          * On demotion, a mapping that hasn't been referenced
 4853                          * is simply destroyed.  To avoid the possibility of a
 4854                          * subsequent page fault on a demoted wired mapping,
 4855                          * always leave its reference bit set.  Moreover,
 4856                          * since the superpage is wired, the current state of
 4857                          * its reference bit won't affect page replacement.
 4858                          */
 4859                         if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PDRSHIFT) ^
 4860                             (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
 4861                             (*pde & PG_W) == 0) {
 4862                                 atomic_clear_int((u_int *)pde, PG_A);
 4863                                 pmap_invalidate_page(pmap, pv->pv_va);
 4864                         }
 4865                         rtval++;
 4866                 }
 4867                 PMAP_UNLOCK(pmap);
 4868                 /* Rotate the PV list if it has more than one entry. */
 4869                 if (TAILQ_NEXT(pv, pv_next) != NULL) {
 4870                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_next);
 4871                         TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next);
 4872                 }
 4873                 if (rtval >= PMAP_TS_REFERENCED_MAX)
 4874                         goto out;
 4875         } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
 4876 small_mappings:
 4877         if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
 4878                 goto out;
 4879         pv = pvf;
 4880         do {
 4881                 pmap = PV_PMAP(pv);
 4882                 PMAP_LOCK(pmap);
 4883                 pde = pmap_pde(pmap, pv->pv_va);
 4884                 KASSERT((*pde & PG_PS) == 0,
 4885                     ("pmap_ts_referenced: found a 4mpage in page %p's pv list",
 4886                     m));
 4887                 pte = pmap_pte_quick(pmap, pv->pv_va);
 4888                 if ((*pte & PG_A) != 0) {
 4889                         atomic_clear_int((u_int *)pte, PG_A);
 4890                         pmap_invalidate_page(pmap, pv->pv_va);
 4891                         rtval++;
 4892                 }
 4893                 PMAP_UNLOCK(pmap);
 4894                 /* Rotate the PV list if it has more than one entry. */
 4895                 if (TAILQ_NEXT(pv, pv_next) != NULL) {
 4896                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_next);
 4897                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next);
 4898                 }
 4899         } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && rtval <
 4900             PMAP_TS_REFERENCED_MAX);
 4901 out:
 4902         sched_unpin();
 4903         rw_wunlock(&pvh_global_lock);
 4904         return (rtval);
 4905 }
 4906 
 4907 /*
 4908  *      Apply the given advice to the specified range of addresses within the
 4909  *      given pmap.  Depending on the advice, clear the referenced and/or
 4910  *      modified flags in each mapping and set the mapped page's dirty field.
 4911  */
 4912 void
 4913 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
 4914 {
 4915         pd_entry_t oldpde, *pde;
 4916         pt_entry_t *pte;
 4917         vm_offset_t pdnxt;
 4918         vm_page_t m;
 4919         boolean_t anychanged, pv_lists_locked;
 4920 
 4921         if (advice != MADV_DONTNEED && advice != MADV_FREE)
 4922                 return;
 4923         if (pmap_is_current(pmap))
 4924                 pv_lists_locked = FALSE;
 4925         else {
 4926                 pv_lists_locked = TRUE;
 4927 resume:
 4928                 rw_wlock(&pvh_global_lock);
 4929                 sched_pin();
 4930         }
 4931         anychanged = FALSE;
 4932         PMAP_LOCK(pmap);
 4933         for (; sva < eva; sva = pdnxt) {
 4934                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 4935                 if (pdnxt < sva)
 4936                         pdnxt = eva;
 4937                 pde = pmap_pde(pmap, sva);
 4938                 oldpde = *pde;
 4939                 if ((oldpde & PG_V) == 0)
 4940                         continue;
 4941                 else if ((oldpde & PG_PS) != 0) {
 4942                         if ((oldpde & PG_MANAGED) == 0)
 4943                                 continue;
 4944                         if (!pv_lists_locked) {
 4945                                 pv_lists_locked = TRUE;
 4946                                 if (!rw_try_wlock(&pvh_global_lock)) {
 4947                                         if (anychanged)
 4948                                                 pmap_invalidate_all(pmap);
 4949                                         PMAP_UNLOCK(pmap);
 4950                                         goto resume;
 4951                                 }
 4952                                 sched_pin();
 4953                         }
 4954                         if (!pmap_demote_pde(pmap, pde, sva)) {
 4955                                 /*
 4956                                  * The large page mapping was destroyed.
 4957                                  */
 4958                                 continue;
 4959                         }
 4960 
 4961                         /*
 4962                          * Unless the page mappings are wired, remove the
 4963                          * mapping to a single page so that a subsequent
 4964                          * access may repromote.  Since the underlying page
 4965                          * table page is fully populated, this removal never
 4966                          * frees a page table page.
 4967                          */
 4968                         if ((oldpde & PG_W) == 0) {
 4969                                 pte = pmap_pte_quick(pmap, sva);
 4970                                 KASSERT((*pte & PG_V) != 0,
 4971                                     ("pmap_advise: invalid PTE"));
 4972                                 pmap_remove_pte(pmap, pte, sva, NULL);
 4973                                 anychanged = TRUE;
 4974                         }
 4975                 }
 4976                 if (pdnxt > eva)
 4977                         pdnxt = eva;
 4978                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 4979                     sva += PAGE_SIZE) {
 4980                         if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED |
 4981                             PG_V))
 4982                                 continue;
 4983                         else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 4984                                 if (advice == MADV_DONTNEED) {
 4985                                         /*
 4986                                          * Future calls to pmap_is_modified()
 4987                                          * can be avoided by making the page
 4988                                          * dirty now.
 4989                                          */
 4990                                         m = PHYS_TO_VM_PAGE(*pte & PG_FRAME);
 4991                                         vm_page_dirty(m);
 4992                                 }
 4993                                 atomic_clear_int((u_int *)pte, PG_M | PG_A);
 4994                         } else if ((*pte & PG_A) != 0)
 4995                                 atomic_clear_int((u_int *)pte, PG_A);
 4996                         else
 4997                                 continue;
 4998                         if ((*pte & PG_G) != 0)
 4999                                 pmap_invalidate_page(pmap, sva);
 5000                         else
 5001                                 anychanged = TRUE;
 5002                 }
 5003         }
 5004         if (anychanged)
 5005                 pmap_invalidate_all(pmap);
 5006         if (pv_lists_locked) {
 5007                 sched_unpin();
 5008                 rw_wunlock(&pvh_global_lock);
 5009         }
 5010         PMAP_UNLOCK(pmap);
 5011 }
 5012 
 5013 /*
 5014  *      Clear the modify bits on the specified physical page.
 5015  */
 5016 void
 5017 pmap_clear_modify(vm_page_t m)
 5018 {
 5019         struct md_page *pvh;
 5020         pv_entry_t next_pv, pv;
 5021         pmap_t pmap;
 5022         pd_entry_t oldpde, *pde;
 5023         pt_entry_t oldpte, *pte;
 5024         vm_offset_t va;
 5025 
 5026         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 5027             ("pmap_clear_modify: page %p is not managed", m));
 5028         VM_OBJECT_ASSERT_WLOCKED(m->object);
 5029         KASSERT(!vm_page_xbusied(m),
 5030             ("pmap_clear_modify: page %p is exclusive busied", m));
 5031 
 5032         /*
 5033          * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
 5034          * If the object containing the page is locked and the page is not
 5035          * exclusive busied, then PGA_WRITEABLE cannot be concurrently set.
 5036          */
 5037         if ((m->aflags & PGA_WRITEABLE) == 0)
 5038                 return;
 5039         rw_wlock(&pvh_global_lock);
 5040         sched_pin();
 5041         if ((m->flags & PG_FICTITIOUS) != 0)
 5042                 goto small_mappings;
 5043         pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m));
 5044         TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) {
 5045                 va = pv->pv_va;
 5046                 pmap = PV_PMAP(pv);
 5047                 PMAP_LOCK(pmap);
 5048                 pde = pmap_pde(pmap, va);
 5049                 oldpde = *pde;
 5050                 if ((oldpde & PG_RW) != 0) {
 5051                         if (pmap_demote_pde(pmap, pde, va)) {
 5052                                 if ((oldpde & PG_W) == 0) {
 5053                                         /*
 5054                                          * Write protect the mapping to a
 5055                                          * single page so that a subsequent
 5056                                          * write access may repromote.
 5057                                          */
 5058                                         va += VM_PAGE_TO_PHYS(m) - (oldpde &
 5059                                             PG_PS_FRAME);
 5060                                         pte = pmap_pte_quick(pmap, va);
 5061                                         oldpte = *pte;
 5062                                         if ((oldpte & PG_V) != 0) {
 5063                                                 /*
 5064                                                  * Regardless of whether a pte is 32 or 64 bits
 5065                                                  * in size, PG_RW and PG_M are among the least
 5066                                                  * significant 32 bits.
 5067                                                  */
 5068                                                 while (!atomic_cmpset_int((u_int *)pte,
 5069                                                     oldpte,
 5070                                                     oldpte & ~(PG_M | PG_RW)))
 5071                                                         oldpte = *pte;
 5072                                                 vm_page_dirty(m);
 5073                                                 pmap_invalidate_page(pmap, va);
 5074                                         }
 5075                                 }
 5076                         }
 5077                 }
 5078                 PMAP_UNLOCK(pmap);
 5079         }
 5080 small_mappings:
 5081         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5082                 pmap = PV_PMAP(pv);
 5083                 PMAP_LOCK(pmap);
 5084                 pde = pmap_pde(pmap, pv->pv_va);
 5085                 KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found"
 5086                     " a 4mpage in page %p's pv list", m));
 5087                 pte = pmap_pte_quick(pmap, pv->pv_va);
 5088                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 5089                         /*
 5090                          * Regardless of whether a pte is 32 or 64 bits
 5091                          * in size, PG_M is among the least significant
 5092                          * 32 bits. 
 5093                          */
 5094                         atomic_clear_int((u_int *)pte, PG_M);
 5095                         pmap_invalidate_page(pmap, pv->pv_va);
 5096                 }
 5097                 PMAP_UNLOCK(pmap);
 5098         }
 5099         sched_unpin();
 5100         rw_wunlock(&pvh_global_lock);
 5101 }
 5102 
 5103 /*
 5104  * Miscellaneous support routines follow
 5105  */
 5106 
 5107 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
 5108 static __inline void
 5109 pmap_pte_attr(pt_entry_t *pte, int cache_bits)
 5110 {
 5111         u_int opte, npte;
 5112 
 5113         /*
 5114          * The cache mode bits are all in the low 32-bits of the
 5115          * PTE, so we can just spin on updating the low 32-bits.
 5116          */
 5117         do {
 5118                 opte = *(u_int *)pte;
 5119                 npte = opte & ~PG_PTE_CACHE;
 5120                 npte |= cache_bits;
 5121         } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
 5122 }
 5123 
 5124 /* Adjust the cache mode for a 2/4MB page mapped via a PDE. */
 5125 static __inline void
 5126 pmap_pde_attr(pd_entry_t *pde, int cache_bits)
 5127 {
 5128         u_int opde, npde;
 5129 
 5130         /*
 5131          * The cache mode bits are all in the low 32-bits of the
 5132          * PDE, so we can just spin on updating the low 32-bits.
 5133          */
 5134         do {
 5135                 opde = *(u_int *)pde;
 5136                 npde = opde & ~PG_PDE_CACHE;
 5137                 npde |= cache_bits;
 5138         } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
 5139 }
 5140 
 5141 /*
 5142  * Map a set of physical memory pages into the kernel virtual
 5143  * address space. Return a pointer to where it is mapped. This
 5144  * routine is intended to be used for mapping device memory,
 5145  * NOT real memory.
 5146  */
 5147 void *
 5148 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 5149 {
 5150         vm_offset_t va, offset;
 5151         vm_size_t tmpsize;
 5152 
 5153         offset = pa & PAGE_MASK;
 5154         size = round_page(offset + size);
 5155         pa = pa & PG_FRAME;
 5156 
 5157         if (pa < KERNLOAD && pa + size <= KERNLOAD)
 5158                 va = KERNBASE + pa;
 5159         else
 5160                 va = kva_alloc(size);
 5161         if (!va)
 5162                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 5163 
 5164         for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
 5165                 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
 5166         pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
 5167         pmap_invalidate_cache_range(va, va + size);
 5168         return ((void *)(va + offset));
 5169 }
 5170 
 5171 void *
 5172 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 5173 {
 5174 
 5175         return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
 5176 }
 5177 
 5178 void *
 5179 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
 5180 {
 5181 
 5182         return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
 5183 }
 5184 
 5185 void
 5186 pmap_unmapdev(vm_offset_t va, vm_size_t size)
 5187 {
 5188         vm_offset_t base, offset;
 5189 
 5190         if (va >= KERNBASE && va + size <= KERNBASE + KERNLOAD)
 5191                 return;
 5192         base = trunc_page(va);
 5193         offset = va & PAGE_MASK;
 5194         size = round_page(offset + size);
 5195         kva_free(base, size);
 5196 }
 5197 
 5198 /*
 5199  * Sets the memory attribute for the specified page.
 5200  */
 5201 void
 5202 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
 5203 {
 5204 
 5205         m->md.pat_mode = ma;
 5206         if ((m->flags & PG_FICTITIOUS) != 0)
 5207                 return;
 5208 
 5209         /*
 5210          * If "m" is a normal page, flush it from the cache.
 5211          * See pmap_invalidate_cache_range().
 5212          *
 5213          * First, try to find an existing mapping of the page by sf
 5214          * buffer. sf_buf_invalidate_cache() modifies mapping and
 5215          * flushes the cache.
 5216          */    
 5217         if (sf_buf_invalidate_cache(m))
 5218                 return;
 5219 
 5220         /*
 5221          * If page is not mapped by sf buffer, but CPU does not
 5222          * support self snoop, map the page transient and do
 5223          * invalidation. In the worst case, whole cache is flushed by
 5224          * pmap_invalidate_cache_range().
 5225          */
 5226         if ((cpu_feature & CPUID_SS) == 0)
 5227                 pmap_flush_page(m);
 5228 }
 5229 
 5230 static void
 5231 pmap_flush_page(vm_page_t m)
 5232 {
 5233         struct sysmaps *sysmaps;
 5234         vm_offset_t sva, eva;
 5235 
 5236         if ((cpu_feature & CPUID_CLFSH) != 0) {
 5237                 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 5238                 mtx_lock(&sysmaps->lock);
 5239                 if (*sysmaps->CMAP2)
 5240                         panic("pmap_flush_page: CMAP2 busy");
 5241                 sched_pin();
 5242                 *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
 5243                     PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0);
 5244                 invlcaddr(sysmaps->CADDR2);
 5245                 sva = (vm_offset_t)sysmaps->CADDR2;
 5246                 eva = sva + PAGE_SIZE;
 5247 
 5248                 /*
 5249                  * Use mfence despite the ordering implied by
 5250                  * mtx_{un,}lock() because clflush is not guaranteed
 5251                  * to be ordered by any other instruction.
 5252                  */
 5253                 mfence();
 5254                 for (; sva < eva; sva += cpu_clflush_line_size)
 5255                         clflush(sva);
 5256                 mfence();
 5257                 *sysmaps->CMAP2 = 0;
 5258                 sched_unpin();
 5259                 mtx_unlock(&sysmaps->lock);
 5260         } else
 5261                 pmap_invalidate_cache();
 5262 }
 5263 
 5264 /*
 5265  * Changes the specified virtual address range's memory type to that given by
 5266  * the parameter "mode".  The specified virtual address range must be
 5267  * completely contained within either the kernel map.
 5268  *
 5269  * Returns zero if the change completed successfully, and either EINVAL or
 5270  * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
 5271  * of the virtual address range was not mapped, and ENOMEM is returned if
 5272  * there was insufficient memory available to complete the change.
 5273  */
 5274 int
 5275 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
 5276 {
 5277         vm_offset_t base, offset, tmpva;
 5278         pd_entry_t *pde;
 5279         pt_entry_t *pte;
 5280         int cache_bits_pte, cache_bits_pde;
 5281         boolean_t changed;
 5282 
 5283         base = trunc_page(va);
 5284         offset = va & PAGE_MASK;
 5285         size = round_page(offset + size);
 5286 
 5287         /*
 5288          * Only supported on kernel virtual addresses above the recursive map.
 5289          */
 5290         if (base < VM_MIN_KERNEL_ADDRESS)
 5291                 return (EINVAL);
 5292 
 5293         cache_bits_pde = pmap_cache_bits(mode, 1);
 5294         cache_bits_pte = pmap_cache_bits(mode, 0);
 5295         changed = FALSE;
 5296 
 5297         /*
 5298          * Pages that aren't mapped aren't supported.  Also break down
 5299          * 2/4MB pages into 4KB pages if required.
 5300          */
 5301         PMAP_LOCK(kernel_pmap);
 5302         for (tmpva = base; tmpva < base + size; ) {
 5303                 pde = pmap_pde(kernel_pmap, tmpva);
 5304                 if (*pde == 0) {
 5305                         PMAP_UNLOCK(kernel_pmap);
 5306                         return (EINVAL);
 5307                 }
 5308                 if (*pde & PG_PS) {
 5309                         /*
 5310                          * If the current 2/4MB page already has
 5311                          * the required memory type, then we need not
 5312                          * demote this page.  Just increment tmpva to
 5313                          * the next 2/4MB page frame.
 5314                          */
 5315                         if ((*pde & PG_PDE_CACHE) == cache_bits_pde) {
 5316                                 tmpva = trunc_4mpage(tmpva) + NBPDR;
 5317                                 continue;
 5318                         }
 5319 
 5320                         /*
 5321                          * If the current offset aligns with a 2/4MB
 5322                          * page frame and there is at least 2/4MB left
 5323                          * within the range, then we need not break
 5324                          * down this page into 4KB pages.
 5325                          */
 5326                         if ((tmpva & PDRMASK) == 0 &&
 5327                             tmpva + PDRMASK < base + size) {
 5328                                 tmpva += NBPDR;
 5329                                 continue;
 5330                         }
 5331                         if (!pmap_demote_pde(kernel_pmap, pde, tmpva)) {
 5332                                 PMAP_UNLOCK(kernel_pmap);
 5333                                 return (ENOMEM);
 5334                         }
 5335                 }
 5336                 pte = vtopte(tmpva);
 5337                 if (*pte == 0) {
 5338                         PMAP_UNLOCK(kernel_pmap);
 5339                         return (EINVAL);
 5340                 }
 5341                 tmpva += PAGE_SIZE;
 5342         }
 5343         PMAP_UNLOCK(kernel_pmap);
 5344 
 5345         /*
 5346          * Ok, all the pages exist, so run through them updating their
 5347          * cache mode if required.
 5348          */
 5349         for (tmpva = base; tmpva < base + size; ) {
 5350                 pde = pmap_pde(kernel_pmap, tmpva);
 5351                 if (*pde & PG_PS) {
 5352                         if ((*pde & PG_PDE_CACHE) != cache_bits_pde) {
 5353                                 pmap_pde_attr(pde, cache_bits_pde);
 5354                                 changed = TRUE;
 5355                         }
 5356                         tmpva = trunc_4mpage(tmpva) + NBPDR;
 5357                 } else {
 5358                         pte = vtopte(tmpva);
 5359                         if ((*pte & PG_PTE_CACHE) != cache_bits_pte) {
 5360                                 pmap_pte_attr(pte, cache_bits_pte);
 5361                                 changed = TRUE;
 5362                         }
 5363                         tmpva += PAGE_SIZE;
 5364                 }
 5365         }
 5366 
 5367         /*
 5368          * Flush CPU caches to make sure any data isn't cached that
 5369          * shouldn't be, etc.
 5370          */
 5371         if (changed) {
 5372                 pmap_invalidate_range(kernel_pmap, base, tmpva);
 5373                 pmap_invalidate_cache_range(base, tmpva);
 5374         }
 5375         return (0);
 5376 }
 5377 
 5378 /*
 5379  * perform the pmap work for mincore
 5380  */
 5381 int
 5382 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
 5383 {
 5384         pd_entry_t *pdep;
 5385         pt_entry_t *ptep, pte;
 5386         vm_paddr_t pa;
 5387         int val;
 5388 
 5389         PMAP_LOCK(pmap);
 5390 retry:
 5391         pdep = pmap_pde(pmap, addr);
 5392         if (*pdep != 0) {
 5393                 if (*pdep & PG_PS) {
 5394                         pte = *pdep;
 5395                         /* Compute the physical address of the 4KB page. */
 5396                         pa = ((*pdep & PG_PS_FRAME) | (addr & PDRMASK)) &
 5397                             PG_FRAME;
 5398                         val = MINCORE_SUPER;
 5399                 } else {
 5400                         ptep = pmap_pte(pmap, addr);
 5401                         pte = *ptep;
 5402                         pmap_pte_release(ptep);
 5403                         pa = pte & PG_FRAME;
 5404                         val = 0;
 5405                 }
 5406         } else {
 5407                 pte = 0;
 5408                 pa = 0;
 5409                 val = 0;
 5410         }
 5411         if ((pte & PG_V) != 0) {
 5412                 val |= MINCORE_INCORE;
 5413                 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 5414                         val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
 5415                 if ((pte & PG_A) != 0)
 5416                         val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
 5417         }
 5418         if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
 5419             (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
 5420             (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
 5421                 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */
 5422                 if (vm_page_pa_tryrelock(pmap, pa, locked_pa))
 5423                         goto retry;
 5424         } else
 5425                 PA_UNLOCK_COND(*locked_pa);
 5426         PMAP_UNLOCK(pmap);
 5427         return (val);
 5428 }
 5429 
 5430 void
 5431 pmap_activate(struct thread *td)
 5432 {
 5433         pmap_t  pmap, oldpmap;
 5434         u_int   cpuid;
 5435         u_int32_t  cr3;
 5436 
 5437         critical_enter();
 5438         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 5439         oldpmap = PCPU_GET(curpmap);
 5440         cpuid = PCPU_GET(cpuid);
 5441 #if defined(SMP)
 5442         CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
 5443         CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
 5444 #else
 5445         CPU_CLR(cpuid, &oldpmap->pm_active);
 5446         CPU_SET(cpuid, &pmap->pm_active);
 5447 #endif
 5448 #ifdef PAE
 5449         cr3 = vtophys(pmap->pm_pdpt);
 5450 #else
 5451         cr3 = vtophys(pmap->pm_pdir);
 5452 #endif
 5453         /*
 5454          * pmap_activate is for the current thread on the current cpu
 5455          */
 5456         td->td_pcb->pcb_cr3 = cr3;
 5457         load_cr3(cr3);
 5458         PCPU_SET(curpmap, pmap);
 5459         critical_exit();
 5460 }
 5461 
 5462 void
 5463 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
 5464 {
 5465 }
 5466 
 5467 /*
 5468  *      Increase the starting virtual address of the given mapping if a
 5469  *      different alignment might result in more superpage mappings.
 5470  */
 5471 void
 5472 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
 5473     vm_offset_t *addr, vm_size_t size)
 5474 {
 5475         vm_offset_t superpage_offset;
 5476 
 5477         if (size < NBPDR)
 5478                 return;
 5479         if (object != NULL && (object->flags & OBJ_COLORED) != 0)
 5480                 offset += ptoa(object->pg_color);
 5481         superpage_offset = offset & PDRMASK;
 5482         if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR ||
 5483             (*addr & PDRMASK) == superpage_offset)
 5484                 return;
 5485         if ((*addr & PDRMASK) < superpage_offset)
 5486                 *addr = (*addr & ~PDRMASK) + superpage_offset;
 5487         else
 5488                 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
 5489 }
 5490 
 5491 
 5492 #if defined(PMAP_DEBUG)
 5493 pmap_pid_dump(int pid)
 5494 {
 5495         pmap_t pmap;
 5496         struct proc *p;
 5497         int npte = 0;
 5498         int index;
 5499 
 5500         sx_slock(&allproc_lock);
 5501         FOREACH_PROC_IN_SYSTEM(p) {
 5502                 if (p->p_pid != pid)
 5503                         continue;
 5504 
 5505                 if (p->p_vmspace) {
 5506                         int i,j;
 5507                         index = 0;
 5508                         pmap = vmspace_pmap(p->p_vmspace);
 5509                         for (i = 0; i < NPDEPTD; i++) {
 5510                                 pd_entry_t *pde;
 5511                                 pt_entry_t *pte;
 5512                                 vm_offset_t base = i << PDRSHIFT;
 5513                                 
 5514                                 pde = &pmap->pm_pdir[i];
 5515                                 if (pde && pmap_pde_v(pde)) {
 5516                                         for (j = 0; j < NPTEPG; j++) {
 5517                                                 vm_offset_t va = base + (j << PAGE_SHIFT);
 5518                                                 if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) {
 5519                                                         if (index) {
 5520                                                                 index = 0;
 5521                                                                 printf("\n");
 5522                                                         }
 5523                                                         sx_sunlock(&allproc_lock);
 5524                                                         return (npte);
 5525                                                 }
 5526                                                 pte = pmap_pte(pmap, va);
 5527                                                 if (pte && pmap_pte_v(pte)) {
 5528                                                         pt_entry_t pa;
 5529                                                         vm_page_t m;
 5530                                                         pa = *pte;
 5531                                                         m = PHYS_TO_VM_PAGE(pa & PG_FRAME);
 5532                                                         printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x",
 5533                                                                 va, pa, m->hold_count, m->wire_count, m->flags);
 5534                                                         npte++;
 5535                                                         index++;
 5536                                                         if (index >= 2) {
 5537                                                                 index = 0;
 5538                                                                 printf("\n");
 5539                                                         } else {
 5540                                                                 printf(" ");
 5541                                                         }
 5542                                                 }
 5543                                         }
 5544                                 }
 5545                         }
 5546                 }
 5547         }
 5548         sx_sunlock(&allproc_lock);
 5549         return (npte);
 5550 }
 5551 #endif
 5552 
 5553 #if defined(DEBUG)
 5554 
 5555 static void     pads(pmap_t pm);
 5556 void            pmap_pvdump(vm_paddr_t pa);
 5557 
 5558 /* print address space of pmap*/
 5559 static void
 5560 pads(pmap_t pm)
 5561 {
 5562         int i, j;
 5563         vm_paddr_t va;
 5564         pt_entry_t *ptep;
 5565 
 5566         if (pm == kernel_pmap)
 5567                 return;
 5568         for (i = 0; i < NPDEPTD; i++)
 5569                 if (pm->pm_pdir[i])
 5570                         for (j = 0; j < NPTEPG; j++) {
 5571                                 va = (i << PDRSHIFT) + (j << PAGE_SHIFT);
 5572                                 if (pm == kernel_pmap && va < KERNBASE)
 5573                                         continue;
 5574                                 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS)
 5575                                         continue;
 5576                                 ptep = pmap_pte(pm, va);
 5577                                 if (pmap_pte_v(ptep))
 5578                                         printf("%x:%x ", va, *ptep);
 5579                         };
 5580 
 5581 }
 5582 
 5583 void
 5584 pmap_pvdump(vm_paddr_t pa)
 5585 {
 5586         pv_entry_t pv;
 5587         pmap_t pmap;
 5588         vm_page_t m;
 5589 
 5590         printf("pa %x", pa);
 5591         m = PHYS_TO_VM_PAGE(pa);
 5592         TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) {
 5593                 pmap = PV_PMAP(pv);
 5594                 printf(" -> pmap %p, va %x", (void *)pmap, pv->pv_va);
 5595                 pads(pmap);
 5596         }
 5597         printf(" ");
 5598 }
 5599 #endif

Cache object: 070ad3fc030c6082e28e5fde4c0b3fc8


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.