The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/pmap.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2003 Peter Wemm
    9  * All rights reserved.
   10  * Copyright (c) 2005 Alan L. Cox <alc@cs.rice.edu>
   11  * All rights reserved.
   12  *
   13  * This code is derived from software contributed to Berkeley by
   14  * the Systems Programming Group of the University of Utah Computer
   15  * Science Department and William Jolitz of UUNET Technologies Inc.
   16  *
   17  * Redistribution and use in source and binary forms, with or without
   18  * modification, are permitted provided that the following conditions
   19  * are met:
   20  * 1. Redistributions of source code must retain the above copyright
   21  *    notice, this list of conditions and the following disclaimer.
   22  * 2. Redistributions in binary form must reproduce the above copyright
   23  *    notice, this list of conditions and the following disclaimer in the
   24  *    documentation and/or other materials provided with the distribution.
   25  * 3. All advertising materials mentioning features or use of this software
   26  *    must display the following acknowledgement:
   27  *      This product includes software developed by the University of
   28  *      California, Berkeley and its contributors.
   29  * 4. Neither the name of the University nor the names of its contributors
   30  *    may be used to endorse or promote products derived from this software
   31  *    without specific prior written permission.
   32  *
   33  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   34  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   35  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   36  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   37  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   38  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   39  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   40  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   41  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   42  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   43  * SUCH DAMAGE.
   44  *
   45  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   46  */
   47 /*-
   48  * Copyright (c) 2003 Networks Associates Technology, Inc.
   49  * All rights reserved.
   50  *
   51  * This software was developed for the FreeBSD Project by Jake Burkholder,
   52  * Safeport Network Services, and Network Associates Laboratories, the
   53  * Security Research Division of Network Associates, Inc. under
   54  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   55  * CHATS research program.
   56  *
   57  * Redistribution and use in source and binary forms, with or without
   58  * modification, are permitted provided that the following conditions
   59  * are met:
   60  * 1. Redistributions of source code must retain the above copyright
   61  *    notice, this list of conditions and the following disclaimer.
   62  * 2. Redistributions in binary form must reproduce the above copyright
   63  *    notice, this list of conditions and the following disclaimer in the
   64  *    documentation and/or other materials provided with the distribution.
   65  *
   66  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   67  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   68  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   69  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   70  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   71  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   72  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   73  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   74  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   75  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   76  * SUCH DAMAGE.
   77  */
   78 
   79 #include <sys/cdefs.h>
   80 __FBSDID("$FreeBSD$");
   81 
   82 /*
   83  *      Manages physical address maps.
   84  *
   85  *      In addition to hardware address maps, this
   86  *      module is called upon to provide software-use-only
   87  *      maps which may or may not be stored in the same
   88  *      form as hardware maps.  These pseudo-maps are
   89  *      used to store intermediate results from copy
   90  *      operations to and from address spaces.
   91  *
   92  *      Since the information managed by this module is
   93  *      also stored by the logical address mapping module,
   94  *      this module may throw away valid virtual-to-physical
   95  *      mappings at almost any time.  However, invalidations
   96  *      of virtual-to-physical mappings must be done as
   97  *      requested.
   98  *
   99  *      In order to cope with hardware architectures which
  100  *      make virtual-to-physical map invalidates expensive,
  101  *      this module may delay invalidate or reduced protection
  102  *      operations until such time as they are actually
  103  *      necessary.  This module is given full information as
  104  *      to which processors are currently using which maps,
  105  *      and to when physical maps must be made correct.
  106  */
  107 
  108 #include "opt_msgbuf.h"
  109 #include "opt_pmap.h"
  110 
  111 #include <sys/param.h>
  112 #include <sys/systm.h>
  113 #include <sys/kernel.h>
  114 #include <sys/lock.h>
  115 #include <sys/malloc.h>
  116 #include <sys/mman.h>
  117 #include <sys/msgbuf.h>
  118 #include <sys/mutex.h>
  119 #include <sys/proc.h>
  120 #include <sys/sx.h>
  121 #include <sys/vmmeter.h>
  122 #include <sys/sched.h>
  123 #include <sys/sysctl.h>
  124 #ifdef SMP
  125 #include <sys/smp.h>
  126 #endif
  127 
  128 #include <vm/vm.h>
  129 #include <vm/vm_param.h>
  130 #include <vm/vm_kern.h>
  131 #include <vm/vm_page.h>
  132 #include <vm/vm_map.h>
  133 #include <vm/vm_object.h>
  134 #include <vm/vm_extern.h>
  135 #include <vm/vm_pageout.h>
  136 #include <vm/vm_pager.h>
  137 #include <vm/uma.h>
  138 
  139 #include <machine/cpu.h>
  140 #include <machine/cputypes.h>
  141 #include <machine/md_var.h>
  142 #include <machine/pcb.h>
  143 #include <machine/specialreg.h>
  144 #ifdef SMP
  145 #include <machine/smp.h>
  146 #endif
  147 
  148 #ifndef PMAP_SHPGPERPROC
  149 #define PMAP_SHPGPERPROC 200
  150 #endif
  151 
  152 #if defined(DIAGNOSTIC)
  153 #define PMAP_DIAGNOSTIC
  154 #endif
  155 
  156 #if !defined(PMAP_DIAGNOSTIC)
  157 #define PMAP_INLINE     __gnu89_inline
  158 #else
  159 #define PMAP_INLINE
  160 #endif
  161 
  162 #define PV_STATS
  163 #ifdef PV_STATS
  164 #define PV_STAT(x)      do { x ; } while (0)
  165 #else
  166 #define PV_STAT(x)      do { } while (0)
  167 #endif
  168 
  169 struct pmap kernel_pmap_store;
  170 
  171 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  172 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  173 
  174 static int nkpt;
  175 static int ndmpdp;
  176 static vm_paddr_t dmaplimit;
  177 vm_offset_t kernel_vm_end;
  178 pt_entry_t pg_nx;
  179 
  180 static u_int64_t        KPTphys;        /* phys addr of kernel level 1 */
  181 static u_int64_t        KPDphys;        /* phys addr of kernel level 2 */
  182 u_int64_t               KPDPphys;       /* phys addr of kernel level 3 */
  183 u_int64_t               KPML4phys;      /* phys addr of kernel level 4 */
  184 
  185 static u_int64_t        DMPDphys;       /* phys addr of direct mapped level 2 */
  186 static u_int64_t        DMPDPphys;      /* phys addr of direct mapped level 3 */
  187 
  188 /*
  189  * Data for the pv entry allocation mechanism
  190  */
  191 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  192 static int shpgperproc = PMAP_SHPGPERPROC;
  193 
  194 /*
  195  * All those kernel PT submaps that BSD is so fond of
  196  */
  197 pt_entry_t *CMAP1 = 0;
  198 caddr_t CADDR1 = 0;
  199 struct msgbuf *msgbufp = 0;
  200 
  201 /*
  202  * Crashdump maps.
  203  */
  204 static caddr_t crashdumpmap;
  205 
  206 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  207 static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try);
  208 
  209 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  210     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  211 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
  212                 vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free);
  213 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
  214     vm_page_t *free);
  215 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  216                 vm_offset_t va);
  217 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  218 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  219     vm_page_t m);
  220 
  221 static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags);
  222 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
  223 
  224 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
  225 static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
  226                 vm_page_t* free);
  227 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
  228 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
  229 
  230 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  231 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  232 
  233 /*
  234  * Move the kernel virtual free pointer to the next
  235  * 2MB.  This is used to help improve performance
  236  * by using a large (2MB) page for much of the kernel
  237  * (.text, .data, .bss)
  238  */
  239 static vm_offset_t
  240 pmap_kmem_choose(vm_offset_t addr)
  241 {
  242         vm_offset_t newaddr = addr;
  243 
  244         newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
  245         return newaddr;
  246 }
  247 
  248 /********************/
  249 /* Inline functions */
  250 /********************/
  251 
  252 /* Return a non-clipped PD index for a given VA */
  253 static __inline vm_pindex_t
  254 pmap_pde_pindex(vm_offset_t va)
  255 {
  256         return va >> PDRSHIFT;
  257 }
  258 
  259 
  260 /* Return various clipped indexes for a given VA */
  261 static __inline vm_pindex_t
  262 pmap_pte_index(vm_offset_t va)
  263 {
  264 
  265         return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
  266 }
  267 
  268 static __inline vm_pindex_t
  269 pmap_pde_index(vm_offset_t va)
  270 {
  271 
  272         return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
  273 }
  274 
  275 static __inline vm_pindex_t
  276 pmap_pdpe_index(vm_offset_t va)
  277 {
  278 
  279         return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
  280 }
  281 
  282 static __inline vm_pindex_t
  283 pmap_pml4e_index(vm_offset_t va)
  284 {
  285 
  286         return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
  287 }
  288 
  289 /* Return a pointer to the PML4 slot that corresponds to a VA */
  290 static __inline pml4_entry_t *
  291 pmap_pml4e(pmap_t pmap, vm_offset_t va)
  292 {
  293 
  294         if (!pmap)
  295                 return NULL;
  296         return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
  297 }
  298 
  299 /* Return a pointer to the PDP slot that corresponds to a VA */
  300 static __inline pdp_entry_t *
  301 pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
  302 {
  303         pdp_entry_t *pdpe;
  304 
  305         pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
  306         return (&pdpe[pmap_pdpe_index(va)]);
  307 }
  308 
  309 /* Return a pointer to the PDP slot that corresponds to a VA */
  310 static __inline pdp_entry_t *
  311 pmap_pdpe(pmap_t pmap, vm_offset_t va)
  312 {
  313         pml4_entry_t *pml4e;
  314 
  315         pml4e = pmap_pml4e(pmap, va);
  316         if (pml4e == NULL || (*pml4e & PG_V) == 0)
  317                 return NULL;
  318         return (pmap_pml4e_to_pdpe(pml4e, va));
  319 }
  320 
  321 /* Return a pointer to the PD slot that corresponds to a VA */
  322 static __inline pd_entry_t *
  323 pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
  324 {
  325         pd_entry_t *pde;
  326 
  327         pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
  328         return (&pde[pmap_pde_index(va)]);
  329 }
  330 
  331 /* Return a pointer to the PD slot that corresponds to a VA */
  332 static __inline pd_entry_t *
  333 pmap_pde(pmap_t pmap, vm_offset_t va)
  334 {
  335         pdp_entry_t *pdpe;
  336 
  337         pdpe = pmap_pdpe(pmap, va);
  338         if (pdpe == NULL || (*pdpe & PG_V) == 0)
  339                  return NULL;
  340         return (pmap_pdpe_to_pde(pdpe, va));
  341 }
  342 
  343 /* Return a pointer to the PT slot that corresponds to a VA */
  344 static __inline pt_entry_t *
  345 pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
  346 {
  347         pt_entry_t *pte;
  348 
  349         pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
  350         return (&pte[pmap_pte_index(va)]);
  351 }
  352 
  353 /* Return a pointer to the PT slot that corresponds to a VA */
  354 static __inline pt_entry_t *
  355 pmap_pte(pmap_t pmap, vm_offset_t va)
  356 {
  357         pd_entry_t *pde;
  358 
  359         pde = pmap_pde(pmap, va);
  360         if (pde == NULL || (*pde & PG_V) == 0)
  361                 return NULL;
  362         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  363                 return ((pt_entry_t *)pde);
  364         return (pmap_pde_to_pte(pde, va));
  365 }
  366 
  367 
  368 static __inline pt_entry_t *
  369 pmap_pte_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *ptepde)
  370 {
  371         pd_entry_t *pde;
  372 
  373         pde = pmap_pde(pmap, va);
  374         if (pde == NULL || (*pde & PG_V) == 0)
  375                 return NULL;
  376         *ptepde = *pde;
  377         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  378                 return ((pt_entry_t *)pde);
  379         return (pmap_pde_to_pte(pde, va));
  380 }
  381 
  382 
  383 PMAP_INLINE pt_entry_t *
  384 vtopte(vm_offset_t va)
  385 {
  386         u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  387 
  388         return (PTmap + ((va >> PAGE_SHIFT) & mask));
  389 }
  390 
  391 static __inline pd_entry_t *
  392 vtopde(vm_offset_t va)
  393 {
  394         u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  395 
  396         return (PDmap + ((va >> PDRSHIFT) & mask));
  397 }
  398 
  399 static u_int64_t
  400 allocpages(vm_paddr_t *firstaddr, int n)
  401 {
  402         u_int64_t ret;
  403 
  404         ret = *firstaddr;
  405         bzero((void *)ret, n * PAGE_SIZE);
  406         *firstaddr += n * PAGE_SIZE;
  407         return (ret);
  408 }
  409 
  410 static void
  411 create_pagetables(vm_paddr_t *firstaddr)
  412 {
  413         int i;
  414 
  415         /* Allocate pages */
  416         KPTphys = allocpages(firstaddr, NKPT);
  417         KPML4phys = allocpages(firstaddr, 1);
  418         KPDPphys = allocpages(firstaddr, NKPML4E);
  419         KPDphys = allocpages(firstaddr, NKPDPE);
  420 
  421         ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
  422         if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
  423                 ndmpdp = 4;
  424         DMPDPphys = allocpages(firstaddr, NDMPML4E);
  425         DMPDphys = allocpages(firstaddr, ndmpdp);
  426         dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
  427 
  428         /* Fill in the underlying page table pages */
  429         /* Read-only from zero to physfree */
  430         /* XXX not fully used, underneath 2M pages */
  431         for (i = 0; (i << PAGE_SHIFT) < *firstaddr; i++) {
  432                 ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
  433                 ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V | PG_G;
  434         }
  435 
  436         /* Now map the page tables at their location within PTmap */
  437         for (i = 0; i < NKPT; i++) {
  438                 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
  439                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
  440         }
  441 
  442         /* Map from zero to end of allocations under 2M pages */
  443         /* This replaces some of the KPTphys entries above */
  444         for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) {
  445                 ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
  446                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  447         }
  448 
  449         /* And connect up the PD to the PDP */
  450         for (i = 0; i < NKPDPE; i++) {
  451                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys + (i << PAGE_SHIFT);
  452                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
  453         }
  454 
  455 
  456         /* Now set up the direct map space using 2MB pages */
  457         for (i = 0; i < NPDEPG * ndmpdp; i++) {
  458                 ((pd_entry_t *)DMPDphys)[i] = (vm_paddr_t)i << PDRSHIFT;
  459                 ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  460         }
  461 
  462         /* And the direct map space's PDP */
  463         for (i = 0; i < ndmpdp; i++) {
  464                 ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + (i << PAGE_SHIFT);
  465                 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
  466         }
  467 
  468         /* And recursively map PML4 to itself in order to get PTmap */
  469         ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
  470         ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
  471 
  472         /* Connect the Direct Map slot up to the PML4 */
  473         ((pdp_entry_t *)KPML4phys)[DMPML4I] = DMPDPphys;
  474         ((pdp_entry_t *)KPML4phys)[DMPML4I] |= PG_RW | PG_V | PG_U;
  475 
  476         /* Connect the KVA slot up to the PML4 */
  477         ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
  478         ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
  479 }
  480 
  481 /*
  482  *      Bootstrap the system enough to run with virtual memory.
  483  *
  484  *      On amd64 this is called after mapping has already been enabled
  485  *      and just syncs the pmap module with what has already been done.
  486  *      [We can't call it easily with mapping off since the kernel is not
  487  *      mapped with PA == VA, hence we would have to relocate every address
  488  *      from the linked base (virtual) address "KERNBASE" to the actual
  489  *      (physical) address starting relative to 0]
  490  */
  491 void
  492 pmap_bootstrap(vm_paddr_t *firstaddr)
  493 {
  494         vm_offset_t va;
  495         pt_entry_t *pte, *unused;
  496 
  497         /*
  498          * Create an initial set of page tables to run the kernel in.
  499          */
  500         create_pagetables(firstaddr);
  501 
  502         virtual_avail = (vm_offset_t) KERNBASE + *firstaddr;
  503         virtual_avail = pmap_kmem_choose(virtual_avail);
  504 
  505         virtual_end = VM_MAX_KERNEL_ADDRESS;
  506 
  507 
  508         /* XXX do %cr0 as well */
  509         load_cr4(rcr4() | CR4_PGE | CR4_PSE);
  510         load_cr3(KPML4phys);
  511 
  512         /*
  513          * Initialize the kernel pmap (which is statically allocated).
  514          */
  515         PMAP_LOCK_INIT(kernel_pmap);
  516         kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + KPML4phys);
  517         kernel_pmap->pm_active = -1;    /* don't allow deactivation */
  518         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  519         nkpt = NKPT;
  520 
  521         /*
  522          * Reserve some special page table entries/VA space for temporary
  523          * mapping of pages.
  524          */
  525 #define SYSMAP(c, p, v, n)      \
  526         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  527 
  528         va = virtual_avail;
  529         pte = vtopte(va);
  530 
  531         /*
  532          * CMAP1 is only used for the memory test.
  533          */
  534         SYSMAP(caddr_t, CMAP1, CADDR1, 1)
  535 
  536         /*
  537          * Crashdump maps.
  538          */
  539         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  540 
  541         /*
  542          * msgbufp is used to map the system message buffer.
  543          */
  544         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(MSGBUF_SIZE)))
  545 
  546         virtual_avail = va;
  547 
  548         *CMAP1 = 0;
  549 
  550         invltlb();
  551 
  552         /* Initialize the PAT MSR. */
  553         pmap_init_pat();
  554 }
  555 
  556 /*
  557  * Setup the PAT MSR.
  558  */
  559 void
  560 pmap_init_pat(void)
  561 {
  562         uint64_t pat_msr;
  563 
  564         /* Bail if this CPU doesn't implement PAT. */
  565         if (!(cpu_feature & CPUID_PAT))
  566                 panic("no PAT??");
  567 
  568 #ifdef PAT_WORKS
  569         /*
  570          * Leave the indices 0-3 at the default of WB, WT, UC, and UC-.
  571          * Program 4 and 5 as WP and WC.
  572          * Leave 6 and 7 as UC and UC-.
  573          */
  574         pat_msr = rdmsr(MSR_PAT);
  575         pat_msr &= ~(PAT_MASK(4) | PAT_MASK(5));
  576         pat_msr |= PAT_VALUE(4, PAT_WRITE_PROTECTED) |
  577             PAT_VALUE(5, PAT_WRITE_COMBINING);
  578 #else
  579         /*
  580          * Due to some Intel errata, we can only safely use the lower 4
  581          * PAT entries.  Thus, just replace PAT Index 2 with WC instead
  582          * of UC-.
  583          *
  584          *   Intel Pentium III Processor Specification Update
  585          * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
  586          * or Mode C Paging)
  587          *
  588          *   Intel Pentium IV  Processor Specification Update
  589          * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
  590          */
  591         pat_msr = rdmsr(MSR_PAT);
  592         pat_msr &= ~PAT_MASK(2);
  593         pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  594 #endif
  595         wrmsr(MSR_PAT, pat_msr);
  596 }
  597 
  598 /*
  599  *      Initialize a vm_page's machine-dependent fields.
  600  */
  601 void
  602 pmap_page_init(vm_page_t m)
  603 {
  604 
  605         TAILQ_INIT(&m->md.pv_list);
  606         m->md.pv_list_count = 0;
  607 }
  608 
  609 /*
  610  *      Initialize the pmap module.
  611  *      Called by vm_init, to initialize any structures that the pmap
  612  *      system needs to map virtual memory.
  613  */
  614 void
  615 pmap_init(void)
  616 {
  617 
  618         /*
  619          * Initialize the address space (zone) for the pv entries.  Set a
  620          * high water mark so that the system can recover from excessive
  621          * numbers of pv entries.
  622          */
  623         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  624         pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  625         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  626         pv_entry_high_water = 9 * (pv_entry_max / 10);
  627 }
  628 
  629 SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  630 static int
  631 pmap_pventry_proc(SYSCTL_HANDLER_ARGS)
  632 {
  633         int error;
  634 
  635         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
  636         if (error == 0 && req->newptr) {
  637                 shpgperproc = (pv_entry_max - cnt.v_page_count) / maxproc;
  638                 pv_entry_high_water = 9 * (pv_entry_max / 10);
  639         }
  640         return (error);
  641 }
  642 SYSCTL_PROC(_vm_pmap, OID_AUTO, pv_entry_max, CTLTYPE_INT|CTLFLAG_RW, 
  643     &pv_entry_max, 0, pmap_pventry_proc, "IU", "Max number of PV entries");
  644 
  645 static int
  646 pmap_shpgperproc_proc(SYSCTL_HANDLER_ARGS)
  647 {
  648         int error;
  649 
  650         error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
  651         if (error == 0 && req->newptr) {
  652                 pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  653                 pv_entry_high_water = 9 * (pv_entry_max / 10);
  654         }
  655         return (error);
  656 }
  657 SYSCTL_PROC(_vm_pmap, OID_AUTO, shpgperproc, CTLTYPE_INT|CTLFLAG_RW, 
  658     &shpgperproc, 0, pmap_shpgperproc_proc, "IU", "Page share factor per proc");
  659 
  660 
  661 /***************************************************
  662  * Low level helper routines.....
  663  ***************************************************/
  664 
  665 /*
  666  * Determine the appropriate bits to set in a PTE or PDE for a specified
  667  * caching mode.
  668  */
  669 static int
  670 pmap_cache_bits(int mode, boolean_t is_pde)
  671 {
  672         int pat_flag, pat_index, cache_bits;
  673 
  674         /* The PAT bit is different for PTE's and PDE's. */
  675         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
  676 
  677         /* If we don't support PAT, map extended modes to older ones. */
  678         if (!(cpu_feature & CPUID_PAT)) {
  679                 switch (mode) {
  680                 case PAT_UNCACHEABLE:
  681                 case PAT_WRITE_THROUGH:
  682                 case PAT_WRITE_BACK:
  683                         break;
  684                 case PAT_UNCACHED:
  685                 case PAT_WRITE_COMBINING:
  686                 case PAT_WRITE_PROTECTED:
  687                         mode = PAT_UNCACHEABLE;
  688                         break;
  689                 }
  690         }
  691         
  692         /* Map the caching mode to a PAT index. */
  693         switch (mode) {
  694 #ifdef PAT_WORKS
  695         case PAT_UNCACHEABLE:
  696                 pat_index = 3;
  697                 break;
  698         case PAT_WRITE_THROUGH:
  699                 pat_index = 1;
  700                 break;
  701         case PAT_WRITE_BACK:
  702                 pat_index = 0;
  703                 break;
  704         case PAT_UNCACHED:
  705                 pat_index = 2;
  706                 break;
  707         case PAT_WRITE_COMBINING:
  708                 pat_index = 5;
  709                 break;
  710         case PAT_WRITE_PROTECTED:
  711                 pat_index = 4;
  712                 break;
  713 #else
  714         case PAT_UNCACHED:
  715         case PAT_UNCACHEABLE:
  716         case PAT_WRITE_PROTECTED:
  717                 pat_index = 3;
  718                 break;
  719         case PAT_WRITE_THROUGH:
  720                 pat_index = 1;
  721                 break;
  722         case PAT_WRITE_BACK:
  723                 pat_index = 0;
  724                 break;
  725         case PAT_WRITE_COMBINING:
  726                 pat_index = 2;
  727                 break;
  728 #endif
  729         default:
  730                 panic("Unknown caching mode %d\n", mode);
  731         }       
  732 
  733         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
  734         cache_bits = 0;
  735         if (pat_index & 0x4)
  736                 cache_bits |= pat_flag;
  737         if (pat_index & 0x2)
  738                 cache_bits |= PG_NC_PCD;
  739         if (pat_index & 0x1)
  740                 cache_bits |= PG_NC_PWT;
  741         return (cache_bits);
  742 }
  743 #ifdef SMP
  744 /*
  745  * For SMP, these functions have to use the IPI mechanism for coherence.
  746  *
  747  * N.B.: Before calling any of the following TLB invalidation functions,
  748  * the calling processor must ensure that all stores updating a non-
  749  * kernel page table are globally performed.  Otherwise, another
  750  * processor could cache an old, pre-update entry without being
  751  * invalidated.  This can happen one of two ways: (1) The pmap becomes
  752  * active on another processor after its pm_active field is checked by
  753  * one of the following functions but before a store updating the page
  754  * table is globally performed. (2) The pmap becomes active on another
  755  * processor before its pm_active field is checked but due to
  756  * speculative loads one of the following functions stills reads the
  757  * pmap as inactive on the other processor.
  758  * 
  759  * The kernel page table is exempt because its pm_active field is
  760  * immutable.  The kernel page table is always active on every
  761  * processor.
  762  */
  763 void
  764 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  765 {
  766         u_int cpumask;
  767         u_int other_cpus;
  768 
  769         sched_pin();
  770         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  771                 invlpg(va);
  772                 smp_invlpg(va);
  773         } else {
  774                 cpumask = PCPU_GET(cpumask);
  775                 other_cpus = PCPU_GET(other_cpus);
  776                 if (pmap->pm_active & cpumask)
  777                         invlpg(va);
  778                 if (pmap->pm_active & other_cpus)
  779                         smp_masked_invlpg(pmap->pm_active & other_cpus, va);
  780         }
  781         sched_unpin();
  782 }
  783 
  784 void
  785 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  786 {
  787         u_int cpumask;
  788         u_int other_cpus;
  789         vm_offset_t addr;
  790 
  791         sched_pin();
  792         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  793                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  794                         invlpg(addr);
  795                 smp_invlpg_range(sva, eva);
  796         } else {
  797                 cpumask = PCPU_GET(cpumask);
  798                 other_cpus = PCPU_GET(other_cpus);
  799                 if (pmap->pm_active & cpumask)
  800                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
  801                                 invlpg(addr);
  802                 if (pmap->pm_active & other_cpus)
  803                         smp_masked_invlpg_range(pmap->pm_active & other_cpus,
  804                             sva, eva);
  805         }
  806         sched_unpin();
  807 }
  808 
  809 void
  810 pmap_invalidate_all(pmap_t pmap)
  811 {
  812         u_int cpumask;
  813         u_int other_cpus;
  814 
  815         sched_pin();
  816         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  817                 invltlb();
  818                 smp_invltlb();
  819         } else {
  820                 cpumask = PCPU_GET(cpumask);
  821                 other_cpus = PCPU_GET(other_cpus);
  822                 if (pmap->pm_active & cpumask)
  823                         invltlb();
  824                 if (pmap->pm_active & other_cpus)
  825                         smp_masked_invltlb(pmap->pm_active & other_cpus);
  826         }
  827         sched_unpin();
  828 }
  829 
  830 void
  831 pmap_invalidate_cache(void)
  832 {
  833 
  834         sched_pin();
  835         wbinvd();
  836         smp_cache_flush();
  837         sched_unpin();
  838 }
  839 #else /* !SMP */
  840 /*
  841  * Normal, non-SMP, invalidation functions.
  842  * We inline these within pmap.c for speed.
  843  */
  844 PMAP_INLINE void
  845 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  846 {
  847 
  848         if (pmap == kernel_pmap || pmap->pm_active)
  849                 invlpg(va);
  850 }
  851 
  852 PMAP_INLINE void
  853 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  854 {
  855         vm_offset_t addr;
  856 
  857         if (pmap == kernel_pmap || pmap->pm_active)
  858                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  859                         invlpg(addr);
  860 }
  861 
  862 PMAP_INLINE void
  863 pmap_invalidate_all(pmap_t pmap)
  864 {
  865 
  866         if (pmap == kernel_pmap || pmap->pm_active)
  867                 invltlb();
  868 }
  869 
  870 PMAP_INLINE void
  871 pmap_invalidate_cache(void)
  872 {
  873 
  874         wbinvd();
  875 }
  876 #endif /* !SMP */
  877 
  878 /*
  879  * Are we current address space or kernel?
  880  */
  881 static __inline int
  882 pmap_is_current(pmap_t pmap)
  883 {
  884         return (pmap == kernel_pmap ||
  885             (pmap->pm_pml4[PML4PML4I] & PG_FRAME) == (PML4pml4e[0] & PG_FRAME));
  886 }
  887 
  888 /*
  889  *      Routine:        pmap_extract
  890  *      Function:
  891  *              Extract the physical page address associated
  892  *              with the given map/virtual_address pair.
  893  */
  894 vm_paddr_t 
  895 pmap_extract(pmap_t pmap, vm_offset_t va)
  896 {
  897         vm_paddr_t rtval;
  898         pt_entry_t *pte;
  899         pd_entry_t pde, *pdep;
  900 
  901         rtval = 0;
  902         PMAP_LOCK(pmap);
  903         pdep = pmap_pde(pmap, va);
  904         if (pdep != NULL) {
  905                 pde = *pdep;
  906                 if (pde) {
  907                         if ((pde & PG_PS) != 0) {
  908                                 rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
  909                                 PMAP_UNLOCK(pmap);
  910                                 return rtval;
  911                         }
  912                         pte = pmap_pde_to_pte(pdep, va);
  913                         rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
  914                 }
  915         }
  916         PMAP_UNLOCK(pmap);
  917         return (rtval);
  918 }
  919 
  920 /*
  921  *      Routine:        pmap_extract_and_hold
  922  *      Function:
  923  *              Atomically extract and hold the physical page
  924  *              with the given pmap and virtual address pair
  925  *              if that mapping permits the given protection.
  926  */
  927 vm_page_t
  928 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
  929 {
  930         pd_entry_t pde, *pdep;
  931         pt_entry_t pte;
  932         vm_page_t m;
  933 
  934         m = NULL;
  935         vm_page_lock_queues();
  936         PMAP_LOCK(pmap);
  937         pdep = pmap_pde(pmap, va);
  938         if (pdep != NULL && (pde = *pdep)) {
  939                 if (pde & PG_PS) {
  940                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
  941                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
  942                                     (va & PDRMASK));
  943                                 vm_page_hold(m);
  944                         }
  945                 } else {
  946                         pte = *pmap_pde_to_pte(pdep, va);
  947                         if ((pte & PG_V) &&
  948                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
  949                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
  950                                 vm_page_hold(m);
  951                         }
  952                 }
  953         }
  954         vm_page_unlock_queues();
  955         PMAP_UNLOCK(pmap);
  956         return (m);
  957 }
  958 
  959 vm_paddr_t
  960 pmap_kextract(vm_offset_t va)
  961 {
  962         pd_entry_t *pde;
  963         vm_paddr_t pa;
  964 
  965         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
  966                 pa = DMAP_TO_PHYS(va);
  967         } else {
  968                 pde = vtopde(va);
  969                 if (*pde & PG_PS) {
  970                         pa = (*pde & PG_PS_FRAME) | (va & PDRMASK);
  971                 } else {
  972                         pa = *vtopte(va);
  973                         pa = (pa & PG_FRAME) | (va & PAGE_MASK);
  974                 }
  975         }
  976         return pa;
  977 }
  978 
  979 /***************************************************
  980  * Low level mapping routines.....
  981  ***************************************************/
  982 
  983 /*
  984  * Add a wired page to the kva.
  985  * Note: not SMP coherent.
  986  */
  987 PMAP_INLINE void 
  988 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
  989 {
  990         pt_entry_t *pte;
  991 
  992         pte = vtopte(va);
  993         pte_store(pte, pa | PG_RW | PG_V | PG_G);
  994 }
  995 
  996 PMAP_INLINE void 
  997 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
  998 {
  999         pt_entry_t *pte;
 1000 
 1001         pte = vtopte(va);
 1002         pte_store(pte, pa | PG_RW | PG_V | PG_G | pmap_cache_bits(mode, 0));
 1003 }
 1004 
 1005 /*
 1006  * Remove a page from the kernel pagetables.
 1007  * Note: not SMP coherent.
 1008  */
 1009 PMAP_INLINE void
 1010 pmap_kremove(vm_offset_t va)
 1011 {
 1012         pt_entry_t *pte;
 1013 
 1014         pte = vtopte(va);
 1015         pte_clear(pte);
 1016 }
 1017 
 1018 /*
 1019  *      Used to map a range of physical addresses into kernel
 1020  *      virtual address space.
 1021  *
 1022  *      The value passed in '*virt' is a suggested virtual address for
 1023  *      the mapping. Architectures which can support a direct-mapped
 1024  *      physical to virtual region can return the appropriate address
 1025  *      within that region, leaving '*virt' unchanged. Other
 1026  *      architectures should map the pages starting at '*virt' and
 1027  *      update '*virt' with the first usable address after the mapped
 1028  *      region.
 1029  */
 1030 vm_offset_t
 1031 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 1032 {
 1033         return PHYS_TO_DMAP(start);
 1034 }
 1035 
 1036 
 1037 /*
 1038  * Add a list of wired pages to the kva
 1039  * this routine is only used for temporary
 1040  * kernel mappings that do not need to have
 1041  * page modification or references recorded.
 1042  * Note that old mappings are simply written
 1043  * over.  The page *must* be wired.
 1044  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1045  */
 1046 void
 1047 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 1048 {
 1049         pt_entry_t *endpte, oldpte, *pte;
 1050 
 1051         oldpte = 0;
 1052         pte = vtopte(sva);
 1053         endpte = pte + count;
 1054         while (pte < endpte) {
 1055                 oldpte |= *pte;
 1056                 pte_store(pte, VM_PAGE_TO_PHYS(*ma) | PG_G | PG_RW | PG_V);
 1057                 pte++;
 1058                 ma++;
 1059         }
 1060         if ((oldpte & PG_V) != 0)
 1061                 pmap_invalidate_range(kernel_pmap, sva, sva + count *
 1062                     PAGE_SIZE);
 1063 }
 1064 
 1065 /*
 1066  * This routine tears out page mappings from the
 1067  * kernel -- it is meant only for temporary mappings.
 1068  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1069  */
 1070 void
 1071 pmap_qremove(vm_offset_t sva, int count)
 1072 {
 1073         vm_offset_t va;
 1074 
 1075         va = sva;
 1076         while (count-- > 0) {
 1077                 pmap_kremove(va);
 1078                 va += PAGE_SIZE;
 1079         }
 1080         pmap_invalidate_range(kernel_pmap, sva, va);
 1081 }
 1082 
 1083 /***************************************************
 1084  * Page table page management routines.....
 1085  ***************************************************/
 1086 static __inline void
 1087 pmap_free_zero_pages(vm_page_t free)
 1088 {
 1089         vm_page_t m;
 1090 
 1091         while (free != NULL) {
 1092                 m = free;
 1093                 free = m->right;
 1094                 vm_page_free_zero(m);
 1095         }
 1096 }
 1097 
 1098 /*
 1099  * This routine unholds page table pages, and if the hold count
 1100  * drops to zero, then it decrements the wire count.
 1101  */
 1102 static __inline int
 1103 pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free)
 1104 {
 1105 
 1106         --m->wire_count;
 1107         if (m->wire_count == 0)
 1108                 return _pmap_unwire_pte_hold(pmap, va, m, free);
 1109         else
 1110                 return 0;
 1111 }
 1112 
 1113 static int 
 1114 _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, 
 1115     vm_page_t *free)
 1116 {
 1117         vm_offset_t pteva;
 1118 
 1119         /*
 1120          * unmap the page table page
 1121          */
 1122         if (m->pindex >= (NUPDE + NUPDPE)) {
 1123                 /* PDP page */
 1124                 pml4_entry_t *pml4;
 1125                 pml4 = pmap_pml4e(pmap, va);
 1126                 pteva = (vm_offset_t) PDPmap + amd64_ptob(m->pindex - (NUPDE + NUPDPE));
 1127                 *pml4 = 0;
 1128         } else if (m->pindex >= NUPDE) {
 1129                 /* PD page */
 1130                 pdp_entry_t *pdp;
 1131                 pdp = pmap_pdpe(pmap, va);
 1132                 pteva = (vm_offset_t) PDmap + amd64_ptob(m->pindex - NUPDE);
 1133                 *pdp = 0;
 1134         } else {
 1135                 /* PTE page */
 1136                 pd_entry_t *pd;
 1137                 pd = pmap_pde(pmap, va);
 1138                 pteva = (vm_offset_t) PTmap + amd64_ptob(m->pindex);
 1139                 *pd = 0;
 1140         }
 1141         --pmap->pm_stats.resident_count;
 1142         if (m->pindex < NUPDE) {
 1143                 /* We just released a PT, unhold the matching PD */
 1144                 vm_page_t pdpg;
 1145 
 1146                 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
 1147                 pmap_unwire_pte_hold(pmap, va, pdpg, free);
 1148         }
 1149         if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 1150                 /* We just released a PD, unhold the matching PDP */
 1151                 vm_page_t pdppg;
 1152 
 1153                 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
 1154                 pmap_unwire_pte_hold(pmap, va, pdppg, free);
 1155         }
 1156 
 1157         /*
 1158          * This is a release store so that the ordinary store unmapping
 1159          * the page table page is globally performed before TLB shoot-
 1160          * down is begun.
 1161          */
 1162         atomic_subtract_rel_int(&cnt.v_wire_count, 1);
 1163 
 1164         /*
 1165          * Do an invltlb to make the invalidated mapping
 1166          * take effect immediately.
 1167          */
 1168         pmap_invalidate_page(pmap, pteva);
 1169 
 1170         /* 
 1171          * Put page on a list so that it is released after
 1172          * *ALL* TLB shootdown is done
 1173          */
 1174         m->right = *free;
 1175         *free = m;
 1176         
 1177         return 1;
 1178 }
 1179 
 1180 /*
 1181  * After removing a page table entry, this routine is used to
 1182  * conditionally free the page, and manage the hold/wire counts.
 1183  */
 1184 static int
 1185 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, vm_page_t *free)
 1186 {
 1187         vm_page_t mpte;
 1188 
 1189         if (va >= VM_MAXUSER_ADDRESS)
 1190                 return 0;
 1191         KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 1192         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1193         return pmap_unwire_pte_hold(pmap, va, mpte, free);
 1194 }
 1195 
 1196 void
 1197 pmap_pinit0(pmap_t pmap)
 1198 {
 1199 
 1200         PMAP_LOCK_INIT(pmap);
 1201         pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + KPML4phys);
 1202         pmap->pm_active = 0;
 1203         TAILQ_INIT(&pmap->pm_pvchunk);
 1204         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1205 }
 1206 
 1207 /*
 1208  * Initialize a preallocated and zeroed pmap structure,
 1209  * such as one in a vmspace structure.
 1210  */
 1211 int
 1212 pmap_pinit(pmap_t pmap)
 1213 {
 1214         vm_page_t pml4pg;
 1215         static vm_pindex_t color;
 1216 
 1217         PMAP_LOCK_INIT(pmap);
 1218 
 1219         /*
 1220          * allocate the page directory page
 1221          */
 1222         while ((pml4pg = vm_page_alloc(NULL, color++, VM_ALLOC_NOOBJ |
 1223             VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
 1224                 VM_WAIT;
 1225 
 1226         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
 1227 
 1228         if ((pml4pg->flags & PG_ZERO) == 0)
 1229                 pagezero(pmap->pm_pml4);
 1230 
 1231         /* Wire in kernel global address entries. */
 1232         pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
 1233         pmap->pm_pml4[DMPML4I] = DMPDPphys | PG_RW | PG_V | PG_U;
 1234 
 1235         /* install self-referential address mapping entry(s) */
 1236         pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
 1237 
 1238         pmap->pm_active = 0;
 1239         TAILQ_INIT(&pmap->pm_pvchunk);
 1240         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1241 
 1242         return (1);
 1243 }
 1244 
 1245 /*
 1246  * this routine is called if the page table page is not
 1247  * mapped correctly.
 1248  *
 1249  * Note: If a page allocation fails at page table level two or three,
 1250  * one or two pages may be held during the wait, only to be released
 1251  * afterwards.  This conservative approach is easily argued to avoid
 1252  * race conditions.
 1253  */
 1254 static vm_page_t
 1255 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 1256 {
 1257         vm_page_t m, pdppg, pdpg;
 1258 
 1259         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1260             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1261             ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1262 
 1263         /*
 1264          * Allocate a page table page.
 1265          */
 1266         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1267             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1268                 if (flags & M_WAITOK) {
 1269                         PMAP_UNLOCK(pmap);
 1270                         vm_page_unlock_queues();
 1271                         VM_WAIT;
 1272                         vm_page_lock_queues();
 1273                         PMAP_LOCK(pmap);
 1274                 }
 1275 
 1276                 /*
 1277                  * Indicate the need to retry.  While waiting, the page table
 1278                  * page may have been allocated.
 1279                  */
 1280                 return (NULL);
 1281         }
 1282         if ((m->flags & PG_ZERO) == 0)
 1283                 pmap_zero_page(m);
 1284 
 1285         /*
 1286          * Map the pagetable page into the process address space, if
 1287          * it isn't already there.
 1288          */
 1289 
 1290         pmap->pm_stats.resident_count++;
 1291 
 1292         if (ptepindex >= (NUPDE + NUPDPE)) {
 1293                 pml4_entry_t *pml4;
 1294                 vm_pindex_t pml4index;
 1295 
 1296                 /* Wire up a new PDPE page */
 1297                 pml4index = ptepindex - (NUPDE + NUPDPE);
 1298                 pml4 = &pmap->pm_pml4[pml4index];
 1299                 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1300 
 1301         } else if (ptepindex >= NUPDE) {
 1302                 vm_pindex_t pml4index;
 1303                 vm_pindex_t pdpindex;
 1304                 pml4_entry_t *pml4;
 1305                 pdp_entry_t *pdp;
 1306 
 1307                 /* Wire up a new PDE page */
 1308                 pdpindex = ptepindex - NUPDE;
 1309                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1310 
 1311                 pml4 = &pmap->pm_pml4[pml4index];
 1312                 if ((*pml4 & PG_V) == 0) {
 1313                         /* Have to allocate a new pdp, recurse */
 1314                         if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
 1315                             flags) == NULL) {
 1316                                 --m->wire_count;
 1317                                 vm_page_free(m);
 1318                                 return (NULL);
 1319                         }
 1320                 } else {
 1321                         /* Add reference to pdp page */
 1322                         pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
 1323                         pdppg->wire_count++;
 1324                 }
 1325                 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1326 
 1327                 /* Now find the pdp page */
 1328                 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1329                 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1330 
 1331         } else {
 1332                 vm_pindex_t pml4index;
 1333                 vm_pindex_t pdpindex;
 1334                 pml4_entry_t *pml4;
 1335                 pdp_entry_t *pdp;
 1336                 pd_entry_t *pd;
 1337 
 1338                 /* Wire up a new PTE page */
 1339                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1340                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1341 
 1342                 /* First, find the pdp and check that its valid. */
 1343                 pml4 = &pmap->pm_pml4[pml4index];
 1344                 if ((*pml4 & PG_V) == 0) {
 1345                         /* Have to allocate a new pd, recurse */
 1346                         if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1347                             flags) == NULL) {
 1348                                 --m->wire_count;
 1349                                 vm_page_free(m);
 1350                                 return (NULL);
 1351                         }
 1352                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1353                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1354                 } else {
 1355                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1356                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1357                         if ((*pdp & PG_V) == 0) {
 1358                                 /* Have to allocate a new pd, recurse */
 1359                                 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1360                                     flags) == NULL) {
 1361                                         --m->wire_count;
 1362                                         vm_page_free(m);
 1363                                         return (NULL);
 1364                                 }
 1365                         } else {
 1366                                 /* Add reference to the pd page */
 1367                                 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
 1368                                 pdpg->wire_count++;
 1369                         }
 1370                 }
 1371                 pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
 1372 
 1373                 /* Now we know where the page directory page is */
 1374                 pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
 1375                 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1376         }
 1377 
 1378         return m;
 1379 }
 1380 
 1381 static vm_page_t
 1382 pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags)
 1383 {
 1384         vm_pindex_t pdpindex, ptepindex;
 1385         pdp_entry_t *pdpe;
 1386         vm_page_t pdpg;
 1387 
 1388         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1389             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1390             ("pmap_allocpde: flags is neither M_NOWAIT nor M_WAITOK"));
 1391 retry:
 1392         pdpe = pmap_pdpe(pmap, va);
 1393         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 1394                 /* Add a reference to the pd page. */
 1395                 pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
 1396                 pdpg->wire_count++;
 1397         } else {
 1398                 /* Allocate a pd page. */
 1399                 ptepindex = pmap_pde_pindex(va);
 1400                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1401                 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, flags);
 1402                 if (pdpg == NULL && (flags & M_WAITOK))
 1403                         goto retry;
 1404         }
 1405         return (pdpg);
 1406 }
 1407 
 1408 static vm_page_t
 1409 pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
 1410 {
 1411         vm_pindex_t ptepindex;
 1412         pd_entry_t *pd;
 1413         vm_page_t m, free;
 1414 
 1415         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1416             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1417             ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1418 
 1419         /*
 1420          * Calculate pagetable page index
 1421          */
 1422         ptepindex = pmap_pde_pindex(va);
 1423 retry:
 1424         /*
 1425          * Get the page directory entry
 1426          */
 1427         pd = pmap_pde(pmap, va);
 1428 
 1429         /*
 1430          * This supports switching from a 2MB page to a
 1431          * normal 4K page.
 1432          */
 1433         if (pd != 0 && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
 1434                 *pd = 0;
 1435                 pd = 0;
 1436                 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1437                 free = NULL;
 1438                 pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va), &free);
 1439                 pmap_invalidate_all(kernel_pmap);
 1440                 pmap_free_zero_pages(free);
 1441         }
 1442 
 1443         /*
 1444          * If the page table page is mapped, we just increment the
 1445          * hold count, and activate it.
 1446          */
 1447         if (pd != 0 && (*pd & PG_V) != 0) {
 1448                 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
 1449                 m->wire_count++;
 1450         } else {
 1451                 /*
 1452                  * Here if the pte page isn't mapped, or if it has been
 1453                  * deallocated.
 1454                  */
 1455                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1456                 if (m == NULL && (flags & M_WAITOK))
 1457                         goto retry;
 1458         }
 1459         return (m);
 1460 }
 1461 
 1462 
 1463 /***************************************************
 1464  * Pmap allocation/deallocation routines.
 1465  ***************************************************/
 1466 
 1467 /*
 1468  * Release any resources held by the given physical map.
 1469  * Called when a pmap initialized by pmap_pinit is being released.
 1470  * Should only be called if the map contains no valid mappings.
 1471  */
 1472 void
 1473 pmap_release(pmap_t pmap)
 1474 {
 1475         vm_page_t m;
 1476 
 1477         KASSERT(pmap->pm_stats.resident_count == 0,
 1478             ("pmap_release: pmap resident count %ld != 0",
 1479             pmap->pm_stats.resident_count));
 1480 
 1481         m = PHYS_TO_VM_PAGE(pmap->pm_pml4[PML4PML4I] & PG_FRAME);
 1482 
 1483         pmap->pm_pml4[KPML4I] = 0;      /* KVA */
 1484         pmap->pm_pml4[DMPML4I] = 0;     /* Direct Map */
 1485         pmap->pm_pml4[PML4PML4I] = 0;   /* Recursive Mapping */
 1486 
 1487         m->wire_count--;
 1488         atomic_subtract_int(&cnt.v_wire_count, 1);
 1489         vm_page_free_zero(m);
 1490         PMAP_LOCK_DESTROY(pmap);
 1491 }
 1492 
 1493 static int
 1494 kvm_size(SYSCTL_HANDLER_ARGS)
 1495 {
 1496         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
 1497 
 1498         return sysctl_handle_long(oidp, &ksize, 0, req);
 1499 }
 1500 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 1501     0, 0, kvm_size, "LU", "Size of KVM");
 1502 
 1503 static int
 1504 kvm_free(SYSCTL_HANDLER_ARGS)
 1505 {
 1506         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 1507 
 1508         return sysctl_handle_long(oidp, &kfree, 0, req);
 1509 }
 1510 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 1511     0, 0, kvm_free, "LU", "Amount of KVM free");
 1512 
 1513 /*
 1514  * grow the number of kernel page table entries, if needed
 1515  */
 1516 void
 1517 pmap_growkernel(vm_offset_t addr)
 1518 {
 1519         vm_paddr_t paddr;
 1520         vm_page_t nkpg;
 1521         pd_entry_t *pde, newpdir;
 1522         pdp_entry_t newpdp;
 1523 
 1524         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1525         if (kernel_vm_end == 0) {
 1526                 kernel_vm_end = KERNBASE;
 1527                 nkpt = 0;
 1528                 while ((*pmap_pde(kernel_pmap, kernel_vm_end) & PG_V) != 0) {
 1529                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1530                         nkpt++;
 1531                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1532                                 kernel_vm_end = kernel_map->max_offset;
 1533                                 break;                       
 1534                         }
 1535                 }
 1536         }
 1537         addr = roundup2(addr, PAGE_SIZE * NPTEPG);
 1538         if (addr - 1 >= kernel_map->max_offset)
 1539                 addr = kernel_map->max_offset;
 1540         while (kernel_vm_end < addr) {
 1541                 pde = pmap_pde(kernel_pmap, kernel_vm_end);
 1542                 if (pde == NULL) {
 1543                         /* We need a new PDP entry */
 1544                         nkpg = vm_page_alloc(NULL, nkpt,
 1545                             VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 1546                         if (!nkpg)
 1547                                 panic("pmap_growkernel: no memory to grow kernel");
 1548                         pmap_zero_page(nkpg);
 1549                         paddr = VM_PAGE_TO_PHYS(nkpg);
 1550                         newpdp = (pdp_entry_t)
 1551                                 (paddr | PG_V | PG_RW | PG_A | PG_M);
 1552                         *pmap_pdpe(kernel_pmap, kernel_vm_end) = newpdp;
 1553                         continue; /* try again */
 1554                 }
 1555                 if ((*pde & PG_V) != 0) {
 1556                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1557                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1558                                 kernel_vm_end = kernel_map->max_offset;
 1559                                 break;                       
 1560                         }
 1561                         continue;
 1562                 }
 1563 
 1564                 /*
 1565                  * This index is bogus, but out of the way
 1566                  */
 1567                 nkpg = vm_page_alloc(NULL, nkpt,
 1568                     VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 1569                 if (!nkpg)
 1570                         panic("pmap_growkernel: no memory to grow kernel");
 1571 
 1572                 nkpt++;
 1573 
 1574                 pmap_zero_page(nkpg);
 1575                 paddr = VM_PAGE_TO_PHYS(nkpg);
 1576                 newpdir = (pd_entry_t) (paddr | PG_V | PG_RW | PG_A | PG_M);
 1577                 *pmap_pde(kernel_pmap, kernel_vm_end) = newpdir;
 1578 
 1579                 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1580                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1581                         kernel_vm_end = kernel_map->max_offset;
 1582                         break;                       
 1583                 }
 1584         }
 1585 }
 1586 
 1587 
 1588 /***************************************************
 1589  * page management routines.
 1590  ***************************************************/
 1591 
 1592 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 1593 CTASSERT(_NPCM == 3);
 1594 CTASSERT(_NPCPV == 168);
 1595 
 1596 static __inline struct pv_chunk *
 1597 pv_to_chunk(pv_entry_t pv)
 1598 {
 1599 
 1600         return (struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK);
 1601 }
 1602 
 1603 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 1604 
 1605 #define PC_FREE0        0xfffffffffffffffful
 1606 #define PC_FREE1        0xfffffffffffffffful
 1607 #define PC_FREE2        0x000000fffffffffful
 1608 
 1609 static uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 };
 1610 
 1611 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 1612         "Current number of pv entries");
 1613 
 1614 #ifdef PV_STATS
 1615 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 1616 
 1617 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 1618         "Current number of pv entry chunks");
 1619 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 1620         "Current number of pv entry chunks allocated");
 1621 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 1622         "Current number of pv entry chunks frees");
 1623 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 1624         "Number of times tried to get a chunk page but failed.");
 1625 
 1626 static long pv_entry_frees, pv_entry_allocs;
 1627 static int pv_entry_spare;
 1628 
 1629 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 1630         "Current number of pv entry frees");
 1631 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 1632         "Current number of pv entry allocs");
 1633 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 1634         "Current number of spare pv entries");
 1635 
 1636 static int pmap_collect_inactive, pmap_collect_active;
 1637 
 1638 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_collect_inactive, CTLFLAG_RD, &pmap_collect_inactive, 0,
 1639         "Current number times pmap_collect called on inactive queue");
 1640 SYSCTL_INT(_vm_pmap, OID_AUTO, pmap_collect_active, CTLFLAG_RD, &pmap_collect_active, 0,
 1641         "Current number times pmap_collect called on active queue");
 1642 #endif
 1643 
 1644 /*
 1645  * We are in a serious low memory condition.  Resort to
 1646  * drastic measures to free some pages so we can allocate
 1647  * another pv entry chunk.  This is normally called to
 1648  * unmap inactive pages, and if necessary, active pages.
 1649  */
 1650 static void
 1651 pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 1652 {
 1653         pd_entry_t ptepde;
 1654         pmap_t pmap;
 1655         pt_entry_t *pte, tpte;
 1656         pv_entry_t next_pv, pv;
 1657         vm_offset_t va;
 1658         vm_page_t m, free;
 1659 
 1660         TAILQ_FOREACH(m, &vpq->pl, pageq) {
 1661                 if (m->hold_count || m->busy)
 1662                         continue;
 1663                 TAILQ_FOREACH_SAFE(pv, &m->md.pv_list, pv_list, next_pv) {
 1664                         va = pv->pv_va;
 1665                         pmap = PV_PMAP(pv);
 1666                         /* Avoid deadlock and lock recursion. */
 1667                         if (pmap > locked_pmap)
 1668                                 PMAP_LOCK(pmap);
 1669                         else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap))
 1670                                 continue;
 1671                         pmap->pm_stats.resident_count--;
 1672                         pte = pmap_pte_pde(pmap, va, &ptepde);
 1673                         if (pte == NULL) {
 1674                                 panic("null pte in pmap_collect");
 1675                         }
 1676                         tpte = pte_load_clear(pte);
 1677                         KASSERT((tpte & PG_W) == 0,
 1678                             ("pmap_collect: wired pte %#lx", tpte));
 1679                         if (tpte & PG_A)
 1680                                 vm_page_flag_set(m, PG_REFERENCED);
 1681                         if (tpte & PG_M) {
 1682                                 KASSERT((tpte & PG_RW),
 1683         ("pmap_collect: modified page not writable: va: %#lx, pte: %#lx",
 1684                                     va, tpte));
 1685                                 vm_page_dirty(m);
 1686                         }
 1687                         free = NULL;
 1688                         pmap_unuse_pt(pmap, va, ptepde, &free);
 1689                         pmap_invalidate_page(pmap, va);
 1690                         pmap_free_zero_pages(free);
 1691                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1692                         if (TAILQ_EMPTY(&m->md.pv_list))
 1693                                 vm_page_flag_clear(m, PG_WRITEABLE);
 1694                         m->md.pv_list_count--;
 1695                         free_pv_entry(pmap, pv);
 1696                         if (pmap != locked_pmap)
 1697                                 PMAP_UNLOCK(pmap);
 1698                 }
 1699         }
 1700 }
 1701 
 1702 
 1703 /*
 1704  * free the pv_entry back to the free list
 1705  */
 1706 static void
 1707 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 1708 {
 1709         vm_page_t m;
 1710         struct pv_chunk *pc;
 1711         int idx, field, bit;
 1712 
 1713         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1714         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1715         PV_STAT(pv_entry_frees++);
 1716         PV_STAT(pv_entry_spare++);
 1717         pv_entry_count--;
 1718         pc = pv_to_chunk(pv);
 1719         idx = pv - &pc->pc_pventry[0];
 1720         field = idx / 64;
 1721         bit = idx % 64;
 1722         pc->pc_map[field] |= 1ul << bit;
 1723         /* move to head of list */
 1724         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 1725         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 1726         if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 ||
 1727             pc->pc_map[2] != PC_FREE2)
 1728                 return;
 1729         PV_STAT(pv_entry_spare -= _NPCPV);
 1730         PV_STAT(pc_chunk_count--);
 1731         PV_STAT(pc_chunk_frees++);
 1732         /* entire chunk is free, return it */
 1733         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 1734         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 1735         dump_drop_page(m->phys_addr);
 1736         vm_page_free(m);
 1737 }
 1738 
 1739 /*
 1740  * get a new pv_entry, allocating a block from the system
 1741  * when needed.
 1742  */
 1743 static pv_entry_t
 1744 get_pv_entry(pmap_t pmap, int try)
 1745 {
 1746         static const struct timeval printinterval = { 60, 0 };
 1747         static struct timeval lastprint;
 1748         static vm_pindex_t colour;
 1749         int bit, field;
 1750         pv_entry_t pv;
 1751         struct pv_chunk *pc;
 1752         vm_page_t m;
 1753 
 1754         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1755         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1756         PV_STAT(pv_entry_allocs++);
 1757         pv_entry_count++;
 1758         if (pv_entry_count > pv_entry_high_water)
 1759                 if (ratecheck(&lastprint, &printinterval))
 1760                         printf("Approaching the limit on PV entries, consider "
 1761                             "increasing either the vm.pmap.shpgperproc or the "
 1762                             "vm.pmap.pv_entry_max sysctl.\n");
 1763         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 1764         if (pc != NULL) {
 1765                 for (field = 0; field < _NPCM; field++) {
 1766                         if (pc->pc_map[field]) {
 1767                                 bit = bsfq(pc->pc_map[field]);
 1768                                 break;
 1769                         }
 1770                 }
 1771                 if (field < _NPCM) {
 1772                         pv = &pc->pc_pventry[field * 64 + bit];
 1773                         pc->pc_map[field] &= ~(1ul << bit);
 1774                         /* If this was the last item, move it to tail */
 1775                         if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 &&
 1776                             pc->pc_map[2] == 0) {
 1777                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 1778                                 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 1779                         }
 1780                         PV_STAT(pv_entry_spare--);
 1781                         return (pv);
 1782                 }
 1783         }
 1784         /* No free items, allocate another chunk */
 1785         m = vm_page_alloc(NULL, colour, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
 1786         if (m == NULL) {
 1787                 if (try) {
 1788                         pv_entry_count--;
 1789                         PV_STAT(pc_chunk_tryfail++);
 1790                         return (NULL);
 1791                 }
 1792                 /*
 1793                  * Reclaim pv entries: At first, destroy mappings to inactive
 1794                  * pages.  After that, if a pv chunk entry is still needed,
 1795                  * destroy mappings to active pages.
 1796                  */
 1797                 PV_STAT(pmap_collect_inactive++);
 1798                 pmap_collect(pmap, &vm_page_queues[PQ_INACTIVE]);
 1799                 m = vm_page_alloc(NULL, colour,
 1800                     VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
 1801                 if (m == NULL) {
 1802                         PV_STAT(pmap_collect_active++);
 1803                         pmap_collect(pmap, &vm_page_queues[PQ_ACTIVE]);
 1804                         m = vm_page_alloc(NULL, colour,
 1805                             VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ);
 1806                         if (m == NULL)
 1807                                 panic("get_pv_entry: increase vm.pmap.shpgperproc");
 1808                 }
 1809         }
 1810         PV_STAT(pc_chunk_count++);
 1811         PV_STAT(pc_chunk_allocs++);
 1812         colour++;
 1813         dump_add_page(m->phys_addr);
 1814         pc = (void *)PHYS_TO_DMAP(m->phys_addr);
 1815         pc->pc_pmap = pmap;
 1816         pc->pc_map[0] = PC_FREE0 & ~1ul;        /* preallocated bit 0 */
 1817         pc->pc_map[1] = PC_FREE1;
 1818         pc->pc_map[2] = PC_FREE2;
 1819         pv = &pc->pc_pventry[0];
 1820         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 1821         PV_STAT(pv_entry_spare += _NPCPV - 1);
 1822         return (pv);
 1823 }
 1824 
 1825 static void
 1826 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 1827 {
 1828         pv_entry_t pv;
 1829 
 1830         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1831         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1832         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 1833                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) 
 1834                         break;
 1835         }
 1836         KASSERT(pv != NULL, ("pmap_remove_entry: pv not found"));
 1837         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1838         m->md.pv_list_count--;
 1839         if (TAILQ_EMPTY(&m->md.pv_list))
 1840                 vm_page_flag_clear(m, PG_WRITEABLE);
 1841         free_pv_entry(pmap, pv);
 1842 }
 1843 
 1844 /*
 1845  * Create a pv entry for page at pa for
 1846  * (pmap, va).
 1847  */
 1848 static void
 1849 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1850 {
 1851         pv_entry_t pv;
 1852 
 1853         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1854         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1855         pv = get_pv_entry(pmap, FALSE);
 1856         pv->pv_va = va;
 1857         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 1858         m->md.pv_list_count++;
 1859 }
 1860 
 1861 /*
 1862  * Conditionally create a pv entry.
 1863  */
 1864 static boolean_t
 1865 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1866 {
 1867         pv_entry_t pv;
 1868 
 1869         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1870         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1871         if (pv_entry_count < pv_entry_high_water && 
 1872             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 1873                 pv->pv_va = va;
 1874                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 1875                 m->md.pv_list_count++;
 1876                 return (TRUE);
 1877         } else
 1878                 return (FALSE);
 1879 }
 1880 
 1881 /*
 1882  * pmap_remove_pte: do the things to unmap a page in a process
 1883  */
 1884 static int
 1885 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 
 1886     pd_entry_t ptepde, vm_page_t *free)
 1887 {
 1888         pt_entry_t oldpte;
 1889         vm_page_t m;
 1890 
 1891         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1892         oldpte = pte_load_clear(ptq);
 1893         if (oldpte & PG_W)
 1894                 pmap->pm_stats.wired_count -= 1;
 1895         /*
 1896          * Machines that don't support invlpg, also don't support
 1897          * PG_G.
 1898          */
 1899         if (oldpte & PG_G)
 1900                 pmap_invalidate_page(kernel_pmap, va);
 1901         pmap->pm_stats.resident_count -= 1;
 1902         if (oldpte & PG_MANAGED) {
 1903                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 1904                 if (oldpte & PG_M) {
 1905                         KASSERT((oldpte & PG_RW),
 1906         ("pmap_remove_pte: modified page not writable: va: %#lx, pte: %#lx",
 1907                             va, oldpte));
 1908                         vm_page_dirty(m);
 1909                 }
 1910                 if (oldpte & PG_A)
 1911                         vm_page_flag_set(m, PG_REFERENCED);
 1912                 pmap_remove_entry(pmap, m, va);
 1913         }
 1914         return (pmap_unuse_pt(pmap, va, ptepde, free));
 1915 }
 1916 
 1917 /*
 1918  * Remove a single page from a process address space
 1919  */
 1920 static void
 1921 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, vm_page_t *free)
 1922 {
 1923         pt_entry_t *pte;
 1924 
 1925         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1926         if ((*pde & PG_V) == 0)
 1927                 return;
 1928         pte = pmap_pde_to_pte(pde, va);
 1929         if ((*pte & PG_V) == 0)
 1930                 return;
 1931         pmap_remove_pte(pmap, pte, va, *pde, free);
 1932         pmap_invalidate_page(pmap, va);
 1933 }
 1934 
 1935 /*
 1936  *      Remove the given range of addresses from the specified map.
 1937  *
 1938  *      It is assumed that the start and end are properly
 1939  *      rounded to the page size.
 1940  */
 1941 void
 1942 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1943 {
 1944         vm_offset_t va_next;
 1945         pml4_entry_t *pml4e;
 1946         pdp_entry_t *pdpe;
 1947         pd_entry_t ptpaddr, *pde;
 1948         pt_entry_t *pte;
 1949         vm_page_t free = NULL;
 1950         int anyvalid;
 1951 
 1952         /*
 1953          * Perform an unsynchronized read.  This is, however, safe.
 1954          */
 1955         if (pmap->pm_stats.resident_count == 0)
 1956                 return;
 1957 
 1958         anyvalid = 0;
 1959 
 1960         vm_page_lock_queues();
 1961         PMAP_LOCK(pmap);
 1962 
 1963         /*
 1964          * special handling of removing one page.  a very
 1965          * common operation and easy to short circuit some
 1966          * code.
 1967          */
 1968         if (sva + PAGE_SIZE == eva) {
 1969                 pde = pmap_pde(pmap, sva);
 1970                 if (pde && (*pde & PG_PS) == 0) {
 1971                         pmap_remove_page(pmap, sva, pde, &free);
 1972                         goto out;
 1973                 }
 1974         }
 1975 
 1976         for (; sva < eva; sva = va_next) {
 1977 
 1978                 if (pmap->pm_stats.resident_count == 0)
 1979                         break;
 1980 
 1981                 pml4e = pmap_pml4e(pmap, sva);
 1982                 if ((*pml4e & PG_V) == 0) {
 1983                         va_next = (sva + NBPML4) & ~PML4MASK;
 1984                         continue;
 1985                 }
 1986 
 1987                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 1988                 if ((*pdpe & PG_V) == 0) {
 1989                         va_next = (sva + NBPDP) & ~PDPMASK;
 1990                         continue;
 1991                 }
 1992 
 1993                 /*
 1994                  * Calculate index for next page table.
 1995                  */
 1996                 va_next = (sva + NBPDR) & ~PDRMASK;
 1997 
 1998                 pde = pmap_pdpe_to_pde(pdpe, sva);
 1999                 ptpaddr = *pde;
 2000 
 2001                 /*
 2002                  * Weed out invalid mappings.
 2003                  */
 2004                 if (ptpaddr == 0)
 2005                         continue;
 2006 
 2007                 /*
 2008                  * Check for large page.
 2009                  */
 2010                 if ((ptpaddr & PG_PS) != 0) {
 2011                         *pde = 0;
 2012                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 2013                         pmap_unuse_pt(pmap, sva, *pdpe, &free);
 2014                         anyvalid = 1;
 2015                         continue;
 2016                 }
 2017 
 2018                 /*
 2019                  * Limit our scan to either the end of the va represented
 2020                  * by the current page table page, or to the end of the
 2021                  * range being removed.
 2022                  */
 2023                 if (va_next > eva)
 2024                         va_next = eva;
 2025 
 2026                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 2027                     sva += PAGE_SIZE) {
 2028                         if (*pte == 0)
 2029                                 continue;
 2030 
 2031                         /*
 2032                          * The TLB entry for a PG_G mapping is invalidated
 2033                          * by pmap_remove_pte().
 2034                          */
 2035                         if ((*pte & PG_G) == 0)
 2036                                 anyvalid = 1;
 2037                         if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free))
 2038                                 break;
 2039                 }
 2040         }
 2041 out:
 2042         if (anyvalid)
 2043                 pmap_invalidate_all(pmap);
 2044         vm_page_unlock_queues();        
 2045         PMAP_UNLOCK(pmap);
 2046         pmap_free_zero_pages(free);
 2047 }
 2048 
 2049 /*
 2050  *      Routine:        pmap_remove_all
 2051  *      Function:
 2052  *              Removes this physical page from
 2053  *              all physical maps in which it resides.
 2054  *              Reflects back modify bits to the pager.
 2055  *
 2056  *      Notes:
 2057  *              Original versions of this routine were very
 2058  *              inefficient because they iteratively called
 2059  *              pmap_remove (slow...)
 2060  */
 2061 
 2062 void
 2063 pmap_remove_all(vm_page_t m)
 2064 {
 2065         pv_entry_t pv;
 2066         pmap_t pmap;
 2067         pt_entry_t *pte, tpte;
 2068         pd_entry_t ptepde;
 2069         vm_page_t free;
 2070 
 2071 #if defined(PMAP_DIAGNOSTIC)
 2072         /*
 2073          * XXX This makes pmap_remove_all() illegal for non-managed pages!
 2074          */
 2075         if (m->flags & PG_FICTITIOUS) {
 2076                 panic("pmap_remove_all: illegal for unmanaged page, va: 0x%lx",
 2077                     VM_PAGE_TO_PHYS(m));
 2078         }
 2079 #endif
 2080         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2081         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 2082                 pmap = PV_PMAP(pv);
 2083                 PMAP_LOCK(pmap);
 2084                 pmap->pm_stats.resident_count--;
 2085                 pte = pmap_pte_pde(pmap, pv->pv_va, &ptepde);
 2086                 if (pte == NULL) {
 2087                         panic("null pte in pmap_remove_all");
 2088                 }
 2089                 tpte = pte_load_clear(pte);
 2090                 if (tpte & PG_W)
 2091                         pmap->pm_stats.wired_count--;
 2092                 if (tpte & PG_A)
 2093                         vm_page_flag_set(m, PG_REFERENCED);
 2094 
 2095                 /*
 2096                  * Update the vm_page_t clean and reference bits.
 2097                  */
 2098                 if (tpte & PG_M) {
 2099                         KASSERT((tpte & PG_RW),
 2100         ("pmap_remove_all: modified page not writable: va: %#lx, pte: %#lx",
 2101                             pv->pv_va, tpte));
 2102                         vm_page_dirty(m);
 2103                 }
 2104                 free = NULL;
 2105                 pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
 2106                 pmap_invalidate_page(pmap, pv->pv_va);
 2107                 pmap_free_zero_pages(free);
 2108                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2109                 m->md.pv_list_count--;
 2110                 free_pv_entry(pmap, pv);
 2111                 PMAP_UNLOCK(pmap);
 2112         }
 2113         vm_page_flag_clear(m, PG_WRITEABLE);
 2114 }
 2115 
 2116 /*
 2117  *      Set the physical protection on the
 2118  *      specified range of this map as requested.
 2119  */
 2120 void
 2121 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 2122 {
 2123         vm_offset_t va_next;
 2124         pml4_entry_t *pml4e;
 2125         pdp_entry_t *pdpe;
 2126         pd_entry_t ptpaddr, *pde;
 2127         pt_entry_t *pte;
 2128         int anychanged;
 2129 
 2130         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 2131                 pmap_remove(pmap, sva, eva);
 2132                 return;
 2133         }
 2134 
 2135         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 2136             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 2137                 return;
 2138 
 2139         anychanged = 0;
 2140 
 2141         vm_page_lock_queues();
 2142         PMAP_LOCK(pmap);
 2143         for (; sva < eva; sva = va_next) {
 2144 
 2145                 pml4e = pmap_pml4e(pmap, sva);
 2146                 if ((*pml4e & PG_V) == 0) {
 2147                         va_next = (sva + NBPML4) & ~PML4MASK;
 2148                         continue;
 2149                 }
 2150 
 2151                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 2152                 if ((*pdpe & PG_V) == 0) {
 2153                         va_next = (sva + NBPDP) & ~PDPMASK;
 2154                         continue;
 2155                 }
 2156 
 2157                 va_next = (sva + NBPDR) & ~PDRMASK;
 2158 
 2159                 pde = pmap_pdpe_to_pde(pdpe, sva);
 2160                 ptpaddr = *pde;
 2161 
 2162                 /*
 2163                  * Weed out invalid mappings.
 2164                  */
 2165                 if (ptpaddr == 0)
 2166                         continue;
 2167 
 2168                 /*
 2169                  * Check for large page.
 2170                  */
 2171                 if ((ptpaddr & PG_PS) != 0) {
 2172                         if ((prot & VM_PROT_WRITE) == 0)
 2173                                 *pde &= ~(PG_M|PG_RW);
 2174                         if ((prot & VM_PROT_EXECUTE) == 0)
 2175                                 *pde |= pg_nx;
 2176                         anychanged = 1;
 2177                         continue;
 2178                 }
 2179 
 2180                 if (va_next > eva)
 2181                         va_next = eva;
 2182 
 2183                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 2184                     sva += PAGE_SIZE) {
 2185                         pt_entry_t obits, pbits;
 2186                         vm_page_t m;
 2187 
 2188 retry:
 2189                         obits = pbits = *pte;
 2190                         if ((pbits & PG_V) == 0)
 2191                                 continue;
 2192                         if (pbits & PG_MANAGED) {
 2193                                 m = NULL;
 2194                                 if (pbits & PG_A) {
 2195                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 2196                                         vm_page_flag_set(m, PG_REFERENCED);
 2197                                         pbits &= ~PG_A;
 2198                                 }
 2199                                 if ((pbits & PG_M) != 0) {
 2200                                         if (m == NULL)
 2201                                                 m = PHYS_TO_VM_PAGE(pbits &
 2202                                                     PG_FRAME);
 2203                                         vm_page_dirty(m);
 2204                                 }
 2205                         }
 2206 
 2207                         if ((prot & VM_PROT_WRITE) == 0)
 2208                                 pbits &= ~(PG_RW | PG_M);
 2209                         if ((prot & VM_PROT_EXECUTE) == 0)
 2210                                 pbits |= pg_nx;
 2211 
 2212                         if (pbits != obits) {
 2213                                 if (!atomic_cmpset_long(pte, obits, pbits))
 2214                                         goto retry;
 2215                                 if (obits & PG_G)
 2216                                         pmap_invalidate_page(pmap, sva);
 2217                                 else
 2218                                         anychanged = 1;
 2219                         }
 2220                 }
 2221         }
 2222         if (anychanged)
 2223                 pmap_invalidate_all(pmap);
 2224         vm_page_unlock_queues();
 2225         PMAP_UNLOCK(pmap);
 2226 }
 2227 
 2228 /*
 2229  *      Insert the given physical page (p) at
 2230  *      the specified virtual address (v) in the
 2231  *      target physical map with the protection requested.
 2232  *
 2233  *      If specified, the page will be wired down, meaning
 2234  *      that the related pte can not be reclaimed.
 2235  *
 2236  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 2237  *      or lose information.  That is, this routine must actually
 2238  *      insert this page into the given map NOW.
 2239  */
 2240 void
 2241 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 2242            boolean_t wired)
 2243 {
 2244         vm_paddr_t pa;
 2245         pd_entry_t *pde;
 2246         pt_entry_t *pte;
 2247         vm_paddr_t opa;
 2248         pt_entry_t origpte, newpte;
 2249         vm_page_t mpte, om;
 2250         boolean_t invlva;
 2251 
 2252         va = trunc_page(va);
 2253 #ifdef PMAP_DIAGNOSTIC
 2254         if (va > VM_MAX_KERNEL_ADDRESS)
 2255                 panic("pmap_enter: toobig");
 2256         if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
 2257                 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", va);
 2258 #endif
 2259 
 2260         mpte = NULL;
 2261 
 2262         vm_page_lock_queues();
 2263         PMAP_LOCK(pmap);
 2264 
 2265         /*
 2266          * In the case that a page table page is not
 2267          * resident, we are creating it here.
 2268          */
 2269         if (va < VM_MAXUSER_ADDRESS) {
 2270                 mpte = pmap_allocpte(pmap, va, M_WAITOK);
 2271         }
 2272 #if 0 && defined(PMAP_DIAGNOSTIC)
 2273         else {
 2274                 pd_entry_t *pdeaddr = pmap_pde(pmap, va);
 2275                 origpte = *pdeaddr;
 2276                 if ((origpte & PG_V) == 0) { 
 2277                         panic("pmap_enter: invalid kernel page table page, pde=%p, va=%p\n",
 2278                                 origpte, va);
 2279                 }
 2280         }
 2281 #endif
 2282 
 2283         pde = pmap_pde(pmap, va);
 2284         if (pde != NULL) {
 2285                 if ((*pde & PG_PS) != 0)
 2286                         panic("pmap_enter: attempted pmap_enter on 2MB page");
 2287                 pte = pmap_pde_to_pte(pde, va);
 2288         } else
 2289                 pte = NULL;
 2290 
 2291         /*
 2292          * Page Directory table entry not valid, we need a new PT page
 2293          */
 2294         if (pte == NULL)
 2295                 panic("pmap_enter: invalid page directory va=%#lx\n", va);
 2296 
 2297         pa = VM_PAGE_TO_PHYS(m);
 2298         om = NULL;
 2299         origpte = *pte;
 2300         opa = origpte & PG_FRAME;
 2301 
 2302         /*
 2303          * Mapping has not changed, must be protection or wiring change.
 2304          */
 2305         if (origpte && (opa == pa)) {
 2306                 /*
 2307                  * Wiring change, just update stats. We don't worry about
 2308                  * wiring PT pages as they remain resident as long as there
 2309                  * are valid mappings in them. Hence, if a user page is wired,
 2310                  * the PT page will be also.
 2311                  */
 2312                 if (wired && ((origpte & PG_W) == 0))
 2313                         pmap->pm_stats.wired_count++;
 2314                 else if (!wired && (origpte & PG_W))
 2315                         pmap->pm_stats.wired_count--;
 2316 
 2317                 /*
 2318                  * Remove extra pte reference
 2319                  */
 2320                 if (mpte)
 2321                         mpte->wire_count--;
 2322 
 2323                 /*
 2324                  * We might be turning off write access to the page,
 2325                  * so we go ahead and sense modify status.
 2326                  */
 2327                 if (origpte & PG_MANAGED) {
 2328                         om = m;
 2329                         pa |= PG_MANAGED;
 2330                 }
 2331                 goto validate;
 2332         } 
 2333         /*
 2334          * Mapping has changed, invalidate old range and fall through to
 2335          * handle validating new mapping.
 2336          */
 2337         if (opa) {
 2338                 if (origpte & PG_W)
 2339                         pmap->pm_stats.wired_count--;
 2340                 if (origpte & PG_MANAGED) {
 2341                         om = PHYS_TO_VM_PAGE(opa);
 2342                         pmap_remove_entry(pmap, om, va);
 2343                 }
 2344                 if (mpte != NULL) {
 2345                         mpte->wire_count--;
 2346                         KASSERT(mpte->wire_count > 0,
 2347                             ("pmap_enter: missing reference to page table page,"
 2348                              " va: 0x%lx", va));
 2349                 }
 2350         } else
 2351                 pmap->pm_stats.resident_count++;
 2352 
 2353         /*
 2354          * Enter on the PV list if part of our managed memory.
 2355          */
 2356         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
 2357                 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
 2358                     ("pmap_enter: managed mapping within the clean submap"));
 2359                 pmap_insert_entry(pmap, va, m);
 2360                 pa |= PG_MANAGED;
 2361         }
 2362 
 2363         /*
 2364          * Increment counters
 2365          */
 2366         if (wired)
 2367                 pmap->pm_stats.wired_count++;
 2368 
 2369 validate:
 2370         /*
 2371          * Now validate mapping with desired protection/wiring.
 2372          */
 2373         newpte = (pt_entry_t)(pa | PG_V);
 2374         if ((prot & VM_PROT_WRITE) != 0) {
 2375                 newpte |= PG_RW;
 2376                 vm_page_flag_set(m, PG_WRITEABLE);
 2377         }
 2378         if ((prot & VM_PROT_EXECUTE) == 0)
 2379                 newpte |= pg_nx;
 2380         if (wired)
 2381                 newpte |= PG_W;
 2382         if (va < VM_MAXUSER_ADDRESS)
 2383                 newpte |= PG_U;
 2384         if (pmap == kernel_pmap)
 2385                 newpte |= PG_G;
 2386 
 2387         /*
 2388          * if the mapping or permission bits are different, we need
 2389          * to update the pte.
 2390          */
 2391         if ((origpte & ~(PG_M|PG_A)) != newpte) {
 2392                 if (origpte & PG_V) {
 2393                         invlva = FALSE;
 2394                         origpte = pte_load_store(pte, newpte | PG_A);
 2395                         if (origpte & PG_A) {
 2396                                 if (origpte & PG_MANAGED)
 2397                                         vm_page_flag_set(om, PG_REFERENCED);
 2398                                 if (opa != VM_PAGE_TO_PHYS(m) || ((origpte &
 2399                                     PG_NX) == 0 && (newpte & PG_NX)))
 2400                                         invlva = TRUE;
 2401                         }
 2402                         if (origpte & PG_M) {
 2403                                 KASSERT((origpte & PG_RW),
 2404         ("pmap_enter: modified page not writable: va: %#lx, pte: %#lx",
 2405                                     va, origpte));
 2406                                 if ((origpte & PG_MANAGED) != 0)
 2407                                         vm_page_dirty(om);
 2408                                 if ((newpte & PG_RW) == 0)
 2409                                         invlva = TRUE;
 2410                         }
 2411                         if (invlva)
 2412                                 pmap_invalidate_page(pmap, va);
 2413                 } else
 2414                         pte_store(pte, newpte | PG_A);
 2415         }
 2416         vm_page_unlock_queues();
 2417         PMAP_UNLOCK(pmap);
 2418 }
 2419 
 2420 /*
 2421  * Maps a sequence of resident pages belonging to the same object.
 2422  * The sequence begins with the given page m_start.  This page is
 2423  * mapped at the given virtual address start.  Each subsequent page is
 2424  * mapped at a virtual address that is offset from start by the same
 2425  * amount as the page is offset from m_start within the object.  The
 2426  * last page in the sequence is the page with the largest offset from
 2427  * m_start that can be mapped at a virtual address less than the given
 2428  * virtual address end.  Not every virtual page between start and end
 2429  * is mapped; only those for which a resident page exists with the
 2430  * corresponding offset from m_start are mapped.
 2431  */
 2432 void
 2433 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 2434     vm_page_t m_start, vm_prot_t prot)
 2435 {
 2436         vm_page_t m, mpte;
 2437         vm_pindex_t diff, psize;
 2438 
 2439         VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
 2440         psize = atop(end - start);
 2441         mpte = NULL;
 2442         m = m_start;
 2443         PMAP_LOCK(pmap);
 2444         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 2445                 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m,
 2446                     prot, mpte);
 2447                 m = TAILQ_NEXT(m, listq);
 2448         }
 2449         PMAP_UNLOCK(pmap);
 2450 }
 2451 
 2452 /*
 2453  * this code makes some *MAJOR* assumptions:
 2454  * 1. Current pmap & pmap exists.
 2455  * 2. Not wired.
 2456  * 3. Read access.
 2457  * 4. No page table pages.
 2458  * but is *MUCH* faster than pmap_enter...
 2459  */
 2460 
 2461 void
 2462 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 2463 {
 2464 
 2465         PMAP_LOCK(pmap);
 2466         (void) pmap_enter_quick_locked(pmap, va, m, prot, NULL);
 2467         PMAP_UNLOCK(pmap);
 2468 }
 2469 
 2470 static vm_page_t
 2471 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 2472     vm_prot_t prot, vm_page_t mpte)
 2473 {
 2474         vm_page_t free;
 2475         pt_entry_t *pte;
 2476         vm_paddr_t pa;
 2477 
 2478         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 2479             (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
 2480             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 2481         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2482         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2483 
 2484         /*
 2485          * In the case that a page table page is not
 2486          * resident, we are creating it here.
 2487          */
 2488         if (va < VM_MAXUSER_ADDRESS) {
 2489                 vm_pindex_t ptepindex;
 2490                 pd_entry_t *ptepa;
 2491 
 2492                 /*
 2493                  * Calculate pagetable page index
 2494                  */
 2495                 ptepindex = pmap_pde_pindex(va);
 2496                 if (mpte && (mpte->pindex == ptepindex)) {
 2497                         mpte->wire_count++;
 2498                 } else {
 2499                         /*
 2500                          * Get the page directory entry
 2501                          */
 2502                         ptepa = pmap_pde(pmap, va);
 2503 
 2504                         /*
 2505                          * If the page table page is mapped, we just increment
 2506                          * the hold count, and activate it.
 2507                          */
 2508                         if (ptepa && (*ptepa & PG_V) != 0) {
 2509                                 if (*ptepa & PG_PS)
 2510                                         panic("pmap_enter_quick: unexpected mapping into 2MB page");
 2511                                 mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
 2512                                 mpte->wire_count++;
 2513                         } else {
 2514                                 mpte = _pmap_allocpte(pmap, ptepindex,
 2515                                     M_NOWAIT);
 2516                                 if (mpte == NULL)
 2517                                         return (mpte);
 2518                         }
 2519                 }
 2520         } else {
 2521                 mpte = NULL;
 2522         }
 2523 
 2524         /*
 2525          * This call to vtopte makes the assumption that we are
 2526          * entering the page into the current pmap.  In order to support
 2527          * quick entry into any pmap, one would likely use pmap_pte.
 2528          * But that isn't as quick as vtopte.
 2529          */
 2530         pte = vtopte(va);
 2531         if (*pte) {
 2532                 if (mpte != NULL) {
 2533                         mpte->wire_count--;
 2534                         mpte = NULL;
 2535                 }
 2536                 return (mpte);
 2537         }
 2538 
 2539         /*
 2540          * Enter on the PV list if part of our managed memory.
 2541          */
 2542         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 &&
 2543             !pmap_try_insert_pv_entry(pmap, va, m)) {
 2544                 if (mpte != NULL) {
 2545                         free = NULL;
 2546                         if (pmap_unwire_pte_hold(pmap, va, mpte, &free)) {
 2547                                 pmap_invalidate_page(pmap, va);
 2548                                 pmap_free_zero_pages(free);
 2549                         }
 2550                         mpte = NULL;
 2551                 }
 2552                 return (mpte);
 2553         }
 2554 
 2555         /*
 2556          * Increment counters
 2557          */
 2558         pmap->pm_stats.resident_count++;
 2559 
 2560         pa = VM_PAGE_TO_PHYS(m);
 2561         if ((prot & VM_PROT_EXECUTE) == 0)
 2562                 pa |= pg_nx;
 2563 
 2564         /*
 2565          * Now validate mapping with RO protection
 2566          */
 2567         if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED))
 2568                 pte_store(pte, pa | PG_V | PG_U);
 2569         else
 2570                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 2571         return mpte;
 2572 }
 2573 
 2574 /*
 2575  * Make a temporary mapping for a physical address.  This is only intended
 2576  * to be used for panic dumps.
 2577  */
 2578 void *
 2579 pmap_kenter_temporary(vm_paddr_t pa, int i)
 2580 {
 2581         vm_offset_t va;
 2582 
 2583         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 2584         pmap_kenter(va, pa);
 2585         invlpg(va);
 2586         return ((void *)crashdumpmap);
 2587 }
 2588 
 2589 /*
 2590  * This code maps large physical mmap regions into the
 2591  * processor address space.  Note that some shortcuts
 2592  * are taken, but the code works.
 2593  */
 2594 void
 2595 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
 2596                     vm_object_t object, vm_pindex_t pindex,
 2597                     vm_size_t size)
 2598 {
 2599         vm_offset_t va;
 2600         vm_page_t p, pdpg;
 2601 
 2602         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 2603         KASSERT(object->type == OBJT_DEVICE,
 2604             ("pmap_object_init_pt: non-device object"));
 2605         if (((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0)) {
 2606                 vm_page_t m[1];
 2607                 pd_entry_t ptepa, *pde;
 2608 
 2609                 PMAP_LOCK(pmap);
 2610                 pde = pmap_pde(pmap, addr);
 2611                 if (pde != 0 && (*pde & PG_V) != 0)
 2612                         goto out;
 2613                 PMAP_UNLOCK(pmap);
 2614 retry:
 2615                 p = vm_page_lookup(object, pindex);
 2616                 if (p != NULL) {
 2617                         if (vm_page_sleep_if_busy(p, FALSE, "init4p"))
 2618                                 goto retry;
 2619                 } else {
 2620                         p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
 2621                         if (p == NULL)
 2622                                 return;
 2623                         m[0] = p;
 2624 
 2625                         if (vm_pager_get_pages(object, m, 1, 0) != VM_PAGER_OK) {
 2626                                 vm_page_lock_queues();
 2627                                 vm_page_free(p);
 2628                                 vm_page_unlock_queues();
 2629                                 return;
 2630                         }
 2631 
 2632                         p = vm_page_lookup(object, pindex);
 2633                         vm_page_lock_queues();
 2634                         vm_page_wakeup(p);
 2635                         vm_page_unlock_queues();
 2636                 }
 2637 
 2638                 ptepa = VM_PAGE_TO_PHYS(p);
 2639                 if (ptepa & (NBPDR - 1))
 2640                         return;
 2641 
 2642                 p->valid = VM_PAGE_BITS_ALL;
 2643 
 2644                 PMAP_LOCK(pmap);
 2645                 for (va = addr; va < addr + size; va += NBPDR) {
 2646                         while ((pdpg =
 2647                             pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) {
 2648                                 PMAP_UNLOCK(pmap);
 2649                                 vm_page_lock_queues();
 2650                                 vm_page_busy(p);
 2651                                 vm_page_unlock_queues();
 2652                                 VM_OBJECT_UNLOCK(object);
 2653                                 VM_WAIT;
 2654                                 VM_OBJECT_LOCK(object);
 2655                                 vm_page_lock_queues();
 2656                                 vm_page_wakeup(p);
 2657                                 vm_page_unlock_queues();
 2658                                 PMAP_LOCK(pmap);
 2659                         }
 2660                         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
 2661                         pde = &pde[pmap_pde_index(va)];
 2662                         if ((*pde & PG_V) == 0) {
 2663                                 pde_store(pde, ptepa | PG_PS | PG_M | PG_A |
 2664                                     PG_U | PG_RW | PG_V);
 2665                                 pmap->pm_stats.resident_count +=
 2666                                     NBPDR / PAGE_SIZE;
 2667                         } else {
 2668                                 pdpg->wire_count--;
 2669                                 KASSERT(pdpg->wire_count > 0,
 2670                                     ("pmap_object_init_pt: missing reference "
 2671                                      "to page directory page, va: 0x%lx", va));
 2672                         }
 2673                         ptepa += NBPDR;
 2674                 }
 2675                 pmap_invalidate_all(pmap);
 2676 out:
 2677                 PMAP_UNLOCK(pmap);
 2678         }
 2679 }
 2680 
 2681 /*
 2682  *      Routine:        pmap_change_wiring
 2683  *      Function:       Change the wiring attribute for a map/virtual-address
 2684  *                      pair.
 2685  *      In/out conditions:
 2686  *                      The mapping must already exist in the pmap.
 2687  */
 2688 void
 2689 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 2690 {
 2691         pt_entry_t *pte;
 2692 
 2693         /*
 2694          * Wiring is not a hardware characteristic so there is no need to
 2695          * invalidate TLB.
 2696          */
 2697         PMAP_LOCK(pmap);
 2698         pte = pmap_pte(pmap, va);
 2699         if (wired && (*pte & PG_W) == 0) {
 2700                 pmap->pm_stats.wired_count++;
 2701                 atomic_set_long(pte, PG_W);
 2702         } else if (!wired && (*pte & PG_W) != 0) {
 2703                 pmap->pm_stats.wired_count--;
 2704                 atomic_clear_long(pte, PG_W);
 2705         }
 2706         PMAP_UNLOCK(pmap);
 2707 }
 2708 
 2709 
 2710 
 2711 /*
 2712  *      Copy the range specified by src_addr/len
 2713  *      from the source map to the range dst_addr/len
 2714  *      in the destination map.
 2715  *
 2716  *      This routine is only advisory and need not do anything.
 2717  */
 2718 
 2719 void
 2720 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 2721           vm_offset_t src_addr)
 2722 {
 2723         vm_page_t   free;
 2724         vm_offset_t addr;
 2725         vm_offset_t end_addr = src_addr + len;
 2726         vm_offset_t va_next;
 2727 
 2728         if (dst_addr != src_addr)
 2729                 return;
 2730 
 2731         if (!pmap_is_current(src_pmap))
 2732                 return;
 2733 
 2734         vm_page_lock_queues();
 2735         if (dst_pmap < src_pmap) {
 2736                 PMAP_LOCK(dst_pmap);
 2737                 PMAP_LOCK(src_pmap);
 2738         } else {
 2739                 PMAP_LOCK(src_pmap);
 2740                 PMAP_LOCK(dst_pmap);
 2741         }
 2742         for (addr = src_addr; addr < end_addr; addr = va_next) {
 2743                 pt_entry_t *src_pte, *dst_pte;
 2744                 vm_page_t dstmpde, dstmpte, srcmpte;
 2745                 pml4_entry_t *pml4e;
 2746                 pdp_entry_t *pdpe;
 2747                 pd_entry_t srcptepaddr, *pde;
 2748 
 2749                 if (addr >= UPT_MIN_ADDRESS)
 2750                         panic("pmap_copy: invalid to pmap_copy page tables");
 2751 
 2752                 pml4e = pmap_pml4e(src_pmap, addr);
 2753                 if ((*pml4e & PG_V) == 0) {
 2754                         va_next = (addr + NBPML4) & ~PML4MASK;
 2755                         continue;
 2756                 }
 2757 
 2758                 pdpe = pmap_pml4e_to_pdpe(pml4e, addr);
 2759                 if ((*pdpe & PG_V) == 0) {
 2760                         va_next = (addr + NBPDP) & ~PDPMASK;
 2761                         continue;
 2762                 }
 2763 
 2764                 va_next = (addr + NBPDR) & ~PDRMASK;
 2765 
 2766                 pde = pmap_pdpe_to_pde(pdpe, addr);
 2767                 srcptepaddr = *pde;
 2768                 if (srcptepaddr == 0)
 2769                         continue;
 2770                         
 2771                 if (srcptepaddr & PG_PS) {
 2772                         dstmpde = pmap_allocpde(dst_pmap, addr, M_NOWAIT);
 2773                         if (dstmpde == NULL)
 2774                                 break;
 2775                         pde = (pd_entry_t *)
 2776                             PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpde));
 2777                         pde = &pde[pmap_pde_index(addr)];
 2778                         if (*pde == 0) {
 2779                                 *pde = srcptepaddr & ~PG_W;
 2780                                 dst_pmap->pm_stats.resident_count +=
 2781                                     NBPDR / PAGE_SIZE;
 2782                         } else
 2783                                 dstmpde->wire_count--;
 2784                         continue;
 2785                 }
 2786 
 2787                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 2788                 if (srcmpte->wire_count == 0)
 2789                         panic("pmap_copy: source page table page is unused");
 2790 
 2791                 if (va_next > end_addr)
 2792                         va_next = end_addr;
 2793 
 2794                 src_pte = vtopte(addr);
 2795                 while (addr < va_next) {
 2796                         pt_entry_t ptetemp;
 2797                         ptetemp = *src_pte;
 2798                         /*
 2799                          * we only virtual copy managed pages
 2800                          */
 2801                         if ((ptetemp & PG_MANAGED) != 0) {
 2802                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 2803                                     M_NOWAIT);
 2804                                 if (dstmpte == NULL)
 2805                                         break;
 2806                                 dst_pte = (pt_entry_t *)
 2807                                     PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
 2808                                 dst_pte = &dst_pte[pmap_pte_index(addr)];
 2809                                 if (*dst_pte == 0 &&
 2810                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 2811                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
 2812                                         /*
 2813                                          * Clear the wired, modified, and
 2814                                          * accessed (referenced) bits
 2815                                          * during the copy.
 2816                                          */
 2817                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 2818                                             PG_A);
 2819                                         dst_pmap->pm_stats.resident_count++;
 2820                                 } else {
 2821                                         free = NULL;
 2822                                         if (pmap_unwire_pte_hold(dst_pmap,
 2823                                             addr, dstmpte, &free)) {
 2824                                                 pmap_invalidate_page(dst_pmap,
 2825                                                     addr);
 2826                                                 pmap_free_zero_pages(free);
 2827                                         }
 2828                                 }
 2829                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 2830                                         break;
 2831                         }
 2832                         addr += PAGE_SIZE;
 2833                         src_pte++;
 2834                 }
 2835         }
 2836         vm_page_unlock_queues();
 2837         PMAP_UNLOCK(src_pmap);
 2838         PMAP_UNLOCK(dst_pmap);
 2839 }       
 2840 
 2841 /*
 2842  *      pmap_zero_page zeros the specified hardware page by mapping 
 2843  *      the page into KVM and using bzero to clear its contents.
 2844  */
 2845 void
 2846 pmap_zero_page(vm_page_t m)
 2847 {
 2848         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2849 
 2850         pagezero((void *)va);
 2851 }
 2852 
 2853 /*
 2854  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 2855  *      the page into KVM and using bzero to clear its contents.
 2856  *
 2857  *      off and size may not cover an area beyond a single hardware page.
 2858  */
 2859 void
 2860 pmap_zero_page_area(vm_page_t m, int off, int size)
 2861 {
 2862         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2863 
 2864         if (off == 0 && size == PAGE_SIZE)
 2865                 pagezero((void *)va);
 2866         else
 2867                 bzero((char *)va + off, size);
 2868 }
 2869 
 2870 /*
 2871  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 2872  *      the page into KVM and using bzero to clear its contents.  This
 2873  *      is intended to be called from the vm_pagezero process only and
 2874  *      outside of Giant.
 2875  */
 2876 void
 2877 pmap_zero_page_idle(vm_page_t m)
 2878 {
 2879         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2880 
 2881         pagezero((void *)va);
 2882 }
 2883 
 2884 /*
 2885  *      pmap_copy_page copies the specified (machine independent)
 2886  *      page by mapping the page into virtual memory and using
 2887  *      bcopy to copy the page, one machine dependent page at a
 2888  *      time.
 2889  */
 2890 void
 2891 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 2892 {
 2893         vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
 2894         vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
 2895 
 2896         pagecopy((void *)src, (void *)dst);
 2897 }
 2898 
 2899 /*
 2900  * Returns true if the pmap's pv is one of the first
 2901  * 16 pvs linked to from this page.  This count may
 2902  * be changed upwards or downwards in the future; it
 2903  * is only necessary that true be returned for a small
 2904  * subset of pmaps for proper page aging.
 2905  */
 2906 boolean_t
 2907 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 2908 {
 2909         pv_entry_t pv;
 2910         int loops = 0;
 2911 
 2912         if (m->flags & PG_FICTITIOUS)
 2913                 return FALSE;
 2914 
 2915         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2916         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2917                 if (PV_PMAP(pv) == pmap) {
 2918                         return TRUE;
 2919                 }
 2920                 loops++;
 2921                 if (loops >= 16)
 2922                         break;
 2923         }
 2924         return (FALSE);
 2925 }
 2926 
 2927 /*
 2928  * Remove all pages from specified address space
 2929  * this aids process exit speeds.  Also, this code
 2930  * is special cased for current process only, but
 2931  * can have the more generic (and slightly slower)
 2932  * mode enabled.  This is much faster than pmap_remove
 2933  * in the case of running down an entire address space.
 2934  */
 2935 void
 2936 pmap_remove_pages(pmap_t pmap)
 2937 {
 2938         pt_entry_t *pte, tpte;
 2939         vm_page_t m, free = NULL;
 2940         pv_entry_t pv;
 2941         struct pv_chunk *pc, *npc;
 2942         int field, idx;
 2943         int64_t bit;
 2944         uint64_t inuse, bitmask;
 2945         int allfree;
 2946 
 2947         if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
 2948                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 2949                 return;
 2950         }
 2951         vm_page_lock_queues();
 2952         PMAP_LOCK(pmap);
 2953         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 2954                 allfree = 1;
 2955                 for (field = 0; field < _NPCM; field++) {
 2956                         inuse = (~(pc->pc_map[field])) & pc_freemask[field];
 2957                         while (inuse != 0) {
 2958                                 bit = bsfq(inuse);
 2959                                 bitmask = 1UL << bit;
 2960                                 idx = field * 64 + bit;
 2961                                 pv = &pc->pc_pventry[idx];
 2962                                 inuse &= ~bitmask;
 2963 
 2964                                 pte = vtopte(pv->pv_va);
 2965                                 tpte = *pte;
 2966 
 2967                                 if (tpte == 0) {
 2968                                         printf(
 2969                                             "TPTE at %p  IS ZERO @ VA %08lx\n",
 2970                                             pte, pv->pv_va);
 2971                                         panic("bad pte");
 2972                                 }
 2973 
 2974 /*
 2975  * We cannot remove wired pages from a process' mapping at this time
 2976  */
 2977                                 if (tpte & PG_W) {
 2978                                         allfree = 0;
 2979                                         continue;
 2980                                 }
 2981 
 2982                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2983                                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 2984                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 2985                                     m, (uintmax_t)m->phys_addr,
 2986                                     (uintmax_t)tpte));
 2987 
 2988                                 KASSERT(m < &vm_page_array[vm_page_array_size],
 2989                                         ("pmap_remove_pages: bad tpte %#jx",
 2990                                         (uintmax_t)tpte));
 2991 
 2992                                 pmap->pm_stats.resident_count--;
 2993 
 2994                                 pte_clear(pte);
 2995 
 2996                                 /*
 2997                                  * Update the vm_page_t clean/reference bits.
 2998                                  */
 2999                                 if (tpte & PG_M)
 3000                                         vm_page_dirty(m);
 3001 
 3002                                 /* Mark free */
 3003                                 PV_STAT(pv_entry_frees++);
 3004                                 PV_STAT(pv_entry_spare++);
 3005                                 pv_entry_count--;
 3006                                 pc->pc_map[field] |= bitmask;
 3007                                 m->md.pv_list_count--;
 3008                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 3009                                 if (TAILQ_EMPTY(&m->md.pv_list))
 3010                                         vm_page_flag_clear(m, PG_WRITEABLE);
 3011                                 pmap_unuse_pt(pmap, pv->pv_va,
 3012                                     *vtopde(pv->pv_va), &free);
 3013                         }
 3014                 }
 3015                 if (allfree) {
 3016                         PV_STAT(pv_entry_spare -= _NPCPV);
 3017                         PV_STAT(pc_chunk_count--);
 3018                         PV_STAT(pc_chunk_frees++);
 3019                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3020                         m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 3021                         dump_drop_page(m->phys_addr);
 3022                         vm_page_free(m);
 3023                 }
 3024         }
 3025         pmap_invalidate_all(pmap);
 3026         vm_page_unlock_queues();
 3027         PMAP_UNLOCK(pmap);
 3028         pmap_free_zero_pages(free);
 3029 }
 3030 
 3031 /*
 3032  *      pmap_is_modified:
 3033  *
 3034  *      Return whether or not the specified physical page was modified
 3035  *      in any physical maps.
 3036  */
 3037 boolean_t
 3038 pmap_is_modified(vm_page_t m)
 3039 {
 3040         pv_entry_t pv;
 3041         pt_entry_t *pte;
 3042         pmap_t pmap;
 3043         boolean_t rv;
 3044 
 3045         rv = FALSE;
 3046         if (m->flags & PG_FICTITIOUS)
 3047                 return (rv);
 3048 
 3049         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3050         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3051                 pmap = PV_PMAP(pv);
 3052                 PMAP_LOCK(pmap);
 3053                 pte = pmap_pte(pmap, pv->pv_va);
 3054                 rv = (*pte & PG_M) != 0;
 3055                 PMAP_UNLOCK(pmap);
 3056                 if (rv)
 3057                         break;
 3058         }
 3059         return (rv);
 3060 }
 3061 
 3062 /*
 3063  *      pmap_is_prefaultable:
 3064  *
 3065  *      Return whether or not the specified virtual address is elgible
 3066  *      for prefault.
 3067  */
 3068 boolean_t
 3069 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 3070 {
 3071         pd_entry_t *pde;
 3072         pt_entry_t *pte;
 3073         boolean_t rv;
 3074 
 3075         rv = FALSE;
 3076         PMAP_LOCK(pmap);
 3077         pde = pmap_pde(pmap, addr);
 3078         if (pde != NULL && (*pde & PG_V)) {
 3079                 pte = vtopte(addr);
 3080                 rv = (*pte & PG_V) == 0;
 3081         }
 3082         PMAP_UNLOCK(pmap);
 3083         return (rv);
 3084 }
 3085 
 3086 /*
 3087  * Clear the write and modified bits in each of the given page's mappings.
 3088  */
 3089 void
 3090 pmap_remove_write(vm_page_t m)
 3091 {
 3092         pv_entry_t pv;
 3093         pmap_t pmap;
 3094         pt_entry_t oldpte, *pte;
 3095 
 3096         if ((m->flags & PG_FICTITIOUS) != 0 ||
 3097             (m->flags & PG_WRITEABLE) == 0)
 3098                 return;
 3099         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3100         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3101                 pmap = PV_PMAP(pv);
 3102                 PMAP_LOCK(pmap);
 3103                 pte = pmap_pte(pmap, pv->pv_va);
 3104 retry:
 3105                 oldpte = *pte;
 3106                 if (oldpte & PG_RW) {
 3107                         if (!atomic_cmpset_long(pte, oldpte, oldpte &
 3108                             ~(PG_RW | PG_M)))
 3109                                 goto retry;
 3110                         if ((oldpte & PG_M) != 0)
 3111                                 vm_page_dirty(m);
 3112                         pmap_invalidate_page(pmap, pv->pv_va);
 3113                 }
 3114                 PMAP_UNLOCK(pmap);
 3115         }
 3116         vm_page_flag_clear(m, PG_WRITEABLE);
 3117 }
 3118 
 3119 /*
 3120  *      pmap_ts_referenced:
 3121  *
 3122  *      Return a count of reference bits for a page, clearing those bits.
 3123  *      It is not necessary for every reference bit to be cleared, but it
 3124  *      is necessary that 0 only be returned when there are truly no
 3125  *      reference bits set.
 3126  *
 3127  *      XXX: The exact number of bits to check and clear is a matter that
 3128  *      should be tested and standardized at some point in the future for
 3129  *      optimal aging of shared pages.
 3130  */
 3131 int
 3132 pmap_ts_referenced(vm_page_t m)
 3133 {
 3134         pv_entry_t pv, pvf, pvn;
 3135         pmap_t pmap;
 3136         pt_entry_t *pte;
 3137         int rtval = 0;
 3138 
 3139         if (m->flags & PG_FICTITIOUS)
 3140                 return (rtval);
 3141         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3142         if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 3143                 pvf = pv;
 3144                 do {
 3145                         pvn = TAILQ_NEXT(pv, pv_list);
 3146                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 3147                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 3148                         pmap = PV_PMAP(pv);
 3149                         PMAP_LOCK(pmap);
 3150                         pte = pmap_pte(pmap, pv->pv_va);
 3151                         if ((*pte & PG_A) != 0) {
 3152                                 atomic_clear_long(pte, PG_A);
 3153                                 pmap_invalidate_page(pmap, pv->pv_va);
 3154                                 rtval++;
 3155                                 if (rtval > 4)
 3156                                         pvn = NULL;
 3157                         }
 3158                         PMAP_UNLOCK(pmap);
 3159                 } while ((pv = pvn) != NULL && pv != pvf);
 3160         }
 3161         return (rtval);
 3162 }
 3163 
 3164 /*
 3165  *      Clear the modify bits on the specified physical page.
 3166  */
 3167 void
 3168 pmap_clear_modify(vm_page_t m)
 3169 {
 3170         pv_entry_t pv;
 3171         pmap_t pmap;
 3172         pt_entry_t *pte;
 3173 
 3174         if ((m->flags & PG_FICTITIOUS) != 0)
 3175                 return;
 3176         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3177         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3178                 pmap = PV_PMAP(pv);
 3179                 PMAP_LOCK(pmap);
 3180                 pte = pmap_pte(pmap, pv->pv_va);
 3181                 if (*pte & PG_M) {
 3182                         atomic_clear_long(pte, PG_M);
 3183                         pmap_invalidate_page(pmap, pv->pv_va);
 3184                 }
 3185                 PMAP_UNLOCK(pmap);
 3186         }
 3187 }
 3188 
 3189 /*
 3190  *      pmap_clear_reference:
 3191  *
 3192  *      Clear the reference bit on the specified physical page.
 3193  */
 3194 void
 3195 pmap_clear_reference(vm_page_t m)
 3196 {
 3197         pv_entry_t pv;
 3198         pmap_t pmap;
 3199         pt_entry_t *pte;
 3200 
 3201         if ((m->flags & PG_FICTITIOUS) != 0)
 3202                 return;
 3203         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 3204         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3205                 pmap = PV_PMAP(pv);
 3206                 PMAP_LOCK(pmap);
 3207                 pte = pmap_pte(pmap, pv->pv_va);
 3208                 if (*pte & PG_A) {
 3209                         atomic_clear_long(pte, PG_A);
 3210                         pmap_invalidate_page(pmap, pv->pv_va);
 3211                 }
 3212                 PMAP_UNLOCK(pmap);
 3213         }
 3214 }
 3215 
 3216 /*
 3217  * Miscellaneous support routines follow
 3218  */
 3219 
 3220 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
 3221 static __inline void
 3222 pmap_pte_attr(vm_offset_t va, int mode)
 3223 {
 3224         pt_entry_t *pte;
 3225         u_int opte, npte;
 3226 
 3227         pte = vtopte(va);
 3228 
 3229         /*
 3230          * The cache mode bits are all in the low 32-bits of the
 3231          * PTE, so we can just spin on updating the low 32-bits.
 3232          */
 3233         do {
 3234                 opte = *(u_int *)pte;
 3235                 npte = opte & ~(PG_PTE_PAT | PG_NC_PCD | PG_NC_PWT);
 3236                 npte |= pmap_cache_bits(mode, 0);
 3237         } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
 3238 }
 3239 
 3240 /* Adjust the cache mode for a 2MB page mapped via a PDE. */
 3241 static __inline void
 3242 pmap_pde_attr(vm_offset_t va, int mode)
 3243 {
 3244         pd_entry_t *pde;
 3245         u_int opde, npde;
 3246 
 3247         pde = pmap_pde(kernel_pmap, va);
 3248 
 3249         /*
 3250          * The cache mode bits are all in the low 32-bits of the
 3251          * PDE, so we can just spin on updating the low 32-bits.
 3252          */
 3253         do {
 3254                 opde = *(u_int *)pde;
 3255                 npde = opde & ~(PG_PDE_PAT | PG_NC_PCD | PG_NC_PWT);
 3256                 npde |= pmap_cache_bits(mode, 1);
 3257         } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
 3258 }
 3259 
 3260 /*
 3261  * Map a set of physical memory pages into the kernel virtual
 3262  * address space. Return a pointer to where it is mapped. This
 3263  * routine is intended to be used for mapping device memory,
 3264  * NOT real memory.
 3265  */
 3266 void *
 3267 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 3268 {
 3269         vm_offset_t va, tmpva, offset;
 3270 
 3271         /*
 3272          * If this fits within the direct map window and use WB caching
 3273          * mode, use the direct map.
 3274          */
 3275         if (pa < dmaplimit && (pa + size) < dmaplimit && mode == PAT_WRITE_BACK)
 3276                 return ((void *)PHYS_TO_DMAP(pa));
 3277         offset = pa & PAGE_MASK;
 3278         size = roundup(offset + size, PAGE_SIZE);
 3279         va = kmem_alloc_nofault(kernel_map, size);
 3280         if (!va)
 3281                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 3282         pa = trunc_page(pa);
 3283         for (tmpva = va; size > 0; ) {
 3284                 pmap_kenter_attr(tmpva, pa, mode);
 3285                 size -= PAGE_SIZE;
 3286                 tmpva += PAGE_SIZE;
 3287                 pa += PAGE_SIZE;
 3288         }
 3289         pmap_invalidate_range(kernel_pmap, va, tmpva);
 3290         pmap_invalidate_cache();
 3291         return ((void *)(va + offset));
 3292 }
 3293 
 3294 void *
 3295 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 3296 {
 3297 
 3298         return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
 3299 }
 3300 
 3301 void *
 3302 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
 3303 {
 3304 
 3305         return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
 3306 }
 3307 
 3308 void
 3309 pmap_unmapdev(vm_offset_t va, vm_size_t size)
 3310 {
 3311         vm_offset_t base, offset, tmpva;
 3312 
 3313         /* If we gave a direct map region in pmap_mapdev, do nothing */
 3314         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
 3315                 return;
 3316         base = trunc_page(va);
 3317         offset = va & PAGE_MASK;
 3318         size = roundup(offset + size, PAGE_SIZE);
 3319         for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE)
 3320                 pmap_kremove(tmpva);
 3321         pmap_invalidate_range(kernel_pmap, va, tmpva);
 3322         kmem_free(kernel_map, base, size);
 3323 }
 3324 
 3325 int
 3326 pmap_change_attr(va, size, mode)
 3327         vm_offset_t va;
 3328         vm_size_t size;
 3329         int mode;
 3330 {
 3331         vm_offset_t base, offset, tmpva;
 3332         pd_entry_t *pde;
 3333         pt_entry_t *pte;
 3334 
 3335         base = trunc_page(va);
 3336         offset = va & PAGE_MASK;
 3337         size = roundup(offset + size, PAGE_SIZE);
 3338 
 3339         /* Only supported on kernel virtual addresses. */
 3340         if (base <= VM_MAXUSER_ADDRESS)
 3341                 return (EINVAL);
 3342 
 3343         /*
 3344          * XXX: We have to support tearing 2MB pages down into 4k pages if
 3345          * needed here.
 3346          */
 3347         /* Pages that aren't mapped aren't supported. */
 3348         for (tmpva = base; tmpva < (base + size); ) {
 3349                 pde = pmap_pde(kernel_pmap, tmpva);
 3350                 if (*pde == 0)
 3351                         return (EINVAL);
 3352                 if (*pde & PG_PS) {
 3353                         /* Handle 2MB pages that are completely contained. */
 3354                         if (size >= NBPDR) {
 3355                                 tmpva += NBPDR;
 3356                                 continue;
 3357                         }
 3358                         return (EINVAL);
 3359                 }
 3360                 pte = vtopte(va);
 3361                 if (*pte == 0)
 3362                         return (EINVAL);
 3363                 tmpva += PAGE_SIZE;
 3364         }
 3365 
 3366         /*
 3367          * Ok, all the pages exist, so run through them updating their
 3368          * cache mode.
 3369          */
 3370         for (tmpva = base; size > 0; ) {
 3371                 pde = pmap_pde(kernel_pmap, tmpva);
 3372                 if (*pde & PG_PS) {
 3373                         pmap_pde_attr(tmpva, mode);
 3374                         tmpva += NBPDR;
 3375                         size -= NBPDR;
 3376                 } else {
 3377                         pmap_pte_attr(tmpva, mode);
 3378                         tmpva += PAGE_SIZE;
 3379                         size -= PAGE_SIZE;
 3380                 }
 3381         }
 3382 
 3383         /*
 3384          * Flush CPU caches to make sure any data isn't cached that shouldn't
 3385          * be, etc.
 3386          */    
 3387         pmap_invalidate_range(kernel_pmap, base, tmpva);
 3388         pmap_invalidate_cache();
 3389         return (0);
 3390 }
 3391 
 3392 /*
 3393  * perform the pmap work for mincore
 3394  */
 3395 int
 3396 pmap_mincore(pmap_t pmap, vm_offset_t addr)
 3397 {
 3398         pt_entry_t *ptep, pte;
 3399         vm_page_t m;
 3400         int val = 0;
 3401         
 3402         PMAP_LOCK(pmap);
 3403         ptep = pmap_pte(pmap, addr);
 3404         pte = (ptep != NULL) ? *ptep : 0;
 3405         PMAP_UNLOCK(pmap);
 3406 
 3407         if (pte != 0) {
 3408                 vm_paddr_t pa;
 3409 
 3410                 val = MINCORE_INCORE;
 3411                 if ((pte & PG_MANAGED) == 0)
 3412                         return val;
 3413 
 3414                 pa = pte & PG_FRAME;
 3415 
 3416                 m = PHYS_TO_VM_PAGE(pa);
 3417 
 3418                 /*
 3419                  * Modified by us
 3420                  */
 3421                 if (pte & PG_M)
 3422                         val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
 3423                 else {
 3424                         /*
 3425                          * Modified by someone else
 3426                          */
 3427                         vm_page_lock_queues();
 3428                         if (m->dirty || pmap_is_modified(m))
 3429                                 val |= MINCORE_MODIFIED_OTHER;
 3430                         vm_page_unlock_queues();
 3431                 }
 3432                 /*
 3433                  * Referenced by us
 3434                  */
 3435                 if (pte & PG_A)
 3436                         val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
 3437                 else {
 3438                         /*
 3439                          * Referenced by someone else
 3440                          */
 3441                         vm_page_lock_queues();
 3442                         if ((m->flags & PG_REFERENCED) ||
 3443                             pmap_ts_referenced(m)) {
 3444                                 val |= MINCORE_REFERENCED_OTHER;
 3445                                 vm_page_flag_set(m, PG_REFERENCED);
 3446                         }
 3447                         vm_page_unlock_queues();
 3448                 }
 3449         } 
 3450         return val;
 3451 }
 3452 
 3453 void
 3454 pmap_activate(struct thread *td)
 3455 {
 3456         pmap_t  pmap, oldpmap;
 3457         u_int64_t  cr3;
 3458 
 3459         critical_enter();
 3460         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 3461         oldpmap = PCPU_GET(curpmap);
 3462 #ifdef SMP
 3463 if (oldpmap)    /* XXX FIXME */
 3464         atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
 3465         atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
 3466 #else
 3467 if (oldpmap)    /* XXX FIXME */
 3468         oldpmap->pm_active &= ~PCPU_GET(cpumask);
 3469         pmap->pm_active |= PCPU_GET(cpumask);
 3470 #endif
 3471         cr3 = vtophys(pmap->pm_pml4);
 3472         td->td_pcb->pcb_cr3 = cr3;
 3473         load_cr3(cr3);
 3474         critical_exit();
 3475 }
 3476 
 3477 vm_offset_t
 3478 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
 3479 {
 3480 
 3481         if ((obj == NULL) || (size < NBPDR) || (obj->type != OBJT_DEVICE)) {
 3482                 return addr;
 3483         }
 3484 
 3485         addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
 3486         return addr;
 3487 }

Cache object: 8d9343566b232477b562abd441d93160


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.