The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/pmap.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2003 Peter Wemm
    9  * All rights reserved.
   10  *
   11  * This code is derived from software contributed to Berkeley by
   12  * the Systems Programming Group of the University of Utah Computer
   13  * Science Department and William Jolitz of UUNET Technologies Inc.
   14  *
   15  * Redistribution and use in source and binary forms, with or without
   16  * modification, are permitted provided that the following conditions
   17  * are met:
   18  * 1. Redistributions of source code must retain the above copyright
   19  *    notice, this list of conditions and the following disclaimer.
   20  * 2. Redistributions in binary form must reproduce the above copyright
   21  *    notice, this list of conditions and the following disclaimer in the
   22  *    documentation and/or other materials provided with the distribution.
   23  * 3. All advertising materials mentioning features or use of this software
   24  *    must display the following acknowledgement:
   25  *      This product includes software developed by the University of
   26  *      California, Berkeley and its contributors.
   27  * 4. Neither the name of the University nor the names of its contributors
   28  *    may be used to endorse or promote products derived from this software
   29  *    without specific prior written permission.
   30  *
   31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   41  * SUCH DAMAGE.
   42  *
   43  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   44  */
   45 /*-
   46  * Copyright (c) 2003 Networks Associates Technology, Inc.
   47  * All rights reserved.
   48  *
   49  * This software was developed for the FreeBSD Project by Jake Burkholder,
   50  * Safeport Network Services, and Network Associates Laboratories, the
   51  * Security Research Division of Network Associates, Inc. under
   52  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   53  * CHATS research program.
   54  *
   55  * Redistribution and use in source and binary forms, with or without
   56  * modification, are permitted provided that the following conditions
   57  * are met:
   58  * 1. Redistributions of source code must retain the above copyright
   59  *    notice, this list of conditions and the following disclaimer.
   60  * 2. Redistributions in binary form must reproduce the above copyright
   61  *    notice, this list of conditions and the following disclaimer in the
   62  *    documentation and/or other materials provided with the distribution.
   63  *
   64  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   65  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   66  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   67  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   68  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   69  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   70  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   71  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   72  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   73  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   74  * SUCH DAMAGE.
   75  */
   76 
   77 #include <sys/cdefs.h>
   78 __FBSDID("$FreeBSD: releng/5.4/sys/amd64/amd64/pmap.c 145335 2005-04-20 19:11:07Z cvs2svn $");
   79 
   80 /*
   81  *      Manages physical address maps.
   82  *
   83  *      In addition to hardware address maps, this
   84  *      module is called upon to provide software-use-only
   85  *      maps which may or may not be stored in the same
   86  *      form as hardware maps.  These pseudo-maps are
   87  *      used to store intermediate results from copy
   88  *      operations to and from address spaces.
   89  *
   90  *      Since the information managed by this module is
   91  *      also stored by the logical address mapping module,
   92  *      this module may throw away valid virtual-to-physical
   93  *      mappings at almost any time.  However, invalidations
   94  *      of virtual-to-physical mappings must be done as
   95  *      requested.
   96  *
   97  *      In order to cope with hardware architectures which
   98  *      make virtual-to-physical map invalidates expensive,
   99  *      this module may delay invalidate or reduced protection
  100  *      operations until such time as they are actually
  101  *      necessary.  This module is given full information as
  102  *      to which processors are currently using which maps,
  103  *      and to when physical maps must be made correct.
  104  */
  105 
  106 #include "opt_msgbuf.h"
  107 #include "opt_kstack_pages.h"
  108 
  109 #include <sys/param.h>
  110 #include <sys/systm.h>
  111 #include <sys/kernel.h>
  112 #include <sys/lock.h>
  113 #include <sys/malloc.h>
  114 #include <sys/mman.h>
  115 #include <sys/msgbuf.h>
  116 #include <sys/mutex.h>
  117 #include <sys/proc.h>
  118 #include <sys/sx.h>
  119 #include <sys/vmmeter.h>
  120 #include <sys/sched.h>
  121 #include <sys/sysctl.h>
  122 #ifdef SMP
  123 #include <sys/smp.h>
  124 #endif
  125 
  126 #include <vm/vm.h>
  127 #include <vm/vm_param.h>
  128 #include <vm/vm_kern.h>
  129 #include <vm/vm_page.h>
  130 #include <vm/vm_map.h>
  131 #include <vm/vm_object.h>
  132 #include <vm/vm_extern.h>
  133 #include <vm/vm_pageout.h>
  134 #include <vm/vm_pager.h>
  135 #include <vm/uma.h>
  136 
  137 #include <machine/cpu.h>
  138 #include <machine/cputypes.h>
  139 #include <machine/md_var.h>
  140 #include <machine/pcb.h>
  141 #include <machine/specialreg.h>
  142 #ifdef SMP
  143 #include <machine/smp.h>
  144 #endif
  145 
  146 #ifndef PMAP_SHPGPERPROC
  147 #define PMAP_SHPGPERPROC 200
  148 #endif
  149 
  150 #if defined(DIAGNOSTIC)
  151 #define PMAP_DIAGNOSTIC
  152 #endif
  153 
  154 #define MINPV 2048
  155 
  156 #if !defined(PMAP_DIAGNOSTIC)
  157 #define PMAP_INLINE __inline
  158 #else
  159 #define PMAP_INLINE
  160 #endif
  161 
  162 struct pmap kernel_pmap_store;
  163 
  164 vm_paddr_t avail_start;         /* PA of first available physical page */
  165 vm_paddr_t avail_end;           /* PA of last available physical page */
  166 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  167 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  168 static boolean_t pmap_initialized = FALSE;      /* Has pmap_init completed? */
  169 
  170 static int nkpt;
  171 static int ndmpdp;
  172 static vm_paddr_t dmaplimit;
  173 vm_offset_t kernel_vm_end;
  174 pt_entry_t pg_nx;
  175 
  176 static u_int64_t        KPTphys;        /* phys addr of kernel level 1 */
  177 static u_int64_t        KPDphys;        /* phys addr of kernel level 2 */
  178 static u_int64_t        KPDPphys;       /* phys addr of kernel level 3 */
  179 u_int64_t               KPML4phys;      /* phys addr of kernel level 4 */
  180 
  181 static u_int64_t        DMPDphys;       /* phys addr of direct mapped level 2 */
  182 static u_int64_t        DMPDPphys;      /* phys addr of direct mapped level 3 */
  183 
  184 /*
  185  * Data for the pv entry allocation mechanism
  186  */
  187 static uma_zone_t pvzone;
  188 static struct vm_object pvzone_obj;
  189 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  190 int pmap_pagedaemon_waken;
  191 
  192 /*
  193  * All those kernel PT submaps that BSD is so fond of
  194  */
  195 pt_entry_t *CMAP1 = 0;
  196 caddr_t CADDR1 = 0;
  197 struct msgbuf *msgbufp = 0;
  198 
  199 /*
  200  * Crashdump maps.
  201  */
  202 static caddr_t crashdumpmap;
  203 
  204 static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
  205 static pv_entry_t get_pv_entry(void);
  206 static void     pmap_clear_ptes(vm_page_t m, long bit);
  207 
  208 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
  209                 vm_offset_t sva, pd_entry_t ptepde);
  210 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va);
  211 static int pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  212                 vm_offset_t va, pd_entry_t ptepde);
  213 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  214 
  215 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
  216 
  217 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
  218 static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m);
  219 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t);
  220 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
  221 
  222 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  223 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  224 
  225 /*
  226  * Move the kernel virtual free pointer to the next
  227  * 2MB.  This is used to help improve performance
  228  * by using a large (2MB) page for much of the kernel
  229  * (.text, .data, .bss)
  230  */
  231 static vm_offset_t
  232 pmap_kmem_choose(vm_offset_t addr)
  233 {
  234         vm_offset_t newaddr = addr;
  235 
  236         newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
  237         return newaddr;
  238 }
  239 
  240 /********************/
  241 /* Inline functions */
  242 /********************/
  243 
  244 /* Return a non-clipped PD index for a given VA */
  245 static __inline vm_pindex_t
  246 pmap_pde_pindex(vm_offset_t va)
  247 {
  248         return va >> PDRSHIFT;
  249 }
  250 
  251 
  252 /* Return various clipped indexes for a given VA */
  253 static __inline vm_pindex_t
  254 pmap_pte_index(vm_offset_t va)
  255 {
  256 
  257         return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
  258 }
  259 
  260 static __inline vm_pindex_t
  261 pmap_pde_index(vm_offset_t va)
  262 {
  263 
  264         return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
  265 }
  266 
  267 static __inline vm_pindex_t
  268 pmap_pdpe_index(vm_offset_t va)
  269 {
  270 
  271         return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
  272 }
  273 
  274 static __inline vm_pindex_t
  275 pmap_pml4e_index(vm_offset_t va)
  276 {
  277 
  278         return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
  279 }
  280 
  281 /* Return a pointer to the PML4 slot that corresponds to a VA */
  282 static __inline pml4_entry_t *
  283 pmap_pml4e(pmap_t pmap, vm_offset_t va)
  284 {
  285 
  286         if (!pmap)
  287                 return NULL;
  288         return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
  289 }
  290 
  291 /* Return a pointer to the PDP slot that corresponds to a VA */
  292 static __inline pdp_entry_t *
  293 pmap_pdpe(pmap_t pmap, vm_offset_t va)
  294 {
  295         pml4_entry_t *pml4e;
  296         pdp_entry_t *pdpe;
  297 
  298         pml4e = pmap_pml4e(pmap, va);
  299         if (pml4e == NULL || (*pml4e & PG_V) == 0)
  300                 return NULL;
  301         pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
  302         return (&pdpe[pmap_pdpe_index(va)]);
  303 }
  304 
  305 /* Return a pointer to the PD slot that corresponds to a VA */
  306 static __inline pd_entry_t *
  307 pmap_pde(pmap_t pmap, vm_offset_t va)
  308 {
  309         pdp_entry_t *pdpe;
  310         pd_entry_t *pde;
  311 
  312         pdpe = pmap_pdpe(pmap, va);
  313         if (pdpe == NULL || (*pdpe & PG_V) == 0)
  314                  return NULL;
  315         pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
  316         return (&pde[pmap_pde_index(va)]);
  317 }
  318 
  319 /* Return a pointer to the PT slot that corresponds to a VA */
  320 static __inline pt_entry_t *
  321 pmap_pte(pmap_t pmap, vm_offset_t va)
  322 {
  323         pd_entry_t *pde;
  324         pt_entry_t *pte;
  325 
  326         pde = pmap_pde(pmap, va);
  327         if (pde == NULL || (*pde & PG_V) == 0)
  328                 return NULL;
  329         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  330                 return ((pt_entry_t *)pde);
  331         pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
  332         return (&pte[pmap_pte_index(va)]);
  333 }
  334 
  335 
  336 static __inline pt_entry_t *
  337 pmap_pte_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *ptepde)
  338 {
  339         pd_entry_t *pde;
  340         pt_entry_t *pte;
  341 
  342         pde = pmap_pde(pmap, va);
  343         if (pde == NULL || (*pde & PG_V) == 0)
  344                 return NULL;
  345         *ptepde = *pde;
  346         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  347                 return ((pt_entry_t *)pde);
  348         pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
  349         return (&pte[pmap_pte_index(va)]);
  350 }
  351 
  352 
  353 PMAP_INLINE pt_entry_t *
  354 vtopte(vm_offset_t va)
  355 {
  356         u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  357 
  358         return (PTmap + ((va >> PAGE_SHIFT) & mask));
  359 }
  360 
  361 static __inline pd_entry_t *
  362 vtopde(vm_offset_t va)
  363 {
  364         u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  365 
  366         return (PDmap + ((va >> PDRSHIFT) & mask));
  367 }
  368 
  369 static u_int64_t
  370 allocpages(int n)
  371 {
  372         u_int64_t ret;
  373 
  374         ret = avail_start;
  375         bzero((void *)ret, n * PAGE_SIZE);
  376         avail_start += n * PAGE_SIZE;
  377         return (ret);
  378 }
  379 
  380 static void
  381 create_pagetables(void)
  382 {
  383         int i;
  384 
  385         /* Allocate pages */
  386         KPTphys = allocpages(NKPT);
  387         KPML4phys = allocpages(1);
  388         KPDPphys = allocpages(NKPML4E);
  389         KPDphys = allocpages(NKPDPE);
  390 
  391         ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
  392         if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
  393                 ndmpdp = 4;
  394         DMPDPphys = allocpages(NDMPML4E);
  395         DMPDphys = allocpages(ndmpdp);
  396         dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
  397 
  398         /* Fill in the underlying page table pages */
  399         /* Read-only from zero to physfree */
  400         /* XXX not fully used, underneath 2M pages */
  401         for (i = 0; (i << PAGE_SHIFT) < avail_start; i++) {
  402                 ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
  403                 ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V | PG_G;
  404         }
  405 
  406         /* Now map the page tables at their location within PTmap */
  407         for (i = 0; i < NKPT; i++) {
  408                 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
  409                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
  410         }
  411 
  412         /* Map from zero to end of allocations under 2M pages */
  413         /* This replaces some of the KPTphys entries above */
  414         for (i = 0; (i << PDRSHIFT) < avail_start; i++) {
  415                 ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
  416                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  417         }
  418 
  419         /* And connect up the PD to the PDP */
  420         for (i = 0; i < NKPDPE; i++) {
  421                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys + (i << PAGE_SHIFT);
  422                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
  423         }
  424 
  425 
  426         /* Now set up the direct map space using 2MB pages */
  427         for (i = 0; i < NPDEPG * ndmpdp; i++) {
  428                 ((pd_entry_t *)DMPDphys)[i] = (vm_paddr_t)i << PDRSHIFT;
  429                 ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  430         }
  431 
  432         /* And the direct map space's PDP */
  433         for (i = 0; i < ndmpdp; i++) {
  434                 ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + (i << PAGE_SHIFT);
  435                 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
  436         }
  437 
  438         /* And recursively map PML4 to itself in order to get PTmap */
  439         ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
  440         ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
  441 
  442         /* Connect the Direct Map slot up to the PML4 */
  443         ((pdp_entry_t *)KPML4phys)[DMPML4I] = DMPDPphys;
  444         ((pdp_entry_t *)KPML4phys)[DMPML4I] |= PG_RW | PG_V | PG_U;
  445 
  446         /* Connect the KVA slot up to the PML4 */
  447         ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
  448         ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
  449 }
  450 
  451 /*
  452  *      Bootstrap the system enough to run with virtual memory.
  453  *
  454  *      On amd64 this is called after mapping has already been enabled
  455  *      and just syncs the pmap module with what has already been done.
  456  *      [We can't call it easily with mapping off since the kernel is not
  457  *      mapped with PA == VA, hence we would have to relocate every address
  458  *      from the linked base (virtual) address "KERNBASE" to the actual
  459  *      (physical) address starting relative to 0]
  460  */
  461 void
  462 pmap_bootstrap(firstaddr)
  463         vm_paddr_t *firstaddr;
  464 {
  465         vm_offset_t va;
  466         pt_entry_t *pte, *unused;
  467 
  468         avail_start = *firstaddr;
  469 
  470         /*
  471          * Create an initial set of page tables to run the kernel in.
  472          */
  473         create_pagetables();
  474         *firstaddr = avail_start;
  475 
  476         virtual_avail = (vm_offset_t) KERNBASE + avail_start;
  477         virtual_avail = pmap_kmem_choose(virtual_avail);
  478 
  479         virtual_end = VM_MAX_KERNEL_ADDRESS;
  480 
  481 
  482         /* XXX do %cr0 as well */
  483         load_cr4(rcr4() | CR4_PGE | CR4_PSE);
  484         load_cr3(KPML4phys);
  485 
  486         /*
  487          * Initialize the kernel pmap (which is statically allocated).
  488          */
  489         PMAP_LOCK_INIT(kernel_pmap);
  490         kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + KPML4phys);
  491         kernel_pmap->pm_active = -1;    /* don't allow deactivation */
  492         TAILQ_INIT(&kernel_pmap->pm_pvlist);
  493         nkpt = NKPT;
  494 
  495         /*
  496          * Reserve some special page table entries/VA space for temporary
  497          * mapping of pages.
  498          */
  499 #define SYSMAP(c, p, v, n)      \
  500         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  501 
  502         va = virtual_avail;
  503         pte = vtopte(va);
  504 
  505         /*
  506          * CMAP1 is only used for the memory test.
  507          */
  508         SYSMAP(caddr_t, CMAP1, CADDR1, 1)
  509 
  510         /*
  511          * Crashdump maps.
  512          */
  513         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  514 
  515         /*
  516          * msgbufp is used to map the system message buffer.
  517          */
  518         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(MSGBUF_SIZE)))
  519 
  520         virtual_avail = va;
  521 
  522         *CMAP1 = 0;
  523 
  524         invltlb();
  525 }
  526 
  527 /*
  528  *      Initialize the pmap module.
  529  *      Called by vm_init, to initialize any structures that the pmap
  530  *      system needs to map virtual memory.
  531  *      pmap_init has been enhanced to support in a fairly consistant
  532  *      way, discontiguous physical memory.
  533  */
  534 void
  535 pmap_init(void)
  536 {
  537         int i;
  538 
  539         /*
  540          * Allocate memory for random pmap data structures.  Includes the
  541          * pv_head_table.
  542          */
  543 
  544         for(i = 0; i < vm_page_array_size; i++) {
  545                 vm_page_t m;
  546 
  547                 m = &vm_page_array[i];
  548                 TAILQ_INIT(&m->md.pv_list);
  549                 m->md.pv_list_count = 0;
  550         }
  551 
  552         /*
  553          * init the pv free list
  554          */
  555         pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, 
  556             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
  557         uma_prealloc(pvzone, MINPV);
  558 
  559         /*
  560          * Now it is safe to enable pv_table recording.
  561          */
  562         pmap_initialized = TRUE;
  563 }
  564 
  565 /*
  566  * Initialize the address space (zone) for the pv_entries.  Set a
  567  * high water mark so that the system can recover from excessive
  568  * numbers of pv entries.
  569  */
  570 void
  571 pmap_init2()
  572 {
  573         int shpgperproc = PMAP_SHPGPERPROC;
  574 
  575         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  576         pv_entry_max = shpgperproc * maxproc + vm_page_array_size;
  577         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  578         pv_entry_high_water = 9 * (pv_entry_max / 10);
  579         uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max);
  580 }
  581 
  582 
  583 /***************************************************
  584  * Low level helper routines.....
  585  ***************************************************/
  586 
  587 #if defined(PMAP_DIAGNOSTIC)
  588 
  589 /*
  590  * This code checks for non-writeable/modified pages.
  591  * This should be an invalid condition.
  592  */
  593 static int
  594 pmap_nw_modified(pt_entry_t ptea)
  595 {
  596         int pte;
  597 
  598         pte = (int) ptea;
  599 
  600         if ((pte & (PG_M|PG_RW)) == PG_M)
  601                 return 1;
  602         else
  603                 return 0;
  604 }
  605 #endif
  606 
  607 
  608 /*
  609  * this routine defines the region(s) of memory that should
  610  * not be tested for the modified bit.
  611  */
  612 static PMAP_INLINE int
  613 pmap_track_modified(vm_offset_t va)
  614 {
  615         if ((va < kmi.clean_sva) || (va >= kmi.clean_eva)) 
  616                 return 1;
  617         else
  618                 return 0;
  619 }
  620 
  621 #ifdef SMP
  622 /*
  623  * For SMP, these functions have to use the IPI mechanism for coherence.
  624  */
  625 void
  626 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  627 {
  628         u_int cpumask;
  629         u_int other_cpus;
  630 
  631         if (smp_started) {
  632                 if (!(read_rflags() & PSL_I))
  633                         panic("%s: interrupts disabled", __func__);
  634                 mtx_lock_spin(&smp_ipi_mtx);
  635         } else
  636                 critical_enter();
  637         /*
  638          * We need to disable interrupt preemption but MUST NOT have
  639          * interrupts disabled here.
  640          * XXX we may need to hold schedlock to get a coherent pm_active
  641          * XXX critical sections disable interrupts again
  642          */
  643         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  644                 invlpg(va);
  645                 smp_invlpg(va);
  646         } else {
  647                 cpumask = PCPU_GET(cpumask);
  648                 other_cpus = PCPU_GET(other_cpus);
  649                 if (pmap->pm_active & cpumask)
  650                         invlpg(va);
  651                 if (pmap->pm_active & other_cpus)
  652                         smp_masked_invlpg(pmap->pm_active & other_cpus, va);
  653         }
  654         if (smp_started)
  655                 mtx_unlock_spin(&smp_ipi_mtx);
  656         else
  657                 critical_exit();
  658 }
  659 
  660 void
  661 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  662 {
  663         u_int cpumask;
  664         u_int other_cpus;
  665         vm_offset_t addr;
  666 
  667         if (smp_started) {
  668                 if (!(read_rflags() & PSL_I))
  669                         panic("%s: interrupts disabled", __func__);
  670                 mtx_lock_spin(&smp_ipi_mtx);
  671         } else
  672                 critical_enter();
  673         /*
  674          * We need to disable interrupt preemption but MUST NOT have
  675          * interrupts disabled here.
  676          * XXX we may need to hold schedlock to get a coherent pm_active
  677          * XXX critical sections disable interrupts again
  678          */
  679         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  680                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  681                         invlpg(addr);
  682                 smp_invlpg_range(sva, eva);
  683         } else {
  684                 cpumask = PCPU_GET(cpumask);
  685                 other_cpus = PCPU_GET(other_cpus);
  686                 if (pmap->pm_active & cpumask)
  687                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
  688                                 invlpg(addr);
  689                 if (pmap->pm_active & other_cpus)
  690                         smp_masked_invlpg_range(pmap->pm_active & other_cpus,
  691                             sva, eva);
  692         }
  693         if (smp_started)
  694                 mtx_unlock_spin(&smp_ipi_mtx);
  695         else
  696                 critical_exit();
  697 }
  698 
  699 void
  700 pmap_invalidate_all(pmap_t pmap)
  701 {
  702         u_int cpumask;
  703         u_int other_cpus;
  704 
  705         if (smp_started) {
  706                 if (!(read_rflags() & PSL_I))
  707                         panic("%s: interrupts disabled", __func__);
  708                 mtx_lock_spin(&smp_ipi_mtx);
  709         } else
  710                 critical_enter();
  711         /*
  712          * We need to disable interrupt preemption but MUST NOT have
  713          * interrupts disabled here.
  714          * XXX we may need to hold schedlock to get a coherent pm_active
  715          * XXX critical sections disable interrupts again
  716          */
  717         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  718                 invltlb();
  719                 smp_invltlb();
  720         } else {
  721                 cpumask = PCPU_GET(cpumask);
  722                 other_cpus = PCPU_GET(other_cpus);
  723                 if (pmap->pm_active & cpumask)
  724                         invltlb();
  725                 if (pmap->pm_active & other_cpus)
  726                         smp_masked_invltlb(pmap->pm_active & other_cpus);
  727         }
  728         if (smp_started)
  729                 mtx_unlock_spin(&smp_ipi_mtx);
  730         else
  731                 critical_exit();
  732 }
  733 #else /* !SMP */
  734 /*
  735  * Normal, non-SMP, invalidation functions.
  736  * We inline these within pmap.c for speed.
  737  */
  738 PMAP_INLINE void
  739 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  740 {
  741 
  742         if (pmap == kernel_pmap || pmap->pm_active)
  743                 invlpg(va);
  744 }
  745 
  746 PMAP_INLINE void
  747 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  748 {
  749         vm_offset_t addr;
  750 
  751         if (pmap == kernel_pmap || pmap->pm_active)
  752                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  753                         invlpg(addr);
  754 }
  755 
  756 PMAP_INLINE void
  757 pmap_invalidate_all(pmap_t pmap)
  758 {
  759 
  760         if (pmap == kernel_pmap || pmap->pm_active)
  761                 invltlb();
  762 }
  763 #endif /* !SMP */
  764 
  765 /*
  766  * Are we current address space or kernel?
  767  */
  768 static __inline int
  769 pmap_is_current(pmap_t pmap)
  770 {
  771         return (pmap == kernel_pmap ||
  772             (pmap->pm_pml4[PML4PML4I] & PG_FRAME) == (PML4pml4e[0] & PG_FRAME));
  773 }
  774 
  775 /*
  776  *      Routine:        pmap_extract
  777  *      Function:
  778  *              Extract the physical page address associated
  779  *              with the given map/virtual_address pair.
  780  */
  781 vm_paddr_t 
  782 pmap_extract(pmap_t pmap, vm_offset_t va)
  783 {
  784         vm_paddr_t rtval;
  785         pt_entry_t *pte;
  786         pd_entry_t pde, *pdep;
  787 
  788         rtval = 0;
  789         PMAP_LOCK(pmap);
  790         pdep = pmap_pde(pmap, va);
  791         if (pdep != NULL) {
  792                 pde = *pdep;
  793                 if (pde) {
  794                         if ((pde & PG_PS) != 0) {
  795                                 rtval = (pde & ~PDRMASK) | (va & PDRMASK);
  796                                 PMAP_UNLOCK(pmap);
  797                                 return rtval;
  798                         }
  799                         pte = pmap_pte(pmap, va);
  800                         rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
  801                 }
  802         }
  803         PMAP_UNLOCK(pmap);
  804         return (rtval);
  805 }
  806 
  807 /*
  808  *      Routine:        pmap_extract_and_hold
  809  *      Function:
  810  *              Atomically extract and hold the physical page
  811  *              with the given pmap and virtual address pair
  812  *              if that mapping permits the given protection.
  813  */
  814 vm_page_t
  815 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
  816 {
  817         pd_entry_t pde, *pdep;
  818         pt_entry_t pte;
  819         vm_page_t m;
  820 
  821         m = NULL;
  822         vm_page_lock_queues();
  823         PMAP_LOCK(pmap);
  824         pdep = pmap_pde(pmap, va);
  825         if (pdep != NULL && (pde = *pdep)) {
  826                 if (pde & PG_PS) {
  827                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
  828                                 m = PHYS_TO_VM_PAGE((pde & ~PDRMASK) |
  829                                     (va & PDRMASK));
  830                                 vm_page_hold(m);
  831                         }
  832                 } else {
  833                         pte = *pmap_pte(pmap, va);
  834                         if ((pte & PG_V) &&
  835                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
  836                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
  837                                 vm_page_hold(m);
  838                         }
  839                 }
  840         }
  841         vm_page_unlock_queues();
  842         PMAP_UNLOCK(pmap);
  843         return (m);
  844 }
  845 
  846 vm_paddr_t
  847 pmap_kextract(vm_offset_t va)
  848 {
  849         pd_entry_t *pde;
  850         vm_paddr_t pa;
  851 
  852         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
  853                 pa = DMAP_TO_PHYS(va);
  854         } else {
  855                 pde = pmap_pde(kernel_pmap, va);
  856                 if (*pde & PG_PS) {
  857                         pa = (*pde & ~(NBPDR - 1)) | (va & (NBPDR - 1));
  858                 } else {
  859                         pa = *vtopte(va);
  860                         pa = (pa & PG_FRAME) | (va & PAGE_MASK);
  861                 }
  862         }
  863         return pa;
  864 }
  865 
  866 /***************************************************
  867  * Low level mapping routines.....
  868  ***************************************************/
  869 
  870 /*
  871  * Add a wired page to the kva.
  872  * Note: not SMP coherent.
  873  */
  874 PMAP_INLINE void 
  875 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
  876 {
  877         pt_entry_t *pte;
  878 
  879         pte = vtopte(va);
  880         pte_store(pte, pa | PG_RW | PG_V | PG_G);
  881 }
  882 
  883 /*
  884  * Remove a page from the kernel pagetables.
  885  * Note: not SMP coherent.
  886  */
  887 PMAP_INLINE void
  888 pmap_kremove(vm_offset_t va)
  889 {
  890         pt_entry_t *pte;
  891 
  892         pte = vtopte(va);
  893         pte_clear(pte);
  894 }
  895 
  896 /*
  897  *      Used to map a range of physical addresses into kernel
  898  *      virtual address space.
  899  *
  900  *      The value passed in '*virt' is a suggested virtual address for
  901  *      the mapping. Architectures which can support a direct-mapped
  902  *      physical to virtual region can return the appropriate address
  903  *      within that region, leaving '*virt' unchanged. Other
  904  *      architectures should map the pages starting at '*virt' and
  905  *      update '*virt' with the first usable address after the mapped
  906  *      region.
  907  */
  908 vm_offset_t
  909 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
  910 {
  911         return PHYS_TO_DMAP(start);
  912 }
  913 
  914 
  915 /*
  916  * Add a list of wired pages to the kva
  917  * this routine is only used for temporary
  918  * kernel mappings that do not need to have
  919  * page modification or references recorded.
  920  * Note that old mappings are simply written
  921  * over.  The page *must* be wired.
  922  * Note: SMP coherent.  Uses a ranged shootdown IPI.
  923  */
  924 void
  925 pmap_qenter(vm_offset_t sva, vm_page_t *m, int count)
  926 {
  927         vm_offset_t va;
  928 
  929         va = sva;
  930         while (count-- > 0) {
  931                 pmap_kenter(va, VM_PAGE_TO_PHYS(*m));
  932                 va += PAGE_SIZE;
  933                 m++;
  934         }
  935         pmap_invalidate_range(kernel_pmap, sva, va);
  936 }
  937 
  938 /*
  939  * This routine tears out page mappings from the
  940  * kernel -- it is meant only for temporary mappings.
  941  * Note: SMP coherent.  Uses a ranged shootdown IPI.
  942  */
  943 void
  944 pmap_qremove(vm_offset_t sva, int count)
  945 {
  946         vm_offset_t va;
  947 
  948         va = sva;
  949         while (count-- > 0) {
  950                 pmap_kremove(va);
  951                 va += PAGE_SIZE;
  952         }
  953         pmap_invalidate_range(kernel_pmap, sva, va);
  954 }
  955 
  956 /***************************************************
  957  * Page table page management routines.....
  958  ***************************************************/
  959 
  960 /*
  961  * This routine unholds page table pages, and if the hold count
  962  * drops to zero, then it decrements the wire count.
  963  */
  964 static PMAP_INLINE int
  965 pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
  966 {
  967 
  968         --m->wire_count;
  969         if (m->wire_count == 0)
  970                 return _pmap_unwire_pte_hold(pmap, va, m);
  971         else
  972                 return 0;
  973 }
  974 
  975 static int 
  976 _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
  977 {
  978         vm_offset_t pteva;
  979 
  980         /*
  981          * unmap the page table page
  982          */
  983         if (m->pindex >= (NUPDE + NUPDPE)) {
  984                 /* PDP page */
  985                 pml4_entry_t *pml4;
  986                 pml4 = pmap_pml4e(pmap, va);
  987                 pteva = (vm_offset_t) PDPmap + amd64_ptob(m->pindex - (NUPDE + NUPDPE));
  988                 *pml4 = 0;
  989         } else if (m->pindex >= NUPDE) {
  990                 /* PD page */
  991                 pdp_entry_t *pdp;
  992                 pdp = pmap_pdpe(pmap, va);
  993                 pteva = (vm_offset_t) PDmap + amd64_ptob(m->pindex - NUPDE);
  994                 *pdp = 0;
  995         } else {
  996                 /* PTE page */
  997                 pd_entry_t *pd;
  998                 pd = pmap_pde(pmap, va);
  999                 pteva = (vm_offset_t) PTmap + amd64_ptob(m->pindex);
 1000                 *pd = 0;
 1001         }
 1002         --pmap->pm_stats.resident_count;
 1003         if (m->pindex < NUPDE) {
 1004                 /* We just released a PT, unhold the matching PD */
 1005                 vm_page_t pdpg;
 1006 
 1007                 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
 1008                 pmap_unwire_pte_hold(pmap, va, pdpg);
 1009         }
 1010         if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 1011                 /* We just released a PD, unhold the matching PDP */
 1012                 vm_page_t pdppg;
 1013 
 1014                 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
 1015                 pmap_unwire_pte_hold(pmap, va, pdppg);
 1016         }
 1017 
 1018         /*
 1019          * Do an invltlb to make the invalidated mapping
 1020          * take effect immediately.
 1021          */
 1022         pmap_invalidate_page(pmap, pteva);
 1023 
 1024         vm_page_free_zero(m);
 1025         atomic_subtract_int(&cnt.v_wire_count, 1);
 1026         return 1;
 1027 }
 1028 
 1029 /*
 1030  * After removing a page table entry, this routine is used to
 1031  * conditionally free the page, and manage the hold/wire counts.
 1032  */
 1033 static int
 1034 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde)
 1035 {
 1036         vm_page_t mpte;
 1037 
 1038         if (va >= VM_MAXUSER_ADDRESS)
 1039                 return 0;
 1040         KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 1041         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1042         return pmap_unwire_pte_hold(pmap, va, mpte);
 1043 }
 1044 
 1045 void
 1046 pmap_pinit0(pmap)
 1047         struct pmap *pmap;
 1048 {
 1049 
 1050         PMAP_LOCK_INIT(pmap);
 1051         pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + KPML4phys);
 1052         pmap->pm_active = 0;
 1053         TAILQ_INIT(&pmap->pm_pvlist);
 1054         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1055 }
 1056 
 1057 /*
 1058  * Initialize a preallocated and zeroed pmap structure,
 1059  * such as one in a vmspace structure.
 1060  */
 1061 void
 1062 pmap_pinit(pmap)
 1063         register struct pmap *pmap;
 1064 {
 1065         vm_page_t pml4pg;
 1066         static vm_pindex_t color;
 1067 
 1068         PMAP_LOCK_INIT(pmap);
 1069 
 1070         /*
 1071          * allocate the page directory page
 1072          */
 1073         while ((pml4pg = vm_page_alloc(NULL, color++, VM_ALLOC_NOOBJ |
 1074             VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
 1075                 VM_WAIT;
 1076 
 1077         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
 1078 
 1079         if ((pml4pg->flags & PG_ZERO) == 0)
 1080                 pagezero(pmap->pm_pml4);
 1081 
 1082         /* Wire in kernel global address entries. */
 1083         pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
 1084         pmap->pm_pml4[DMPML4I] = DMPDPphys | PG_RW | PG_V | PG_U;
 1085 
 1086         /* install self-referential address mapping entry(s) */
 1087         pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
 1088 
 1089         pmap->pm_active = 0;
 1090         TAILQ_INIT(&pmap->pm_pvlist);
 1091         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1092 }
 1093 
 1094 /*
 1095  * this routine is called if the page table page is not
 1096  * mapped correctly.
 1097  *
 1098  * Note: If a page allocation fails at page table level two or three,
 1099  * one or two pages may be held during the wait, only to be released
 1100  * afterwards.  This conservative approach is easily argued to avoid
 1101  * race conditions.
 1102  */
 1103 static vm_page_t
 1104 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 1105 {
 1106         vm_page_t m, pdppg, pdpg;
 1107 
 1108         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1109             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1110             ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1111 
 1112         /*
 1113          * Allocate a page table page.
 1114          */
 1115         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1116             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1117                 if (flags & M_WAITOK) {
 1118                         PMAP_UNLOCK(pmap);
 1119                         vm_page_unlock_queues();
 1120                         VM_WAIT;
 1121                         vm_page_lock_queues();
 1122                         PMAP_LOCK(pmap);
 1123                 }
 1124 
 1125                 /*
 1126                  * Indicate the need to retry.  While waiting, the page table
 1127                  * page may have been allocated.
 1128                  */
 1129                 return (NULL);
 1130         }
 1131         if ((m->flags & PG_ZERO) == 0)
 1132                 pmap_zero_page(m);
 1133 
 1134         /*
 1135          * Map the pagetable page into the process address space, if
 1136          * it isn't already there.
 1137          */
 1138 
 1139         pmap->pm_stats.resident_count++;
 1140 
 1141         if (ptepindex >= (NUPDE + NUPDPE)) {
 1142                 pml4_entry_t *pml4;
 1143                 vm_pindex_t pml4index;
 1144 
 1145                 /* Wire up a new PDPE page */
 1146                 pml4index = ptepindex - (NUPDE + NUPDPE);
 1147                 pml4 = &pmap->pm_pml4[pml4index];
 1148                 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1149 
 1150         } else if (ptepindex >= NUPDE) {
 1151                 vm_pindex_t pml4index;
 1152                 vm_pindex_t pdpindex;
 1153                 pml4_entry_t *pml4;
 1154                 pdp_entry_t *pdp;
 1155 
 1156                 /* Wire up a new PDE page */
 1157                 pdpindex = ptepindex - NUPDE;
 1158                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1159 
 1160                 pml4 = &pmap->pm_pml4[pml4index];
 1161                 if ((*pml4 & PG_V) == 0) {
 1162                         /* Have to allocate a new pdp, recurse */
 1163                         if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
 1164                             flags) == NULL) {
 1165                                 --m->wire_count;
 1166                                 vm_page_free(m);
 1167                                 return (NULL);
 1168                         }
 1169                 } else {
 1170                         /* Add reference to pdp page */
 1171                         pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
 1172                         pdppg->wire_count++;
 1173                 }
 1174                 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1175 
 1176                 /* Now find the pdp page */
 1177                 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1178                 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1179 
 1180         } else {
 1181                 vm_pindex_t pml4index;
 1182                 vm_pindex_t pdpindex;
 1183                 pml4_entry_t *pml4;
 1184                 pdp_entry_t *pdp;
 1185                 pd_entry_t *pd;
 1186 
 1187                 /* Wire up a new PTE page */
 1188                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1189                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1190 
 1191                 /* First, find the pdp and check that its valid. */
 1192                 pml4 = &pmap->pm_pml4[pml4index];
 1193                 if ((*pml4 & PG_V) == 0) {
 1194                         /* Have to allocate a new pd, recurse */
 1195                         if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1196                             flags) == NULL) {
 1197                                 --m->wire_count;
 1198                                 vm_page_free(m);
 1199                                 return (NULL);
 1200                         }
 1201                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1202                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1203                 } else {
 1204                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1205                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1206                         if ((*pdp & PG_V) == 0) {
 1207                                 /* Have to allocate a new pd, recurse */
 1208                                 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1209                                     flags) == NULL) {
 1210                                         --m->wire_count;
 1211                                         vm_page_free(m);
 1212                                         return (NULL);
 1213                                 }
 1214                         } else {
 1215                                 /* Add reference to the pd page */
 1216                                 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
 1217                                 pdpg->wire_count++;
 1218                         }
 1219                 }
 1220                 pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
 1221 
 1222                 /* Now we know where the page directory page is */
 1223                 pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
 1224                 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1225         }
 1226 
 1227         return m;
 1228 }
 1229 
 1230 static vm_page_t
 1231 pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
 1232 {
 1233         vm_pindex_t ptepindex;
 1234         pd_entry_t *pd;
 1235         vm_page_t m;
 1236 
 1237         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1238             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1239             ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1240 
 1241         /*
 1242          * Calculate pagetable page index
 1243          */
 1244         ptepindex = pmap_pde_pindex(va);
 1245 retry:
 1246         /*
 1247          * Get the page directory entry
 1248          */
 1249         pd = pmap_pde(pmap, va);
 1250 
 1251         /*
 1252          * This supports switching from a 2MB page to a
 1253          * normal 4K page.
 1254          */
 1255         if (pd != 0 && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
 1256                 *pd = 0;
 1257                 pd = 0;
 1258                 pmap_invalidate_all(kernel_pmap);
 1259         }
 1260 
 1261         /*
 1262          * If the page table page is mapped, we just increment the
 1263          * hold count, and activate it.
 1264          */
 1265         if (pd != 0 && (*pd & PG_V) != 0) {
 1266                 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
 1267                 m->wire_count++;
 1268         } else {
 1269                 /*
 1270                  * Here if the pte page isn't mapped, or if it has been
 1271                  * deallocated.
 1272                  */
 1273                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1274                 if (m == NULL && (flags & M_WAITOK))
 1275                         goto retry;
 1276         }
 1277         return (m);
 1278 }
 1279 
 1280 
 1281 /***************************************************
 1282  * Pmap allocation/deallocation routines.
 1283  ***************************************************/
 1284 
 1285 /*
 1286  * Release any resources held by the given physical map.
 1287  * Called when a pmap initialized by pmap_pinit is being released.
 1288  * Should only be called if the map contains no valid mappings.
 1289  */
 1290 void
 1291 pmap_release(pmap_t pmap)
 1292 {
 1293         vm_page_t m;
 1294 
 1295         KASSERT(pmap->pm_stats.resident_count == 0,
 1296             ("pmap_release: pmap resident count %ld != 0",
 1297             pmap->pm_stats.resident_count));
 1298 
 1299         m = PHYS_TO_VM_PAGE(pmap->pm_pml4[PML4PML4I] & PG_FRAME);
 1300 
 1301         pmap->pm_pml4[KPML4I] = 0;      /* KVA */
 1302         pmap->pm_pml4[DMPML4I] = 0;     /* Direct Map */
 1303         pmap->pm_pml4[PML4PML4I] = 0;   /* Recursive Mapping */
 1304 
 1305         vm_page_lock_queues();
 1306         m->wire_count--;
 1307         atomic_subtract_int(&cnt.v_wire_count, 1);
 1308         vm_page_free_zero(m);
 1309         vm_page_unlock_queues();
 1310         PMAP_LOCK_DESTROY(pmap);
 1311 }
 1312 
 1313 static int
 1314 kvm_size(SYSCTL_HANDLER_ARGS)
 1315 {
 1316         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
 1317 
 1318         return sysctl_handle_long(oidp, &ksize, 0, req);
 1319 }
 1320 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 1321     0, 0, kvm_size, "IU", "Size of KVM");
 1322 
 1323 static int
 1324 kvm_free(SYSCTL_HANDLER_ARGS)
 1325 {
 1326         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 1327 
 1328         return sysctl_handle_long(oidp, &kfree, 0, req);
 1329 }
 1330 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 1331     0, 0, kvm_free, "IU", "Amount of KVM free");
 1332 
 1333 /*
 1334  * grow the number of kernel page table entries, if needed
 1335  */
 1336 void
 1337 pmap_growkernel(vm_offset_t addr)
 1338 {
 1339         vm_paddr_t paddr;
 1340         vm_page_t nkpg;
 1341         pd_entry_t *pde, newpdir;
 1342         pdp_entry_t newpdp;
 1343 
 1344         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1345         if (kernel_vm_end == 0) {
 1346                 kernel_vm_end = KERNBASE;
 1347                 nkpt = 0;
 1348                 while ((*pmap_pde(kernel_pmap, kernel_vm_end) & PG_V) != 0) {
 1349                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1350                         nkpt++;
 1351                 }
 1352         }
 1353         addr = roundup2(addr, PAGE_SIZE * NPTEPG);
 1354         while (kernel_vm_end < addr) {
 1355                 pde = pmap_pde(kernel_pmap, kernel_vm_end);
 1356                 if (pde == NULL) {
 1357                         /* We need a new PDP entry */
 1358                         nkpg = vm_page_alloc(NULL, nkpt,
 1359                             VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 1360                         if (!nkpg)
 1361                                 panic("pmap_growkernel: no memory to grow kernel");
 1362                         pmap_zero_page(nkpg);
 1363                         paddr = VM_PAGE_TO_PHYS(nkpg);
 1364                         newpdp = (pdp_entry_t)
 1365                                 (paddr | PG_V | PG_RW | PG_A | PG_M);
 1366                         *pmap_pdpe(kernel_pmap, kernel_vm_end) = newpdp;
 1367                         continue; /* try again */
 1368                 }
 1369                 if ((*pde & PG_V) != 0) {
 1370                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1371                         continue;
 1372                 }
 1373 
 1374                 /*
 1375                  * This index is bogus, but out of the way
 1376                  */
 1377                 nkpg = vm_page_alloc(NULL, nkpt,
 1378                     VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 1379                 if (!nkpg)
 1380                         panic("pmap_growkernel: no memory to grow kernel");
 1381 
 1382                 nkpt++;
 1383 
 1384                 pmap_zero_page(nkpg);
 1385                 paddr = VM_PAGE_TO_PHYS(nkpg);
 1386                 newpdir = (pd_entry_t) (paddr | PG_V | PG_RW | PG_A | PG_M);
 1387                 *pmap_pde(kernel_pmap, kernel_vm_end) = newpdir;
 1388 
 1389                 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1390         }
 1391 }
 1392 
 1393 
 1394 /***************************************************
 1395  * page management routines.
 1396  ***************************************************/
 1397 
 1398 /*
 1399  * free the pv_entry back to the free list
 1400  */
 1401 static PMAP_INLINE void
 1402 free_pv_entry(pv_entry_t pv)
 1403 {
 1404         pv_entry_count--;
 1405         uma_zfree(pvzone, pv);
 1406 }
 1407 
 1408 /*
 1409  * get a new pv_entry, allocating a block from the system
 1410  * when needed.
 1411  * the memory allocation is performed bypassing the malloc code
 1412  * because of the possibility of allocations at interrupt time.
 1413  */
 1414 static pv_entry_t
 1415 get_pv_entry(void)
 1416 {
 1417         pv_entry_count++;
 1418         if (pv_entry_high_water &&
 1419                 (pv_entry_count > pv_entry_high_water) &&
 1420                 (pmap_pagedaemon_waken == 0)) {
 1421                 pmap_pagedaemon_waken = 1;
 1422                 wakeup (&vm_pages_needed);
 1423         }
 1424         return uma_zalloc(pvzone, M_NOWAIT);
 1425 }
 1426 
 1427 
 1428 static int
 1429 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va, pd_entry_t ptepde)
 1430 {
 1431         pv_entry_t pv;
 1432         int rtval;
 1433 
 1434         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1435         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1436         if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
 1437                 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 1438                         if (pmap == pv->pv_pmap && va == pv->pv_va) 
 1439                                 break;
 1440                 }
 1441         } else {
 1442                 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
 1443                         if (va == pv->pv_va) 
 1444                                 break;
 1445                 }
 1446         }
 1447 
 1448         rtval = 0;
 1449         if (pv) {
 1450                 rtval = pmap_unuse_pt(pmap, va, ptepde);
 1451                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1452                 m->md.pv_list_count--;
 1453                 if (TAILQ_FIRST(&m->md.pv_list) == NULL)
 1454                         vm_page_flag_clear(m, PG_WRITEABLE);
 1455 
 1456                 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
 1457                 free_pv_entry(pv);
 1458         }
 1459                         
 1460         return rtval;
 1461 }
 1462 
 1463 /*
 1464  * Create a pv entry for page at pa for
 1465  * (pmap, va).
 1466  */
 1467 static void
 1468 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1469 {
 1470         pv_entry_t pv;
 1471 
 1472         pv = get_pv_entry();
 1473         pv->pv_va = va;
 1474         pv->pv_pmap = pmap;
 1475 
 1476         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1477         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1478         TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
 1479         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 1480         m->md.pv_list_count++;
 1481 }
 1482 
 1483 /*
 1484  * pmap_remove_pte: do the things to unmap a page in a process
 1485  */
 1486 static int
 1487 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t ptepde)
 1488 {
 1489         pt_entry_t oldpte;
 1490         vm_page_t m;
 1491 
 1492         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1493         oldpte = pte_load_clear(ptq);
 1494         if (oldpte & PG_W)
 1495                 pmap->pm_stats.wired_count -= 1;
 1496         /*
 1497          * Machines that don't support invlpg, also don't support
 1498          * PG_G.
 1499          */
 1500         if (oldpte & PG_G)
 1501                 pmap_invalidate_page(kernel_pmap, va);
 1502         pmap->pm_stats.resident_count -= 1;
 1503         if (oldpte & PG_MANAGED) {
 1504                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 1505                 if (oldpte & PG_M) {
 1506 #if defined(PMAP_DIAGNOSTIC)
 1507                         if (pmap_nw_modified((pt_entry_t) oldpte)) {
 1508                                 printf(
 1509         "pmap_remove: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
 1510                                     va, oldpte);
 1511                         }
 1512 #endif
 1513                         if (pmap_track_modified(va))
 1514                                 vm_page_dirty(m);
 1515                 }
 1516                 if (oldpte & PG_A)
 1517                         vm_page_flag_set(m, PG_REFERENCED);
 1518                 return pmap_remove_entry(pmap, m, va, ptepde);
 1519         } else {
 1520                 return pmap_unuse_pt(pmap, va, ptepde);
 1521         }
 1522 }
 1523 
 1524 /*
 1525  * Remove a single page from a process address space
 1526  */
 1527 static void
 1528 pmap_remove_page(pmap_t pmap, vm_offset_t va)
 1529 {
 1530         pd_entry_t ptepde;
 1531         pt_entry_t *pte;
 1532 
 1533         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1534         pte = pmap_pte_pde(pmap, va, &ptepde);
 1535         if (pte == NULL || (*pte & PG_V) == 0)
 1536                 return;
 1537         pmap_remove_pte(pmap, pte, va, ptepde);
 1538         pmap_invalidate_page(pmap, va);
 1539 }
 1540 
 1541 /*
 1542  *      Remove the given range of addresses from the specified map.
 1543  *
 1544  *      It is assumed that the start and end are properly
 1545  *      rounded to the page size.
 1546  */
 1547 void
 1548 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1549 {
 1550         vm_offset_t va_next;
 1551         pml4_entry_t *pml4e;
 1552         pdp_entry_t *pdpe;
 1553         pd_entry_t ptpaddr, *pde;
 1554         pt_entry_t *pte;
 1555         int anyvalid;
 1556 
 1557         /*
 1558          * Perform an unsynchronized read.  This is, however, safe.
 1559          */
 1560         if (pmap->pm_stats.resident_count == 0)
 1561                 return;
 1562 
 1563         anyvalid = 0;
 1564 
 1565         vm_page_lock_queues();
 1566         PMAP_LOCK(pmap);
 1567 
 1568         /*
 1569          * special handling of removing one page.  a very
 1570          * common operation and easy to short circuit some
 1571          * code.
 1572          */
 1573         if (sva + PAGE_SIZE == eva) {
 1574                 pde = pmap_pde(pmap, sva);
 1575                 if (pde && (*pde & PG_PS) == 0) {
 1576                         pmap_remove_page(pmap, sva);
 1577                         goto out;
 1578                 }
 1579         }
 1580 
 1581         for (; sva < eva; sva = va_next) {
 1582 
 1583                 if (pmap->pm_stats.resident_count == 0)
 1584                         break;
 1585 
 1586                 pml4e = pmap_pml4e(pmap, sva);
 1587                 if (pml4e == 0) {
 1588                         va_next = (sva + NBPML4) & ~PML4MASK;
 1589                         continue;
 1590                 }
 1591 
 1592                 pdpe = pmap_pdpe(pmap, sva);
 1593                 if (pdpe == 0) {
 1594                         va_next = (sva + NBPDP) & ~PDPMASK;
 1595                         continue;
 1596                 }
 1597 
 1598                 /*
 1599                  * Calculate index for next page table.
 1600                  */
 1601                 va_next = (sva + NBPDR) & ~PDRMASK;
 1602 
 1603                 pde = pmap_pde(pmap, sva);
 1604                 if (pde == 0)
 1605                         continue;
 1606                 ptpaddr = *pde;
 1607 
 1608                 /*
 1609                  * Weed out invalid mappings. Note: we assume that the page
 1610                  * directory table is always allocated, and in kernel virtual.
 1611                  */
 1612                 if (ptpaddr == 0)
 1613                         continue;
 1614 
 1615                 /*
 1616                  * Check for large page.
 1617                  */
 1618                 if ((ptpaddr & PG_PS) != 0) {
 1619                         *pde = 0;
 1620                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1621                         anyvalid = 1;
 1622                         continue;
 1623                 }
 1624 
 1625                 /*
 1626                  * Limit our scan to either the end of the va represented
 1627                  * by the current page table page, or to the end of the
 1628                  * range being removed.
 1629                  */
 1630                 if (va_next > eva)
 1631                         va_next = eva;
 1632 
 1633                 for (; sva != va_next; sva += PAGE_SIZE) {
 1634                         pte = pmap_pte(pmap, sva);
 1635                         if (pte == NULL || *pte == 0)
 1636                                 continue;
 1637                         anyvalid = 1;
 1638                         if (pmap_remove_pte(pmap, pte, sva, ptpaddr))
 1639                                 break;
 1640                 }
 1641         }
 1642 out:
 1643         vm_page_unlock_queues();
 1644         if (anyvalid)
 1645                 pmap_invalidate_all(pmap);
 1646         PMAP_UNLOCK(pmap);
 1647 }
 1648 
 1649 /*
 1650  *      Routine:        pmap_remove_all
 1651  *      Function:
 1652  *              Removes this physical page from
 1653  *              all physical maps in which it resides.
 1654  *              Reflects back modify bits to the pager.
 1655  *
 1656  *      Notes:
 1657  *              Original versions of this routine were very
 1658  *              inefficient because they iteratively called
 1659  *              pmap_remove (slow...)
 1660  */
 1661 
 1662 void
 1663 pmap_remove_all(vm_page_t m)
 1664 {
 1665         register pv_entry_t pv;
 1666         pt_entry_t *pte, tpte;
 1667         pd_entry_t ptepde;
 1668 
 1669 #if defined(PMAP_DIAGNOSTIC)
 1670         /*
 1671          * XXX This makes pmap_remove_all() illegal for non-managed pages!
 1672          */
 1673         if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) {
 1674                 panic("pmap_remove_all: illegal for unmanaged page, va: 0x%lx",
 1675                     VM_PAGE_TO_PHYS(m));
 1676         }
 1677 #endif
 1678         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1679         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 1680                 PMAP_LOCK(pv->pv_pmap);
 1681                 pv->pv_pmap->pm_stats.resident_count--;
 1682                 pte = pmap_pte_pde(pv->pv_pmap, pv->pv_va, &ptepde);
 1683                 tpte = pte_load_clear(pte);
 1684                 if (tpte & PG_W)
 1685                         pv->pv_pmap->pm_stats.wired_count--;
 1686                 if (tpte & PG_A)
 1687                         vm_page_flag_set(m, PG_REFERENCED);
 1688 
 1689                 /*
 1690                  * Update the vm_page_t clean and reference bits.
 1691                  */
 1692                 if (tpte & PG_M) {
 1693 #if defined(PMAP_DIAGNOSTIC)
 1694                         if (pmap_nw_modified((pt_entry_t) tpte)) {
 1695                                 printf(
 1696         "pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
 1697                                     pv->pv_va, tpte);
 1698                         }
 1699 #endif
 1700                         if (pmap_track_modified(pv->pv_va))
 1701                                 vm_page_dirty(m);
 1702                 }
 1703                 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 1704                 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
 1705                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1706                 m->md.pv_list_count--;
 1707                 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, ptepde);
 1708                 PMAP_UNLOCK(pv->pv_pmap);
 1709                 free_pv_entry(pv);
 1710         }
 1711         vm_page_flag_clear(m, PG_WRITEABLE);
 1712 }
 1713 
 1714 /*
 1715  *      Set the physical protection on the
 1716  *      specified range of this map as requested.
 1717  */
 1718 void
 1719 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 1720 {
 1721         vm_offset_t va_next;
 1722         pml4_entry_t *pml4e;
 1723         pdp_entry_t *pdpe;
 1724         pd_entry_t ptpaddr, *pde;
 1725         int anychanged;
 1726 
 1727         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 1728                 pmap_remove(pmap, sva, eva);
 1729                 return;
 1730         }
 1731 
 1732         if (prot & VM_PROT_WRITE)
 1733                 return;
 1734 
 1735         anychanged = 0;
 1736 
 1737         vm_page_lock_queues();
 1738         PMAP_LOCK(pmap);
 1739         for (; sva < eva; sva = va_next) {
 1740 
 1741                 pml4e = pmap_pml4e(pmap, sva);
 1742                 if (pml4e == 0) {
 1743                         va_next = (sva + NBPML4) & ~PML4MASK;
 1744                         continue;
 1745                 }
 1746 
 1747                 pdpe = pmap_pdpe(pmap, sva);
 1748                 if (pdpe == 0) {
 1749                         va_next = (sva + NBPDP) & ~PDPMASK;
 1750                         continue;
 1751                 }
 1752 
 1753                 va_next = (sva + NBPDR) & ~PDRMASK;
 1754 
 1755                 pde = pmap_pde(pmap, sva);
 1756                 if (pde == NULL)
 1757                         continue;
 1758                 ptpaddr = *pde;
 1759 
 1760                 /*
 1761                  * Weed out invalid mappings. Note: we assume that the page
 1762                  * directory table is always allocated, and in kernel virtual.
 1763                  */
 1764                 if (ptpaddr == 0)
 1765                         continue;
 1766 
 1767                 /*
 1768                  * Check for large page.
 1769                  */
 1770                 if ((ptpaddr & PG_PS) != 0) {
 1771                         *pde &= ~(PG_M|PG_RW);
 1772                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1773                         anychanged = 1;
 1774                         continue;
 1775                 }
 1776 
 1777                 if (va_next > eva)
 1778                         va_next = eva;
 1779 
 1780                 for (; sva != va_next; sva += PAGE_SIZE) {
 1781                         pt_entry_t obits, pbits;
 1782                         pt_entry_t *pte;
 1783                         vm_page_t m;
 1784 
 1785                         pte = pmap_pte(pmap, sva);
 1786                         if (pte == NULL)
 1787                                 continue;
 1788 retry:
 1789                         obits = pbits = *pte;
 1790                         if (pbits & PG_MANAGED) {
 1791                                 m = NULL;
 1792                                 if (pbits & PG_A) {
 1793                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 1794                                         vm_page_flag_set(m, PG_REFERENCED);
 1795                                         pbits &= ~PG_A;
 1796                                 }
 1797                                 if ((pbits & PG_M) != 0 &&
 1798                                     pmap_track_modified(sva)) {
 1799                                         if (m == NULL)
 1800                                                 m = PHYS_TO_VM_PAGE(pbits &
 1801                                                     PG_FRAME);
 1802                                         vm_page_dirty(m);
 1803                                 }
 1804                         }
 1805 
 1806                         pbits &= ~(PG_RW | PG_M);
 1807 
 1808                         if (pbits != obits) {
 1809                                 if (!atomic_cmpset_long(pte, obits, pbits))
 1810                                         goto retry;
 1811                                 if (obits & PG_G)
 1812                                         pmap_invalidate_page(pmap, sva);
 1813                                 else
 1814                                         anychanged = 1;
 1815                         }
 1816                 }
 1817         }
 1818         vm_page_unlock_queues();
 1819         if (anychanged)
 1820                 pmap_invalidate_all(pmap);
 1821         PMAP_UNLOCK(pmap);
 1822 }
 1823 
 1824 /*
 1825  *      Insert the given physical page (p) at
 1826  *      the specified virtual address (v) in the
 1827  *      target physical map with the protection requested.
 1828  *
 1829  *      If specified, the page will be wired down, meaning
 1830  *      that the related pte can not be reclaimed.
 1831  *
 1832  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 1833  *      or lose information.  That is, this routine must actually
 1834  *      insert this page into the given map NOW.
 1835  */
 1836 void
 1837 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 1838            boolean_t wired)
 1839 {
 1840         vm_paddr_t pa;
 1841         register pt_entry_t *pte;
 1842         vm_paddr_t opa;
 1843         pd_entry_t ptepde;
 1844         pt_entry_t origpte, newpte;
 1845         vm_page_t mpte, om;
 1846 
 1847         va = trunc_page(va);
 1848 #ifdef PMAP_DIAGNOSTIC
 1849         if (va > VM_MAX_KERNEL_ADDRESS)
 1850                 panic("pmap_enter: toobig");
 1851         if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
 1852                 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", va);
 1853 #endif
 1854 
 1855         mpte = NULL;
 1856 
 1857         vm_page_lock_queues();
 1858         PMAP_LOCK(pmap);
 1859 
 1860         /*
 1861          * In the case that a page table page is not
 1862          * resident, we are creating it here.
 1863          */
 1864         if (va < VM_MAXUSER_ADDRESS) {
 1865                 mpte = pmap_allocpte(pmap, va, M_WAITOK);
 1866         }
 1867 #if 0 && defined(PMAP_DIAGNOSTIC)
 1868         else {
 1869                 pd_entry_t *pdeaddr = pmap_pde(pmap, va);
 1870                 origpte = *pdeaddr;
 1871                 if ((origpte & PG_V) == 0) { 
 1872                         panic("pmap_enter: invalid kernel page table page, pde=%p, va=%p\n",
 1873                                 origpte, va);
 1874                 }
 1875         }
 1876 #endif
 1877 
 1878         pte = pmap_pte_pde(pmap, va, &ptepde);
 1879 
 1880         /*
 1881          * Page Directory table entry not valid, we need a new PT page
 1882          */
 1883         if (pte == NULL)
 1884                 panic("pmap_enter: invalid page directory va=%#lx\n", va);
 1885 
 1886         pa = VM_PAGE_TO_PHYS(m);
 1887         om = NULL;
 1888         origpte = *pte;
 1889         opa = origpte & PG_FRAME;
 1890 
 1891         if (origpte & PG_PS)
 1892                 panic("pmap_enter: attempted pmap_enter on 2MB page");
 1893 
 1894         /*
 1895          * Mapping has not changed, must be protection or wiring change.
 1896          */
 1897         if (origpte && (opa == pa)) {
 1898                 /*
 1899                  * Wiring change, just update stats. We don't worry about
 1900                  * wiring PT pages as they remain resident as long as there
 1901                  * are valid mappings in them. Hence, if a user page is wired,
 1902                  * the PT page will be also.
 1903                  */
 1904                 if (wired && ((origpte & PG_W) == 0))
 1905                         pmap->pm_stats.wired_count++;
 1906                 else if (!wired && (origpte & PG_W))
 1907                         pmap->pm_stats.wired_count--;
 1908 
 1909 #if defined(PMAP_DIAGNOSTIC)
 1910                 if (pmap_nw_modified((pt_entry_t) origpte)) {
 1911                         printf(
 1912         "pmap_enter: modified page not writable: va: 0x%lx, pte: 0x%lx\n",
 1913                             va, origpte);
 1914                 }
 1915 #endif
 1916 
 1917                 /*
 1918                  * Remove extra pte reference
 1919                  */
 1920                 if (mpte)
 1921                         mpte->wire_count--;
 1922 
 1923                 /*
 1924                  * We might be turning off write access to the page,
 1925                  * so we go ahead and sense modify status.
 1926                  */
 1927                 if (origpte & PG_MANAGED) {
 1928                         om = m;
 1929                         pa |= PG_MANAGED;
 1930                 }
 1931                 goto validate;
 1932         } 
 1933         /*
 1934          * Mapping has changed, invalidate old range and fall through to
 1935          * handle validating new mapping.
 1936          */
 1937         if (opa) {
 1938                 int err;
 1939                 if (origpte & PG_W)
 1940                         pmap->pm_stats.wired_count--;
 1941                 if (origpte & PG_MANAGED) {
 1942                         om = PHYS_TO_VM_PAGE(opa);
 1943                         err = pmap_remove_entry(pmap, om, va, ptepde);
 1944                 } else
 1945                         err = pmap_unuse_pt(pmap, va, ptepde);
 1946                 if (err)
 1947                         panic("pmap_enter: pte vanished, va: 0x%lx", va);
 1948         } else
 1949                 pmap->pm_stats.resident_count++;
 1950 
 1951         /*
 1952          * Enter on the PV list if part of our managed memory. Note that we
 1953          * raise IPL while manipulating pv_table since pmap_enter can be
 1954          * called at interrupt time.
 1955          */
 1956         if (pmap_initialized && 
 1957             (m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0) {
 1958                 pmap_insert_entry(pmap, va, m);
 1959                 pa |= PG_MANAGED;
 1960         }
 1961 
 1962         /*
 1963          * Increment counters
 1964          */
 1965         if (wired)
 1966                 pmap->pm_stats.wired_count++;
 1967 
 1968 validate:
 1969         /*
 1970          * Now validate mapping with desired protection/wiring.
 1971          */
 1972         newpte = (pt_entry_t)(pa | PG_V);
 1973         if ((prot & VM_PROT_WRITE) != 0)
 1974                 newpte |= PG_RW;
 1975         if ((prot & VM_PROT_EXECUTE) == 0)
 1976                 newpte |= pg_nx;
 1977         if (wired)
 1978                 newpte |= PG_W;
 1979         if (va < VM_MAXUSER_ADDRESS)
 1980                 newpte |= PG_U;
 1981         if (pmap == kernel_pmap)
 1982                 newpte |= PG_G;
 1983 
 1984         /*
 1985          * if the mapping or permission bits are different, we need
 1986          * to update the pte.
 1987          */
 1988         if ((origpte & ~(PG_M|PG_A)) != newpte) {
 1989                 if (origpte & PG_MANAGED) {
 1990                         origpte = pte_load_store(pte, newpte | PG_A);
 1991                         if ((origpte & PG_M) && pmap_track_modified(va))
 1992                                 vm_page_dirty(om);
 1993                         if (origpte & PG_A)
 1994                                 vm_page_flag_set(om, PG_REFERENCED);
 1995                 } else
 1996                         pte_store(pte, newpte | PG_A);
 1997                 if (origpte) {
 1998                         pmap_invalidate_page(pmap, va);
 1999                 }
 2000         }
 2001         vm_page_unlock_queues();
 2002         PMAP_UNLOCK(pmap);
 2003 }
 2004 
 2005 /*
 2006  * this code makes some *MAJOR* assumptions:
 2007  * 1. Current pmap & pmap exists.
 2008  * 2. Not wired.
 2009  * 3. Read access.
 2010  * 4. No page table pages.
 2011  * 5. Tlbflush is deferred to calling procedure.
 2012  * 6. Page IS managed.
 2013  * but is *MUCH* faster than pmap_enter...
 2014  */
 2015 
 2016 vm_page_t
 2017 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 2018 {
 2019         pt_entry_t *pte;
 2020         vm_paddr_t pa;
 2021 
 2022         vm_page_lock_queues();
 2023         PMAP_LOCK(pmap);
 2024 
 2025         /*
 2026          * In the case that a page table page is not
 2027          * resident, we are creating it here.
 2028          */
 2029         if (va < VM_MAXUSER_ADDRESS) {
 2030                 vm_pindex_t ptepindex;
 2031                 pd_entry_t *ptepa;
 2032 
 2033                 /*
 2034                  * Calculate pagetable page index
 2035                  */
 2036                 ptepindex = pmap_pde_pindex(va);
 2037                 if (mpte && (mpte->pindex == ptepindex)) {
 2038                         mpte->wire_count++;
 2039                 } else {
 2040         retry:
 2041                         /*
 2042                          * Get the page directory entry
 2043                          */
 2044                         ptepa = pmap_pde(pmap, va);
 2045 
 2046                         /*
 2047                          * If the page table page is mapped, we just increment
 2048                          * the hold count, and activate it.
 2049                          */
 2050                         if (ptepa && (*ptepa & PG_V) != 0) {
 2051                                 if (*ptepa & PG_PS)
 2052                                         panic("pmap_enter_quick: unexpected mapping into 2MB page");
 2053                                 mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
 2054                                 mpte->wire_count++;
 2055                         } else {
 2056                                 mpte = _pmap_allocpte(pmap, ptepindex,
 2057                                     M_WAITOK);
 2058                                 if (mpte == NULL)
 2059                                         goto retry;
 2060                         }
 2061                 }
 2062         } else {
 2063                 mpte = NULL;
 2064         }
 2065 
 2066         /*
 2067          * This call to vtopte makes the assumption that we are
 2068          * entering the page into the current pmap.  In order to support
 2069          * quick entry into any pmap, one would likely use pmap_pte.
 2070          * But that isn't as quick as vtopte.
 2071          */
 2072         pte = vtopte(va);
 2073         if (*pte) {
 2074                 if (mpte != NULL) {
 2075                         pmap_unwire_pte_hold(pmap, va, mpte);
 2076                         mpte = NULL;
 2077                 }
 2078                 goto out;
 2079         }
 2080 
 2081         /*
 2082          * Enter on the PV list if part of our managed memory. Note that we
 2083          * raise IPL while manipulating pv_table since pmap_enter can be
 2084          * called at interrupt time.
 2085          */
 2086         if ((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0)
 2087                 pmap_insert_entry(pmap, va, m);
 2088 
 2089         /*
 2090          * Increment counters
 2091          */
 2092         pmap->pm_stats.resident_count++;
 2093 
 2094         pa = VM_PAGE_TO_PHYS(m);
 2095 
 2096         /*
 2097          * Now validate mapping with RO protection
 2098          */
 2099         if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED))
 2100                 pte_store(pte, pa | PG_V | PG_U);
 2101         else
 2102                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 2103 out:
 2104         vm_page_unlock_queues();
 2105         PMAP_UNLOCK(pmap);
 2106         return mpte;
 2107 }
 2108 
 2109 /*
 2110  * Make a temporary mapping for a physical address.  This is only intended
 2111  * to be used for panic dumps.
 2112  */
 2113 void *
 2114 pmap_kenter_temporary(vm_paddr_t pa, int i)
 2115 {
 2116         vm_offset_t va;
 2117 
 2118         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 2119         pmap_kenter(va, pa);
 2120         invlpg(va);
 2121         return ((void *)crashdumpmap);
 2122 }
 2123 
 2124 /*
 2125  * This code maps large physical mmap regions into the
 2126  * processor address space.  Note that some shortcuts
 2127  * are taken, but the code works.
 2128  */
 2129 void
 2130 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
 2131                     vm_object_t object, vm_pindex_t pindex,
 2132                     vm_size_t size)
 2133 {
 2134         vm_page_t p;
 2135 
 2136         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 2137         KASSERT(object->type == OBJT_DEVICE,
 2138             ("pmap_object_init_pt: non-device object"));
 2139         if (((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0)) {
 2140                 int i;
 2141                 vm_page_t m[1];
 2142                 int npdes;
 2143                 pd_entry_t ptepa, *pde;
 2144 
 2145                 PMAP_LOCK(pmap);
 2146                 pde = pmap_pde(pmap, addr);
 2147                 if (pde != 0 && (*pde & PG_V) != 0)
 2148                         goto out;
 2149                 PMAP_UNLOCK(pmap);
 2150 retry:
 2151                 p = vm_page_lookup(object, pindex);
 2152                 if (p != NULL) {
 2153                         vm_page_lock_queues();
 2154                         if (vm_page_sleep_if_busy(p, FALSE, "init4p"))
 2155                                 goto retry;
 2156                 } else {
 2157                         p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
 2158                         if (p == NULL)
 2159                                 return;
 2160                         m[0] = p;
 2161 
 2162                         if (vm_pager_get_pages(object, m, 1, 0) != VM_PAGER_OK) {
 2163                                 vm_page_lock_queues();
 2164                                 vm_page_free(p);
 2165                                 vm_page_unlock_queues();
 2166                                 return;
 2167                         }
 2168 
 2169                         p = vm_page_lookup(object, pindex);
 2170                         vm_page_lock_queues();
 2171                         vm_page_wakeup(p);
 2172                 }
 2173                 vm_page_unlock_queues();
 2174 
 2175                 ptepa = VM_PAGE_TO_PHYS(p);
 2176                 if (ptepa & (NBPDR - 1))
 2177                         return;
 2178 
 2179                 p->valid = VM_PAGE_BITS_ALL;
 2180 
 2181                 PMAP_LOCK(pmap);
 2182                 pmap->pm_stats.resident_count += size >> PAGE_SHIFT;
 2183                 npdes = size >> PDRSHIFT;
 2184                 for(i = 0; i < npdes; i++) {
 2185                         pde_store(pde, ptepa | PG_U | PG_RW | PG_V | PG_PS);
 2186                         ptepa += NBPDR;
 2187                         pde++;
 2188                 }
 2189                 pmap_invalidate_all(pmap);
 2190 out:
 2191                 PMAP_UNLOCK(pmap);
 2192         }
 2193 }
 2194 
 2195 /*
 2196  *      Routine:        pmap_change_wiring
 2197  *      Function:       Change the wiring attribute for a map/virtual-address
 2198  *                      pair.
 2199  *      In/out conditions:
 2200  *                      The mapping must already exist in the pmap.
 2201  */
 2202 void
 2203 pmap_change_wiring(pmap, va, wired)
 2204         register pmap_t pmap;
 2205         vm_offset_t va;
 2206         boolean_t wired;
 2207 {
 2208         register pt_entry_t *pte;
 2209 
 2210         /*
 2211          * Wiring is not a hardware characteristic so there is no need to
 2212          * invalidate TLB.
 2213          */
 2214         PMAP_LOCK(pmap);
 2215         pte = pmap_pte(pmap, va);
 2216         if (wired && (*pte & PG_W) == 0) {
 2217                 pmap->pm_stats.wired_count++;
 2218                 atomic_set_long(pte, PG_W);
 2219         } else if (!wired && (*pte & PG_W) != 0) {
 2220                 pmap->pm_stats.wired_count--;
 2221                 atomic_clear_long(pte, PG_W);
 2222         }
 2223         PMAP_UNLOCK(pmap);
 2224 }
 2225 
 2226 
 2227 
 2228 /*
 2229  *      Copy the range specified by src_addr/len
 2230  *      from the source map to the range dst_addr/len
 2231  *      in the destination map.
 2232  *
 2233  *      This routine is only advisory and need not do anything.
 2234  */
 2235 
 2236 void
 2237 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 2238           vm_offset_t src_addr)
 2239 {
 2240         vm_offset_t addr;
 2241         vm_offset_t end_addr = src_addr + len;
 2242         vm_offset_t va_next;
 2243         vm_page_t m;
 2244 
 2245         if (dst_addr != src_addr)
 2246                 return;
 2247 
 2248         if (!pmap_is_current(src_pmap))
 2249                 return;
 2250 
 2251         vm_page_lock_queues();
 2252         if (dst_pmap < src_pmap) {
 2253                 PMAP_LOCK(dst_pmap);
 2254                 PMAP_LOCK(src_pmap);
 2255         } else {
 2256                 PMAP_LOCK(src_pmap);
 2257                 PMAP_LOCK(dst_pmap);
 2258         }
 2259         for (addr = src_addr; addr < end_addr; addr = va_next) {
 2260                 pt_entry_t *src_pte, *dst_pte;
 2261                 vm_page_t dstmpte, srcmpte;
 2262                 pml4_entry_t *pml4e;
 2263                 pdp_entry_t *pdpe;
 2264                 pd_entry_t srcptepaddr, *pde;
 2265 
 2266                 if (addr >= UPT_MIN_ADDRESS)
 2267                         panic("pmap_copy: invalid to pmap_copy page tables");
 2268 
 2269                 /*
 2270                  * Don't let optional prefaulting of pages make us go
 2271                  * way below the low water mark of free pages or way
 2272                  * above high water mark of used pv entries.
 2273                  */
 2274                 if (cnt.v_free_count < cnt.v_free_reserved ||
 2275                     pv_entry_count > pv_entry_high_water)
 2276                         break;
 2277                 
 2278                 pml4e = pmap_pml4e(src_pmap, addr);
 2279                 if (pml4e == 0) {
 2280                         va_next = (addr + NBPML4) & ~PML4MASK;
 2281                         continue;
 2282                 }
 2283 
 2284                 pdpe = pmap_pdpe(src_pmap, addr);
 2285                 if (pdpe == 0) {
 2286                         va_next = (addr + NBPDP) & ~PDPMASK;
 2287                         continue;
 2288                 }
 2289 
 2290                 va_next = (addr + NBPDR) & ~PDRMASK;
 2291 
 2292                 pde = pmap_pde(src_pmap, addr);
 2293                 if (pde)
 2294                         srcptepaddr = *pde;
 2295                 else
 2296                         continue;
 2297                 if (srcptepaddr == 0)
 2298                         continue;
 2299                         
 2300                 if (srcptepaddr & PG_PS) {
 2301                         pde = pmap_pde(dst_pmap, addr);
 2302                         if (pde == 0) {
 2303                                 /*
 2304                                  * XXX should do an allocpte here to
 2305                                  * instantiate the pde
 2306                                  */
 2307                                 continue;
 2308                         }
 2309                         if (*pde == 0) {
 2310                                 *pde = srcptepaddr;
 2311                                 dst_pmap->pm_stats.resident_count +=
 2312                                     NBPDR / PAGE_SIZE;
 2313                         }
 2314                         continue;
 2315                 }
 2316 
 2317                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 2318                 if (srcmpte->wire_count == 0)
 2319                         panic("pmap_copy: source page table page is unused");
 2320 
 2321                 if (va_next > end_addr)
 2322                         va_next = end_addr;
 2323 
 2324                 src_pte = vtopte(addr);
 2325                 while (addr < va_next) {
 2326                         pt_entry_t ptetemp;
 2327                         ptetemp = *src_pte;
 2328                         /*
 2329                          * we only virtual copy managed pages
 2330                          */
 2331                         if ((ptetemp & PG_MANAGED) != 0) {
 2332                                 /*
 2333                                  * We have to check after allocpte for the
 2334                                  * pte still being around...  allocpte can
 2335                                  * block.
 2336                                  */
 2337                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 2338                                     M_NOWAIT);
 2339                                 if (dstmpte == NULL)
 2340                                         break;
 2341                                 dst_pte = pmap_pte(dst_pmap, addr);
 2342                                 if (*dst_pte == 0) {
 2343                                         /*
 2344                                          * Clear the modified and
 2345                                          * accessed (referenced) bits
 2346                                          * during the copy.
 2347                                          */
 2348                                         m = PHYS_TO_VM_PAGE(ptetemp & PG_FRAME);
 2349                                         *dst_pte = ptetemp & ~(PG_M | PG_A);
 2350                                         dst_pmap->pm_stats.resident_count++;
 2351                                         pmap_insert_entry(dst_pmap, addr, m);
 2352                                 } else
 2353                                         pmap_unwire_pte_hold(dst_pmap, addr, dstmpte);
 2354                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 2355                                         break;
 2356                         }
 2357                         addr += PAGE_SIZE;
 2358                         src_pte++;
 2359                 }
 2360         }
 2361         vm_page_unlock_queues();
 2362         PMAP_UNLOCK(src_pmap);
 2363         PMAP_UNLOCK(dst_pmap);
 2364 }       
 2365 
 2366 /*
 2367  *      pmap_zero_page zeros the specified hardware page by mapping 
 2368  *      the page into KVM and using bzero to clear its contents.
 2369  */
 2370 void
 2371 pmap_zero_page(vm_page_t m)
 2372 {
 2373         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2374 
 2375         pagezero((void *)va);
 2376 }
 2377 
 2378 /*
 2379  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 2380  *      the page into KVM and using bzero to clear its contents.
 2381  *
 2382  *      off and size may not cover an area beyond a single hardware page.
 2383  */
 2384 void
 2385 pmap_zero_page_area(vm_page_t m, int off, int size)
 2386 {
 2387         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2388 
 2389         if (off == 0 && size == PAGE_SIZE)
 2390                 pagezero((void *)va);
 2391         else
 2392                 bzero((char *)va + off, size);
 2393 }
 2394 
 2395 /*
 2396  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 2397  *      the page into KVM and using bzero to clear its contents.  This
 2398  *      is intended to be called from the vm_pagezero process only and
 2399  *      outside of Giant.
 2400  */
 2401 void
 2402 pmap_zero_page_idle(vm_page_t m)
 2403 {
 2404         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2405 
 2406         pagezero((void *)va);
 2407 }
 2408 
 2409 /*
 2410  *      pmap_copy_page copies the specified (machine independent)
 2411  *      page by mapping the page into virtual memory and using
 2412  *      bcopy to copy the page, one machine dependent page at a
 2413  *      time.
 2414  */
 2415 void
 2416 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 2417 {
 2418         vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
 2419         vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
 2420 
 2421         pagecopy((void *)src, (void *)dst);
 2422 }
 2423 
 2424 /*
 2425  * Returns true if the pmap's pv is one of the first
 2426  * 16 pvs linked to from this page.  This count may
 2427  * be changed upwards or downwards in the future; it
 2428  * is only necessary that true be returned for a small
 2429  * subset of pmaps for proper page aging.
 2430  */
 2431 boolean_t
 2432 pmap_page_exists_quick(pmap, m)
 2433         pmap_t pmap;
 2434         vm_page_t m;
 2435 {
 2436         pv_entry_t pv;
 2437         int loops = 0;
 2438 
 2439         if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
 2440                 return FALSE;
 2441 
 2442         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2443         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2444                 if (pv->pv_pmap == pmap) {
 2445                         return TRUE;
 2446                 }
 2447                 loops++;
 2448                 if (loops >= 16)
 2449                         break;
 2450         }
 2451         return (FALSE);
 2452 }
 2453 
 2454 #define PMAP_REMOVE_PAGES_CURPROC_ONLY
 2455 /*
 2456  * Remove all pages from specified address space
 2457  * this aids process exit speeds.  Also, this code
 2458  * is special cased for current process only, but
 2459  * can have the more generic (and slightly slower)
 2460  * mode enabled.  This is much faster than pmap_remove
 2461  * in the case of running down an entire address space.
 2462  */
 2463 void
 2464 pmap_remove_pages(pmap, sva, eva)
 2465         pmap_t pmap;
 2466         vm_offset_t sva, eva;
 2467 {
 2468         pt_entry_t *pte, tpte;
 2469         vm_page_t m;
 2470         pv_entry_t pv, npv;
 2471 
 2472 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
 2473         if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
 2474                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 2475                 return;
 2476         }
 2477 #endif
 2478         vm_page_lock_queues();
 2479         PMAP_LOCK(pmap);
 2480         for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
 2481 
 2482                 if (pv->pv_va >= eva || pv->pv_va < sva) {
 2483                         npv = TAILQ_NEXT(pv, pv_plist);
 2484                         continue;
 2485                 }
 2486 
 2487 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
 2488                 pte = vtopte(pv->pv_va);
 2489 #else
 2490                 pte = pmap_pte(pmap, pv->pv_va);
 2491 #endif
 2492                 tpte = *pte;
 2493 
 2494                 if (tpte == 0) {
 2495                         printf("TPTE at %p  IS ZERO @ VA %08lx\n",
 2496                                                         pte, pv->pv_va);
 2497                         panic("bad pte");
 2498                 }
 2499 
 2500 /*
 2501  * We cannot remove wired pages from a process' mapping at this time
 2502  */
 2503                 if (tpte & PG_W) {
 2504                         npv = TAILQ_NEXT(pv, pv_plist);
 2505                         continue;
 2506                 }
 2507 
 2508                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2509                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 2510                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 2511                     m, (uintmax_t)m->phys_addr, (uintmax_t)tpte));
 2512 
 2513                 KASSERT(m < &vm_page_array[vm_page_array_size],
 2514                         ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte));
 2515 
 2516                 pmap->pm_stats.resident_count--;
 2517 
 2518                 pte_clear(pte);
 2519 
 2520                 /*
 2521                  * Update the vm_page_t clean and reference bits.
 2522                  */
 2523                 if (tpte & PG_M) {
 2524                         vm_page_dirty(m);
 2525                 }
 2526 
 2527                 npv = TAILQ_NEXT(pv, pv_plist);
 2528                 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
 2529 
 2530                 m->md.pv_list_count--;
 2531                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2532                 if (TAILQ_EMPTY(&m->md.pv_list))
 2533                         vm_page_flag_clear(m, PG_WRITEABLE);
 2534 
 2535                 pmap_unuse_pt(pmap, pv->pv_va, *vtopde(pv->pv_va));
 2536                 free_pv_entry(pv);
 2537         }
 2538         pmap_invalidate_all(pmap);
 2539         PMAP_UNLOCK(pmap);
 2540         vm_page_unlock_queues();
 2541 }
 2542 
 2543 /*
 2544  *      pmap_is_modified:
 2545  *
 2546  *      Return whether or not the specified physical page was modified
 2547  *      in any physical maps.
 2548  */
 2549 boolean_t
 2550 pmap_is_modified(vm_page_t m)
 2551 {
 2552         pv_entry_t pv;
 2553         pt_entry_t *pte;
 2554         boolean_t rv;
 2555 
 2556         rv = FALSE;
 2557         if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
 2558                 return (rv);
 2559 
 2560         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2561         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2562                 /*
 2563                  * if the bit being tested is the modified bit, then
 2564                  * mark clean_map and ptes as never
 2565                  * modified.
 2566                  */
 2567                 if (!pmap_track_modified(pv->pv_va))
 2568                         continue;
 2569 #if defined(PMAP_DIAGNOSTIC)
 2570                 if (!pv->pv_pmap) {
 2571                         printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va);
 2572                         continue;
 2573                 }
 2574 #endif
 2575                 PMAP_LOCK(pv->pv_pmap);
 2576                 pte = pmap_pte(pv->pv_pmap, pv->pv_va);
 2577                 rv = (*pte & PG_M) != 0;
 2578                 PMAP_UNLOCK(pv->pv_pmap);
 2579                 if (rv)
 2580                         break;
 2581         }
 2582         return (rv);
 2583 }
 2584 
 2585 /*
 2586  *      pmap_is_prefaultable:
 2587  *
 2588  *      Return whether or not the specified virtual address is elgible
 2589  *      for prefault.
 2590  */
 2591 boolean_t
 2592 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 2593 {
 2594         pd_entry_t *pde;
 2595         pt_entry_t *pte;
 2596         boolean_t rv;
 2597 
 2598         rv = FALSE;
 2599         PMAP_LOCK(pmap);
 2600         pde = pmap_pde(pmap, addr);
 2601         if (pde != NULL && (*pde & PG_V)) {
 2602                 pte = vtopte(addr);
 2603                 rv = (*pte & PG_V) == 0;
 2604         }
 2605         PMAP_UNLOCK(pmap);
 2606         return (rv);
 2607 }
 2608 
 2609 /*
 2610  *      Clear the given bit in each of the given page's ptes.
 2611  */
 2612 static __inline void
 2613 pmap_clear_ptes(vm_page_t m, long bit)
 2614 {
 2615         register pv_entry_t pv;
 2616         pt_entry_t pbits, *pte;
 2617 
 2618         if (!pmap_initialized || (m->flags & PG_FICTITIOUS) ||
 2619             (bit == PG_RW && (m->flags & PG_WRITEABLE) == 0))
 2620                 return;
 2621 
 2622         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2623         /*
 2624          * Loop over all current mappings setting/clearing as appropos If
 2625          * setting RO do we need to clear the VAC?
 2626          */
 2627         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2628                 /*
 2629                  * don't write protect pager mappings
 2630                  */
 2631                 if (bit == PG_RW) {
 2632                         if (!pmap_track_modified(pv->pv_va))
 2633                                 continue;
 2634                 }
 2635 
 2636 #if defined(PMAP_DIAGNOSTIC)
 2637                 if (!pv->pv_pmap) {
 2638                         printf("Null pmap (cb) at va: 0x%lx\n", pv->pv_va);
 2639                         continue;
 2640                 }
 2641 #endif
 2642 
 2643                 PMAP_LOCK(pv->pv_pmap);
 2644                 pte = pmap_pte(pv->pv_pmap, pv->pv_va);
 2645 retry:
 2646                 pbits = *pte;
 2647                 if (pbits & bit) {
 2648                         if (bit == PG_RW) {
 2649                                 if (!atomic_cmpset_long(pte, pbits,
 2650                                     pbits & ~(PG_RW | PG_M)))
 2651                                         goto retry;
 2652                                 if (pbits & PG_M) {
 2653                                         vm_page_dirty(m);
 2654                                 }
 2655                         } else {
 2656                                 atomic_clear_long(pte, bit);
 2657                         }
 2658                         pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 2659                 }
 2660                 PMAP_UNLOCK(pv->pv_pmap);
 2661         }
 2662         if (bit == PG_RW)
 2663                 vm_page_flag_clear(m, PG_WRITEABLE);
 2664 }
 2665 
 2666 /*
 2667  *      pmap_page_protect:
 2668  *
 2669  *      Lower the permission for all mappings to a given page.
 2670  */
 2671 void
 2672 pmap_page_protect(vm_page_t m, vm_prot_t prot)
 2673 {
 2674         if ((prot & VM_PROT_WRITE) == 0) {
 2675                 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
 2676                         pmap_clear_ptes(m, PG_RW);
 2677                 } else {
 2678                         pmap_remove_all(m);
 2679                 }
 2680         }
 2681 }
 2682 
 2683 /*
 2684  *      pmap_ts_referenced:
 2685  *
 2686  *      Return a count of reference bits for a page, clearing those bits.
 2687  *      It is not necessary for every reference bit to be cleared, but it
 2688  *      is necessary that 0 only be returned when there are truly no
 2689  *      reference bits set.
 2690  *
 2691  *      XXX: The exact number of bits to check and clear is a matter that
 2692  *      should be tested and standardized at some point in the future for
 2693  *      optimal aging of shared pages.
 2694  */
 2695 int
 2696 pmap_ts_referenced(vm_page_t m)
 2697 {
 2698         register pv_entry_t pv, pvf, pvn;
 2699         pt_entry_t *pte;
 2700         pt_entry_t v;
 2701         int rtval = 0;
 2702 
 2703         if (!pmap_initialized || (m->flags & PG_FICTITIOUS))
 2704                 return (rtval);
 2705 
 2706         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2707         if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 2708 
 2709                 pvf = pv;
 2710 
 2711                 do {
 2712                         pvn = TAILQ_NEXT(pv, pv_list);
 2713 
 2714                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2715 
 2716                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2717 
 2718                         if (!pmap_track_modified(pv->pv_va))
 2719                                 continue;
 2720 
 2721                         PMAP_LOCK(pv->pv_pmap);
 2722                         pte = pmap_pte(pv->pv_pmap, pv->pv_va);
 2723 
 2724                         if (pte && ((v = pte_load(pte)) & PG_A) != 0) {
 2725                                 atomic_clear_long(pte, PG_A);
 2726                                 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 2727 
 2728                                 rtval++;
 2729                                 if (rtval > 4) {
 2730                                         PMAP_UNLOCK(pv->pv_pmap);
 2731                                         break;
 2732                                 }
 2733                         }
 2734                         PMAP_UNLOCK(pv->pv_pmap);
 2735                 } while ((pv = pvn) != NULL && pv != pvf);
 2736         }
 2737 
 2738         return (rtval);
 2739 }
 2740 
 2741 /*
 2742  *      Clear the modify bits on the specified physical page.
 2743  */
 2744 void
 2745 pmap_clear_modify(vm_page_t m)
 2746 {
 2747         pmap_clear_ptes(m, PG_M);
 2748 }
 2749 
 2750 /*
 2751  *      pmap_clear_reference:
 2752  *
 2753  *      Clear the reference bit on the specified physical page.
 2754  */
 2755 void
 2756 pmap_clear_reference(vm_page_t m)
 2757 {
 2758         pmap_clear_ptes(m, PG_A);
 2759 }
 2760 
 2761 /*
 2762  * Miscellaneous support routines follow
 2763  */
 2764 
 2765 /*
 2766  * Map a set of physical memory pages into the kernel virtual
 2767  * address space. Return a pointer to where it is mapped. This
 2768  * routine is intended to be used for mapping device memory,
 2769  * NOT real memory.
 2770  */
 2771 void *
 2772 pmap_mapdev(pa, size)
 2773         vm_paddr_t pa;
 2774         vm_size_t size;
 2775 {
 2776         vm_offset_t va, tmpva, offset;
 2777 
 2778         /* If this fits within the direct map window, use it */
 2779         if (pa < dmaplimit && (pa + size) < dmaplimit)
 2780                 return ((void *)PHYS_TO_DMAP(pa));
 2781         offset = pa & PAGE_MASK;
 2782         size = roundup(offset + size, PAGE_SIZE);
 2783         va = kmem_alloc_nofault(kernel_map, size);
 2784         if (!va)
 2785                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 2786         pa = trunc_page(pa);
 2787         for (tmpva = va; size > 0; ) {
 2788                 pmap_kenter(tmpva, pa);
 2789                 size -= PAGE_SIZE;
 2790                 tmpva += PAGE_SIZE;
 2791                 pa += PAGE_SIZE;
 2792         }
 2793         pmap_invalidate_range(kernel_pmap, va, tmpva);
 2794         return ((void *)(va + offset));
 2795 }
 2796 
 2797 void
 2798 pmap_unmapdev(va, size)
 2799         vm_offset_t va;
 2800         vm_size_t size;
 2801 {
 2802         vm_offset_t base, offset, tmpva;
 2803 
 2804         /* If we gave a direct map region in pmap_mapdev, do nothing */
 2805         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
 2806                 return;
 2807         base = trunc_page(va);
 2808         offset = va & PAGE_MASK;
 2809         size = roundup(offset + size, PAGE_SIZE);
 2810         for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE)
 2811                 pmap_kremove(tmpva);
 2812         pmap_invalidate_range(kernel_pmap, va, tmpva);
 2813         kmem_free(kernel_map, base, size);
 2814 }
 2815 
 2816 /*
 2817  * perform the pmap work for mincore
 2818  */
 2819 int
 2820 pmap_mincore(pmap, addr)
 2821         pmap_t pmap;
 2822         vm_offset_t addr;
 2823 {
 2824         pt_entry_t *ptep, pte;
 2825         vm_page_t m;
 2826         int val = 0;
 2827         
 2828         PMAP_LOCK(pmap);
 2829         ptep = pmap_pte(pmap, addr);
 2830         pte = (ptep != NULL) ? *ptep : 0;
 2831         PMAP_UNLOCK(pmap);
 2832 
 2833         if (pte != 0) {
 2834                 vm_paddr_t pa;
 2835 
 2836                 val = MINCORE_INCORE;
 2837                 if ((pte & PG_MANAGED) == 0)
 2838                         return val;
 2839 
 2840                 pa = pte & PG_FRAME;
 2841 
 2842                 m = PHYS_TO_VM_PAGE(pa);
 2843 
 2844                 /*
 2845                  * Modified by us
 2846                  */
 2847                 if (pte & PG_M)
 2848                         val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
 2849                 else {
 2850                         /*
 2851                          * Modified by someone else
 2852                          */
 2853                         vm_page_lock_queues();
 2854                         if (m->dirty || pmap_is_modified(m))
 2855                                 val |= MINCORE_MODIFIED_OTHER;
 2856                         vm_page_unlock_queues();
 2857                 }
 2858                 /*
 2859                  * Referenced by us
 2860                  */
 2861                 if (pte & PG_A)
 2862                         val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
 2863                 else {
 2864                         /*
 2865                          * Referenced by someone else
 2866                          */
 2867                         vm_page_lock_queues();
 2868                         if ((m->flags & PG_REFERENCED) ||
 2869                             pmap_ts_referenced(m)) {
 2870                                 val |= MINCORE_REFERENCED_OTHER;
 2871                                 vm_page_flag_set(m, PG_REFERENCED);
 2872                         }
 2873                         vm_page_unlock_queues();
 2874                 }
 2875         } 
 2876         return val;
 2877 }
 2878 
 2879 void
 2880 pmap_activate(struct thread *td)
 2881 {
 2882         struct proc *p = td->td_proc;
 2883         pmap_t  pmap, oldpmap;
 2884         u_int64_t  cr3;
 2885 
 2886         critical_enter();
 2887         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 2888         oldpmap = PCPU_GET(curpmap);
 2889 #ifdef SMP
 2890 if (oldpmap)    /* XXX FIXME */
 2891         atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
 2892         atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
 2893 #else
 2894 if (oldpmap)    /* XXX FIXME */
 2895         oldpmap->pm_active &= ~PCPU_GET(cpumask);
 2896         pmap->pm_active |= PCPU_GET(cpumask);
 2897 #endif
 2898         cr3 = vtophys(pmap->pm_pml4);
 2899         /* XXXKSE this is wrong.
 2900          * pmap_activate is for the current thread on the current cpu
 2901          */
 2902         if (p->p_flag & P_SA) {
 2903                 /* Make sure all other cr3 entries are updated. */
 2904                 /* what if they are running?  XXXKSE (maybe abort them) */
 2905                 FOREACH_THREAD_IN_PROC(p, td) {
 2906                         td->td_pcb->pcb_cr3 = cr3;
 2907                 }
 2908         } else {
 2909                 td->td_pcb->pcb_cr3 = cr3;
 2910         }
 2911         load_cr3(cr3);
 2912         critical_exit();
 2913 }
 2914 
 2915 vm_offset_t
 2916 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
 2917 {
 2918 
 2919         if ((obj == NULL) || (size < NBPDR) || (obj->type != OBJT_DEVICE)) {
 2920                 return addr;
 2921         }
 2922 
 2923         addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
 2924         return addr;
 2925 }

Cache object: 8a39646c080d69bc8f68539d2fc3f1c0


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.