The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/amd64/amd64/pmap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2003 Peter Wemm
    9  * All rights reserved.
   10  * Copyright (c) 2005 Alan L. Cox <alc@cs.rice.edu>
   11  * All rights reserved.
   12  *
   13  * This code is derived from software contributed to Berkeley by
   14  * the Systems Programming Group of the University of Utah Computer
   15  * Science Department and William Jolitz of UUNET Technologies Inc.
   16  *
   17  * Redistribution and use in source and binary forms, with or without
   18  * modification, are permitted provided that the following conditions
   19  * are met:
   20  * 1. Redistributions of source code must retain the above copyright
   21  *    notice, this list of conditions and the following disclaimer.
   22  * 2. Redistributions in binary form must reproduce the above copyright
   23  *    notice, this list of conditions and the following disclaimer in the
   24  *    documentation and/or other materials provided with the distribution.
   25  * 3. All advertising materials mentioning features or use of this software
   26  *    must display the following acknowledgement:
   27  *      This product includes software developed by the University of
   28  *      California, Berkeley and its contributors.
   29  * 4. Neither the name of the University nor the names of its contributors
   30  *    may be used to endorse or promote products derived from this software
   31  *    without specific prior written permission.
   32  *
   33  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   34  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   35  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   36  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   37  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   38  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   39  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   40  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   41  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   42  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   43  * SUCH DAMAGE.
   44  *
   45  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   46  */
   47 /*-
   48  * Copyright (c) 2003 Networks Associates Technology, Inc.
   49  * All rights reserved.
   50  *
   51  * This software was developed for the FreeBSD Project by Jake Burkholder,
   52  * Safeport Network Services, and Network Associates Laboratories, the
   53  * Security Research Division of Network Associates, Inc. under
   54  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   55  * CHATS research program.
   56  *
   57  * Redistribution and use in source and binary forms, with or without
   58  * modification, are permitted provided that the following conditions
   59  * are met:
   60  * 1. Redistributions of source code must retain the above copyright
   61  *    notice, this list of conditions and the following disclaimer.
   62  * 2. Redistributions in binary form must reproduce the above copyright
   63  *    notice, this list of conditions and the following disclaimer in the
   64  *    documentation and/or other materials provided with the distribution.
   65  *
   66  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   67  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   68  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   69  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   70  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   71  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   72  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   73  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   74  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   75  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   76  * SUCH DAMAGE.
   77  */
   78 
   79 #include <sys/cdefs.h>
   80 __FBSDID("$FreeBSD$");
   81 
   82 /*
   83  *      Manages physical address maps.
   84  *
   85  *      In addition to hardware address maps, this
   86  *      module is called upon to provide software-use-only
   87  *      maps which may or may not be stored in the same
   88  *      form as hardware maps.  These pseudo-maps are
   89  *      used to store intermediate results from copy
   90  *      operations to and from address spaces.
   91  *
   92  *      Since the information managed by this module is
   93  *      also stored by the logical address mapping module,
   94  *      this module may throw away valid virtual-to-physical
   95  *      mappings at almost any time.  However, invalidations
   96  *      of virtual-to-physical mappings must be done as
   97  *      requested.
   98  *
   99  *      In order to cope with hardware architectures which
  100  *      make virtual-to-physical map invalidates expensive,
  101  *      this module may delay invalidate or reduced protection
  102  *      operations until such time as they are actually
  103  *      necessary.  This module is given full information as
  104  *      to which processors are currently using which maps,
  105  *      and to when physical maps must be made correct.
  106  */
  107 
  108 #include "opt_msgbuf.h"
  109 #include "opt_pmap.h"
  110 
  111 #include <sys/param.h>
  112 #include <sys/systm.h>
  113 #include <sys/kernel.h>
  114 #include <sys/lock.h>
  115 #include <sys/malloc.h>
  116 #include <sys/mman.h>
  117 #include <sys/msgbuf.h>
  118 #include <sys/mutex.h>
  119 #include <sys/proc.h>
  120 #include <sys/sx.h>
  121 #include <sys/vmmeter.h>
  122 #include <sys/sched.h>
  123 #include <sys/sysctl.h>
  124 #ifdef SMP
  125 #include <sys/smp.h>
  126 #endif
  127 
  128 #include <vm/vm.h>
  129 #include <vm/vm_param.h>
  130 #include <vm/vm_kern.h>
  131 #include <vm/vm_page.h>
  132 #include <vm/vm_map.h>
  133 #include <vm/vm_object.h>
  134 #include <vm/vm_extern.h>
  135 #include <vm/vm_pageout.h>
  136 #include <vm/vm_pager.h>
  137 #include <vm/uma.h>
  138 
  139 #include <machine/cpu.h>
  140 #include <machine/cputypes.h>
  141 #include <machine/md_var.h>
  142 #include <machine/pcb.h>
  143 #include <machine/specialreg.h>
  144 #ifdef SMP
  145 #include <machine/smp.h>
  146 #endif
  147 
  148 #ifndef PMAP_SHPGPERPROC
  149 #define PMAP_SHPGPERPROC 200
  150 #endif
  151 
  152 #if defined(DIAGNOSTIC)
  153 #define PMAP_DIAGNOSTIC
  154 #endif
  155 
  156 #if !defined(PMAP_DIAGNOSTIC)
  157 #define PMAP_INLINE __inline
  158 #else
  159 #define PMAP_INLINE
  160 #endif
  161 
  162 struct pmap kernel_pmap_store;
  163 
  164 vm_paddr_t avail_start;         /* PA of first available physical page */
  165 vm_paddr_t avail_end;           /* PA of last available physical page */
  166 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  167 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  168 
  169 static int nkpt;
  170 static int ndmpdp;
  171 static vm_paddr_t dmaplimit;
  172 vm_offset_t kernel_vm_end;
  173 pt_entry_t pg_nx;
  174 
  175 static u_int64_t        KPTphys;        /* phys addr of kernel level 1 */
  176 static u_int64_t        KPDphys;        /* phys addr of kernel level 2 */
  177 u_int64_t               KPDPphys;       /* phys addr of kernel level 3 */
  178 u_int64_t               KPML4phys;      /* phys addr of kernel level 4 */
  179 
  180 static u_int64_t        DMPDphys;       /* phys addr of direct mapped level 2 */
  181 static u_int64_t        DMPDPphys;      /* phys addr of direct mapped level 3 */
  182 
  183 /*
  184  * Data for the pv entry allocation mechanism
  185  */
  186 static uma_zone_t pvzone;
  187 static struct vm_object pvzone_obj;
  188 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  189 int pmap_pagedaemon_waken;
  190 
  191 /*
  192  * All those kernel PT submaps that BSD is so fond of
  193  */
  194 pt_entry_t *CMAP1 = 0;
  195 caddr_t CADDR1 = 0;
  196 struct msgbuf *msgbufp = 0;
  197 
  198 /*
  199  * Crashdump maps.
  200  */
  201 static caddr_t crashdumpmap;
  202 
  203 static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
  204 static pv_entry_t get_pv_entry(void);
  205 static void     pmap_clear_ptes(vm_page_t m, long bit);
  206 
  207 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  208     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  209 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
  210                 vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free);
  211 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde);
  212 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  213                 vm_offset_t va);
  214 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  215 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  216     vm_page_t m);
  217 
  218 static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags);
  219 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
  220 
  221 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
  222 static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
  223                 vm_page_t* free);
  224 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
  225 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
  226 
  227 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  228 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  229 
  230 /*
  231  * Move the kernel virtual free pointer to the next
  232  * 2MB.  This is used to help improve performance
  233  * by using a large (2MB) page for much of the kernel
  234  * (.text, .data, .bss)
  235  */
  236 static vm_offset_t
  237 pmap_kmem_choose(vm_offset_t addr)
  238 {
  239         vm_offset_t newaddr = addr;
  240 
  241         newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
  242         return newaddr;
  243 }
  244 
  245 /********************/
  246 /* Inline functions */
  247 /********************/
  248 
  249 /* Return a non-clipped PD index for a given VA */
  250 static __inline vm_pindex_t
  251 pmap_pde_pindex(vm_offset_t va)
  252 {
  253         return va >> PDRSHIFT;
  254 }
  255 
  256 
  257 /* Return various clipped indexes for a given VA */
  258 static __inline vm_pindex_t
  259 pmap_pte_index(vm_offset_t va)
  260 {
  261 
  262         return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1));
  263 }
  264 
  265 static __inline vm_pindex_t
  266 pmap_pde_index(vm_offset_t va)
  267 {
  268 
  269         return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1));
  270 }
  271 
  272 static __inline vm_pindex_t
  273 pmap_pdpe_index(vm_offset_t va)
  274 {
  275 
  276         return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1));
  277 }
  278 
  279 static __inline vm_pindex_t
  280 pmap_pml4e_index(vm_offset_t va)
  281 {
  282 
  283         return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1));
  284 }
  285 
  286 /* Return a pointer to the PML4 slot that corresponds to a VA */
  287 static __inline pml4_entry_t *
  288 pmap_pml4e(pmap_t pmap, vm_offset_t va)
  289 {
  290 
  291         if (!pmap)
  292                 return NULL;
  293         return (&pmap->pm_pml4[pmap_pml4e_index(va)]);
  294 }
  295 
  296 /* Return a pointer to the PDP slot that corresponds to a VA */
  297 static __inline pdp_entry_t *
  298 pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va)
  299 {
  300         pdp_entry_t *pdpe;
  301 
  302         pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME);
  303         return (&pdpe[pmap_pdpe_index(va)]);
  304 }
  305 
  306 /* Return a pointer to the PDP slot that corresponds to a VA */
  307 static __inline pdp_entry_t *
  308 pmap_pdpe(pmap_t pmap, vm_offset_t va)
  309 {
  310         pml4_entry_t *pml4e;
  311 
  312         pml4e = pmap_pml4e(pmap, va);
  313         if (pml4e == NULL || (*pml4e & PG_V) == 0)
  314                 return NULL;
  315         return (pmap_pml4e_to_pdpe(pml4e, va));
  316 }
  317 
  318 /* Return a pointer to the PD slot that corresponds to a VA */
  319 static __inline pd_entry_t *
  320 pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va)
  321 {
  322         pd_entry_t *pde;
  323 
  324         pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME);
  325         return (&pde[pmap_pde_index(va)]);
  326 }
  327 
  328 /* Return a pointer to the PD slot that corresponds to a VA */
  329 static __inline pd_entry_t *
  330 pmap_pde(pmap_t pmap, vm_offset_t va)
  331 {
  332         pdp_entry_t *pdpe;
  333 
  334         pdpe = pmap_pdpe(pmap, va);
  335         if (pdpe == NULL || (*pdpe & PG_V) == 0)
  336                  return NULL;
  337         return (pmap_pdpe_to_pde(pdpe, va));
  338 }
  339 
  340 /* Return a pointer to the PT slot that corresponds to a VA */
  341 static __inline pt_entry_t *
  342 pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
  343 {
  344         pt_entry_t *pte;
  345 
  346         pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME);
  347         return (&pte[pmap_pte_index(va)]);
  348 }
  349 
  350 /* Return a pointer to the PT slot that corresponds to a VA */
  351 static __inline pt_entry_t *
  352 pmap_pte(pmap_t pmap, vm_offset_t va)
  353 {
  354         pd_entry_t *pde;
  355 
  356         pde = pmap_pde(pmap, va);
  357         if (pde == NULL || (*pde & PG_V) == 0)
  358                 return NULL;
  359         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  360                 return ((pt_entry_t *)pde);
  361         return (pmap_pde_to_pte(pde, va));
  362 }
  363 
  364 
  365 static __inline pt_entry_t *
  366 pmap_pte_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *ptepde)
  367 {
  368         pd_entry_t *pde;
  369 
  370         pde = pmap_pde(pmap, va);
  371         if (pde == NULL || (*pde & PG_V) == 0)
  372                 return NULL;
  373         *ptepde = *pde;
  374         if ((*pde & PG_PS) != 0)        /* compat with i386 pmap_pte() */
  375                 return ((pt_entry_t *)pde);
  376         return (pmap_pde_to_pte(pde, va));
  377 }
  378 
  379 
  380 PMAP_INLINE pt_entry_t *
  381 vtopte(vm_offset_t va)
  382 {
  383         u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  384 
  385         return (PTmap + ((va >> PAGE_SHIFT) & mask));
  386 }
  387 
  388 static __inline pd_entry_t *
  389 vtopde(vm_offset_t va)
  390 {
  391         u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1);
  392 
  393         return (PDmap + ((va >> PDRSHIFT) & mask));
  394 }
  395 
  396 static u_int64_t
  397 allocpages(int n)
  398 {
  399         u_int64_t ret;
  400 
  401         ret = avail_start;
  402         bzero((void *)ret, n * PAGE_SIZE);
  403         avail_start += n * PAGE_SIZE;
  404         return (ret);
  405 }
  406 
  407 static void
  408 create_pagetables(void)
  409 {
  410         int i;
  411 
  412         /* Allocate pages */
  413         KPTphys = allocpages(NKPT);
  414         KPML4phys = allocpages(1);
  415         KPDPphys = allocpages(NKPML4E);
  416         KPDphys = allocpages(NKPDPE);
  417 
  418         ndmpdp = (ptoa(Maxmem) + NBPDP - 1) >> PDPSHIFT;
  419         if (ndmpdp < 4)         /* Minimum 4GB of dirmap */
  420                 ndmpdp = 4;
  421         DMPDPphys = allocpages(NDMPML4E);
  422         DMPDphys = allocpages(ndmpdp);
  423         dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT;
  424 
  425         /* Fill in the underlying page table pages */
  426         /* Read-only from zero to physfree */
  427         /* XXX not fully used, underneath 2M pages */
  428         for (i = 0; (i << PAGE_SHIFT) < avail_start; i++) {
  429                 ((pt_entry_t *)KPTphys)[i] = i << PAGE_SHIFT;
  430                 ((pt_entry_t *)KPTphys)[i] |= PG_RW | PG_V | PG_G;
  431         }
  432 
  433         /* Now map the page tables at their location within PTmap */
  434         for (i = 0; i < NKPT; i++) {
  435                 ((pd_entry_t *)KPDphys)[i] = KPTphys + (i << PAGE_SHIFT);
  436                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V;
  437         }
  438 
  439         /* Map from zero to end of allocations under 2M pages */
  440         /* This replaces some of the KPTphys entries above */
  441         for (i = 0; (i << PDRSHIFT) < avail_start; i++) {
  442                 ((pd_entry_t *)KPDphys)[i] = i << PDRSHIFT;
  443                 ((pd_entry_t *)KPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  444         }
  445 
  446         /* And connect up the PD to the PDP */
  447         for (i = 0; i < NKPDPE; i++) {
  448                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] = KPDphys + (i << PAGE_SHIFT);
  449                 ((pdp_entry_t *)KPDPphys)[i + KPDPI] |= PG_RW | PG_V | PG_U;
  450         }
  451 
  452 
  453         /* Now set up the direct map space using 2MB pages */
  454         for (i = 0; i < NPDEPG * ndmpdp; i++) {
  455                 ((pd_entry_t *)DMPDphys)[i] = (vm_paddr_t)i << PDRSHIFT;
  456                 ((pd_entry_t *)DMPDphys)[i] |= PG_RW | PG_V | PG_PS | PG_G;
  457         }
  458 
  459         /* And the direct map space's PDP */
  460         for (i = 0; i < ndmpdp; i++) {
  461                 ((pdp_entry_t *)DMPDPphys)[i] = DMPDphys + (i << PAGE_SHIFT);
  462                 ((pdp_entry_t *)DMPDPphys)[i] |= PG_RW | PG_V | PG_U;
  463         }
  464 
  465         /* And recursively map PML4 to itself in order to get PTmap */
  466         ((pdp_entry_t *)KPML4phys)[PML4PML4I] = KPML4phys;
  467         ((pdp_entry_t *)KPML4phys)[PML4PML4I] |= PG_RW | PG_V | PG_U;
  468 
  469         /* Connect the Direct Map slot up to the PML4 */
  470         ((pdp_entry_t *)KPML4phys)[DMPML4I] = DMPDPphys;
  471         ((pdp_entry_t *)KPML4phys)[DMPML4I] |= PG_RW | PG_V | PG_U;
  472 
  473         /* Connect the KVA slot up to the PML4 */
  474         ((pdp_entry_t *)KPML4phys)[KPML4I] = KPDPphys;
  475         ((pdp_entry_t *)KPML4phys)[KPML4I] |= PG_RW | PG_V | PG_U;
  476 }
  477 
  478 /*
  479  *      Bootstrap the system enough to run with virtual memory.
  480  *
  481  *      On amd64 this is called after mapping has already been enabled
  482  *      and just syncs the pmap module with what has already been done.
  483  *      [We can't call it easily with mapping off since the kernel is not
  484  *      mapped with PA == VA, hence we would have to relocate every address
  485  *      from the linked base (virtual) address "KERNBASE" to the actual
  486  *      (physical) address starting relative to 0]
  487  */
  488 void
  489 pmap_bootstrap(firstaddr)
  490         vm_paddr_t *firstaddr;
  491 {
  492         vm_offset_t va;
  493         pt_entry_t *pte, *unused;
  494 
  495         avail_start = *firstaddr;
  496 
  497         /*
  498          * Create an initial set of page tables to run the kernel in.
  499          */
  500         create_pagetables();
  501         *firstaddr = avail_start;
  502 
  503         virtual_avail = (vm_offset_t) KERNBASE + avail_start;
  504         virtual_avail = pmap_kmem_choose(virtual_avail);
  505 
  506         virtual_end = VM_MAX_KERNEL_ADDRESS;
  507 
  508 
  509         /* XXX do %cr0 as well */
  510         load_cr4(rcr4() | CR4_PGE | CR4_PSE);
  511         load_cr3(KPML4phys);
  512 
  513         /*
  514          * Initialize the kernel pmap (which is statically allocated).
  515          */
  516         PMAP_LOCK_INIT(kernel_pmap);
  517         kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + KPML4phys);
  518         kernel_pmap->pm_active = -1;    /* don't allow deactivation */
  519         TAILQ_INIT(&kernel_pmap->pm_pvlist);
  520         nkpt = NKPT;
  521 
  522         /*
  523          * Reserve some special page table entries/VA space for temporary
  524          * mapping of pages.
  525          */
  526 #define SYSMAP(c, p, v, n)      \
  527         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  528 
  529         va = virtual_avail;
  530         pte = vtopte(va);
  531 
  532         /*
  533          * CMAP1 is only used for the memory test.
  534          */
  535         SYSMAP(caddr_t, CMAP1, CADDR1, 1)
  536 
  537         /*
  538          * Crashdump maps.
  539          */
  540         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  541 
  542         /*
  543          * msgbufp is used to map the system message buffer.
  544          */
  545         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(MSGBUF_SIZE)))
  546 
  547         virtual_avail = va;
  548 
  549         *CMAP1 = 0;
  550 
  551         invltlb();
  552 
  553         /* Initialize the PAT MSR. */
  554         pmap_init_pat();
  555 }
  556 
  557 /*
  558  * Setup the PAT MSR.
  559  */
  560 void
  561 pmap_init_pat(void)
  562 {
  563         uint64_t pat_msr;
  564 
  565         /* Bail if this CPU doesn't implement PAT. */
  566         if (!(cpu_feature & CPUID_PAT))
  567                 panic("no PAT??");
  568 
  569 #ifdef PAT_WORKS
  570         /*
  571          * Leave the indices 0-3 at the default of WB, WT, UC, and UC-.
  572          * Program 4 and 5 as WP and WC.
  573          * Leave 6 and 7 as UC and UC-.
  574          */
  575         pat_msr = rdmsr(MSR_PAT);
  576         pat_msr &= ~(PAT_MASK(4) | PAT_MASK(5));
  577         pat_msr |= PAT_VALUE(4, PAT_WRITE_PROTECTED) |
  578             PAT_VALUE(5, PAT_WRITE_COMBINING);
  579 #else
  580         /*
  581          * Due to some Intel errata, we can only safely use the lower 4
  582          * PAT entries.  Thus, just replace PAT Index 2 with WC instead
  583          * of UC-.
  584          *
  585          *   Intel Pentium III Processor Specification Update
  586          * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
  587          * or Mode C Paging)
  588          *
  589          *   Intel Pentium IV  Processor Specification Update
  590          * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
  591          */
  592         pat_msr = rdmsr(MSR_PAT);
  593         pat_msr &= ~PAT_MASK(2);
  594         pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  595 #endif
  596         wrmsr(MSR_PAT, pat_msr);
  597 }
  598 
  599 /*
  600  *      Initialize a vm_page's machine-dependent fields.
  601  */
  602 void
  603 pmap_page_init(vm_page_t m)
  604 {
  605 
  606         TAILQ_INIT(&m->md.pv_list);
  607         m->md.pv_list_count = 0;
  608 }
  609 
  610 /*
  611  *      Initialize the pmap module.
  612  *      Called by vm_init, to initialize any structures that the pmap
  613  *      system needs to map virtual memory.
  614  */
  615 void
  616 pmap_init(void)
  617 {
  618         int shpgperproc = PMAP_SHPGPERPROC;
  619 
  620         /*
  621          * Initialize the address space (zone) for the pv entries.  Set a
  622          * high water mark so that the system can recover from excessive
  623          * numbers of pv entries.
  624          */
  625         pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 
  626             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
  627         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  628         pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  629         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  630         pv_entry_high_water = 9 * (pv_entry_max / 10);
  631         uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max);
  632 }
  633 
  634 void
  635 pmap_init2()
  636 {
  637 }
  638 
  639 
  640 /***************************************************
  641  * Low level helper routines.....
  642  ***************************************************/
  643 
  644 /*
  645  * Determine the appropriate bits to set in a PTE or PDE for a specified
  646  * caching mode.
  647  */
  648 static int
  649 pmap_cache_bits(int mode, boolean_t is_pde)
  650 {
  651         int pat_flag, pat_index, cache_bits;
  652 
  653         /* The PAT bit is different for PTE's and PDE's. */
  654         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
  655 
  656         /* If we don't support PAT, map extended modes to older ones. */
  657         if (!(cpu_feature & CPUID_PAT)) {
  658                 switch (mode) {
  659                 case PAT_UNCACHEABLE:
  660                 case PAT_WRITE_THROUGH:
  661                 case PAT_WRITE_BACK:
  662                         break;
  663                 case PAT_UNCACHED:
  664                 case PAT_WRITE_COMBINING:
  665                 case PAT_WRITE_PROTECTED:
  666                         mode = PAT_UNCACHEABLE;
  667                         break;
  668                 }
  669         }
  670         
  671         /* Map the caching mode to a PAT index. */
  672         switch (mode) {
  673 #ifdef PAT_WORKS
  674         case PAT_UNCACHEABLE:
  675                 pat_index = 3;
  676                 break;
  677         case PAT_WRITE_THROUGH:
  678                 pat_index = 1;
  679                 break;
  680         case PAT_WRITE_BACK:
  681                 pat_index = 0;
  682                 break;
  683         case PAT_UNCACHED:
  684                 pat_index = 2;
  685                 break;
  686         case PAT_WRITE_COMBINING:
  687                 pat_index = 5;
  688                 break;
  689         case PAT_WRITE_PROTECTED:
  690                 pat_index = 4;
  691                 break;
  692 #else
  693         case PAT_UNCACHED:
  694         case PAT_UNCACHEABLE:
  695         case PAT_WRITE_PROTECTED:
  696                 pat_index = 3;
  697                 break;
  698         case PAT_WRITE_THROUGH:
  699                 pat_index = 1;
  700                 break;
  701         case PAT_WRITE_BACK:
  702                 pat_index = 0;
  703                 break;
  704         case PAT_WRITE_COMBINING:
  705                 pat_index = 2;
  706                 break;
  707 #endif
  708         default:
  709                 panic("Unknown caching mode %d\n", mode);
  710         }       
  711 
  712         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
  713         cache_bits = 0;
  714         if (pat_index & 0x4)
  715                 cache_bits |= pat_flag;
  716         if (pat_index & 0x2)
  717                 cache_bits |= PG_NC_PCD;
  718         if (pat_index & 0x1)
  719                 cache_bits |= PG_NC_PWT;
  720         return (cache_bits);
  721 }
  722 #ifdef SMP
  723 /*
  724  * For SMP, these functions have to use the IPI mechanism for coherence.
  725  *
  726  * N.B.: Before calling any of the following TLB invalidation functions,
  727  * the calling processor must ensure that all stores updating a non-
  728  * kernel page table are globally performed.  Otherwise, another
  729  * processor could cache an old, pre-update entry without being
  730  * invalidated.  This can happen one of two ways: (1) The pmap becomes
  731  * active on another processor after its pm_active field is checked by
  732  * one of the following functions but before a store updating the page
  733  * table is globally performed. (2) The pmap becomes active on another
  734  * processor before its pm_active field is checked but due to
  735  * speculative loads one of the following functions stills reads the
  736  * pmap as inactive on the other processor.
  737  * 
  738  * The kernel page table is exempt because its pm_active field is
  739  * immutable.  The kernel page table is always active on every
  740  * processor.
  741  */
  742 void
  743 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  744 {
  745         u_int cpumask;
  746         u_int other_cpus;
  747 
  748         sched_pin();
  749         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  750                 invlpg(va);
  751                 smp_invlpg(va);
  752         } else {
  753                 cpumask = PCPU_GET(cpumask);
  754                 other_cpus = PCPU_GET(other_cpus);
  755                 if (pmap->pm_active & cpumask)
  756                         invlpg(va);
  757                 if (pmap->pm_active & other_cpus)
  758                         smp_masked_invlpg(pmap->pm_active & other_cpus, va);
  759         }
  760         sched_unpin();
  761 }
  762 
  763 void
  764 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  765 {
  766         u_int cpumask;
  767         u_int other_cpus;
  768         vm_offset_t addr;
  769 
  770         sched_pin();
  771         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  772                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  773                         invlpg(addr);
  774                 smp_invlpg_range(sva, eva);
  775         } else {
  776                 cpumask = PCPU_GET(cpumask);
  777                 other_cpus = PCPU_GET(other_cpus);
  778                 if (pmap->pm_active & cpumask)
  779                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
  780                                 invlpg(addr);
  781                 if (pmap->pm_active & other_cpus)
  782                         smp_masked_invlpg_range(pmap->pm_active & other_cpus,
  783                             sva, eva);
  784         }
  785         sched_unpin();
  786 }
  787 
  788 void
  789 pmap_invalidate_all(pmap_t pmap)
  790 {
  791         u_int cpumask;
  792         u_int other_cpus;
  793 
  794         sched_pin();
  795         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  796                 invltlb();
  797                 smp_invltlb();
  798         } else {
  799                 cpumask = PCPU_GET(cpumask);
  800                 other_cpus = PCPU_GET(other_cpus);
  801                 if (pmap->pm_active & cpumask)
  802                         invltlb();
  803                 if (pmap->pm_active & other_cpus)
  804                         smp_masked_invltlb(pmap->pm_active & other_cpus);
  805         }
  806         sched_unpin();
  807 }
  808 
  809 void
  810 pmap_invalidate_cache(void)
  811 {
  812 
  813         sched_pin();
  814         wbinvd();
  815         smp_cache_flush();
  816         sched_unpin();
  817 }
  818 #else /* !SMP */
  819 /*
  820  * Normal, non-SMP, invalidation functions.
  821  * We inline these within pmap.c for speed.
  822  */
  823 PMAP_INLINE void
  824 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  825 {
  826 
  827         if (pmap == kernel_pmap || pmap->pm_active)
  828                 invlpg(va);
  829 }
  830 
  831 PMAP_INLINE void
  832 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  833 {
  834         vm_offset_t addr;
  835 
  836         if (pmap == kernel_pmap || pmap->pm_active)
  837                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  838                         invlpg(addr);
  839 }
  840 
  841 PMAP_INLINE void
  842 pmap_invalidate_all(pmap_t pmap)
  843 {
  844 
  845         if (pmap == kernel_pmap || pmap->pm_active)
  846                 invltlb();
  847 }
  848 
  849 PMAP_INLINE void
  850 pmap_invalidate_cache(void)
  851 {
  852 
  853         wbinvd();
  854 }
  855 #endif /* !SMP */
  856 
  857 /*
  858  * Are we current address space or kernel?
  859  */
  860 static __inline int
  861 pmap_is_current(pmap_t pmap)
  862 {
  863         return (pmap == kernel_pmap ||
  864             (pmap->pm_pml4[PML4PML4I] & PG_FRAME) == (PML4pml4e[0] & PG_FRAME));
  865 }
  866 
  867 /*
  868  *      Routine:        pmap_extract
  869  *      Function:
  870  *              Extract the physical page address associated
  871  *              with the given map/virtual_address pair.
  872  */
  873 vm_paddr_t 
  874 pmap_extract(pmap_t pmap, vm_offset_t va)
  875 {
  876         vm_paddr_t rtval;
  877         pt_entry_t *pte;
  878         pd_entry_t pde, *pdep;
  879 
  880         rtval = 0;
  881         PMAP_LOCK(pmap);
  882         pdep = pmap_pde(pmap, va);
  883         if (pdep != NULL) {
  884                 pde = *pdep;
  885                 if (pde) {
  886                         if ((pde & PG_PS) != 0) {
  887                                 rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
  888                                 PMAP_UNLOCK(pmap);
  889                                 return rtval;
  890                         }
  891                         pte = pmap_pde_to_pte(pdep, va);
  892                         rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
  893                 }
  894         }
  895         PMAP_UNLOCK(pmap);
  896         return (rtval);
  897 }
  898 
  899 /*
  900  *      Routine:        pmap_extract_and_hold
  901  *      Function:
  902  *              Atomically extract and hold the physical page
  903  *              with the given pmap and virtual address pair
  904  *              if that mapping permits the given protection.
  905  */
  906 vm_page_t
  907 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
  908 {
  909         pd_entry_t pde, *pdep;
  910         pt_entry_t pte;
  911         vm_page_t m;
  912 
  913         m = NULL;
  914         vm_page_lock_queues();
  915         PMAP_LOCK(pmap);
  916         pdep = pmap_pde(pmap, va);
  917         if (pdep != NULL && (pde = *pdep)) {
  918                 if (pde & PG_PS) {
  919                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
  920                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
  921                                     (va & PDRMASK));
  922                                 vm_page_hold(m);
  923                         }
  924                 } else {
  925                         pte = *pmap_pde_to_pte(pdep, va);
  926                         if ((pte & PG_V) &&
  927                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
  928                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
  929                                 vm_page_hold(m);
  930                         }
  931                 }
  932         }
  933         vm_page_unlock_queues();
  934         PMAP_UNLOCK(pmap);
  935         return (m);
  936 }
  937 
  938 vm_paddr_t
  939 pmap_kextract(vm_offset_t va)
  940 {
  941         pd_entry_t *pde;
  942         vm_paddr_t pa;
  943 
  944         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) {
  945                 pa = DMAP_TO_PHYS(va);
  946         } else {
  947                 pde = vtopde(va);
  948                 if (*pde & PG_PS) {
  949                         pa = (*pde & PG_PS_FRAME) | (va & PDRMASK);
  950                 } else {
  951                         pa = *vtopte(va);
  952                         pa = (pa & PG_FRAME) | (va & PAGE_MASK);
  953                 }
  954         }
  955         return pa;
  956 }
  957 
  958 /***************************************************
  959  * Low level mapping routines.....
  960  ***************************************************/
  961 
  962 /*
  963  * Add a wired page to the kva.
  964  * Note: not SMP coherent.
  965  */
  966 PMAP_INLINE void 
  967 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
  968 {
  969         pt_entry_t *pte;
  970 
  971         pte = vtopte(va);
  972         pte_store(pte, pa | PG_RW | PG_V | PG_G);
  973 }
  974 
  975 PMAP_INLINE void 
  976 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
  977 {
  978         pt_entry_t *pte;
  979 
  980         pte = vtopte(va);
  981         pte_store(pte, pa | PG_RW | PG_V | PG_G | pmap_cache_bits(mode, 0));
  982 }
  983 
  984 /*
  985  * Remove a page from the kernel pagetables.
  986  * Note: not SMP coherent.
  987  */
  988 PMAP_INLINE void
  989 pmap_kremove(vm_offset_t va)
  990 {
  991         pt_entry_t *pte;
  992 
  993         pte = vtopte(va);
  994         pte_clear(pte);
  995 }
  996 
  997 /*
  998  *      Used to map a range of physical addresses into kernel
  999  *      virtual address space.
 1000  *
 1001  *      The value passed in '*virt' is a suggested virtual address for
 1002  *      the mapping. Architectures which can support a direct-mapped
 1003  *      physical to virtual region can return the appropriate address
 1004  *      within that region, leaving '*virt' unchanged. Other
 1005  *      architectures should map the pages starting at '*virt' and
 1006  *      update '*virt' with the first usable address after the mapped
 1007  *      region.
 1008  */
 1009 vm_offset_t
 1010 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 1011 {
 1012         return PHYS_TO_DMAP(start);
 1013 }
 1014 
 1015 
 1016 /*
 1017  * Add a list of wired pages to the kva
 1018  * this routine is only used for temporary
 1019  * kernel mappings that do not need to have
 1020  * page modification or references recorded.
 1021  * Note that old mappings are simply written
 1022  * over.  The page *must* be wired.
 1023  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1024  */
 1025 void
 1026 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 1027 {
 1028         pt_entry_t *endpte, oldpte, *pte;
 1029 
 1030         oldpte = 0;
 1031         pte = vtopte(sva);
 1032         endpte = pte + count;
 1033         while (pte < endpte) {
 1034                 oldpte |= *pte;
 1035                 pte_store(pte, VM_PAGE_TO_PHYS(*ma) | PG_G | PG_RW | PG_V);
 1036                 pte++;
 1037                 ma++;
 1038         }
 1039         if ((oldpte & PG_V) != 0)
 1040                 pmap_invalidate_range(kernel_pmap, sva, sva + count *
 1041                     PAGE_SIZE);
 1042 }
 1043 
 1044 /*
 1045  * This routine tears out page mappings from the
 1046  * kernel -- it is meant only for temporary mappings.
 1047  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1048  */
 1049 void
 1050 pmap_qremove(vm_offset_t sva, int count)
 1051 {
 1052         vm_offset_t va;
 1053 
 1054         va = sva;
 1055         while (count-- > 0) {
 1056                 pmap_kremove(va);
 1057                 va += PAGE_SIZE;
 1058         }
 1059         pmap_invalidate_range(kernel_pmap, sva, va);
 1060 }
 1061 
 1062 /***************************************************
 1063  * Page table page management routines.....
 1064  ***************************************************/
 1065 static PMAP_INLINE void
 1066 pmap_free_zero_pages(vm_page_t free)
 1067 {
 1068         vm_page_t m;
 1069 
 1070         while (free != NULL) {
 1071                 m = free;
 1072                 free = m->right;
 1073                 vm_page_free_zero(m);
 1074         }
 1075 }
 1076 
 1077 /*
 1078  * This routine unholds page table pages, and if the hold count
 1079  * drops to zero, then it decrements the wire count.
 1080  */
 1081 static PMAP_INLINE int
 1082 pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free)
 1083 {
 1084 
 1085         --m->wire_count;
 1086         if (m->wire_count == 0)
 1087                 return _pmap_unwire_pte_hold(pmap, va, m, free);
 1088         else
 1089                 return 0;
 1090 }
 1091 
 1092 static int 
 1093 _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, 
 1094     vm_page_t *free)
 1095 {
 1096         vm_offset_t pteva;
 1097 
 1098         /*
 1099          * unmap the page table page
 1100          */
 1101         if (m->pindex >= (NUPDE + NUPDPE)) {
 1102                 /* PDP page */
 1103                 pml4_entry_t *pml4;
 1104                 pml4 = pmap_pml4e(pmap, va);
 1105                 pteva = (vm_offset_t) PDPmap + amd64_ptob(m->pindex - (NUPDE + NUPDPE));
 1106                 *pml4 = 0;
 1107         } else if (m->pindex >= NUPDE) {
 1108                 /* PD page */
 1109                 pdp_entry_t *pdp;
 1110                 pdp = pmap_pdpe(pmap, va);
 1111                 pteva = (vm_offset_t) PDmap + amd64_ptob(m->pindex - NUPDE);
 1112                 *pdp = 0;
 1113         } else {
 1114                 /* PTE page */
 1115                 pd_entry_t *pd;
 1116                 pd = pmap_pde(pmap, va);
 1117                 pteva = (vm_offset_t) PTmap + amd64_ptob(m->pindex);
 1118                 *pd = 0;
 1119         }
 1120         --pmap->pm_stats.resident_count;
 1121         if (m->pindex < NUPDE) {
 1122                 /* We just released a PT, unhold the matching PD */
 1123                 vm_page_t pdpg;
 1124 
 1125                 pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
 1126                 pmap_unwire_pte_hold(pmap, va, pdpg, free);
 1127         }
 1128         if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 1129                 /* We just released a PD, unhold the matching PDP */
 1130                 vm_page_t pdppg;
 1131 
 1132                 pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
 1133                 pmap_unwire_pte_hold(pmap, va, pdppg, free);
 1134         }
 1135 
 1136         /*
 1137          * This is a release store so that the ordinary store unmapping
 1138          * the page table page is globally performed before TLB shoot-
 1139          * down is begun.
 1140          */
 1141         atomic_subtract_rel_int(&cnt.v_wire_count, 1);
 1142 
 1143         /*
 1144          * Do an invltlb to make the invalidated mapping
 1145          * take effect immediately.
 1146          */
 1147         pmap_invalidate_page(pmap, pteva);
 1148 
 1149         /* 
 1150          * Put page on a list so that it is released after
 1151          * *ALL* TLB shootdown is done
 1152          */
 1153         m->right = *free;
 1154         *free = m;
 1155         
 1156         return 1;
 1157 }
 1158 
 1159 /*
 1160  * After removing a page table entry, this routine is used to
 1161  * conditionally free the page, and manage the hold/wire counts.
 1162  */
 1163 static int
 1164 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, vm_page_t *free)
 1165 {
 1166         vm_page_t mpte;
 1167 
 1168         if (va >= VM_MAXUSER_ADDRESS)
 1169                 return 0;
 1170         KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 1171         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1172         return pmap_unwire_pte_hold(pmap, va, mpte, free);
 1173 }
 1174 
 1175 void
 1176 pmap_pinit0(pmap)
 1177         struct pmap *pmap;
 1178 {
 1179 
 1180         PMAP_LOCK_INIT(pmap);
 1181         pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + KPML4phys);
 1182         pmap->pm_active = 0;
 1183         TAILQ_INIT(&pmap->pm_pvlist);
 1184         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1185 }
 1186 
 1187 /*
 1188  * Initialize a preallocated and zeroed pmap structure,
 1189  * such as one in a vmspace structure.
 1190  */
 1191 void
 1192 pmap_pinit(pmap)
 1193         register struct pmap *pmap;
 1194 {
 1195         vm_page_t pml4pg;
 1196         static vm_pindex_t color;
 1197 
 1198         PMAP_LOCK_INIT(pmap);
 1199 
 1200         /*
 1201          * allocate the page directory page
 1202          */
 1203         while ((pml4pg = vm_page_alloc(NULL, color++, VM_ALLOC_NOOBJ |
 1204             VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
 1205                 VM_WAIT;
 1206 
 1207         pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pml4pg));
 1208 
 1209         if ((pml4pg->flags & PG_ZERO) == 0)
 1210                 pagezero(pmap->pm_pml4);
 1211 
 1212         /* Wire in kernel global address entries. */
 1213         pmap->pm_pml4[KPML4I] = KPDPphys | PG_RW | PG_V | PG_U;
 1214         pmap->pm_pml4[DMPML4I] = DMPDPphys | PG_RW | PG_V | PG_U;
 1215 
 1216         /* install self-referential address mapping entry(s) */
 1217         pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | PG_V | PG_RW | PG_A | PG_M;
 1218 
 1219         pmap->pm_active = 0;
 1220         TAILQ_INIT(&pmap->pm_pvlist);
 1221         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1222 }
 1223 
 1224 /*
 1225  * this routine is called if the page table page is not
 1226  * mapped correctly.
 1227  *
 1228  * Note: If a page allocation fails at page table level two or three,
 1229  * one or two pages may be held during the wait, only to be released
 1230  * afterwards.  This conservative approach is easily argued to avoid
 1231  * race conditions.
 1232  */
 1233 static vm_page_t
 1234 _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 1235 {
 1236         vm_page_t m, pdppg, pdpg;
 1237 
 1238         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1239             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1240             ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1241 
 1242         /*
 1243          * Allocate a page table page.
 1244          */
 1245         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1246             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1247                 if (flags & M_WAITOK) {
 1248                         PMAP_UNLOCK(pmap);
 1249                         vm_page_unlock_queues();
 1250                         VM_WAIT;
 1251                         vm_page_lock_queues();
 1252                         PMAP_LOCK(pmap);
 1253                 }
 1254 
 1255                 /*
 1256                  * Indicate the need to retry.  While waiting, the page table
 1257                  * page may have been allocated.
 1258                  */
 1259                 return (NULL);
 1260         }
 1261         if ((m->flags & PG_ZERO) == 0)
 1262                 pmap_zero_page(m);
 1263 
 1264         /*
 1265          * Map the pagetable page into the process address space, if
 1266          * it isn't already there.
 1267          */
 1268 
 1269         pmap->pm_stats.resident_count++;
 1270 
 1271         if (ptepindex >= (NUPDE + NUPDPE)) {
 1272                 pml4_entry_t *pml4;
 1273                 vm_pindex_t pml4index;
 1274 
 1275                 /* Wire up a new PDPE page */
 1276                 pml4index = ptepindex - (NUPDE + NUPDPE);
 1277                 pml4 = &pmap->pm_pml4[pml4index];
 1278                 *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1279 
 1280         } else if (ptepindex >= NUPDE) {
 1281                 vm_pindex_t pml4index;
 1282                 vm_pindex_t pdpindex;
 1283                 pml4_entry_t *pml4;
 1284                 pdp_entry_t *pdp;
 1285 
 1286                 /* Wire up a new PDE page */
 1287                 pdpindex = ptepindex - NUPDE;
 1288                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1289 
 1290                 pml4 = &pmap->pm_pml4[pml4index];
 1291                 if ((*pml4 & PG_V) == 0) {
 1292                         /* Have to allocate a new pdp, recurse */
 1293                         if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
 1294                             flags) == NULL) {
 1295                                 --m->wire_count;
 1296                                 vm_page_free(m);
 1297                                 return (NULL);
 1298                         }
 1299                 } else {
 1300                         /* Add reference to pdp page */
 1301                         pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME);
 1302                         pdppg->wire_count++;
 1303                 }
 1304                 pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1305 
 1306                 /* Now find the pdp page */
 1307                 pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1308                 *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1309 
 1310         } else {
 1311                 vm_pindex_t pml4index;
 1312                 vm_pindex_t pdpindex;
 1313                 pml4_entry_t *pml4;
 1314                 pdp_entry_t *pdp;
 1315                 pd_entry_t *pd;
 1316 
 1317                 /* Wire up a new PTE page */
 1318                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1319                 pml4index = pdpindex >> NPML4EPGSHIFT;
 1320 
 1321                 /* First, find the pdp and check that its valid. */
 1322                 pml4 = &pmap->pm_pml4[pml4index];
 1323                 if ((*pml4 & PG_V) == 0) {
 1324                         /* Have to allocate a new pd, recurse */
 1325                         if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1326                             flags) == NULL) {
 1327                                 --m->wire_count;
 1328                                 vm_page_free(m);
 1329                                 return (NULL);
 1330                         }
 1331                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1332                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1333                 } else {
 1334                         pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
 1335                         pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)];
 1336                         if ((*pdp & PG_V) == 0) {
 1337                                 /* Have to allocate a new pd, recurse */
 1338                                 if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 1339                                     flags) == NULL) {
 1340                                         --m->wire_count;
 1341                                         vm_page_free(m);
 1342                                         return (NULL);
 1343                                 }
 1344                         } else {
 1345                                 /* Add reference to the pd page */
 1346                                 pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME);
 1347                                 pdpg->wire_count++;
 1348                         }
 1349                 }
 1350                 pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME);
 1351 
 1352                 /* Now we know where the page directory page is */
 1353                 pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)];
 1354                 *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 1355         }
 1356 
 1357         return m;
 1358 }
 1359 
 1360 static vm_page_t
 1361 pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags)
 1362 {
 1363         vm_pindex_t pdpindex, ptepindex;
 1364         pdp_entry_t *pdpe;
 1365         vm_page_t pdpg;
 1366 
 1367         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1368             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1369             ("pmap_allocpde: flags is neither M_NOWAIT nor M_WAITOK"));
 1370 retry:
 1371         pdpe = pmap_pdpe(pmap, va);
 1372         if (pdpe != NULL && (*pdpe & PG_V) != 0) {
 1373                 /* Add a reference to the pd page. */
 1374                 pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME);
 1375                 pdpg->wire_count++;
 1376         } else {
 1377                 /* Allocate a pd page. */
 1378                 ptepindex = pmap_pde_pindex(va);
 1379                 pdpindex = ptepindex >> NPDPEPGSHIFT;
 1380                 pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, flags);
 1381                 if (pdpg == NULL && (flags & M_WAITOK))
 1382                         goto retry;
 1383         }
 1384         return (pdpg);
 1385 }
 1386 
 1387 static vm_page_t
 1388 pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
 1389 {
 1390         vm_pindex_t ptepindex;
 1391         pd_entry_t *pd;
 1392         vm_page_t m, free;
 1393 
 1394         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1395             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1396             ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1397 
 1398         /*
 1399          * Calculate pagetable page index
 1400          */
 1401         ptepindex = pmap_pde_pindex(va);
 1402 retry:
 1403         /*
 1404          * Get the page directory entry
 1405          */
 1406         pd = pmap_pde(pmap, va);
 1407 
 1408         /*
 1409          * This supports switching from a 2MB page to a
 1410          * normal 4K page.
 1411          */
 1412         if (pd != 0 && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
 1413                 *pd = 0;
 1414                 pd = 0;
 1415                 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1416                 free = NULL;
 1417                 pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va), &free);
 1418                 pmap_invalidate_all(kernel_pmap);
 1419                 pmap_free_zero_pages(free);
 1420         }
 1421 
 1422         /*
 1423          * If the page table page is mapped, we just increment the
 1424          * hold count, and activate it.
 1425          */
 1426         if (pd != 0 && (*pd & PG_V) != 0) {
 1427                 m = PHYS_TO_VM_PAGE(*pd & PG_FRAME);
 1428                 m->wire_count++;
 1429         } else {
 1430                 /*
 1431                  * Here if the pte page isn't mapped, or if it has been
 1432                  * deallocated.
 1433                  */
 1434                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1435                 if (m == NULL && (flags & M_WAITOK))
 1436                         goto retry;
 1437         }
 1438         return (m);
 1439 }
 1440 
 1441 
 1442 /***************************************************
 1443  * Pmap allocation/deallocation routines.
 1444  ***************************************************/
 1445 
 1446 /*
 1447  * Release any resources held by the given physical map.
 1448  * Called when a pmap initialized by pmap_pinit is being released.
 1449  * Should only be called if the map contains no valid mappings.
 1450  */
 1451 void
 1452 pmap_release(pmap_t pmap)
 1453 {
 1454         vm_page_t m;
 1455 
 1456         KASSERT(pmap->pm_stats.resident_count == 0,
 1457             ("pmap_release: pmap resident count %ld != 0",
 1458             pmap->pm_stats.resident_count));
 1459 
 1460         m = PHYS_TO_VM_PAGE(pmap->pm_pml4[PML4PML4I] & PG_FRAME);
 1461 
 1462         pmap->pm_pml4[KPML4I] = 0;      /* KVA */
 1463         pmap->pm_pml4[DMPML4I] = 0;     /* Direct Map */
 1464         pmap->pm_pml4[PML4PML4I] = 0;   /* Recursive Mapping */
 1465 
 1466         vm_page_lock_queues();
 1467         m->wire_count--;
 1468         atomic_subtract_int(&cnt.v_wire_count, 1);
 1469         vm_page_free_zero(m);
 1470         vm_page_unlock_queues();
 1471         PMAP_LOCK_DESTROY(pmap);
 1472 }
 1473 
 1474 static int
 1475 kvm_size(SYSCTL_HANDLER_ARGS)
 1476 {
 1477         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
 1478 
 1479         return sysctl_handle_long(oidp, &ksize, 0, req);
 1480 }
 1481 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 1482     0, 0, kvm_size, "LU", "Size of KVM");
 1483 
 1484 static int
 1485 kvm_free(SYSCTL_HANDLER_ARGS)
 1486 {
 1487         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 1488 
 1489         return sysctl_handle_long(oidp, &kfree, 0, req);
 1490 }
 1491 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 1492     0, 0, kvm_free, "LU", "Amount of KVM free");
 1493 
 1494 /*
 1495  * grow the number of kernel page table entries, if needed
 1496  */
 1497 void
 1498 pmap_growkernel(vm_offset_t addr)
 1499 {
 1500         vm_paddr_t paddr;
 1501         vm_page_t nkpg;
 1502         pd_entry_t *pde, newpdir;
 1503         pdp_entry_t newpdp;
 1504 
 1505         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1506         if (kernel_vm_end == 0) {
 1507                 kernel_vm_end = KERNBASE;
 1508                 nkpt = 0;
 1509                 while ((*pmap_pde(kernel_pmap, kernel_vm_end) & PG_V) != 0) {
 1510                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1511                         nkpt++;
 1512                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1513                                 kernel_vm_end = kernel_map->max_offset;
 1514                                 break;                       
 1515                         }
 1516                 }
 1517         }
 1518         addr = roundup2(addr, PAGE_SIZE * NPTEPG);
 1519         if (addr - 1 >= kernel_map->max_offset)
 1520                 addr = kernel_map->max_offset;
 1521         while (kernel_vm_end < addr) {
 1522                 pde = pmap_pde(kernel_pmap, kernel_vm_end);
 1523                 if (pde == NULL) {
 1524                         /* We need a new PDP entry */
 1525                         nkpg = vm_page_alloc(NULL, nkpt,
 1526                             VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 1527                         if (!nkpg)
 1528                                 panic("pmap_growkernel: no memory to grow kernel");
 1529                         pmap_zero_page(nkpg);
 1530                         paddr = VM_PAGE_TO_PHYS(nkpg);
 1531                         newpdp = (pdp_entry_t)
 1532                                 (paddr | PG_V | PG_RW | PG_A | PG_M);
 1533                         *pmap_pdpe(kernel_pmap, kernel_vm_end) = newpdp;
 1534                         continue; /* try again */
 1535                 }
 1536                 if ((*pde & PG_V) != 0) {
 1537                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1538                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1539                                 kernel_vm_end = kernel_map->max_offset;
 1540                                 break;                       
 1541                         }
 1542                         continue;
 1543                 }
 1544 
 1545                 /*
 1546                  * This index is bogus, but out of the way
 1547                  */
 1548                 nkpg = vm_page_alloc(NULL, nkpt,
 1549                     VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 1550                 if (!nkpg)
 1551                         panic("pmap_growkernel: no memory to grow kernel");
 1552 
 1553                 nkpt++;
 1554 
 1555                 pmap_zero_page(nkpg);
 1556                 paddr = VM_PAGE_TO_PHYS(nkpg);
 1557                 newpdir = (pd_entry_t) (paddr | PG_V | PG_RW | PG_A | PG_M);
 1558                 *pmap_pde(kernel_pmap, kernel_vm_end) = newpdir;
 1559 
 1560                 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1561                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1562                         kernel_vm_end = kernel_map->max_offset;
 1563                         break;                       
 1564                 }
 1565         }
 1566 }
 1567 
 1568 
 1569 /***************************************************
 1570  * page management routines.
 1571  ***************************************************/
 1572 
 1573 /*
 1574  * free the pv_entry back to the free list
 1575  */
 1576 static PMAP_INLINE void
 1577 free_pv_entry(pv_entry_t pv)
 1578 {
 1579         pv_entry_count--;
 1580         uma_zfree(pvzone, pv);
 1581 }
 1582 
 1583 /*
 1584  * get a new pv_entry, allocating a block from the system
 1585  * when needed.
 1586  * the memory allocation is performed bypassing the malloc code
 1587  * because of the possibility of allocations at interrupt time.
 1588  */
 1589 static pv_entry_t
 1590 get_pv_entry(void)
 1591 {
 1592         pv_entry_count++;
 1593         if ((pv_entry_count > pv_entry_high_water) &&
 1594                 (pmap_pagedaemon_waken == 0)) {
 1595                 pmap_pagedaemon_waken = 1;
 1596                 wakeup (&vm_pages_needed);
 1597         }
 1598         return uma_zalloc(pvzone, M_NOWAIT);
 1599 }
 1600 
 1601 
 1602 static void
 1603 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 1604 {
 1605         pv_entry_t pv;
 1606 
 1607         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1608         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1609         if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
 1610                 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 1611                         if (pmap == pv->pv_pmap && va == pv->pv_va) 
 1612                                 break;
 1613                 }
 1614         } else {
 1615                 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
 1616                         if (va == pv->pv_va) 
 1617                                 break;
 1618                 }
 1619         }
 1620         KASSERT(pv != NULL, ("pmap_remove_entry: pv not found"));
 1621         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1622         m->md.pv_list_count--;
 1623         if (TAILQ_EMPTY(&m->md.pv_list))
 1624                 vm_page_flag_clear(m, PG_WRITEABLE);
 1625         TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
 1626         free_pv_entry(pv);
 1627 }
 1628 
 1629 /*
 1630  * Create a pv entry for page at pa for
 1631  * (pmap, va).
 1632  */
 1633 static void
 1634 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1635 {
 1636         pv_entry_t pv;
 1637 
 1638         pv = get_pv_entry();
 1639         if (pv == NULL)
 1640                 panic("no pv entries: increase vm.pmap.shpgperproc");
 1641         pv->pv_va = va;
 1642         pv->pv_pmap = pmap;
 1643 
 1644         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1645         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1646         TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
 1647         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 1648         m->md.pv_list_count++;
 1649 }
 1650 
 1651 /*
 1652  * Conditionally create a pv entry.
 1653  */
 1654 static boolean_t
 1655 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1656 {
 1657         pv_entry_t pv;
 1658 
 1659         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1660         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1661         if (pv_entry_count < pv_entry_high_water && 
 1662             (pv = uma_zalloc(pvzone, M_NOWAIT)) != NULL) {
 1663                 pv_entry_count++;
 1664                 pv->pv_va = va;
 1665                 pv->pv_pmap = pmap;
 1666                 TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
 1667                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 1668                 m->md.pv_list_count++;
 1669                 return (TRUE);
 1670         } else
 1671                 return (FALSE);
 1672 }
 1673 
 1674 /*
 1675  * pmap_remove_pte: do the things to unmap a page in a process
 1676  */
 1677 static int
 1678 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 
 1679     pd_entry_t ptepde, vm_page_t *free)
 1680 {
 1681         pt_entry_t oldpte;
 1682         vm_page_t m;
 1683 
 1684         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1685         oldpte = pte_load_clear(ptq);
 1686         if (oldpte & PG_W)
 1687                 pmap->pm_stats.wired_count -= 1;
 1688         /*
 1689          * Machines that don't support invlpg, also don't support
 1690          * PG_G.
 1691          */
 1692         if (oldpte & PG_G)
 1693                 pmap_invalidate_page(kernel_pmap, va);
 1694         pmap->pm_stats.resident_count -= 1;
 1695         if (oldpte & PG_MANAGED) {
 1696                 m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 1697                 if (oldpte & PG_M) {
 1698                         KASSERT((oldpte & PG_RW),
 1699         ("pmap_remove_pte: modified page not writable: va: %#lx, pte: %#lx",
 1700                             va, oldpte));
 1701                         vm_page_dirty(m);
 1702                 }
 1703                 if (oldpte & PG_A)
 1704                         vm_page_flag_set(m, PG_REFERENCED);
 1705                 pmap_remove_entry(pmap, m, va);
 1706         }
 1707         return (pmap_unuse_pt(pmap, va, ptepde, free));
 1708 }
 1709 
 1710 /*
 1711  * Remove a single page from a process address space
 1712  */
 1713 static void
 1714 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde)
 1715 {
 1716         pt_entry_t *pte;
 1717         vm_page_t free = NULL;
 1718 
 1719         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1720         if ((*pde & PG_V) == 0)
 1721                 return;
 1722         pte = pmap_pde_to_pte(pde, va);
 1723         if ((*pte & PG_V) == 0)
 1724                 return;
 1725         pmap_remove_pte(pmap, pte, va, *pde, &free);
 1726         pmap_invalidate_page(pmap, va);
 1727         pmap_free_zero_pages(free);
 1728 }
 1729 
 1730 /*
 1731  *      Remove the given range of addresses from the specified map.
 1732  *
 1733  *      It is assumed that the start and end are properly
 1734  *      rounded to the page size.
 1735  */
 1736 void
 1737 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1738 {
 1739         vm_offset_t va_next;
 1740         pml4_entry_t *pml4e;
 1741         pdp_entry_t *pdpe;
 1742         pd_entry_t ptpaddr, *pde;
 1743         pt_entry_t *pte;
 1744         vm_page_t free = NULL;
 1745         int anyvalid;
 1746 
 1747         /*
 1748          * Perform an unsynchronized read.  This is, however, safe.
 1749          */
 1750         if (pmap->pm_stats.resident_count == 0)
 1751                 return;
 1752 
 1753         anyvalid = 0;
 1754 
 1755         vm_page_lock_queues();
 1756         PMAP_LOCK(pmap);
 1757 
 1758         /*
 1759          * special handling of removing one page.  a very
 1760          * common operation and easy to short circuit some
 1761          * code.
 1762          */
 1763         if (sva + PAGE_SIZE == eva) {
 1764                 pde = pmap_pde(pmap, sva);
 1765                 if (pde && (*pde & PG_PS) == 0) {
 1766                         pmap_remove_page(pmap, sva, pde);
 1767                         goto out;
 1768                 }
 1769         }
 1770 
 1771         for (; sva < eva; sva = va_next) {
 1772 
 1773                 if (pmap->pm_stats.resident_count == 0)
 1774                         break;
 1775 
 1776                 pml4e = pmap_pml4e(pmap, sva);
 1777                 if ((*pml4e & PG_V) == 0) {
 1778                         va_next = (sva + NBPML4) & ~PML4MASK;
 1779                         continue;
 1780                 }
 1781 
 1782                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 1783                 if ((*pdpe & PG_V) == 0) {
 1784                         va_next = (sva + NBPDP) & ~PDPMASK;
 1785                         continue;
 1786                 }
 1787 
 1788                 /*
 1789                  * Calculate index for next page table.
 1790                  */
 1791                 va_next = (sva + NBPDR) & ~PDRMASK;
 1792 
 1793                 pde = pmap_pdpe_to_pde(pdpe, sva);
 1794                 ptpaddr = *pde;
 1795 
 1796                 /*
 1797                  * Weed out invalid mappings.
 1798                  */
 1799                 if (ptpaddr == 0)
 1800                         continue;
 1801 
 1802                 /*
 1803                  * Check for large page.
 1804                  */
 1805                 if ((ptpaddr & PG_PS) != 0) {
 1806                         *pde = 0;
 1807                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1808                         pmap_unuse_pt(pmap, sva, *pdpe, &free);
 1809                         anyvalid = 1;
 1810                         continue;
 1811                 }
 1812 
 1813                 /*
 1814                  * Limit our scan to either the end of the va represented
 1815                  * by the current page table page, or to the end of the
 1816                  * range being removed.
 1817                  */
 1818                 if (va_next > eva)
 1819                         va_next = eva;
 1820 
 1821                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 1822                     sva += PAGE_SIZE) {
 1823                         if (*pte == 0)
 1824                                 continue;
 1825 
 1826                         /*
 1827                          * The TLB entry for a PG_G mapping is invalidated
 1828                          * by pmap_remove_pte().
 1829                          */
 1830                         if ((*pte & PG_G) == 0)
 1831                                 anyvalid = 1;
 1832                         if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free))
 1833                                 break;
 1834                 }
 1835         }
 1836 out:
 1837         if (anyvalid) {
 1838                 pmap_invalidate_all(pmap);
 1839                 pmap_free_zero_pages(free);
 1840         }
 1841         vm_page_unlock_queues();        
 1842         PMAP_UNLOCK(pmap);
 1843 }
 1844 
 1845 /*
 1846  *      Routine:        pmap_remove_all
 1847  *      Function:
 1848  *              Removes this physical page from
 1849  *              all physical maps in which it resides.
 1850  *              Reflects back modify bits to the pager.
 1851  *
 1852  *      Notes:
 1853  *              Original versions of this routine were very
 1854  *              inefficient because they iteratively called
 1855  *              pmap_remove (slow...)
 1856  */
 1857 
 1858 void
 1859 pmap_remove_all(vm_page_t m)
 1860 {
 1861         register pv_entry_t pv;
 1862         pt_entry_t *pte, tpte;
 1863         pd_entry_t ptepde;
 1864         vm_page_t free;
 1865 
 1866 #if defined(PMAP_DIAGNOSTIC)
 1867         /*
 1868          * XXX This makes pmap_remove_all() illegal for non-managed pages!
 1869          */
 1870         if (m->flags & PG_FICTITIOUS) {
 1871                 panic("pmap_remove_all: illegal for unmanaged page, va: 0x%lx",
 1872                     VM_PAGE_TO_PHYS(m));
 1873         }
 1874 #endif
 1875         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1876         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 1877                 PMAP_LOCK(pv->pv_pmap);
 1878                 pv->pv_pmap->pm_stats.resident_count--;
 1879                 pte = pmap_pte_pde(pv->pv_pmap, pv->pv_va, &ptepde);
 1880                 tpte = pte_load_clear(pte);
 1881                 if (tpte & PG_W)
 1882                         pv->pv_pmap->pm_stats.wired_count--;
 1883                 if (tpte & PG_A)
 1884                         vm_page_flag_set(m, PG_REFERENCED);
 1885 
 1886                 /*
 1887                  * Update the vm_page_t clean and reference bits.
 1888                  */
 1889                 if (tpte & PG_M) {
 1890                         KASSERT((tpte & PG_RW),
 1891         ("pmap_remove_all: modified page not writable: va: %#lx, pte: %#lx",
 1892                             pv->pv_va, tpte));
 1893                         vm_page_dirty(m);
 1894                 }
 1895                 free = NULL;
 1896                 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, ptepde, &free);
 1897                 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 1898                 pmap_free_zero_pages(free);
 1899                 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
 1900                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1901                 m->md.pv_list_count--;
 1902                 PMAP_UNLOCK(pv->pv_pmap);
 1903                 free_pv_entry(pv);
 1904         }
 1905         vm_page_flag_clear(m, PG_WRITEABLE);
 1906 }
 1907 
 1908 /*
 1909  *      Set the physical protection on the
 1910  *      specified range of this map as requested.
 1911  */
 1912 void
 1913 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 1914 {
 1915         vm_offset_t va_next;
 1916         pml4_entry_t *pml4e;
 1917         pdp_entry_t *pdpe;
 1918         pd_entry_t ptpaddr, *pde;
 1919         pt_entry_t *pte;
 1920         int anychanged;
 1921 
 1922         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 1923                 pmap_remove(pmap, sva, eva);
 1924                 return;
 1925         }
 1926 
 1927         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 1928             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 1929                 return;
 1930 
 1931         anychanged = 0;
 1932 
 1933         vm_page_lock_queues();
 1934         PMAP_LOCK(pmap);
 1935         for (; sva < eva; sva = va_next) {
 1936 
 1937                 pml4e = pmap_pml4e(pmap, sva);
 1938                 if ((*pml4e & PG_V) == 0) {
 1939                         va_next = (sva + NBPML4) & ~PML4MASK;
 1940                         continue;
 1941                 }
 1942 
 1943                 pdpe = pmap_pml4e_to_pdpe(pml4e, sva);
 1944                 if ((*pdpe & PG_V) == 0) {
 1945                         va_next = (sva + NBPDP) & ~PDPMASK;
 1946                         continue;
 1947                 }
 1948 
 1949                 va_next = (sva + NBPDR) & ~PDRMASK;
 1950 
 1951                 pde = pmap_pdpe_to_pde(pdpe, sva);
 1952                 ptpaddr = *pde;
 1953 
 1954                 /*
 1955                  * Weed out invalid mappings.
 1956                  */
 1957                 if (ptpaddr == 0)
 1958                         continue;
 1959 
 1960                 /*
 1961                  * Check for large page.
 1962                  */
 1963                 if ((ptpaddr & PG_PS) != 0) {
 1964                         if ((prot & VM_PROT_WRITE) == 0)
 1965                                 *pde &= ~(PG_M|PG_RW);
 1966                         if ((prot & VM_PROT_EXECUTE) == 0)
 1967                                 *pde |= pg_nx;
 1968                         anychanged = 1;
 1969                         continue;
 1970                 }
 1971 
 1972                 if (va_next > eva)
 1973                         va_next = eva;
 1974 
 1975                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 1976                     sva += PAGE_SIZE) {
 1977                         pt_entry_t obits, pbits;
 1978                         vm_page_t m;
 1979 
 1980 retry:
 1981                         obits = pbits = *pte;
 1982                         if ((pbits & PG_V) == 0)
 1983                                 continue;
 1984                         if (pbits & PG_MANAGED) {
 1985                                 m = NULL;
 1986                                 if (pbits & PG_A) {
 1987                                         m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 1988                                         vm_page_flag_set(m, PG_REFERENCED);
 1989                                         pbits &= ~PG_A;
 1990                                 }
 1991                                 if ((pbits & PG_M) != 0) {
 1992                                         if (m == NULL)
 1993                                                 m = PHYS_TO_VM_PAGE(pbits &
 1994                                                     PG_FRAME);
 1995                                         vm_page_dirty(m);
 1996                                 }
 1997                         }
 1998 
 1999                         if ((prot & VM_PROT_WRITE) == 0)
 2000                                 pbits &= ~(PG_RW | PG_M);
 2001                         if ((prot & VM_PROT_EXECUTE) == 0)
 2002                                 pbits |= pg_nx;
 2003 
 2004                         if (pbits != obits) {
 2005                                 if (!atomic_cmpset_long(pte, obits, pbits))
 2006                                         goto retry;
 2007                                 if (obits & PG_G)
 2008                                         pmap_invalidate_page(pmap, sva);
 2009                                 else
 2010                                         anychanged = 1;
 2011                         }
 2012                 }
 2013         }
 2014         if (anychanged)
 2015                 pmap_invalidate_all(pmap);
 2016         vm_page_unlock_queues();
 2017         PMAP_UNLOCK(pmap);
 2018 }
 2019 
 2020 /*
 2021  *      Insert the given physical page (p) at
 2022  *      the specified virtual address (v) in the
 2023  *      target physical map with the protection requested.
 2024  *
 2025  *      If specified, the page will be wired down, meaning
 2026  *      that the related pte can not be reclaimed.
 2027  *
 2028  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 2029  *      or lose information.  That is, this routine must actually
 2030  *      insert this page into the given map NOW.
 2031  */
 2032 void
 2033 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 2034            boolean_t wired)
 2035 {
 2036         vm_paddr_t pa;
 2037         pd_entry_t *pde;
 2038         register pt_entry_t *pte;
 2039         vm_paddr_t opa;
 2040         pt_entry_t origpte, newpte;
 2041         vm_page_t mpte, om;
 2042         boolean_t invlva;
 2043 
 2044         va = trunc_page(va);
 2045 #ifdef PMAP_DIAGNOSTIC
 2046         if (va > VM_MAX_KERNEL_ADDRESS)
 2047                 panic("pmap_enter: toobig");
 2048         if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
 2049                 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", va);
 2050 #endif
 2051 
 2052         mpte = NULL;
 2053 
 2054         vm_page_lock_queues();
 2055         PMAP_LOCK(pmap);
 2056 
 2057         /*
 2058          * In the case that a page table page is not
 2059          * resident, we are creating it here.
 2060          */
 2061         if (va < VM_MAXUSER_ADDRESS) {
 2062                 mpte = pmap_allocpte(pmap, va, M_WAITOK);
 2063         }
 2064 #if 0 && defined(PMAP_DIAGNOSTIC)
 2065         else {
 2066                 pd_entry_t *pdeaddr = pmap_pde(pmap, va);
 2067                 origpte = *pdeaddr;
 2068                 if ((origpte & PG_V) == 0) { 
 2069                         panic("pmap_enter: invalid kernel page table page, pde=%p, va=%p\n",
 2070                                 origpte, va);
 2071                 }
 2072         }
 2073 #endif
 2074 
 2075         pde = pmap_pde(pmap, va);
 2076         if (pde != NULL) {
 2077                 if ((*pde & PG_PS) != 0)
 2078                         panic("pmap_enter: attempted pmap_enter on 2MB page");
 2079                 pte = pmap_pde_to_pte(pde, va);
 2080         } else
 2081                 pte = NULL;
 2082 
 2083         /*
 2084          * Page Directory table entry not valid, we need a new PT page
 2085          */
 2086         if (pte == NULL)
 2087                 panic("pmap_enter: invalid page directory va=%#lx\n", va);
 2088 
 2089         pa = VM_PAGE_TO_PHYS(m);
 2090         om = NULL;
 2091         origpte = *pte;
 2092         opa = origpte & PG_FRAME;
 2093 
 2094         /*
 2095          * Mapping has not changed, must be protection or wiring change.
 2096          */
 2097         if (origpte && (opa == pa)) {
 2098                 /*
 2099                  * Wiring change, just update stats. We don't worry about
 2100                  * wiring PT pages as they remain resident as long as there
 2101                  * are valid mappings in them. Hence, if a user page is wired,
 2102                  * the PT page will be also.
 2103                  */
 2104                 if (wired && ((origpte & PG_W) == 0))
 2105                         pmap->pm_stats.wired_count++;
 2106                 else if (!wired && (origpte & PG_W))
 2107                         pmap->pm_stats.wired_count--;
 2108 
 2109                 /*
 2110                  * Remove extra pte reference
 2111                  */
 2112                 if (mpte)
 2113                         mpte->wire_count--;
 2114 
 2115                 /*
 2116                  * We might be turning off write access to the page,
 2117                  * so we go ahead and sense modify status.
 2118                  */
 2119                 if (origpte & PG_MANAGED) {
 2120                         om = m;
 2121                         pa |= PG_MANAGED;
 2122                 }
 2123                 goto validate;
 2124         } 
 2125         /*
 2126          * Mapping has changed, invalidate old range and fall through to
 2127          * handle validating new mapping.
 2128          */
 2129         if (opa) {
 2130                 if (origpte & PG_W)
 2131                         pmap->pm_stats.wired_count--;
 2132                 if (origpte & PG_MANAGED) {
 2133                         om = PHYS_TO_VM_PAGE(opa);
 2134                         pmap_remove_entry(pmap, om, va);
 2135                 }
 2136                 if (mpte != NULL) {
 2137                         mpte->wire_count--;
 2138                         KASSERT(mpte->wire_count > 0,
 2139                             ("pmap_enter: missing reference to page table page,"
 2140                              " va: 0x%lx", va));
 2141                 }
 2142         } else
 2143                 pmap->pm_stats.resident_count++;
 2144 
 2145         /*
 2146          * Enter on the PV list if part of our managed memory.
 2147          */
 2148         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
 2149                 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
 2150                     ("pmap_enter: managed mapping within the clean submap"));
 2151                 pmap_insert_entry(pmap, va, m);
 2152                 pa |= PG_MANAGED;
 2153         }
 2154 
 2155         /*
 2156          * Increment counters
 2157          */
 2158         if (wired)
 2159                 pmap->pm_stats.wired_count++;
 2160 
 2161 validate:
 2162         /*
 2163          * Now validate mapping with desired protection/wiring.
 2164          */
 2165         newpte = (pt_entry_t)(pa | PG_V);
 2166         if ((prot & VM_PROT_WRITE) != 0)
 2167                 newpte |= PG_RW;
 2168         if ((prot & VM_PROT_EXECUTE) == 0)
 2169                 newpte |= pg_nx;
 2170         if (wired)
 2171                 newpte |= PG_W;
 2172         if (va < VM_MAXUSER_ADDRESS)
 2173                 newpte |= PG_U;
 2174         if (pmap == kernel_pmap)
 2175                 newpte |= PG_G;
 2176 
 2177         /*
 2178          * if the mapping or permission bits are different, we need
 2179          * to update the pte.
 2180          */
 2181         if ((origpte & ~(PG_M|PG_A)) != newpte) {
 2182                 if (origpte & PG_V) {
 2183                         invlva = FALSE;
 2184                         origpte = pte_load_store(pte, newpte | PG_A);
 2185                         if (origpte & PG_A) {
 2186                                 if (origpte & PG_MANAGED)
 2187                                         vm_page_flag_set(om, PG_REFERENCED);
 2188                                 if (opa != VM_PAGE_TO_PHYS(m) || ((origpte &
 2189                                     PG_NX) == 0 && (newpte & PG_NX)))
 2190                                         invlva = TRUE;
 2191                         }
 2192                         if (origpte & PG_M) {
 2193                                 KASSERT((origpte & PG_RW),
 2194         ("pmap_enter: modified page not writable: va: %#lx, pte: %#lx",
 2195                                     va, origpte));
 2196                                 if ((origpte & PG_MANAGED) != 0)
 2197                                         vm_page_dirty(om);
 2198                                 if ((newpte & PG_RW) == 0)
 2199                                         invlva = TRUE;
 2200                         }
 2201                         if (invlva)
 2202                                 pmap_invalidate_page(pmap, va);
 2203                 } else
 2204                         pte_store(pte, newpte | PG_A);
 2205         }
 2206         vm_page_unlock_queues();
 2207         PMAP_UNLOCK(pmap);
 2208 }
 2209 
 2210 /*
 2211  * Maps a sequence of resident pages belonging to the same object.
 2212  * The sequence begins with the given page m_start.  This page is
 2213  * mapped at the given virtual address start.  Each subsequent page is
 2214  * mapped at a virtual address that is offset from start by the same
 2215  * amount as the page is offset from m_start within the object.  The
 2216  * last page in the sequence is the page with the largest offset from
 2217  * m_start that can be mapped at a virtual address less than the given
 2218  * virtual address end.  Not every virtual page between start and end
 2219  * is mapped; only those for which a resident page exists with the
 2220  * corresponding offset from m_start are mapped.
 2221  */
 2222 void
 2223 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 2224     vm_page_t m_start, vm_prot_t prot)
 2225 {
 2226         vm_page_t m, mpte;
 2227         vm_pindex_t diff, psize;
 2228 
 2229         VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
 2230         psize = atop(end - start);
 2231         mpte = NULL;
 2232         m = m_start;
 2233         PMAP_LOCK(pmap);
 2234         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 2235                 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m,
 2236                     prot, mpte);
 2237                 m = TAILQ_NEXT(m, listq);
 2238         }
 2239         PMAP_UNLOCK(pmap);
 2240 }
 2241 
 2242 /*
 2243  * this code makes some *MAJOR* assumptions:
 2244  * 1. Current pmap & pmap exists.
 2245  * 2. Not wired.
 2246  * 3. Read access.
 2247  * 4. No page table pages.
 2248  * but is *MUCH* faster than pmap_enter...
 2249  */
 2250 
 2251 void
 2252 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 2253 {
 2254 
 2255         PMAP_LOCK(pmap);
 2256         (void) pmap_enter_quick_locked(pmap, va, m, prot, NULL);
 2257         PMAP_UNLOCK(pmap);
 2258 }
 2259 
 2260 static vm_page_t
 2261 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 2262     vm_prot_t prot, vm_page_t mpte)
 2263 {
 2264         vm_page_t free;
 2265         pt_entry_t *pte;
 2266         vm_paddr_t pa;
 2267 
 2268         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 2269             (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
 2270             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 2271         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2272         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2273 
 2274         /*
 2275          * In the case that a page table page is not
 2276          * resident, we are creating it here.
 2277          */
 2278         if (va < VM_MAXUSER_ADDRESS) {
 2279                 vm_pindex_t ptepindex;
 2280                 pd_entry_t *ptepa;
 2281 
 2282                 /*
 2283                  * Calculate pagetable page index
 2284                  */
 2285                 ptepindex = pmap_pde_pindex(va);
 2286                 if (mpte && (mpte->pindex == ptepindex)) {
 2287                         mpte->wire_count++;
 2288                 } else {
 2289                         /*
 2290                          * Get the page directory entry
 2291                          */
 2292                         ptepa = pmap_pde(pmap, va);
 2293 
 2294                         /*
 2295                          * If the page table page is mapped, we just increment
 2296                          * the hold count, and activate it.
 2297                          */
 2298                         if (ptepa && (*ptepa & PG_V) != 0) {
 2299                                 if (*ptepa & PG_PS)
 2300                                         panic("pmap_enter_quick: unexpected mapping into 2MB page");
 2301                                 mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
 2302                                 mpte->wire_count++;
 2303                         } else {
 2304                                 mpte = _pmap_allocpte(pmap, ptepindex,
 2305                                     M_NOWAIT);
 2306                                 if (mpte == NULL)
 2307                                         return (mpte);
 2308                         }
 2309                 }
 2310         } else {
 2311                 mpte = NULL;
 2312         }
 2313 
 2314         /*
 2315          * This call to vtopte makes the assumption that we are
 2316          * entering the page into the current pmap.  In order to support
 2317          * quick entry into any pmap, one would likely use pmap_pte.
 2318          * But that isn't as quick as vtopte.
 2319          */
 2320         pte = vtopte(va);
 2321         if (*pte) {
 2322                 if (mpte != NULL) {
 2323                         mpte->wire_count--;
 2324                         mpte = NULL;
 2325                 }
 2326                 return (mpte);
 2327         }
 2328 
 2329         /*
 2330          * Enter on the PV list if part of our managed memory.
 2331          */
 2332         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 &&
 2333             !pmap_try_insert_pv_entry(pmap, va, m)) {
 2334                 if (mpte != NULL) {
 2335                         free = NULL;
 2336                         if (pmap_unwire_pte_hold(pmap, va, mpte, &free)) {
 2337                                 pmap_invalidate_page(pmap, va);
 2338                                 pmap_free_zero_pages(free);
 2339                         }
 2340                         mpte = NULL;
 2341                 }
 2342                 return (mpte);
 2343         }
 2344 
 2345         /*
 2346          * Increment counters
 2347          */
 2348         pmap->pm_stats.resident_count++;
 2349 
 2350         pa = VM_PAGE_TO_PHYS(m);
 2351         if ((prot & VM_PROT_EXECUTE) == 0)
 2352                 pa |= pg_nx;
 2353 
 2354         /*
 2355          * Now validate mapping with RO protection
 2356          */
 2357         if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED))
 2358                 pte_store(pte, pa | PG_V | PG_U);
 2359         else
 2360                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 2361         return mpte;
 2362 }
 2363 
 2364 /*
 2365  * Make a temporary mapping for a physical address.  This is only intended
 2366  * to be used for panic dumps.
 2367  */
 2368 void *
 2369 pmap_kenter_temporary(vm_paddr_t pa, int i)
 2370 {
 2371         vm_offset_t va;
 2372 
 2373         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 2374         pmap_kenter(va, pa);
 2375         invlpg(va);
 2376         return ((void *)crashdumpmap);
 2377 }
 2378 
 2379 /*
 2380  * This code maps large physical mmap regions into the
 2381  * processor address space.  Note that some shortcuts
 2382  * are taken, but the code works.
 2383  */
 2384 void
 2385 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
 2386                     vm_object_t object, vm_pindex_t pindex,
 2387                     vm_size_t size)
 2388 {
 2389         vm_offset_t va;
 2390         vm_page_t p, pdpg;
 2391 
 2392         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 2393         KASSERT(object->type == OBJT_DEVICE,
 2394             ("pmap_object_init_pt: non-device object"));
 2395         if (((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0)) {
 2396                 vm_page_t m[1];
 2397                 pd_entry_t ptepa, *pde;
 2398 
 2399                 PMAP_LOCK(pmap);
 2400                 pde = pmap_pde(pmap, addr);
 2401                 if (pde != 0 && (*pde & PG_V) != 0)
 2402                         goto out;
 2403                 PMAP_UNLOCK(pmap);
 2404 retry:
 2405                 p = vm_page_lookup(object, pindex);
 2406                 if (p != NULL) {
 2407                         vm_page_lock_queues();
 2408                         if (vm_page_sleep_if_busy(p, FALSE, "init4p"))
 2409                                 goto retry;
 2410                 } else {
 2411                         p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
 2412                         if (p == NULL)
 2413                                 return;
 2414                         m[0] = p;
 2415 
 2416                         if (vm_pager_get_pages(object, m, 1, 0) != VM_PAGER_OK) {
 2417                                 vm_page_lock_queues();
 2418                                 vm_page_free(p);
 2419                                 vm_page_unlock_queues();
 2420                                 return;
 2421                         }
 2422 
 2423                         p = vm_page_lookup(object, pindex);
 2424                         vm_page_lock_queues();
 2425                         vm_page_wakeup(p);
 2426                 }
 2427                 vm_page_unlock_queues();
 2428 
 2429                 ptepa = VM_PAGE_TO_PHYS(p);
 2430                 if (ptepa & (NBPDR - 1))
 2431                         return;
 2432 
 2433                 p->valid = VM_PAGE_BITS_ALL;
 2434 
 2435                 PMAP_LOCK(pmap);
 2436                 for (va = addr; va < addr + size; va += NBPDR) {
 2437                         while ((pdpg =
 2438                             pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) {
 2439                                 PMAP_UNLOCK(pmap);
 2440                                 vm_page_lock_queues();
 2441                                 vm_page_busy(p);
 2442                                 vm_page_unlock_queues();
 2443                                 VM_OBJECT_UNLOCK(object);
 2444                                 VM_WAIT;
 2445                                 VM_OBJECT_LOCK(object);
 2446                                 vm_page_lock_queues();
 2447                                 vm_page_wakeup(p);
 2448                                 vm_page_unlock_queues();
 2449                                 PMAP_LOCK(pmap);
 2450                         }
 2451                         pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg));
 2452                         pde = &pde[pmap_pde_index(va)];
 2453                         if ((*pde & PG_V) == 0) {
 2454                                 pde_store(pde, ptepa | PG_PS | PG_M | PG_A |
 2455                                     PG_U | PG_RW | PG_V);
 2456                                 pmap->pm_stats.resident_count +=
 2457                                     NBPDR / PAGE_SIZE;
 2458                         } else {
 2459                                 pdpg->wire_count--;
 2460                                 KASSERT(pdpg->wire_count > 0,
 2461                                     ("pmap_object_init_pt: missing reference "
 2462                                      "to page directory page, va: 0x%lx", va));
 2463                         }
 2464                         ptepa += NBPDR;
 2465                 }
 2466                 pmap_invalidate_all(pmap);
 2467 out:
 2468                 PMAP_UNLOCK(pmap);
 2469         }
 2470 }
 2471 
 2472 /*
 2473  *      Routine:        pmap_change_wiring
 2474  *      Function:       Change the wiring attribute for a map/virtual-address
 2475  *                      pair.
 2476  *      In/out conditions:
 2477  *                      The mapping must already exist in the pmap.
 2478  */
 2479 void
 2480 pmap_change_wiring(pmap, va, wired)
 2481         register pmap_t pmap;
 2482         vm_offset_t va;
 2483         boolean_t wired;
 2484 {
 2485         register pt_entry_t *pte;
 2486 
 2487         /*
 2488          * Wiring is not a hardware characteristic so there is no need to
 2489          * invalidate TLB.
 2490          */
 2491         PMAP_LOCK(pmap);
 2492         pte = pmap_pte(pmap, va);
 2493         if (wired && (*pte & PG_W) == 0) {
 2494                 pmap->pm_stats.wired_count++;
 2495                 atomic_set_long(pte, PG_W);
 2496         } else if (!wired && (*pte & PG_W) != 0) {
 2497                 pmap->pm_stats.wired_count--;
 2498                 atomic_clear_long(pte, PG_W);
 2499         }
 2500         PMAP_UNLOCK(pmap);
 2501 }
 2502 
 2503 
 2504 
 2505 /*
 2506  *      Copy the range specified by src_addr/len
 2507  *      from the source map to the range dst_addr/len
 2508  *      in the destination map.
 2509  *
 2510  *      This routine is only advisory and need not do anything.
 2511  */
 2512 
 2513 void
 2514 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 2515           vm_offset_t src_addr)
 2516 {
 2517         vm_page_t   free;
 2518         vm_offset_t addr;
 2519         vm_offset_t end_addr = src_addr + len;
 2520         vm_offset_t va_next;
 2521 
 2522         if (dst_addr != src_addr)
 2523                 return;
 2524 
 2525         if (!pmap_is_current(src_pmap))
 2526                 return;
 2527 
 2528         vm_page_lock_queues();
 2529         if (dst_pmap < src_pmap) {
 2530                 PMAP_LOCK(dst_pmap);
 2531                 PMAP_LOCK(src_pmap);
 2532         } else {
 2533                 PMAP_LOCK(src_pmap);
 2534                 PMAP_LOCK(dst_pmap);
 2535         }
 2536         for (addr = src_addr; addr < end_addr; addr = va_next) {
 2537                 pt_entry_t *src_pte, *dst_pte;
 2538                 vm_page_t dstmpde, dstmpte, srcmpte;
 2539                 pml4_entry_t *pml4e;
 2540                 pdp_entry_t *pdpe;
 2541                 pd_entry_t srcptepaddr, *pde;
 2542 
 2543                 if (addr >= UPT_MIN_ADDRESS)
 2544                         panic("pmap_copy: invalid to pmap_copy page tables");
 2545 
 2546                 pml4e = pmap_pml4e(src_pmap, addr);
 2547                 if ((*pml4e & PG_V) == 0) {
 2548                         va_next = (addr + NBPML4) & ~PML4MASK;
 2549                         continue;
 2550                 }
 2551 
 2552                 pdpe = pmap_pml4e_to_pdpe(pml4e, addr);
 2553                 if ((*pdpe & PG_V) == 0) {
 2554                         va_next = (addr + NBPDP) & ~PDPMASK;
 2555                         continue;
 2556                 }
 2557 
 2558                 va_next = (addr + NBPDR) & ~PDRMASK;
 2559 
 2560                 pde = pmap_pdpe_to_pde(pdpe, addr);
 2561                 srcptepaddr = *pde;
 2562                 if (srcptepaddr == 0)
 2563                         continue;
 2564                         
 2565                 if (srcptepaddr & PG_PS) {
 2566                         dstmpde = pmap_allocpde(dst_pmap, addr, M_NOWAIT);
 2567                         if (dstmpde == NULL)
 2568                                 break;
 2569                         pde = (pd_entry_t *)
 2570                             PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpde));
 2571                         pde = &pde[pmap_pde_index(addr)];
 2572                         if (*pde == 0) {
 2573                                 *pde = srcptepaddr & ~PG_W;
 2574                                 dst_pmap->pm_stats.resident_count +=
 2575                                     NBPDR / PAGE_SIZE;
 2576                         } else
 2577                                 dstmpde->wire_count--;
 2578                         continue;
 2579                 }
 2580 
 2581                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 2582                 if (srcmpte->wire_count == 0)
 2583                         panic("pmap_copy: source page table page is unused");
 2584 
 2585                 if (va_next > end_addr)
 2586                         va_next = end_addr;
 2587 
 2588                 src_pte = vtopte(addr);
 2589                 while (addr < va_next) {
 2590                         pt_entry_t ptetemp;
 2591                         ptetemp = *src_pte;
 2592                         /*
 2593                          * we only virtual copy managed pages
 2594                          */
 2595                         if ((ptetemp & PG_MANAGED) != 0) {
 2596                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 2597                                     M_NOWAIT);
 2598                                 if (dstmpte == NULL)
 2599                                         break;
 2600                                 dst_pte = (pt_entry_t *)
 2601                                     PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
 2602                                 dst_pte = &dst_pte[pmap_pte_index(addr)];
 2603                                 if (*dst_pte == 0 &&
 2604                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 2605                                     PHYS_TO_VM_PAGE(ptetemp & PG_FRAME))) {
 2606                                         /*
 2607                                          * Clear the wired, modified, and
 2608                                          * accessed (referenced) bits
 2609                                          * during the copy.
 2610                                          */
 2611                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 2612                                             PG_A);
 2613                                         dst_pmap->pm_stats.resident_count++;
 2614                                 } else {
 2615                                         free = NULL;
 2616                                         if (pmap_unwire_pte_hold(dst_pmap,
 2617                                             addr, dstmpte, &free)) {
 2618                                                 pmap_invalidate_page(dst_pmap,
 2619                                                     addr);
 2620                                                 pmap_free_zero_pages(free);
 2621                                         }
 2622                                 }
 2623                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 2624                                         break;
 2625                         }
 2626                         addr += PAGE_SIZE;
 2627                         src_pte++;
 2628                 }
 2629         }
 2630         vm_page_unlock_queues();
 2631         PMAP_UNLOCK(src_pmap);
 2632         PMAP_UNLOCK(dst_pmap);
 2633 }       
 2634 
 2635 /*
 2636  *      pmap_zero_page zeros the specified hardware page by mapping 
 2637  *      the page into KVM and using bzero to clear its contents.
 2638  */
 2639 void
 2640 pmap_zero_page(vm_page_t m)
 2641 {
 2642         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2643 
 2644         pagezero((void *)va);
 2645 }
 2646 
 2647 /*
 2648  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 2649  *      the page into KVM and using bzero to clear its contents.
 2650  *
 2651  *      off and size may not cover an area beyond a single hardware page.
 2652  */
 2653 void
 2654 pmap_zero_page_area(vm_page_t m, int off, int size)
 2655 {
 2656         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2657 
 2658         if (off == 0 && size == PAGE_SIZE)
 2659                 pagezero((void *)va);
 2660         else
 2661                 bzero((char *)va + off, size);
 2662 }
 2663 
 2664 /*
 2665  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 2666  *      the page into KVM and using bzero to clear its contents.  This
 2667  *      is intended to be called from the vm_pagezero process only and
 2668  *      outside of Giant.
 2669  */
 2670 void
 2671 pmap_zero_page_idle(vm_page_t m)
 2672 {
 2673         vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
 2674 
 2675         pagezero((void *)va);
 2676 }
 2677 
 2678 /*
 2679  *      pmap_copy_page copies the specified (machine independent)
 2680  *      page by mapping the page into virtual memory and using
 2681  *      bcopy to copy the page, one machine dependent page at a
 2682  *      time.
 2683  */
 2684 void
 2685 pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 2686 {
 2687         vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc));
 2688         vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst));
 2689 
 2690         pagecopy((void *)src, (void *)dst);
 2691 }
 2692 
 2693 /*
 2694  * Returns true if the pmap's pv is one of the first
 2695  * 16 pvs linked to from this page.  This count may
 2696  * be changed upwards or downwards in the future; it
 2697  * is only necessary that true be returned for a small
 2698  * subset of pmaps for proper page aging.
 2699  */
 2700 boolean_t
 2701 pmap_page_exists_quick(pmap, m)
 2702         pmap_t pmap;
 2703         vm_page_t m;
 2704 {
 2705         pv_entry_t pv;
 2706         int loops = 0;
 2707 
 2708         if (m->flags & PG_FICTITIOUS)
 2709                 return FALSE;
 2710 
 2711         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2712         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2713                 if (pv->pv_pmap == pmap) {
 2714                         return TRUE;
 2715                 }
 2716                 loops++;
 2717                 if (loops >= 16)
 2718                         break;
 2719         }
 2720         return (FALSE);
 2721 }
 2722 
 2723 #define PMAP_REMOVE_PAGES_CURPROC_ONLY
 2724 /*
 2725  * Remove all pages from specified address space
 2726  * this aids process exit speeds.  Also, this code
 2727  * is special cased for current process only, but
 2728  * can have the more generic (and slightly slower)
 2729  * mode enabled.  This is much faster than pmap_remove
 2730  * in the case of running down an entire address space.
 2731  */
 2732 void
 2733 pmap_remove_pages(pmap, sva, eva)
 2734         pmap_t pmap;
 2735         vm_offset_t sva, eva;
 2736 {
 2737         pt_entry_t *pte, tpte;
 2738         vm_page_t m, free = NULL;
 2739         pv_entry_t pv, npv;
 2740 
 2741 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
 2742         if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
 2743                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 2744                 return;
 2745         }
 2746 #endif
 2747         vm_page_lock_queues();
 2748         PMAP_LOCK(pmap);
 2749         for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
 2750 
 2751                 if (pv->pv_va >= eva || pv->pv_va < sva) {
 2752                         npv = TAILQ_NEXT(pv, pv_plist);
 2753                         continue;
 2754                 }
 2755 
 2756 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
 2757                 pte = vtopte(pv->pv_va);
 2758 #else
 2759                 pte = pmap_pte(pmap, pv->pv_va);
 2760 #endif
 2761                 tpte = *pte;
 2762 
 2763                 if (tpte == 0) {
 2764                         printf("TPTE at %p  IS ZERO @ VA %08lx\n",
 2765                                                         pte, pv->pv_va);
 2766                         panic("bad pte");
 2767                 }
 2768 
 2769 /*
 2770  * We cannot remove wired pages from a process' mapping at this time
 2771  */
 2772                 if (tpte & PG_W) {
 2773                         npv = TAILQ_NEXT(pv, pv_plist);
 2774                         continue;
 2775                 }
 2776 
 2777                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2778                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 2779                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 2780                     m, (uintmax_t)m->phys_addr, (uintmax_t)tpte));
 2781 
 2782                 KASSERT(m < &vm_page_array[vm_page_array_size],
 2783                         ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte));
 2784 
 2785                 pmap->pm_stats.resident_count--;
 2786 
 2787                 pte_clear(pte);
 2788 
 2789                 /*
 2790                  * Update the vm_page_t clean and reference bits.
 2791                  */
 2792                 if (tpte & PG_M) {
 2793                         vm_page_dirty(m);
 2794                 }
 2795 
 2796                 npv = TAILQ_NEXT(pv, pv_plist);
 2797                 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
 2798 
 2799                 m->md.pv_list_count--;
 2800                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2801                 if (TAILQ_EMPTY(&m->md.pv_list))
 2802                         vm_page_flag_clear(m, PG_WRITEABLE);
 2803 
 2804                 pmap_unuse_pt(pmap, pv->pv_va, *vtopde(pv->pv_va), &free);
 2805                 free_pv_entry(pv);
 2806         }
 2807         pmap_invalidate_all(pmap);
 2808         pmap_free_zero_pages(free);
 2809         PMAP_UNLOCK(pmap);
 2810         vm_page_unlock_queues();
 2811 }
 2812 
 2813 /*
 2814  *      pmap_is_modified:
 2815  *
 2816  *      Return whether or not the specified physical page was modified
 2817  *      in any physical maps.
 2818  */
 2819 boolean_t
 2820 pmap_is_modified(vm_page_t m)
 2821 {
 2822         pv_entry_t pv;
 2823         pt_entry_t *pte;
 2824         boolean_t rv;
 2825 
 2826         rv = FALSE;
 2827         if (m->flags & PG_FICTITIOUS)
 2828                 return (rv);
 2829 
 2830         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2831         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2832                 PMAP_LOCK(pv->pv_pmap);
 2833                 pte = pmap_pte(pv->pv_pmap, pv->pv_va);
 2834                 rv = (*pte & PG_M) != 0;
 2835                 PMAP_UNLOCK(pv->pv_pmap);
 2836                 if (rv)
 2837                         break;
 2838         }
 2839         return (rv);
 2840 }
 2841 
 2842 /*
 2843  *      pmap_is_prefaultable:
 2844  *
 2845  *      Return whether or not the specified virtual address is elgible
 2846  *      for prefault.
 2847  */
 2848 boolean_t
 2849 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 2850 {
 2851         pd_entry_t *pde;
 2852         pt_entry_t *pte;
 2853         boolean_t rv;
 2854 
 2855         rv = FALSE;
 2856         PMAP_LOCK(pmap);
 2857         pde = pmap_pde(pmap, addr);
 2858         if (pde != NULL && (*pde & PG_V)) {
 2859                 pte = vtopte(addr);
 2860                 rv = (*pte & PG_V) == 0;
 2861         }
 2862         PMAP_UNLOCK(pmap);
 2863         return (rv);
 2864 }
 2865 
 2866 /*
 2867  *      Clear the given bit in each of the given page's ptes.
 2868  */
 2869 static __inline void
 2870 pmap_clear_ptes(vm_page_t m, long bit)
 2871 {
 2872         register pv_entry_t pv;
 2873         pt_entry_t pbits, *pte;
 2874 
 2875         if ((m->flags & PG_FICTITIOUS) ||
 2876             (bit == PG_RW && (m->flags & PG_WRITEABLE) == 0))
 2877                 return;
 2878 
 2879         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2880         /*
 2881          * Loop over all current mappings setting/clearing as appropos If
 2882          * setting RO do we need to clear the VAC?
 2883          */
 2884         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2885                 PMAP_LOCK(pv->pv_pmap);
 2886                 pte = pmap_pte(pv->pv_pmap, pv->pv_va);
 2887 retry:
 2888                 pbits = *pte;
 2889                 if (pbits & bit) {
 2890                         if (bit == PG_RW) {
 2891                                 if (!atomic_cmpset_long(pte, pbits,
 2892                                     pbits & ~(PG_RW | PG_M)))
 2893                                         goto retry;
 2894                                 if (pbits & PG_M) {
 2895                                         vm_page_dirty(m);
 2896                                 }
 2897                         } else {
 2898                                 atomic_clear_long(pte, bit);
 2899                         }
 2900                         pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 2901                 }
 2902                 PMAP_UNLOCK(pv->pv_pmap);
 2903         }
 2904         if (bit == PG_RW)
 2905                 vm_page_flag_clear(m, PG_WRITEABLE);
 2906 }
 2907 
 2908 /*
 2909  *      pmap_page_protect:
 2910  *
 2911  *      Lower the permission for all mappings to a given page.
 2912  */
 2913 void
 2914 pmap_page_protect(vm_page_t m, vm_prot_t prot)
 2915 {
 2916         if ((prot & VM_PROT_WRITE) == 0) {
 2917                 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
 2918                         pmap_clear_ptes(m, PG_RW);
 2919                 } else {
 2920                         pmap_remove_all(m);
 2921                 }
 2922         }
 2923 }
 2924 
 2925 /*
 2926  *      pmap_ts_referenced:
 2927  *
 2928  *      Return a count of reference bits for a page, clearing those bits.
 2929  *      It is not necessary for every reference bit to be cleared, but it
 2930  *      is necessary that 0 only be returned when there are truly no
 2931  *      reference bits set.
 2932  *
 2933  *      XXX: The exact number of bits to check and clear is a matter that
 2934  *      should be tested and standardized at some point in the future for
 2935  *      optimal aging of shared pages.
 2936  */
 2937 int
 2938 pmap_ts_referenced(vm_page_t m)
 2939 {
 2940         register pv_entry_t pv, pvf, pvn;
 2941         pt_entry_t *pte;
 2942         pt_entry_t v;
 2943         int rtval = 0;
 2944 
 2945         if (m->flags & PG_FICTITIOUS)
 2946                 return (rtval);
 2947 
 2948         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2949         if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 2950 
 2951                 pvf = pv;
 2952 
 2953                 do {
 2954                         pvn = TAILQ_NEXT(pv, pv_list);
 2955 
 2956                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2957 
 2958                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2959 
 2960                         PMAP_LOCK(pv->pv_pmap);
 2961                         pte = pmap_pte(pv->pv_pmap, pv->pv_va);
 2962 
 2963                         if (pte && ((v = pte_load(pte)) & PG_A) != 0) {
 2964                                 atomic_clear_long(pte, PG_A);
 2965                                 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 2966 
 2967                                 rtval++;
 2968                                 if (rtval > 4) {
 2969                                         PMAP_UNLOCK(pv->pv_pmap);
 2970                                         break;
 2971                                 }
 2972                         }
 2973                         PMAP_UNLOCK(pv->pv_pmap);
 2974                 } while ((pv = pvn) != NULL && pv != pvf);
 2975         }
 2976 
 2977         return (rtval);
 2978 }
 2979 
 2980 /*
 2981  *      Clear the modify bits on the specified physical page.
 2982  */
 2983 void
 2984 pmap_clear_modify(vm_page_t m)
 2985 {
 2986         pmap_clear_ptes(m, PG_M);
 2987 }
 2988 
 2989 /*
 2990  *      pmap_clear_reference:
 2991  *
 2992  *      Clear the reference bit on the specified physical page.
 2993  */
 2994 void
 2995 pmap_clear_reference(vm_page_t m)
 2996 {
 2997         pmap_clear_ptes(m, PG_A);
 2998 }
 2999 
 3000 /*
 3001  * Miscellaneous support routines follow
 3002  */
 3003 
 3004 /* Adjust the cache mode for a 4KB page mapped via a PTE. */
 3005 static __inline void
 3006 pmap_pte_attr(vm_offset_t va, int mode)
 3007 {
 3008         pt_entry_t *pte;
 3009         u_int opte, npte;
 3010 
 3011         pte = vtopte(va);
 3012 
 3013         /*
 3014          * The cache mode bits are all in the low 32-bits of the
 3015          * PTE, so we can just spin on updating the low 32-bits.
 3016          */
 3017         do {
 3018                 opte = *(u_int *)pte;
 3019                 npte = opte & ~(PG_PTE_PAT | PG_NC_PCD | PG_NC_PWT);
 3020                 npte |= pmap_cache_bits(mode, 0);
 3021         } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte));
 3022 }
 3023 
 3024 /* Adjust the cache mode for a 2MB page mapped via a PDE. */
 3025 static __inline void
 3026 pmap_pde_attr(vm_offset_t va, int mode)
 3027 {
 3028         pd_entry_t *pde;
 3029         u_int opde, npde;
 3030 
 3031         pde = pmap_pde(kernel_pmap, va);
 3032 
 3033         /*
 3034          * The cache mode bits are all in the low 32-bits of the
 3035          * PDE, so we can just spin on updating the low 32-bits.
 3036          */
 3037         do {
 3038                 opde = *(u_int *)pde;
 3039                 npde = opde & ~(PG_PDE_PAT | PG_NC_PCD | PG_NC_PWT);
 3040                 npde |= pmap_cache_bits(mode, 1);
 3041         } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde));
 3042 }
 3043 
 3044 /*
 3045  * Map a set of physical memory pages into the kernel virtual
 3046  * address space. Return a pointer to where it is mapped. This
 3047  * routine is intended to be used for mapping device memory,
 3048  * NOT real memory.
 3049  */
 3050 void *
 3051 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 3052 {
 3053         vm_offset_t va, tmpva, offset;
 3054 
 3055         /*
 3056          * If this fits within the direct map window and use WB caching
 3057          * mode, use the direct map.
 3058          */
 3059         if (pa < dmaplimit && (pa + size) < dmaplimit && mode == PAT_WRITE_BACK)
 3060                 return ((void *)PHYS_TO_DMAP(pa));
 3061         offset = pa & PAGE_MASK;
 3062         size = roundup(offset + size, PAGE_SIZE);
 3063         va = kmem_alloc_nofault(kernel_map, size);
 3064         if (!va)
 3065                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 3066         pa = trunc_page(pa);
 3067         for (tmpva = va; size > 0; ) {
 3068                 pmap_kenter_attr(tmpva, pa, mode);
 3069                 size -= PAGE_SIZE;
 3070                 tmpva += PAGE_SIZE;
 3071                 pa += PAGE_SIZE;
 3072         }
 3073         pmap_invalidate_range(kernel_pmap, va, tmpva);
 3074         pmap_invalidate_cache();
 3075         return ((void *)(va + offset));
 3076 }
 3077 
 3078 void *
 3079 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 3080 {
 3081 
 3082         return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
 3083 }
 3084 
 3085 void *
 3086 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
 3087 {
 3088 
 3089         return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
 3090 }
 3091 
 3092 void
 3093 pmap_unmapdev(va, size)
 3094         vm_offset_t va;
 3095         vm_size_t size;
 3096 {
 3097         vm_offset_t base, offset, tmpva;
 3098 
 3099         /* If we gave a direct map region in pmap_mapdev, do nothing */
 3100         if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS)
 3101                 return;
 3102         base = trunc_page(va);
 3103         offset = va & PAGE_MASK;
 3104         size = roundup(offset + size, PAGE_SIZE);
 3105         for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE)
 3106                 pmap_kremove(tmpva);
 3107         pmap_invalidate_range(kernel_pmap, va, tmpva);
 3108         kmem_free(kernel_map, base, size);
 3109 }
 3110 
 3111 int
 3112 pmap_change_attr(va, size, mode)
 3113         vm_offset_t va;
 3114         vm_size_t size;
 3115         int mode;
 3116 {
 3117         vm_offset_t base, offset, tmpva;
 3118         pd_entry_t *pde;
 3119         pt_entry_t *pte;
 3120 
 3121         base = trunc_page(va);
 3122         offset = va & PAGE_MASK;
 3123         size = roundup(offset + size, PAGE_SIZE);
 3124 
 3125         /* Only supported on kernel virtual addresses. */
 3126         if (base <= VM_MAXUSER_ADDRESS)
 3127                 return (EINVAL);
 3128 
 3129         /*
 3130          * XXX: We have to support tearing 2MB pages down into 4k pages if
 3131          * needed here.
 3132          */
 3133         /* Pages that aren't mapped aren't supported. */
 3134         for (tmpva = base; tmpva < (base + size); ) {
 3135                 pde = pmap_pde(kernel_pmap, tmpva);
 3136                 if (*pde == 0)
 3137                         return (EINVAL);
 3138                 if (*pde & PG_PS) {
 3139                         /* Handle 2MB pages that are completely contained. */
 3140                         if (size >= NBPDR) {
 3141                                 tmpva += NBPDR;
 3142                                 continue;
 3143                         }
 3144                         return (EINVAL);
 3145                 }
 3146                 pte = vtopte(va);
 3147                 if (*pte == 0)
 3148                         return (EINVAL);
 3149                 tmpva += PAGE_SIZE;
 3150         }
 3151 
 3152         /*
 3153          * Ok, all the pages exist, so run through them updating their
 3154          * cache mode.
 3155          */
 3156         for (tmpva = base; size > 0; ) {
 3157                 pde = pmap_pde(kernel_pmap, tmpva);
 3158                 if (*pde & PG_PS) {
 3159                         pmap_pde_attr(tmpva, mode);
 3160                         tmpva += NBPDR;
 3161                         size -= NBPDR;
 3162                 } else {
 3163                         pmap_pte_attr(tmpva, mode);
 3164                         tmpva += PAGE_SIZE;
 3165                         size -= PAGE_SIZE;
 3166                 }
 3167         }
 3168 
 3169         /*
 3170          * Flush CPU caches to make sure any data isn't cached that shouldn't
 3171          * be, etc.
 3172          */    
 3173         pmap_invalidate_range(kernel_pmap, base, tmpva);
 3174         pmap_invalidate_cache();
 3175         return (0);
 3176 }
 3177 
 3178 /*
 3179  * perform the pmap work for mincore
 3180  */
 3181 int
 3182 pmap_mincore(pmap, addr)
 3183         pmap_t pmap;
 3184         vm_offset_t addr;
 3185 {
 3186         pt_entry_t *ptep, pte;
 3187         vm_page_t m;
 3188         int val = 0;
 3189         
 3190         PMAP_LOCK(pmap);
 3191         ptep = pmap_pte(pmap, addr);
 3192         pte = (ptep != NULL) ? *ptep : 0;
 3193         PMAP_UNLOCK(pmap);
 3194 
 3195         if (pte != 0) {
 3196                 vm_paddr_t pa;
 3197 
 3198                 val = MINCORE_INCORE;
 3199                 if ((pte & PG_MANAGED) == 0)
 3200                         return val;
 3201 
 3202                 pa = pte & PG_FRAME;
 3203 
 3204                 m = PHYS_TO_VM_PAGE(pa);
 3205 
 3206                 /*
 3207                  * Modified by us
 3208                  */
 3209                 if (pte & PG_M)
 3210                         val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
 3211                 else {
 3212                         /*
 3213                          * Modified by someone else
 3214                          */
 3215                         vm_page_lock_queues();
 3216                         if (m->dirty || pmap_is_modified(m))
 3217                                 val |= MINCORE_MODIFIED_OTHER;
 3218                         vm_page_unlock_queues();
 3219                 }
 3220                 /*
 3221                  * Referenced by us
 3222                  */
 3223                 if (pte & PG_A)
 3224                         val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
 3225                 else {
 3226                         /*
 3227                          * Referenced by someone else
 3228                          */
 3229                         vm_page_lock_queues();
 3230                         if ((m->flags & PG_REFERENCED) ||
 3231                             pmap_ts_referenced(m)) {
 3232                                 val |= MINCORE_REFERENCED_OTHER;
 3233                                 vm_page_flag_set(m, PG_REFERENCED);
 3234                         }
 3235                         vm_page_unlock_queues();
 3236                 }
 3237         } 
 3238         return val;
 3239 }
 3240 
 3241 void
 3242 pmap_activate(struct thread *td)
 3243 {
 3244         pmap_t  pmap, oldpmap;
 3245         u_int64_t  cr3;
 3246 
 3247         critical_enter();
 3248         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 3249         oldpmap = PCPU_GET(curpmap);
 3250 #ifdef SMP
 3251 if (oldpmap)    /* XXX FIXME */
 3252         atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
 3253         atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
 3254 #else
 3255 if (oldpmap)    /* XXX FIXME */
 3256         oldpmap->pm_active &= ~PCPU_GET(cpumask);
 3257         pmap->pm_active |= PCPU_GET(cpumask);
 3258 #endif
 3259         cr3 = vtophys(pmap->pm_pml4);
 3260         td->td_pcb->pcb_cr3 = cr3;
 3261         load_cr3(cr3);
 3262         critical_exit();
 3263 }
 3264 
 3265 vm_offset_t
 3266 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
 3267 {
 3268 
 3269         if ((obj == NULL) || (size < NBPDR) || (obj->type != OBJT_DEVICE)) {
 3270                 return addr;
 3271         }
 3272 
 3273         addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
 3274         return addr;
 3275 }

Cache object: 4a56d3a40db0ff8e7a881296088d12bd


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.