The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/xen/pmap.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2005 Alan L. Cox <alc@cs.rice.edu>
    9  * All rights reserved.
   10  *
   11  * This code is derived from software contributed to Berkeley by
   12  * the Systems Programming Group of the University of Utah Computer
   13  * Science Department and William Jolitz of UUNET Technologies Inc.
   14  *
   15  * Redistribution and use in source and binary forms, with or without
   16  * modification, are permitted provided that the following conditions
   17  * are met:
   18  * 1. Redistributions of source code must retain the above copyright
   19  *    notice, this list of conditions and the following disclaimer.
   20  * 2. Redistributions in binary form must reproduce the above copyright
   21  *    notice, this list of conditions and the following disclaimer in the
   22  *    documentation and/or other materials provided with the distribution.
   23  * 3. All advertising materials mentioning features or use of this software
   24  *    must display the following acknowledgement:
   25  *      This product includes software developed by the University of
   26  *      California, Berkeley and its contributors.
   27  * 4. Neither the name of the University nor the names of its contributors
   28  *    may be used to endorse or promote products derived from this software
   29  *    without specific prior written permission.
   30  *
   31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   41  * SUCH DAMAGE.
   42  *
   43  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   44  */
   45 /*-
   46  * Copyright (c) 2003 Networks Associates Technology, Inc.
   47  * All rights reserved.
   48  *
   49  * This software was developed for the FreeBSD Project by Jake Burkholder,
   50  * Safeport Network Services, and Network Associates Laboratories, the
   51  * Security Research Division of Network Associates, Inc. under
   52  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   53  * CHATS research program.
   54  *
   55  * Redistribution and use in source and binary forms, with or without
   56  * modification, are permitted provided that the following conditions
   57  * are met:
   58  * 1. Redistributions of source code must retain the above copyright
   59  *    notice, this list of conditions and the following disclaimer.
   60  * 2. Redistributions in binary form must reproduce the above copyright
   61  *    notice, this list of conditions and the following disclaimer in the
   62  *    documentation and/or other materials provided with the distribution.
   63  *
   64  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   65  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   66  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   67  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   68  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   69  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   70  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   71  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   72  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   73  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   74  * SUCH DAMAGE.
   75  */
   76 
   77 #include <sys/cdefs.h>
   78 __FBSDID("$FreeBSD$");
   79 
   80 /*
   81  *      Manages physical address maps.
   82  *
   83  *      In addition to hardware address maps, this
   84  *      module is called upon to provide software-use-only
   85  *      maps which may or may not be stored in the same
   86  *      form as hardware maps.  These pseudo-maps are
   87  *      used to store intermediate results from copy
   88  *      operations to and from address spaces.
   89  *
   90  *      Since the information managed by this module is
   91  *      also stored by the logical address mapping module,
   92  *      this module may throw away valid virtual-to-physical
   93  *      mappings at almost any time.  However, invalidations
   94  *      of virtual-to-physical mappings must be done as
   95  *      requested.
   96  *
   97  *      In order to cope with hardware architectures which
   98  *      make virtual-to-physical map invalidates expensive,
   99  *      this module may delay invalidate or reduced protection
  100  *      operations until such time as they are actually
  101  *      necessary.  This module is given full information as
  102  *      to which processors are currently using which maps,
  103  *      and to when physical maps must be made correct.
  104  */
  105 
  106 #include "opt_cpu.h"
  107 #include "opt_pmap.h"
  108 #include "opt_smp.h"
  109 #include "opt_xbox.h"
  110 
  111 #include <sys/param.h>
  112 #include <sys/systm.h>
  113 #include <sys/kernel.h>
  114 #include <sys/ktr.h>
  115 #include <sys/lock.h>
  116 #include <sys/malloc.h>
  117 #include <sys/mman.h>
  118 #include <sys/msgbuf.h>
  119 #include <sys/mutex.h>
  120 #include <sys/proc.h>
  121 #include <sys/sf_buf.h>
  122 #include <sys/sx.h>
  123 #include <sys/vmmeter.h>
  124 #include <sys/sched.h>
  125 #include <sys/sysctl.h>
  126 #ifdef SMP
  127 #include <sys/smp.h>
  128 #else
  129 #include <sys/cpuset.h>
  130 #endif
  131 
  132 #include <vm/vm.h>
  133 #include <vm/vm_param.h>
  134 #include <vm/vm_kern.h>
  135 #include <vm/vm_page.h>
  136 #include <vm/vm_map.h>
  137 #include <vm/vm_object.h>
  138 #include <vm/vm_extern.h>
  139 #include <vm/vm_pageout.h>
  140 #include <vm/vm_pager.h>
  141 #include <vm/uma.h>
  142 
  143 #include <machine/cpu.h>
  144 #include <machine/cputypes.h>
  145 #include <machine/md_var.h>
  146 #include <machine/pcb.h>
  147 #include <machine/specialreg.h>
  148 #ifdef SMP
  149 #include <machine/smp.h>
  150 #endif
  151 
  152 #ifdef XBOX
  153 #include <machine/xbox.h>
  154 #endif
  155 
  156 #include <xen/interface/xen.h>
  157 #include <xen/hypervisor.h>
  158 #include <machine/xen/hypercall.h>
  159 #include <machine/xen/xenvar.h>
  160 #include <machine/xen/xenfunc.h>
  161 
  162 #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
  163 #define CPU_ENABLE_SSE
  164 #endif
  165 
  166 #ifndef PMAP_SHPGPERPROC
  167 #define PMAP_SHPGPERPROC 200
  168 #endif
  169 
  170 #define DIAGNOSTIC
  171 
  172 #if !defined(DIAGNOSTIC)
  173 #ifdef __GNUC_GNU_INLINE__
  174 #define PMAP_INLINE     __attribute__((__gnu_inline__)) inline
  175 #else
  176 #define PMAP_INLINE     extern inline
  177 #endif
  178 #else
  179 #define PMAP_INLINE
  180 #endif
  181 
  182 #define PV_STATS
  183 #ifdef PV_STATS
  184 #define PV_STAT(x)      do { x ; } while (0)
  185 #else
  186 #define PV_STAT(x)      do { } while (0)
  187 #endif
  188 
  189 /*
  190  * Get PDEs and PTEs for user/kernel address space
  191  */
  192 #define pmap_pde(m, v)  (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
  193 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
  194 
  195 #define pmap_pde_v(pte)         ((*(int *)pte & PG_V) != 0)
  196 #define pmap_pte_w(pte)         ((*(int *)pte & PG_W) != 0)
  197 #define pmap_pte_m(pte)         ((*(int *)pte & PG_M) != 0)
  198 #define pmap_pte_u(pte)         ((*(int *)pte & PG_A) != 0)
  199 #define pmap_pte_v(pte)         ((*(int *)pte & PG_V) != 0)
  200 
  201 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
  202 
  203 #define HAMFISTED_LOCKING
  204 #ifdef HAMFISTED_LOCKING
  205 static struct mtx createdelete_lock;
  206 #endif
  207 
  208 struct pmap kernel_pmap_store;
  209 LIST_HEAD(pmaplist, pmap);
  210 static struct pmaplist allpmaps;
  211 static struct mtx allpmaps_lock;
  212 
  213 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  214 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  215 int pgeflag = 0;                /* PG_G or-in */
  216 int pseflag = 0;                /* PG_PS or-in */
  217 
  218 int nkpt;
  219 vm_offset_t kernel_vm_end;
  220 extern u_int32_t KERNend;
  221 
  222 #ifdef PAE
  223 pt_entry_t pg_nx;
  224 #endif
  225 
  226 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
  227 
  228 static int pat_works;                   /* Is page attribute table sane? */
  229 
  230 /*
  231  * Data for the pv entry allocation mechanism
  232  */
  233 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
  234 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  235 static int shpgperproc = PMAP_SHPGPERPROC;
  236 
  237 struct pv_chunk *pv_chunkbase;          /* KVA block for pv_chunks */
  238 int pv_maxchunks;                       /* How many chunks we have KVA for */
  239 vm_offset_t pv_vafree;                  /* freelist stored in the PTE */
  240 
  241 /*
  242  * All those kernel PT submaps that BSD is so fond of
  243  */
  244 struct sysmaps {
  245         struct  mtx lock;
  246         pt_entry_t *CMAP1;
  247         pt_entry_t *CMAP2;
  248         caddr_t CADDR1;
  249         caddr_t CADDR2;
  250 };
  251 static struct sysmaps sysmaps_pcpu[MAXCPU];
  252 pt_entry_t *CMAP3;
  253 caddr_t ptvmmap = 0;
  254 caddr_t CADDR3;
  255 struct msgbuf *msgbufp = 0;
  256 
  257 /*
  258  * Crashdump maps.
  259  */
  260 static caddr_t crashdumpmap;
  261 
  262 static pt_entry_t *PMAP1 = 0, *PMAP2;
  263 static pt_entry_t *PADDR1 = 0, *PADDR2;
  264 #ifdef SMP
  265 static int PMAP1cpu;
  266 static int PMAP1changedcpu;
  267 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 
  268            &PMAP1changedcpu, 0,
  269            "Number of times pmap_pte_quick changed CPU with same PMAP1");
  270 #endif
  271 static int PMAP1changed;
  272 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 
  273            &PMAP1changed, 0,
  274            "Number of times pmap_pte_quick changed PMAP1");
  275 static int PMAP1unchanged;
  276 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 
  277            &PMAP1unchanged, 0,
  278            "Number of times pmap_pte_quick didn't change PMAP1");
  279 static struct mtx PMAP2mutex;
  280 
  281 static void     free_pv_chunk(struct pv_chunk *pc);
  282 static void     free_pv_entry(pmap_t pmap, pv_entry_t pv);
  283 static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
  284 static void     pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  285 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  286                     vm_offset_t va);
  287 
  288 static vm_page_t pmap_enter_quick_locked(multicall_entry_t **mcl, int *count, pmap_t pmap, vm_offset_t va,
  289     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  290 static void pmap_flush_page(vm_page_t m);
  291 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
  292 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
  293     vm_page_t *free);
  294 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va,
  295     vm_page_t *free);
  296 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  297                                         vm_offset_t va);
  298 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
  299     vm_page_t m);
  300 
  301 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
  302 
  303 static vm_page_t _pmap_allocpte(pmap_t pmap, u_int ptepindex, int flags);
  304 static void _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, vm_page_t *free);
  305 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
  306 static void pmap_pte_release(pt_entry_t *pte);
  307 static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t *);
  308 static boolean_t pmap_is_prefaultable_locked(pmap_t pmap, vm_offset_t addr);
  309 
  310 static __inline void pagezero(void *page);
  311 
  312 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  313 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  314 
  315 /*
  316  * If you get an error here, then you set KVA_PAGES wrong! See the
  317  * description of KVA_PAGES in sys/i386/include/pmap.h. It must be
  318  * multiple of 4 for a normal kernel, or a multiple of 8 for a PAE.
  319  */
  320 CTASSERT(KERNBASE % (1 << 24) == 0);
  321 
  322 void 
  323 pd_set(struct pmap *pmap, int ptepindex, vm_paddr_t val, int type)
  324 {
  325         vm_paddr_t pdir_ma = vtomach(&pmap->pm_pdir[ptepindex]);
  326         
  327         switch (type) {
  328         case SH_PD_SET_VA:
  329 #if 0           
  330                 xen_queue_pt_update(shadow_pdir_ma,
  331                                     xpmap_ptom(val & ~(PG_RW)));
  332 #endif          
  333                 xen_queue_pt_update(pdir_ma,
  334                                     xpmap_ptom(val));   
  335                 break;
  336         case SH_PD_SET_VA_MA:
  337 #if 0           
  338                 xen_queue_pt_update(shadow_pdir_ma,
  339                                     val & ~(PG_RW));
  340 #endif          
  341                 xen_queue_pt_update(pdir_ma, val);      
  342                 break;
  343         case SH_PD_SET_VA_CLEAR:
  344 #if 0
  345                 xen_queue_pt_update(shadow_pdir_ma, 0);
  346 #endif          
  347                 xen_queue_pt_update(pdir_ma, 0);        
  348                 break;
  349         }
  350 }
  351 
  352 /*
  353  *      Bootstrap the system enough to run with virtual memory.
  354  *
  355  *      On the i386 this is called after mapping has already been enabled
  356  *      and just syncs the pmap module with what has already been done.
  357  *      [We can't call it easily with mapping off since the kernel is not
  358  *      mapped with PA == VA, hence we would have to relocate every address
  359  *      from the linked base (virtual) address "KERNBASE" to the actual
  360  *      (physical) address starting relative to 0]
  361  */
  362 void
  363 pmap_bootstrap(vm_paddr_t firstaddr)
  364 {
  365         vm_offset_t va;
  366         pt_entry_t *pte, *unused;
  367         struct sysmaps *sysmaps;
  368         int i;
  369 
  370         /*
  371          * Initialize the first available kernel virtual address.  However,
  372          * using "firstaddr" may waste a few pages of the kernel virtual
  373          * address space, because locore may not have mapped every physical
  374          * page that it allocated.  Preferably, locore would provide a first
  375          * unused virtual address in addition to "firstaddr".
  376          */
  377         virtual_avail = (vm_offset_t) KERNBASE + firstaddr;
  378 
  379         virtual_end = VM_MAX_KERNEL_ADDRESS;
  380 
  381         /*
  382          * Initialize the kernel pmap (which is statically allocated).
  383          */
  384         PMAP_LOCK_INIT(kernel_pmap);
  385         kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD);
  386 #ifdef PAE
  387         kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT);
  388 #endif
  389         CPU_FILL(&kernel_pmap->pm_active);      /* don't allow deactivation */
  390         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  391         LIST_INIT(&allpmaps);
  392         mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN);
  393         mtx_lock_spin(&allpmaps_lock);
  394         LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list);
  395         mtx_unlock_spin(&allpmaps_lock);
  396         if (nkpt == 0)
  397                 nkpt = NKPT;
  398 
  399         /*
  400          * Reserve some special page table entries/VA space for temporary
  401          * mapping of pages.
  402          */
  403 #define SYSMAP(c, p, v, n)      \
  404         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  405 
  406         va = virtual_avail;
  407         pte = vtopte(va);
  408 
  409         /*
  410          * CMAP1/CMAP2 are used for zeroing and copying pages.
  411          * CMAP3 is used for the idle process page zeroing.
  412          */
  413         for (i = 0; i < MAXCPU; i++) {
  414                 sysmaps = &sysmaps_pcpu[i];
  415                 mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF);
  416                 SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1)
  417                 SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1)
  418                 PT_SET_MA(sysmaps->CADDR1, 0);
  419                 PT_SET_MA(sysmaps->CADDR2, 0);
  420         }
  421         SYSMAP(caddr_t, CMAP3, CADDR3, 1)
  422         PT_SET_MA(CADDR3, 0);
  423 
  424         /*
  425          * Crashdump maps.
  426          */
  427         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  428 
  429         /*
  430          * ptvmmap is used for reading arbitrary physical pages via /dev/mem.
  431          */
  432         SYSMAP(caddr_t, unused, ptvmmap, 1)
  433 
  434         /*
  435          * msgbufp is used to map the system message buffer.
  436          */
  437         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(msgbufsize)))
  438 
  439         /*
  440          * PADDR1 and PADDR2 are used by pmap_pte_quick() and pmap_pte(),
  441          * respectively.
  442          */
  443         SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1)
  444         SYSMAP(pt_entry_t *, PMAP2, PADDR2, 1)
  445 
  446         mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
  447 
  448         virtual_avail = va;
  449 
  450         /*
  451          * Leave in place an identity mapping (virt == phys) for the low 1 MB
  452          * physical memory region that is used by the ACPI wakeup code.  This
  453          * mapping must not have PG_G set. 
  454          */
  455 #ifndef XEN
  456         /*
  457          * leave here deliberately to show that this is not supported
  458          */
  459 #ifdef XBOX
  460         /* FIXME: This is gross, but needed for the XBOX. Since we are in such
  461          * an early stadium, we cannot yet neatly map video memory ... :-(
  462          * Better fixes are very welcome! */
  463         if (!arch_i386_is_xbox)
  464 #endif
  465         for (i = 1; i < NKPT; i++)
  466                 PTD[i] = 0;
  467 
  468         /* Initialize the PAT MSR if present. */
  469         pmap_init_pat();
  470 
  471         /* Turn on PG_G on kernel page(s) */
  472         pmap_set_pg();
  473 #endif
  474 
  475 #ifdef HAMFISTED_LOCKING
  476         mtx_init(&createdelete_lock, "pmap create/delete", NULL, MTX_DEF);
  477 #endif
  478 }
  479 
  480 /*
  481  * Setup the PAT MSR.
  482  */
  483 void
  484 pmap_init_pat(void)
  485 {
  486         uint64_t pat_msr;
  487 
  488         /* Bail if this CPU doesn't implement PAT. */
  489         if (!(cpu_feature & CPUID_PAT))
  490                 return;
  491 
  492         if (cpu_vendor_id != CPU_VENDOR_INTEL ||
  493             (CPUID_TO_FAMILY(cpu_id) == 6 && CPUID_TO_MODEL(cpu_id) >= 0xe)) {
  494                 /*
  495                  * Leave the indices 0-3 at the default of WB, WT, UC, and UC-.
  496                  * Program 4 and 5 as WP and WC.
  497                  * Leave 6 and 7 as UC and UC-.
  498                  */
  499                 pat_msr = rdmsr(MSR_PAT);
  500                 pat_msr &= ~(PAT_MASK(4) | PAT_MASK(5));
  501                 pat_msr |= PAT_VALUE(4, PAT_WRITE_PROTECTED) |
  502                     PAT_VALUE(5, PAT_WRITE_COMBINING);
  503                 pat_works = 1;
  504         } else {
  505                 /*
  506                  * Due to some Intel errata, we can only safely use the lower 4
  507                  * PAT entries.  Thus, just replace PAT Index 2 with WC instead
  508                  * of UC-.
  509                  *
  510                  *   Intel Pentium III Processor Specification Update
  511                  * Errata E.27 (Upper Four PAT Entries Not Usable With Mode B
  512                  * or Mode C Paging)
  513                  *
  514                  *   Intel Pentium IV  Processor Specification Update
  515                  * Errata N46 (PAT Index MSB May Be Calculated Incorrectly)
  516                  */
  517                 pat_msr = rdmsr(MSR_PAT);
  518                 pat_msr &= ~PAT_MASK(2);
  519                 pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING);
  520                 pat_works = 0;
  521         }
  522         wrmsr(MSR_PAT, pat_msr);
  523 }
  524 
  525 /*
  526  * Initialize a vm_page's machine-dependent fields.
  527  */
  528 void
  529 pmap_page_init(vm_page_t m)
  530 {
  531 
  532         TAILQ_INIT(&m->md.pv_list);
  533         m->md.pat_mode = PAT_WRITE_BACK;
  534 }
  535 
  536 /*
  537  * ABuse the pte nodes for unmapped kva to thread a kva freelist through.
  538  * Requirements:
  539  *  - Must deal with pages in order to ensure that none of the PG_* bits
  540  *    are ever set, PG_V in particular.
  541  *  - Assumes we can write to ptes without pte_store() atomic ops, even
  542  *    on PAE systems.  This should be ok.
  543  *  - Assumes nothing will ever test these addresses for 0 to indicate
  544  *    no mapping instead of correctly checking PG_V.
  545  *  - Assumes a vm_offset_t will fit in a pte (true for i386).
  546  * Because PG_V is never set, there can be no mappings to invalidate.
  547  */
  548 static int ptelist_count = 0;
  549 static vm_offset_t
  550 pmap_ptelist_alloc(vm_offset_t *head)
  551 {
  552         vm_offset_t va;
  553         vm_offset_t *phead = (vm_offset_t *)*head;
  554         
  555         if (ptelist_count == 0) {
  556                 printf("out of memory!!!!!!\n");
  557                 return (0);     /* Out of memory */
  558         }
  559         ptelist_count--;
  560         va = phead[ptelist_count];
  561         return (va);
  562 }
  563 
  564 static void
  565 pmap_ptelist_free(vm_offset_t *head, vm_offset_t va)
  566 {
  567         vm_offset_t *phead = (vm_offset_t *)*head;
  568 
  569         phead[ptelist_count++] = va;
  570 }
  571 
  572 static void
  573 pmap_ptelist_init(vm_offset_t *head, void *base, int npages)
  574 {
  575         int i, nstackpages;
  576         vm_offset_t va;
  577         vm_page_t m;
  578         
  579         nstackpages = (npages + PAGE_SIZE/sizeof(vm_offset_t) - 1)/ (PAGE_SIZE/sizeof(vm_offset_t));
  580         for (i = 0; i < nstackpages; i++) {
  581                 va = (vm_offset_t)base + i * PAGE_SIZE;
  582                 m = vm_page_alloc(NULL, i,
  583                     VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
  584                     VM_ALLOC_ZERO);
  585                 pmap_qenter(va, &m, 1);
  586         }
  587 
  588         *head = (vm_offset_t)base;
  589         for (i = npages - 1; i >= nstackpages; i--) {
  590                 va = (vm_offset_t)base + i * PAGE_SIZE;
  591                 pmap_ptelist_free(head, va);
  592         }
  593 }
  594 
  595 
  596 /*
  597  *      Initialize the pmap module.
  598  *      Called by vm_init, to initialize any structures that the pmap
  599  *      system needs to map virtual memory.
  600  */
  601 void
  602 pmap_init(void)
  603 {
  604 
  605         /*
  606          * Initialize the address space (zone) for the pv entries.  Set a
  607          * high water mark so that the system can recover from excessive
  608          * numbers of pv entries.
  609          */
  610         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  611         pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  612         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  613         pv_entry_max = roundup(pv_entry_max, _NPCPV);
  614         pv_entry_high_water = 9 * (pv_entry_max / 10);
  615 
  616         pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc);
  617         pv_chunkbase = (struct pv_chunk *)kmem_alloc_nofault(kernel_map,
  618             PAGE_SIZE * pv_maxchunks);
  619         if (pv_chunkbase == NULL)
  620                 panic("pmap_init: not enough kvm for pv chunks");
  621         pmap_ptelist_init(&pv_vafree, pv_chunkbase, pv_maxchunks);
  622 }
  623 
  624 
  625 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0,
  626         "Max number of PV entries");
  627 SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0,
  628         "Page share factor per proc");
  629 
  630 static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0,
  631     "2/4MB page mapping counters");
  632 
  633 static u_long pmap_pde_mappings;
  634 SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD,
  635     &pmap_pde_mappings, 0, "2/4MB page mappings");
  636 
  637 /***************************************************
  638  * Low level helper routines.....
  639  ***************************************************/
  640 
  641 /*
  642  * Determine the appropriate bits to set in a PTE or PDE for a specified
  643  * caching mode.
  644  */
  645 int
  646 pmap_cache_bits(int mode, boolean_t is_pde)
  647 {
  648         int pat_flag, pat_index, cache_bits;
  649 
  650         /* The PAT bit is different for PTE's and PDE's. */
  651         pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
  652 
  653         /* If we don't support PAT, map extended modes to older ones. */
  654         if (!(cpu_feature & CPUID_PAT)) {
  655                 switch (mode) {
  656                 case PAT_UNCACHEABLE:
  657                 case PAT_WRITE_THROUGH:
  658                 case PAT_WRITE_BACK:
  659                         break;
  660                 case PAT_UNCACHED:
  661                 case PAT_WRITE_COMBINING:
  662                 case PAT_WRITE_PROTECTED:
  663                         mode = PAT_UNCACHEABLE;
  664                         break;
  665                 }
  666         }
  667         
  668         /* Map the caching mode to a PAT index. */
  669         if (pat_works) {
  670                 switch (mode) {
  671                         case PAT_UNCACHEABLE:
  672                                 pat_index = 3;
  673                                 break;
  674                         case PAT_WRITE_THROUGH:
  675                                 pat_index = 1;
  676                                 break;
  677                         case PAT_WRITE_BACK:
  678                                 pat_index = 0;
  679                                 break;
  680                         case PAT_UNCACHED:
  681                                 pat_index = 2;
  682                                 break;
  683                         case PAT_WRITE_COMBINING:
  684                                 pat_index = 5;
  685                                 break;
  686                         case PAT_WRITE_PROTECTED:
  687                                 pat_index = 4;
  688                                 break;
  689                         default:
  690                                 panic("Unknown caching mode %d\n", mode);
  691                 }
  692         } else {
  693                 switch (mode) {
  694                         case PAT_UNCACHED:
  695                         case PAT_UNCACHEABLE:
  696                         case PAT_WRITE_PROTECTED:
  697                                 pat_index = 3;
  698                                 break;
  699                         case PAT_WRITE_THROUGH:
  700                                 pat_index = 1;
  701                                 break;
  702                         case PAT_WRITE_BACK:
  703                                 pat_index = 0;
  704                                 break;
  705                         case PAT_WRITE_COMBINING:
  706                                 pat_index = 2;
  707                                 break;
  708                         default:
  709                                 panic("Unknown caching mode %d\n", mode);
  710                 }
  711         }       
  712 
  713         /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */
  714         cache_bits = 0;
  715         if (pat_index & 0x4)
  716                 cache_bits |= pat_flag;
  717         if (pat_index & 0x2)
  718                 cache_bits |= PG_NC_PCD;
  719         if (pat_index & 0x1)
  720                 cache_bits |= PG_NC_PWT;
  721         return (cache_bits);
  722 }
  723 #ifdef SMP
  724 /*
  725  * For SMP, these functions have to use the IPI mechanism for coherence.
  726  *
  727  * N.B.: Before calling any of the following TLB invalidation functions,
  728  * the calling processor must ensure that all stores updating a non-
  729  * kernel page table are globally performed.  Otherwise, another
  730  * processor could cache an old, pre-update entry without being
  731  * invalidated.  This can happen one of two ways: (1) The pmap becomes
  732  * active on another processor after its pm_active field is checked by
  733  * one of the following functions but before a store updating the page
  734  * table is globally performed. (2) The pmap becomes active on another
  735  * processor before its pm_active field is checked but due to
  736  * speculative loads one of the following functions stills reads the
  737  * pmap as inactive on the other processor.
  738  * 
  739  * The kernel page table is exempt because its pm_active field is
  740  * immutable.  The kernel page table is always active on every
  741  * processor.
  742  */
  743 void
  744 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  745 {
  746         cpuset_t other_cpus;
  747         u_int cpuid;
  748 
  749         CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x",
  750             pmap, va);
  751         
  752         sched_pin();
  753         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
  754                 invlpg(va);
  755                 smp_invlpg(va);
  756         } else {
  757                 cpuid = PCPU_GET(cpuid);
  758                 other_cpus = all_cpus;
  759                 CPU_CLR(cpuid, &other_cpus);
  760                 if (CPU_ISSET(cpuid, &pmap->pm_active))
  761                         invlpg(va);
  762                 CPU_AND(&other_cpus, &pmap->pm_active);
  763                 if (!CPU_EMPTY(&other_cpus))
  764                         smp_masked_invlpg(other_cpus, va);
  765         }
  766         sched_unpin();
  767         PT_UPDATES_FLUSH();
  768 }
  769 
  770 void
  771 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  772 {
  773         cpuset_t other_cpus;
  774         vm_offset_t addr;
  775         u_int cpuid;
  776 
  777         CTR3(KTR_PMAP, "pmap_invalidate_page: pmap=%p eva=0x%x sva=0x%x",
  778             pmap, sva, eva);
  779 
  780         sched_pin();
  781         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
  782                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  783                         invlpg(addr);
  784                 smp_invlpg_range(sva, eva);
  785         } else {
  786                 cpuid = PCPU_GET(cpuid);
  787                 other_cpus = all_cpus;
  788                 CPU_CLR(cpuid, &other_cpus);
  789                 if (CPU_ISSET(cpuid, &pmap->pm_active))
  790                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
  791                                 invlpg(addr);
  792                 CPU_AND(&other_cpus, &pmap->pm_active);
  793                 if (!CPU_EMPTY(&other_cpus))
  794                         smp_masked_invlpg_range(other_cpus, sva, eva);
  795         }
  796         sched_unpin();
  797         PT_UPDATES_FLUSH();
  798 }
  799 
  800 void
  801 pmap_invalidate_all(pmap_t pmap)
  802 {
  803         cpuset_t other_cpus;
  804         u_int cpuid;
  805 
  806         CTR1(KTR_PMAP, "pmap_invalidate_page: pmap=%p", pmap);
  807 
  808         sched_pin();
  809         if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
  810                 invltlb();
  811                 smp_invltlb();
  812         } else {
  813                 cpuid = PCPU_GET(cpuid);
  814                 other_cpus = all_cpus;
  815                 CPU_CLR(cpuid, &other_cpus);
  816                 if (CPU_ISSET(cpuid, &pmap->pm_active))
  817                         invltlb();
  818                 CPU_AND(&other_cpus, &pmap->pm_active);
  819                 if (!CPU_EMPTY(&other_cpus))
  820                         smp_masked_invltlb(other_cpus);
  821         }
  822         sched_unpin();
  823 }
  824 
  825 void
  826 pmap_invalidate_cache(void)
  827 {
  828 
  829         sched_pin();
  830         wbinvd();
  831         smp_cache_flush();
  832         sched_unpin();
  833 }
  834 #else /* !SMP */
  835 /*
  836  * Normal, non-SMP, 486+ invalidation functions.
  837  * We inline these within pmap.c for speed.
  838  */
  839 PMAP_INLINE void
  840 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  841 {
  842         CTR2(KTR_PMAP, "pmap_invalidate_page: pmap=%p va=0x%x",
  843             pmap, va);
  844 
  845         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
  846                 invlpg(va);
  847         PT_UPDATES_FLUSH();
  848 }
  849 
  850 PMAP_INLINE void
  851 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  852 {
  853         vm_offset_t addr;
  854 
  855         if (eva - sva > PAGE_SIZE)
  856                 CTR3(KTR_PMAP, "pmap_invalidate_range: pmap=%p sva=0x%x eva=0x%x",
  857                     pmap, sva, eva);
  858 
  859         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
  860                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  861                         invlpg(addr);
  862         PT_UPDATES_FLUSH();
  863 }
  864 
  865 PMAP_INLINE void
  866 pmap_invalidate_all(pmap_t pmap)
  867 {
  868 
  869         CTR1(KTR_PMAP, "pmap_invalidate_all: pmap=%p", pmap);
  870         
  871         if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active))
  872                 invltlb();
  873 }
  874 
  875 PMAP_INLINE void
  876 pmap_invalidate_cache(void)
  877 {
  878 
  879         wbinvd();
  880 }
  881 #endif /* !SMP */
  882 
  883 #define PMAP_CLFLUSH_THRESHOLD  (2 * 1024 * 1024)
  884 
  885 void
  886 pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
  887 {
  888 
  889         KASSERT((sva & PAGE_MASK) == 0,
  890             ("pmap_invalidate_cache_range: sva not page-aligned"));
  891         KASSERT((eva & PAGE_MASK) == 0,
  892             ("pmap_invalidate_cache_range: eva not page-aligned"));
  893 
  894         if (cpu_feature & CPUID_SS)
  895                 ; /* If "Self Snoop" is supported, do nothing. */
  896         else if ((cpu_feature & CPUID_CLFSH) != 0 &&
  897             eva - sva < PMAP_CLFLUSH_THRESHOLD) {
  898 
  899                 /*
  900                  * Otherwise, do per-cache line flush.  Use the mfence
  901                  * instruction to insure that previous stores are
  902                  * included in the write-back.  The processor
  903                  * propagates flush to other processors in the cache
  904                  * coherence domain.
  905                  */
  906                 mfence();
  907                 for (; sva < eva; sva += cpu_clflush_line_size)
  908                         clflush(sva);
  909                 mfence();
  910         } else {
  911 
  912                 /*
  913                  * No targeted cache flush methods are supported by CPU,
  914                  * or the supplied range is bigger than 2MB.
  915                  * Globally invalidate cache.
  916                  */
  917                 pmap_invalidate_cache();
  918         }
  919 }
  920 
  921 void
  922 pmap_invalidate_cache_pages(vm_page_t *pages, int count)
  923 {
  924         int i;
  925 
  926         if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE ||
  927             (cpu_feature & CPUID_CLFSH) == 0) {
  928                 pmap_invalidate_cache();
  929         } else {
  930                 for (i = 0; i < count; i++)
  931                         pmap_flush_page(pages[i]);
  932         }
  933 }
  934 
  935 /*
  936  * Are we current address space or kernel?  N.B. We return FALSE when
  937  * a pmap's page table is in use because a kernel thread is borrowing
  938  * it.  The borrowed page table can change spontaneously, making any
  939  * dependence on its continued use subject to a race condition.
  940  */
  941 static __inline int
  942 pmap_is_current(pmap_t pmap)
  943 {
  944 
  945         return (pmap == kernel_pmap ||
  946             (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) &&
  947             (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME)));
  948 }
  949 
  950 /*
  951  * If the given pmap is not the current or kernel pmap, the returned pte must
  952  * be released by passing it to pmap_pte_release().
  953  */
  954 pt_entry_t *
  955 pmap_pte(pmap_t pmap, vm_offset_t va)
  956 {
  957         pd_entry_t newpf;
  958         pd_entry_t *pde;
  959 
  960         pde = pmap_pde(pmap, va);
  961         if (*pde & PG_PS)
  962                 return (pde);
  963         if (*pde != 0) {
  964                 /* are we current address space or kernel? */
  965                 if (pmap_is_current(pmap))
  966                         return (vtopte(va));
  967                 mtx_lock(&PMAP2mutex);
  968                 newpf = *pde & PG_FRAME;
  969                 if ((*PMAP2 & PG_FRAME) != newpf) {
  970                         vm_page_lock_queues();
  971                         PT_SET_MA(PADDR2, newpf | PG_V | PG_A | PG_M);
  972                         vm_page_unlock_queues();
  973                         CTR3(KTR_PMAP, "pmap_pte: pmap=%p va=0x%x newpte=0x%08x",
  974                             pmap, va, (*PMAP2 & 0xffffffff));
  975                 }
  976                 return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
  977         }
  978         return (NULL);
  979 }
  980 
  981 /*
  982  * Releases a pte that was obtained from pmap_pte().  Be prepared for the pte
  983  * being NULL.
  984  */
  985 static __inline void
  986 pmap_pte_release(pt_entry_t *pte)
  987 {
  988 
  989         if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2) {
  990                 CTR1(KTR_PMAP, "pmap_pte_release: pte=0x%jx",
  991                     *PMAP2);
  992                 vm_page_lock_queues();
  993                 PT_SET_VA(PMAP2, 0, TRUE);
  994                 vm_page_unlock_queues();
  995                 mtx_unlock(&PMAP2mutex);
  996         }
  997 }
  998 
  999 static __inline void
 1000 invlcaddr(void *caddr)
 1001 {
 1002 
 1003         invlpg((u_int)caddr);
 1004         PT_UPDATES_FLUSH();
 1005 }
 1006 
 1007 /*
 1008  * Super fast pmap_pte routine best used when scanning
 1009  * the pv lists.  This eliminates many coarse-grained
 1010  * invltlb calls.  Note that many of the pv list
 1011  * scans are across different pmaps.  It is very wasteful
 1012  * to do an entire invltlb for checking a single mapping.
 1013  *
 1014  * If the given pmap is not the current pmap, vm_page_queue_mtx
 1015  * must be held and curthread pinned to a CPU.
 1016  */
 1017 static pt_entry_t *
 1018 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
 1019 {
 1020         pd_entry_t newpf;
 1021         pd_entry_t *pde;
 1022 
 1023         pde = pmap_pde(pmap, va);
 1024         if (*pde & PG_PS)
 1025                 return (pde);
 1026         if (*pde != 0) {
 1027                 /* are we current address space or kernel? */
 1028                 if (pmap_is_current(pmap))
 1029                         return (vtopte(va));
 1030                 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1031                 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 1032                 newpf = *pde & PG_FRAME;
 1033                 if ((*PMAP1 & PG_FRAME) != newpf) {
 1034                         PT_SET_MA(PADDR1, newpf | PG_V | PG_A | PG_M);
 1035                         CTR3(KTR_PMAP, "pmap_pte_quick: pmap=%p va=0x%x newpte=0x%08x",
 1036                             pmap, va, (u_long)*PMAP1);
 1037                         
 1038 #ifdef SMP
 1039                         PMAP1cpu = PCPU_GET(cpuid);
 1040 #endif
 1041                         PMAP1changed++;
 1042                 } else
 1043 #ifdef SMP
 1044                 if (PMAP1cpu != PCPU_GET(cpuid)) {
 1045                         PMAP1cpu = PCPU_GET(cpuid);
 1046                         invlcaddr(PADDR1);
 1047                         PMAP1changedcpu++;
 1048                 } else
 1049 #endif
 1050                         PMAP1unchanged++;
 1051                 return (PADDR1 + (i386_btop(va) & (NPTEPG - 1)));
 1052         }
 1053         return (0);
 1054 }
 1055 
 1056 /*
 1057  *      Routine:        pmap_extract
 1058  *      Function:
 1059  *              Extract the physical page address associated
 1060  *              with the given map/virtual_address pair.
 1061  */
 1062 vm_paddr_t 
 1063 pmap_extract(pmap_t pmap, vm_offset_t va)
 1064 {
 1065         vm_paddr_t rtval;
 1066         pt_entry_t *pte;
 1067         pd_entry_t pde;
 1068         pt_entry_t pteval;
 1069 
 1070         rtval = 0;
 1071         PMAP_LOCK(pmap);
 1072         pde = pmap->pm_pdir[va >> PDRSHIFT];
 1073         if (pde != 0) {
 1074                 if ((pde & PG_PS) != 0) {
 1075                         rtval = xpmap_mtop(pde & PG_PS_FRAME) | (va & PDRMASK);
 1076                         PMAP_UNLOCK(pmap);
 1077                         return rtval;
 1078                 }
 1079                 pte = pmap_pte(pmap, va);
 1080                 pteval = *pte ? xpmap_mtop(*pte) : 0;
 1081                 rtval = (pteval & PG_FRAME) | (va & PAGE_MASK);
 1082                 pmap_pte_release(pte);
 1083         }
 1084         PMAP_UNLOCK(pmap);
 1085         return (rtval);
 1086 }
 1087 
 1088 /*
 1089  *      Routine:        pmap_extract_ma
 1090  *      Function:
 1091  *              Like pmap_extract, but returns machine address
 1092  */
 1093 vm_paddr_t 
 1094 pmap_extract_ma(pmap_t pmap, vm_offset_t va)
 1095 {
 1096         vm_paddr_t rtval;
 1097         pt_entry_t *pte;
 1098         pd_entry_t pde;
 1099 
 1100         rtval = 0;
 1101         PMAP_LOCK(pmap);
 1102         pde = pmap->pm_pdir[va >> PDRSHIFT];
 1103         if (pde != 0) {
 1104                 if ((pde & PG_PS) != 0) {
 1105                         rtval = (pde & ~PDRMASK) | (va & PDRMASK);
 1106                         PMAP_UNLOCK(pmap);
 1107                         return rtval;
 1108                 }
 1109                 pte = pmap_pte(pmap, va);
 1110                 rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
 1111                 pmap_pte_release(pte);
 1112         }
 1113         PMAP_UNLOCK(pmap);
 1114         return (rtval);
 1115 }
 1116 
 1117 /*
 1118  *      Routine:        pmap_extract_and_hold
 1119  *      Function:
 1120  *              Atomically extract and hold the physical page
 1121  *              with the given pmap and virtual address pair
 1122  *              if that mapping permits the given protection.
 1123  */
 1124 vm_page_t
 1125 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 1126 {
 1127         pd_entry_t pde;
 1128         pt_entry_t pte, *ptep;
 1129         vm_page_t m;
 1130         vm_paddr_t pa;
 1131 
 1132         pa = 0;
 1133         m = NULL;
 1134         PMAP_LOCK(pmap);
 1135 retry:
 1136         pde = PT_GET(pmap_pde(pmap, va));
 1137         if (pde != 0) {
 1138                 if (pde & PG_PS) {
 1139                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
 1140                                 if (vm_page_pa_tryrelock(pmap, (pde &
 1141                                     PG_PS_FRAME) | (va & PDRMASK), &pa))
 1142                                         goto retry;
 1143                                 m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
 1144                                     (va & PDRMASK));
 1145                                 vm_page_hold(m);
 1146                         }
 1147                 } else {
 1148                         ptep = pmap_pte(pmap, va);
 1149                         pte = PT_GET(ptep);
 1150                         pmap_pte_release(ptep);
 1151                         if (pte != 0 &&
 1152                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
 1153                                 if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME,
 1154                                     &pa))
 1155                                         goto retry;
 1156                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
 1157                                 vm_page_hold(m);
 1158                         }
 1159                 }
 1160         }
 1161         PA_UNLOCK_COND(pa);
 1162         PMAP_UNLOCK(pmap);
 1163         return (m);
 1164 }
 1165 
 1166 /***************************************************
 1167  * Low level mapping routines.....
 1168  ***************************************************/
 1169 
 1170 /*
 1171  * Add a wired page to the kva.
 1172  * Note: not SMP coherent.
 1173  *
 1174  * This function may be used before pmap_bootstrap() is called.
 1175  */
 1176 void 
 1177 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
 1178 {
 1179 
 1180         PT_SET_MA(va, xpmap_ptom(pa)| PG_RW | PG_V | pgeflag);
 1181 }
 1182 
 1183 void 
 1184 pmap_kenter_ma(vm_offset_t va, vm_paddr_t ma)
 1185 {
 1186         pt_entry_t *pte;
 1187 
 1188         pte = vtopte(va);
 1189         pte_store_ma(pte, ma | PG_RW | PG_V | pgeflag);
 1190 }
 1191 
 1192 static __inline void
 1193 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode)
 1194 {
 1195 
 1196         PT_SET_MA(va, pa | PG_RW | PG_V | pgeflag | pmap_cache_bits(mode, 0));
 1197 }
 1198 
 1199 /*
 1200  * Remove a page from the kernel pagetables.
 1201  * Note: not SMP coherent.
 1202  *
 1203  * This function may be used before pmap_bootstrap() is called.
 1204  */
 1205 PMAP_INLINE void
 1206 pmap_kremove(vm_offset_t va)
 1207 {
 1208         pt_entry_t *pte;
 1209 
 1210         pte = vtopte(va);
 1211         PT_CLEAR_VA(pte, FALSE);
 1212 }
 1213 
 1214 /*
 1215  *      Used to map a range of physical addresses into kernel
 1216  *      virtual address space.
 1217  *
 1218  *      The value passed in '*virt' is a suggested virtual address for
 1219  *      the mapping. Architectures which can support a direct-mapped
 1220  *      physical to virtual region can return the appropriate address
 1221  *      within that region, leaving '*virt' unchanged. Other
 1222  *      architectures should map the pages starting at '*virt' and
 1223  *      update '*virt' with the first usable address after the mapped
 1224  *      region.
 1225  */
 1226 vm_offset_t
 1227 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
 1228 {
 1229         vm_offset_t va, sva;
 1230 
 1231         va = sva = *virt;
 1232         CTR4(KTR_PMAP, "pmap_map: va=0x%x start=0x%jx end=0x%jx prot=0x%x",
 1233             va, start, end, prot);
 1234         while (start < end) {
 1235                 pmap_kenter(va, start);
 1236                 va += PAGE_SIZE;
 1237                 start += PAGE_SIZE;
 1238         }
 1239         pmap_invalidate_range(kernel_pmap, sva, va);
 1240         *virt = va;
 1241         return (sva);
 1242 }
 1243 
 1244 
 1245 /*
 1246  * Add a list of wired pages to the kva
 1247  * this routine is only used for temporary
 1248  * kernel mappings that do not need to have
 1249  * page modification or references recorded.
 1250  * Note that old mappings are simply written
 1251  * over.  The page *must* be wired.
 1252  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1253  */
 1254 void
 1255 pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
 1256 {
 1257         pt_entry_t *endpte, *pte;
 1258         vm_paddr_t pa;
 1259         vm_offset_t va = sva;
 1260         int mclcount = 0;
 1261         multicall_entry_t mcl[16];
 1262         multicall_entry_t *mclp = mcl;
 1263         int error;
 1264 
 1265         CTR2(KTR_PMAP, "pmap_qenter:sva=0x%x count=%d", va, count);
 1266         pte = vtopte(sva);
 1267         endpte = pte + count;
 1268         while (pte < endpte) {
 1269                 pa = VM_PAGE_TO_MACH(*ma) | pgeflag | PG_RW | PG_V | PG_M | PG_A;
 1270 
 1271                 mclp->op = __HYPERVISOR_update_va_mapping;
 1272                 mclp->args[0] = va;
 1273                 mclp->args[1] = (uint32_t)(pa & 0xffffffff);
 1274                 mclp->args[2] = (uint32_t)(pa >> 32);
 1275                 mclp->args[3] = (*pte & PG_V) ? UVMF_INVLPG|UVMF_ALL : 0;
 1276         
 1277                 va += PAGE_SIZE;
 1278                 pte++;
 1279                 ma++;
 1280                 mclp++;
 1281                 mclcount++;
 1282                 if (mclcount == 16) {
 1283                         error = HYPERVISOR_multicall(mcl, mclcount);
 1284                         mclp = mcl;
 1285                         mclcount = 0;
 1286                         KASSERT(error == 0, ("bad multicall %d", error));
 1287                 }               
 1288         }
 1289         if (mclcount) {
 1290                 error = HYPERVISOR_multicall(mcl, mclcount);
 1291                 KASSERT(error == 0, ("bad multicall %d", error));
 1292         }
 1293         
 1294 #ifdef INVARIANTS
 1295         for (pte = vtopte(sva), mclcount = 0; mclcount < count; mclcount++, pte++)
 1296                 KASSERT(*pte, ("pte not set for va=0x%x", sva + mclcount*PAGE_SIZE));
 1297 #endif  
 1298 }
 1299 
 1300 /*
 1301  * This routine tears out page mappings from the
 1302  * kernel -- it is meant only for temporary mappings.
 1303  * Note: SMP coherent.  Uses a ranged shootdown IPI.
 1304  */
 1305 void
 1306 pmap_qremove(vm_offset_t sva, int count)
 1307 {
 1308         vm_offset_t va;
 1309 
 1310         CTR2(KTR_PMAP, "pmap_qremove: sva=0x%x count=%d", sva, count);
 1311         va = sva;
 1312         vm_page_lock_queues();
 1313         critical_enter();
 1314         while (count-- > 0) {
 1315                 pmap_kremove(va);
 1316                 va += PAGE_SIZE;
 1317         }
 1318         PT_UPDATES_FLUSH();
 1319         pmap_invalidate_range(kernel_pmap, sva, va);
 1320         critical_exit();
 1321         vm_page_unlock_queues();
 1322 }
 1323 
 1324 /***************************************************
 1325  * Page table page management routines.....
 1326  ***************************************************/
 1327 static __inline void
 1328 pmap_free_zero_pages(vm_page_t free)
 1329 {
 1330         vm_page_t m;
 1331 
 1332         while (free != NULL) {
 1333                 m = free;
 1334                 free = m->right;
 1335                 vm_page_free_zero(m);
 1336         }
 1337 }
 1338 
 1339 /*
 1340  * Decrements a page table page's wire count, which is used to record the
 1341  * number of valid page table entries within the page.  If the wire count
 1342  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
 1343  * page table page was unmapped and FALSE otherwise.
 1344  */
 1345 static inline boolean_t
 1346 pmap_unwire_ptp(pmap_t pmap, vm_page_t m, vm_page_t *free)
 1347 {
 1348 
 1349         --m->wire_count;
 1350         if (m->wire_count == 0) {
 1351                 _pmap_unwire_ptp(pmap, m, free);
 1352                 return (TRUE);
 1353         } else
 1354                 return (FALSE);
 1355 }
 1356 
 1357 static void
 1358 _pmap_unwire_ptp(pmap_t pmap, vm_page_t m, vm_page_t *free)
 1359 {
 1360         vm_offset_t pteva;
 1361 
 1362         PT_UPDATES_FLUSH();
 1363         /*
 1364          * unmap the page table page
 1365          */
 1366         xen_pt_unpin(pmap->pm_pdir[m->pindex]);
 1367         /*
 1368          * page *might* contain residual mapping :-/  
 1369          */
 1370         PD_CLEAR_VA(pmap, m->pindex, TRUE);
 1371         pmap_zero_page(m);
 1372         --pmap->pm_stats.resident_count;
 1373 
 1374         /*
 1375          * This is a release store so that the ordinary store unmapping
 1376          * the page table page is globally performed before TLB shoot-
 1377          * down is begun.
 1378          */
 1379         atomic_subtract_rel_int(&cnt.v_wire_count, 1);
 1380 
 1381         /*
 1382          * Do an invltlb to make the invalidated mapping
 1383          * take effect immediately.
 1384          */
 1385         pteva = VM_MAXUSER_ADDRESS + i386_ptob(m->pindex);
 1386         pmap_invalidate_page(pmap, pteva);
 1387 
 1388         /* 
 1389          * Put page on a list so that it is released after
 1390          * *ALL* TLB shootdown is done
 1391          */
 1392         m->right = *free;
 1393         *free = m;
 1394 }
 1395 
 1396 /*
 1397  * After removing a page table entry, this routine is used to
 1398  * conditionally free the page, and manage the hold/wire counts.
 1399  */
 1400 static int
 1401 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t *free)
 1402 {
 1403         pd_entry_t ptepde;
 1404         vm_page_t mpte;
 1405 
 1406         if (va >= VM_MAXUSER_ADDRESS)
 1407                 return (0);
 1408         ptepde = PT_GET(pmap_pde(pmap, va));
 1409         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1410         return (pmap_unwire_ptp(pmap, mpte, free));
 1411 }
 1412 
 1413 /*
 1414  * Initialize the pmap for the swapper process.
 1415  */
 1416 void
 1417 pmap_pinit0(pmap_t pmap)
 1418 {
 1419 
 1420         PMAP_LOCK_INIT(pmap);
 1421         /*
 1422          * Since the page table directory is shared with the kernel pmap,
 1423          * which is already included in the list "allpmaps", this pmap does
 1424          * not need to be inserted into that list.
 1425          */
 1426         pmap->pm_pdir = (pd_entry_t *)(KERNBASE + (vm_offset_t)IdlePTD);
 1427 #ifdef PAE
 1428         pmap->pm_pdpt = (pdpt_entry_t *)(KERNBASE + (vm_offset_t)IdlePDPT);
 1429 #endif
 1430         CPU_ZERO(&pmap->pm_active);
 1431         PCPU_SET(curpmap, pmap);
 1432         TAILQ_INIT(&pmap->pm_pvchunk);
 1433         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1434 }
 1435 
 1436 /*
 1437  * Initialize a preallocated and zeroed pmap structure,
 1438  * such as one in a vmspace structure.
 1439  */
 1440 int
 1441 pmap_pinit(pmap_t pmap)
 1442 {
 1443         vm_page_t m, ptdpg[NPGPTD + 1];
 1444         int npgptd = NPGPTD + 1;
 1445         int i;
 1446 
 1447 #ifdef HAMFISTED_LOCKING
 1448         mtx_lock(&createdelete_lock);
 1449 #endif
 1450 
 1451         PMAP_LOCK_INIT(pmap);
 1452 
 1453         /*
 1454          * No need to allocate page table space yet but we do need a valid
 1455          * page directory table.
 1456          */
 1457         if (pmap->pm_pdir == NULL) {
 1458                 pmap->pm_pdir = (pd_entry_t *)kmem_alloc_nofault(kernel_map,
 1459                     NBPTD);
 1460                 if (pmap->pm_pdir == NULL) {
 1461                         PMAP_LOCK_DESTROY(pmap);
 1462 #ifdef HAMFISTED_LOCKING
 1463                         mtx_unlock(&createdelete_lock);
 1464 #endif
 1465                         return (0);
 1466                 }
 1467 #ifdef PAE
 1468                 pmap->pm_pdpt = (pd_entry_t *)kmem_alloc_nofault(kernel_map, 1);
 1469 #endif
 1470         }
 1471 
 1472         /*
 1473          * allocate the page directory page(s)
 1474          */
 1475         for (i = 0; i < npgptd;) {
 1476                 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
 1477                     VM_ALLOC_WIRED | VM_ALLOC_ZERO);
 1478                 if (m == NULL)
 1479                         VM_WAIT;
 1480                 else {
 1481                         ptdpg[i++] = m;
 1482                 }
 1483         }
 1484 
 1485         pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD);
 1486 
 1487         for (i = 0; i < NPGPTD; i++)
 1488                 if ((ptdpg[i]->flags & PG_ZERO) == 0)
 1489                         pagezero(pmap->pm_pdir + (i * NPDEPG));
 1490 
 1491         mtx_lock_spin(&allpmaps_lock);
 1492         LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);
 1493         /* Copy the kernel page table directory entries. */
 1494         bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t));
 1495         mtx_unlock_spin(&allpmaps_lock);
 1496 
 1497 #ifdef PAE
 1498         pmap_qenter((vm_offset_t)pmap->pm_pdpt, &ptdpg[NPGPTD], 1);
 1499         if ((ptdpg[NPGPTD]->flags & PG_ZERO) == 0)
 1500                 bzero(pmap->pm_pdpt, PAGE_SIZE);
 1501         for (i = 0; i < NPGPTD; i++) {
 1502                 vm_paddr_t ma;
 1503                 
 1504                 ma = VM_PAGE_TO_MACH(ptdpg[i]);
 1505                 pmap->pm_pdpt[i] = ma | PG_V;
 1506 
 1507         }
 1508 #endif  
 1509         for (i = 0; i < NPGPTD; i++) {
 1510                 pt_entry_t *pd;
 1511                 vm_paddr_t ma;
 1512                 
 1513                 ma = VM_PAGE_TO_MACH(ptdpg[i]);
 1514                 pd = pmap->pm_pdir + (i * NPDEPG);
 1515                 PT_SET_MA(pd, *vtopte((vm_offset_t)pd) & ~(PG_M|PG_A|PG_U|PG_RW));
 1516 #if 0           
 1517                 xen_pgd_pin(ma);
 1518 #endif          
 1519         }
 1520         
 1521 #ifdef PAE      
 1522         PT_SET_MA(pmap->pm_pdpt, *vtopte((vm_offset_t)pmap->pm_pdpt) & ~PG_RW);
 1523 #endif
 1524         vm_page_lock_queues();
 1525         xen_flush_queue();
 1526         xen_pgdpt_pin(VM_PAGE_TO_MACH(ptdpg[NPGPTD]));
 1527         for (i = 0; i < NPGPTD; i++) {
 1528                 vm_paddr_t ma = VM_PAGE_TO_MACH(ptdpg[i]);
 1529                 PT_SET_VA_MA(&pmap->pm_pdir[PTDPTDI + i], ma | PG_V | PG_A, FALSE);
 1530         }
 1531         xen_flush_queue();
 1532         vm_page_unlock_queues();
 1533         CPU_ZERO(&pmap->pm_active);
 1534         TAILQ_INIT(&pmap->pm_pvchunk);
 1535         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1536 
 1537 #ifdef HAMFISTED_LOCKING
 1538         mtx_unlock(&createdelete_lock);
 1539 #endif
 1540         return (1);
 1541 }
 1542 
 1543 /*
 1544  * this routine is called if the page table page is not
 1545  * mapped correctly.
 1546  */
 1547 static vm_page_t
 1548 _pmap_allocpte(pmap_t pmap, u_int ptepindex, int flags)
 1549 {
 1550         vm_paddr_t ptema;
 1551         vm_page_t m;
 1552 
 1553         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1554             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1555             ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1556 
 1557         /*
 1558          * Allocate a page table page.
 1559          */
 1560         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1561             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1562                 if (flags & M_WAITOK) {
 1563                         PMAP_UNLOCK(pmap);
 1564                         vm_page_unlock_queues();
 1565                         VM_WAIT;
 1566                         vm_page_lock_queues();
 1567                         PMAP_LOCK(pmap);
 1568                 }
 1569 
 1570                 /*
 1571                  * Indicate the need to retry.  While waiting, the page table
 1572                  * page may have been allocated.
 1573                  */
 1574                 return (NULL);
 1575         }
 1576         if ((m->flags & PG_ZERO) == 0)
 1577                 pmap_zero_page(m);
 1578 
 1579         /*
 1580          * Map the pagetable page into the process address space, if
 1581          * it isn't already there.
 1582          */
 1583 
 1584         pmap->pm_stats.resident_count++;
 1585 
 1586         ptema = VM_PAGE_TO_MACH(m);
 1587         xen_pt_pin(ptema);
 1588         PT_SET_VA_MA(&pmap->pm_pdir[ptepindex],
 1589                 (ptema | PG_U | PG_RW | PG_V | PG_A | PG_M), TRUE);
 1590         
 1591         KASSERT(pmap->pm_pdir[ptepindex],
 1592             ("_pmap_allocpte: ptepindex=%d did not get mapped", ptepindex));
 1593         return (m);
 1594 }
 1595 
 1596 static vm_page_t
 1597 pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
 1598 {
 1599         u_int ptepindex;
 1600         pd_entry_t ptema;
 1601         vm_page_t m;
 1602 
 1603         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1604             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1605             ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1606 
 1607         /*
 1608          * Calculate pagetable page index
 1609          */
 1610         ptepindex = va >> PDRSHIFT;
 1611 retry:
 1612         /*
 1613          * Get the page directory entry
 1614          */
 1615         ptema = pmap->pm_pdir[ptepindex];
 1616 
 1617         /*
 1618          * This supports switching from a 4MB page to a
 1619          * normal 4K page.
 1620          */
 1621         if (ptema & PG_PS) {
 1622                 /*
 1623                  * XXX 
 1624                  */
 1625                 pmap->pm_pdir[ptepindex] = 0;
 1626                 ptema = 0;
 1627                 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1628                 pmap_invalidate_all(kernel_pmap);
 1629         }
 1630 
 1631         /*
 1632          * If the page table page is mapped, we just increment the
 1633          * hold count, and activate it.
 1634          */
 1635         if (ptema & PG_V) {
 1636                 m = PHYS_TO_VM_PAGE(xpmap_mtop(ptema) & PG_FRAME);
 1637                 m->wire_count++;
 1638         } else {
 1639                 /*
 1640                  * Here if the pte page isn't mapped, or if it has
 1641                  * been deallocated. 
 1642                  */
 1643                 CTR3(KTR_PMAP, "pmap_allocpte: pmap=%p va=0x%08x flags=0x%x",
 1644                     pmap, va, flags);
 1645                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1646                 if (m == NULL && (flags & M_WAITOK))
 1647                         goto retry;
 1648 
 1649                 KASSERT(pmap->pm_pdir[ptepindex], ("ptepindex=%d did not get mapped", ptepindex));
 1650         }
 1651         return (m);
 1652 }
 1653 
 1654 
 1655 /***************************************************
 1656 * Pmap allocation/deallocation routines.
 1657  ***************************************************/
 1658 
 1659 #ifdef SMP
 1660 /*
 1661  * Deal with a SMP shootdown of other users of the pmap that we are
 1662  * trying to dispose of.  This can be a bit hairy.
 1663  */
 1664 static cpuset_t *lazymask;
 1665 static u_int lazyptd;
 1666 static volatile u_int lazywait;
 1667 
 1668 void pmap_lazyfix_action(void);
 1669 
 1670 void
 1671 pmap_lazyfix_action(void)
 1672 {
 1673 
 1674 #ifdef COUNT_IPIS
 1675         (*ipi_lazypmap_counts[PCPU_GET(cpuid)])++;
 1676 #endif
 1677         if (rcr3() == lazyptd)
 1678                 load_cr3(PCPU_GET(curpcb)->pcb_cr3);
 1679         CPU_CLR_ATOMIC(PCPU_GET(cpuid), lazymask);
 1680         atomic_store_rel_int(&lazywait, 1);
 1681 }
 1682 
 1683 static void
 1684 pmap_lazyfix_self(u_int cpuid)
 1685 {
 1686 
 1687         if (rcr3() == lazyptd)
 1688                 load_cr3(PCPU_GET(curpcb)->pcb_cr3);
 1689         CPU_CLR_ATOMIC(cpuid, lazymask);
 1690 }
 1691 
 1692 
 1693 static void
 1694 pmap_lazyfix(pmap_t pmap)
 1695 {
 1696         cpuset_t mymask, mask;
 1697         u_int cpuid, spins;
 1698         int lsb;
 1699 
 1700         mask = pmap->pm_active;
 1701         while (!CPU_EMPTY(&mask)) {
 1702                 spins = 50000000;
 1703 
 1704                 /* Find least significant set bit. */
 1705                 lsb = CPU_FFS(&mask);
 1706                 MPASS(lsb != 0);
 1707                 lsb--;
 1708                 CPU_SETOF(lsb, &mask);
 1709                 mtx_lock_spin(&smp_ipi_mtx);
 1710 #ifdef PAE
 1711                 lazyptd = vtophys(pmap->pm_pdpt);
 1712 #else
 1713                 lazyptd = vtophys(pmap->pm_pdir);
 1714 #endif
 1715                 cpuid = PCPU_GET(cpuid);
 1716 
 1717                 /* Use a cpuset just for having an easy check. */
 1718                 CPU_SETOF(cpuid, &mymask);
 1719                 if (!CPU_CMP(&mask, &mymask)) {
 1720                         lazymask = &pmap->pm_active;
 1721                         pmap_lazyfix_self(cpuid);
 1722                 } else {
 1723                         atomic_store_rel_int((u_int *)&lazymask,
 1724                             (u_int)&pmap->pm_active);
 1725                         atomic_store_rel_int(&lazywait, 0);
 1726                         ipi_selected(mask, IPI_LAZYPMAP);
 1727                         while (lazywait == 0) {
 1728                                 ia32_pause();
 1729                                 if (--spins == 0)
 1730                                         break;
 1731                         }
 1732                 }
 1733                 mtx_unlock_spin(&smp_ipi_mtx);
 1734                 if (spins == 0)
 1735                         printf("pmap_lazyfix: spun for 50000000\n");
 1736                 mask = pmap->pm_active;
 1737         }
 1738 }
 1739 
 1740 #else   /* SMP */
 1741 
 1742 /*
 1743  * Cleaning up on uniprocessor is easy.  For various reasons, we're
 1744  * unlikely to have to even execute this code, including the fact
 1745  * that the cleanup is deferred until the parent does a wait(2), which
 1746  * means that another userland process has run.
 1747  */
 1748 static void
 1749 pmap_lazyfix(pmap_t pmap)
 1750 {
 1751         u_int cr3;
 1752 
 1753         cr3 = vtophys(pmap->pm_pdir);
 1754         if (cr3 == rcr3()) {
 1755                 load_cr3(PCPU_GET(curpcb)->pcb_cr3);
 1756                 CPU_CLR(PCPU_GET(cpuid), &pmap->pm_active);
 1757         }
 1758 }
 1759 #endif  /* SMP */
 1760 
 1761 /*
 1762  * Release any resources held by the given physical map.
 1763  * Called when a pmap initialized by pmap_pinit is being released.
 1764  * Should only be called if the map contains no valid mappings.
 1765  */
 1766 void
 1767 pmap_release(pmap_t pmap)
 1768 {
 1769         vm_page_t m, ptdpg[2*NPGPTD+1];
 1770         vm_paddr_t ma;
 1771         int i;
 1772 #ifdef PAE      
 1773         int npgptd = NPGPTD + 1;
 1774 #else
 1775         int npgptd = NPGPTD;
 1776 #endif
 1777 
 1778         KASSERT(pmap->pm_stats.resident_count == 0,
 1779             ("pmap_release: pmap resident count %ld != 0",
 1780             pmap->pm_stats.resident_count));
 1781         PT_UPDATES_FLUSH();
 1782 
 1783 #ifdef HAMFISTED_LOCKING
 1784         mtx_lock(&createdelete_lock);
 1785 #endif
 1786 
 1787         pmap_lazyfix(pmap);
 1788         mtx_lock_spin(&allpmaps_lock);
 1789         LIST_REMOVE(pmap, pm_list);
 1790         mtx_unlock_spin(&allpmaps_lock);
 1791 
 1792         for (i = 0; i < NPGPTD; i++)
 1793                 ptdpg[i] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdir + (i*NPDEPG)) & PG_FRAME);
 1794         pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
 1795 #ifdef PAE
 1796         ptdpg[NPGPTD] = PHYS_TO_VM_PAGE(vtophys(pmap->pm_pdpt));
 1797 #endif  
 1798 
 1799         for (i = 0; i < npgptd; i++) {
 1800                 m = ptdpg[i];
 1801                 ma = VM_PAGE_TO_MACH(m);
 1802                 /* unpinning L1 and L2 treated the same */
 1803 #if 0
 1804                 xen_pgd_unpin(ma);
 1805 #else
 1806                 if (i == NPGPTD)
 1807                         xen_pgd_unpin(ma);
 1808 #endif
 1809 #ifdef PAE
 1810                 if (i < NPGPTD)
 1811                         KASSERT(VM_PAGE_TO_MACH(m) == (pmap->pm_pdpt[i] & PG_FRAME),
 1812                             ("pmap_release: got wrong ptd page"));
 1813 #endif
 1814                 m->wire_count--;
 1815                 atomic_subtract_int(&cnt.v_wire_count, 1);
 1816                 vm_page_free(m);
 1817         }
 1818 #ifdef PAE
 1819         pmap_qremove((vm_offset_t)pmap->pm_pdpt, 1);
 1820 #endif
 1821         PMAP_LOCK_DESTROY(pmap);
 1822 
 1823 #ifdef HAMFISTED_LOCKING
 1824         mtx_unlock(&createdelete_lock);
 1825 #endif
 1826 }
 1827 
 1828 static int
 1829 kvm_size(SYSCTL_HANDLER_ARGS)
 1830 {
 1831         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
 1832 
 1833         return (sysctl_handle_long(oidp, &ksize, 0, req));
 1834 }
 1835 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 1836     0, 0, kvm_size, "IU", "Size of KVM");
 1837 
 1838 static int
 1839 kvm_free(SYSCTL_HANDLER_ARGS)
 1840 {
 1841         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 1842 
 1843         return (sysctl_handle_long(oidp, &kfree, 0, req));
 1844 }
 1845 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 1846     0, 0, kvm_free, "IU", "Amount of KVM free");
 1847 
 1848 /*
 1849  * grow the number of kernel page table entries, if needed
 1850  */
 1851 void
 1852 pmap_growkernel(vm_offset_t addr)
 1853 {
 1854         struct pmap *pmap;
 1855         vm_paddr_t ptppaddr;
 1856         vm_page_t nkpg;
 1857         pd_entry_t newpdir;
 1858 
 1859         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1860         if (kernel_vm_end == 0) {
 1861                 kernel_vm_end = KERNBASE;
 1862                 nkpt = 0;
 1863                 while (pdir_pde(PTD, kernel_vm_end)) {
 1864                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1865                         nkpt++;
 1866                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1867                                 kernel_vm_end = kernel_map->max_offset;
 1868                                 break;
 1869                         }
 1870                 }
 1871         }
 1872         addr = roundup2(addr, NBPDR);
 1873         if (addr - 1 >= kernel_map->max_offset)
 1874                 addr = kernel_map->max_offset;
 1875         while (kernel_vm_end < addr) {
 1876                 if (pdir_pde(PTD, kernel_vm_end)) {
 1877                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 1878                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1879                                 kernel_vm_end = kernel_map->max_offset;
 1880                                 break;
 1881                         }
 1882                         continue;
 1883                 }
 1884 
 1885                 nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDRSHIFT,
 1886                     VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 1887                     VM_ALLOC_ZERO);
 1888                 if (nkpg == NULL)
 1889                         panic("pmap_growkernel: no memory to grow kernel");
 1890 
 1891                 nkpt++;
 1892 
 1893                 if ((nkpg->flags & PG_ZERO) == 0)
 1894                         pmap_zero_page(nkpg);
 1895                 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
 1896                 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
 1897                 vm_page_lock_queues();
 1898                 PD_SET_VA(kernel_pmap, (kernel_vm_end >> PDRSHIFT), newpdir, TRUE);
 1899                 mtx_lock_spin(&allpmaps_lock);
 1900                 LIST_FOREACH(pmap, &allpmaps, pm_list)
 1901                         PD_SET_VA(pmap, (kernel_vm_end >> PDRSHIFT), newpdir, TRUE);
 1902 
 1903                 mtx_unlock_spin(&allpmaps_lock);
 1904                 vm_page_unlock_queues();
 1905 
 1906                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 1907                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1908                         kernel_vm_end = kernel_map->max_offset;
 1909                         break;
 1910                 }
 1911         }
 1912 }
 1913 
 1914 
 1915 /***************************************************
 1916  * page management routines.
 1917  ***************************************************/
 1918 
 1919 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 1920 CTASSERT(_NPCM == 11);
 1921 CTASSERT(_NPCPV == 336);
 1922 
 1923 static __inline struct pv_chunk *
 1924 pv_to_chunk(pv_entry_t pv)
 1925 {
 1926 
 1927         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
 1928 }
 1929 
 1930 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 1931 
 1932 #define PC_FREE0_9      0xfffffffful    /* Free values for index 0 through 9 */
 1933 #define PC_FREE10       0x0000fffful    /* Free values for index 10 */
 1934 
 1935 static const uint32_t pc_freemask[_NPCM] = {
 1936         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 1937         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 1938         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 1939         PC_FREE0_9, PC_FREE10
 1940 };
 1941 
 1942 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 1943         "Current number of pv entries");
 1944 
 1945 #ifdef PV_STATS
 1946 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 1947 
 1948 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 1949         "Current number of pv entry chunks");
 1950 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 1951         "Current number of pv entry chunks allocated");
 1952 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 1953         "Current number of pv entry chunks frees");
 1954 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 1955         "Number of times tried to get a chunk page but failed.");
 1956 
 1957 static long pv_entry_frees, pv_entry_allocs;
 1958 static int pv_entry_spare;
 1959 
 1960 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 1961         "Current number of pv entry frees");
 1962 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 1963         "Current number of pv entry allocs");
 1964 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 1965         "Current number of spare pv entries");
 1966 #endif
 1967 
 1968 /*
 1969  * We are in a serious low memory condition.  Resort to
 1970  * drastic measures to free some pages so we can allocate
 1971  * another pv entry chunk.
 1972  */
 1973 static vm_page_t
 1974 pmap_pv_reclaim(pmap_t locked_pmap)
 1975 {
 1976         struct pch newtail;
 1977         struct pv_chunk *pc;
 1978         pmap_t pmap;
 1979         pt_entry_t *pte, tpte;
 1980         pv_entry_t pv;
 1981         vm_offset_t va;
 1982         vm_page_t free, m, m_pc;
 1983         uint32_t inuse;
 1984         int bit, field, freed;
 1985 
 1986         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
 1987         pmap = NULL;
 1988         free = m_pc = NULL;
 1989         TAILQ_INIT(&newtail);
 1990         while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 ||
 1991             free == NULL)) {
 1992                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 1993                 if (pmap != pc->pc_pmap) {
 1994                         if (pmap != NULL) {
 1995                                 pmap_invalidate_all(pmap);
 1996                                 if (pmap != locked_pmap)
 1997                                         PMAP_UNLOCK(pmap);
 1998                         }
 1999                         pmap = pc->pc_pmap;
 2000                         /* Avoid deadlock and lock recursion. */
 2001                         if (pmap > locked_pmap)
 2002                                 PMAP_LOCK(pmap);
 2003                         else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
 2004                                 pmap = NULL;
 2005                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2006                                 continue;
 2007                         }
 2008                 }
 2009 
 2010                 /*
 2011                  * Destroy every non-wired, 4 KB page mapping in the chunk.
 2012                  */
 2013                 freed = 0;
 2014                 for (field = 0; field < _NPCM; field++) {
 2015                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
 2016                             inuse != 0; inuse &= ~(1UL << bit)) {
 2017                                 bit = bsfl(inuse);
 2018                                 pv = &pc->pc_pventry[field * 32 + bit];
 2019                                 va = pv->pv_va;
 2020                                 pte = pmap_pte(pmap, va);
 2021                                 tpte = *pte;
 2022                                 if ((tpte & PG_W) == 0)
 2023                                         tpte = pte_load_clear(pte);
 2024                                 pmap_pte_release(pte);
 2025                                 if ((tpte & PG_W) != 0)
 2026                                         continue;
 2027                                 KASSERT(tpte != 0,
 2028                                     ("pmap_pv_reclaim: pmap %p va %x zero pte",
 2029                                     pmap, va));
 2030                                 if ((tpte & PG_G) != 0)
 2031                                         pmap_invalidate_page(pmap, va);
 2032                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 2033                                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2034                                         vm_page_dirty(m);
 2035                                 if ((tpte & PG_A) != 0)
 2036                                         vm_page_aflag_set(m, PGA_REFERENCED);
 2037                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2038                                 if (TAILQ_EMPTY(&m->md.pv_list))
 2039                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 2040                                 pc->pc_map[field] |= 1UL << bit;
 2041                                 pmap_unuse_pt(pmap, va, &free);
 2042                                 freed++;
 2043                         }
 2044                 }
 2045                 if (freed == 0) {
 2046                         TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2047                         continue;
 2048                 }
 2049                 /* Every freed mapping is for a 4 KB page. */
 2050                 pmap->pm_stats.resident_count -= freed;
 2051                 PV_STAT(pv_entry_frees += freed);
 2052                 PV_STAT(pv_entry_spare += freed);
 2053                 pv_entry_count -= freed;
 2054                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2055                 for (field = 0; field < _NPCM; field++)
 2056                         if (pc->pc_map[field] != pc_freemask[field]) {
 2057                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2058                                     pc_list);
 2059                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 2060 
 2061                                 /*
 2062                                  * One freed pv entry in locked_pmap is
 2063                                  * sufficient.
 2064                                  */
 2065                                 if (pmap == locked_pmap)
 2066                                         goto out;
 2067                                 break;
 2068                         }
 2069                 if (field == _NPCM) {
 2070                         PV_STAT(pv_entry_spare -= _NPCPV);
 2071                         PV_STAT(pc_chunk_count--);
 2072                         PV_STAT(pc_chunk_frees++);
 2073                         /* Entire chunk is free; return it. */
 2074                         m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2075                         pmap_qremove((vm_offset_t)pc, 1);
 2076                         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2077                         break;
 2078                 }
 2079         }
 2080 out:
 2081         TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
 2082         if (pmap != NULL) {
 2083                 pmap_invalidate_all(pmap);
 2084                 if (pmap != locked_pmap)
 2085                         PMAP_UNLOCK(pmap);
 2086         }
 2087         if (m_pc == NULL && pv_vafree != 0 && free != NULL) {
 2088                 m_pc = free;
 2089                 free = m_pc->right;
 2090                 /* Recycle a freed page table page. */
 2091                 m_pc->wire_count = 1;
 2092                 atomic_add_int(&cnt.v_wire_count, 1);
 2093         }
 2094         pmap_free_zero_pages(free);
 2095         return (m_pc);
 2096 }
 2097 
 2098 /*
 2099  * free the pv_entry back to the free list
 2100  */
 2101 static void
 2102 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 2103 {
 2104         struct pv_chunk *pc;
 2105         int idx, field, bit;
 2106 
 2107         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2108         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2109         PV_STAT(pv_entry_frees++);
 2110         PV_STAT(pv_entry_spare++);
 2111         pv_entry_count--;
 2112         pc = pv_to_chunk(pv);
 2113         idx = pv - &pc->pc_pventry[0];
 2114         field = idx / 32;
 2115         bit = idx % 32;
 2116         pc->pc_map[field] |= 1ul << bit;
 2117         for (idx = 0; idx < _NPCM; idx++)
 2118                 if (pc->pc_map[idx] != pc_freemask[idx]) {
 2119                         /*
 2120                          * 98% of the time, pc is already at the head of the
 2121                          * list.  If it isn't already, move it to the head.
 2122                          */
 2123                         if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
 2124                             pc)) {
 2125                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2126                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 2127                                     pc_list);
 2128                         }
 2129                         return;
 2130                 }
 2131         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2132         free_pv_chunk(pc);
 2133 }
 2134 
 2135 static void
 2136 free_pv_chunk(struct pv_chunk *pc)
 2137 {
 2138         vm_page_t m;
 2139 
 2140         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 2141         PV_STAT(pv_entry_spare -= _NPCPV);
 2142         PV_STAT(pc_chunk_count--);
 2143         PV_STAT(pc_chunk_frees++);
 2144         /* entire chunk is free, return it */
 2145         m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 2146         pmap_qremove((vm_offset_t)pc, 1);
 2147         vm_page_unwire(m, 0);
 2148         vm_page_free(m);
 2149         pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 2150 }
 2151 
 2152 /*
 2153  * get a new pv_entry, allocating a block from the system
 2154  * when needed.
 2155  */
 2156 static pv_entry_t
 2157 get_pv_entry(pmap_t pmap, boolean_t try)
 2158 {
 2159         static const struct timeval printinterval = { 60, 0 };
 2160         static struct timeval lastprint;
 2161         int bit, field;
 2162         pv_entry_t pv;
 2163         struct pv_chunk *pc;
 2164         vm_page_t m;
 2165 
 2166         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2167         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2168         PV_STAT(pv_entry_allocs++);
 2169         pv_entry_count++;
 2170         if (pv_entry_count > pv_entry_high_water)
 2171                 if (ratecheck(&lastprint, &printinterval))
 2172                         printf("Approaching the limit on PV entries, consider "
 2173                             "increasing either the vm.pmap.shpgperproc or the "
 2174                             "vm.pmap.pv_entry_max tunable.\n");
 2175 retry:
 2176         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 2177         if (pc != NULL) {
 2178                 for (field = 0; field < _NPCM; field++) {
 2179                         if (pc->pc_map[field]) {
 2180                                 bit = bsfl(pc->pc_map[field]);
 2181                                 break;
 2182                         }
 2183                 }
 2184                 if (field < _NPCM) {
 2185                         pv = &pc->pc_pventry[field * 32 + bit];
 2186                         pc->pc_map[field] &= ~(1ul << bit);
 2187                         /* If this was the last item, move it to tail */
 2188                         for (field = 0; field < _NPCM; field++)
 2189                                 if (pc->pc_map[field] != 0) {
 2190                                         PV_STAT(pv_entry_spare--);
 2191                                         return (pv);    /* not full, return */
 2192                                 }
 2193                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2194                         TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 2195                         PV_STAT(pv_entry_spare--);
 2196                         return (pv);
 2197                 }
 2198         }
 2199         /*
 2200          * Access to the ptelist "pv_vafree" is synchronized by the page
 2201          * queues lock.  If "pv_vafree" is currently non-empty, it will
 2202          * remain non-empty until pmap_ptelist_alloc() completes.
 2203          */
 2204         if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 2205             VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) {
 2206                 if (try) {
 2207                         pv_entry_count--;
 2208                         PV_STAT(pc_chunk_tryfail++);
 2209                         return (NULL);
 2210                 }
 2211                 m = pmap_pv_reclaim(pmap);
 2212                 if (m == NULL)
 2213                         goto retry;
 2214         }
 2215         PV_STAT(pc_chunk_count++);
 2216         PV_STAT(pc_chunk_allocs++);
 2217         pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree);
 2218         pmap_qenter((vm_offset_t)pc, &m, 1);
 2219         if ((m->flags & PG_ZERO) == 0)
 2220                 pagezero(pc);
 2221         pc->pc_pmap = pmap;
 2222         pc->pc_map[0] = pc_freemask[0] & ~1ul;  /* preallocated bit 0 */
 2223         for (field = 1; field < _NPCM; field++)
 2224                 pc->pc_map[field] = pc_freemask[field];
 2225         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 2226         pv = &pc->pc_pventry[0];
 2227         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 2228         PV_STAT(pv_entry_spare += _NPCPV - 1);
 2229         return (pv);
 2230 }
 2231 
 2232 static __inline pv_entry_t
 2233 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2234 {
 2235         pv_entry_t pv;
 2236 
 2237         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2238         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 2239                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 2240                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
 2241                         break;
 2242                 }
 2243         }
 2244         return (pv);
 2245 }
 2246 
 2247 static void
 2248 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 2249 {
 2250         pv_entry_t pv;
 2251 
 2252         pv = pmap_pvh_remove(pvh, pmap, va);
 2253         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found"));
 2254         free_pv_entry(pmap, pv);
 2255 }
 2256 
 2257 static void
 2258 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 2259 {
 2260 
 2261         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2262         pmap_pvh_free(&m->md, pmap, va);
 2263         if (TAILQ_EMPTY(&m->md.pv_list))
 2264                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 2265 }
 2266 
 2267 /*
 2268  * Conditionally create a pv entry.
 2269  */
 2270 static boolean_t
 2271 pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 2272 {
 2273         pv_entry_t pv;
 2274 
 2275         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2276         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2277         if (pv_entry_count < pv_entry_high_water && 
 2278             (pv = get_pv_entry(pmap, TRUE)) != NULL) {
 2279                 pv->pv_va = va;
 2280                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2281                 return (TRUE);
 2282         } else
 2283                 return (FALSE);
 2284 }
 2285 
 2286 /*
 2287  * pmap_remove_pte: do the things to unmap a page in a process
 2288  */
 2289 static int
 2290 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, vm_page_t *free)
 2291 {
 2292         pt_entry_t oldpte;
 2293         vm_page_t m;
 2294 
 2295         CTR3(KTR_PMAP, "pmap_remove_pte: pmap=%p *ptq=0x%x va=0x%x",
 2296             pmap, (u_long)*ptq, va);
 2297         
 2298         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2299         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2300         oldpte = *ptq;
 2301         PT_SET_VA_MA(ptq, 0, TRUE);
 2302         KASSERT(oldpte != 0,
 2303             ("pmap_remove_pte: pmap %p va %x zero pte", pmap, va));
 2304         if (oldpte & PG_W)
 2305                 pmap->pm_stats.wired_count -= 1;
 2306         /*
 2307          * Machines that don't support invlpg, also don't support
 2308          * PG_G.
 2309          */
 2310         if (oldpte & PG_G)
 2311                 pmap_invalidate_page(kernel_pmap, va);
 2312         pmap->pm_stats.resident_count -= 1;
 2313         if (oldpte & PG_MANAGED) {
 2314                 m = PHYS_TO_VM_PAGE(xpmap_mtop(oldpte) & PG_FRAME);
 2315                 if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2316                         vm_page_dirty(m);
 2317                 if (oldpte & PG_A)
 2318                         vm_page_aflag_set(m, PGA_REFERENCED);
 2319                 pmap_remove_entry(pmap, m, va);
 2320         }
 2321         return (pmap_unuse_pt(pmap, va, free));
 2322 }
 2323 
 2324 /*
 2325  * Remove a single page from a process address space
 2326  */
 2327 static void
 2328 pmap_remove_page(pmap_t pmap, vm_offset_t va, vm_page_t *free)
 2329 {
 2330         pt_entry_t *pte;
 2331 
 2332         CTR2(KTR_PMAP, "pmap_remove_page: pmap=%p va=0x%x",
 2333             pmap, va);
 2334         
 2335         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2336         KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 2337         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2338         if ((pte = pmap_pte_quick(pmap, va)) == NULL || (*pte & PG_V) == 0)
 2339                 return;
 2340         pmap_remove_pte(pmap, pte, va, free);
 2341         pmap_invalidate_page(pmap, va);
 2342         if (*PMAP1)
 2343                 PT_SET_MA(PADDR1, 0);
 2344 
 2345 }
 2346 
 2347 /*
 2348  *      Remove the given range of addresses from the specified map.
 2349  *
 2350  *      It is assumed that the start and end are properly
 2351  *      rounded to the page size.
 2352  */
 2353 void
 2354 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 2355 {
 2356         vm_offset_t pdnxt;
 2357         pd_entry_t ptpaddr;
 2358         pt_entry_t *pte;
 2359         vm_page_t free = NULL;
 2360         int anyvalid;
 2361 
 2362         CTR3(KTR_PMAP, "pmap_remove: pmap=%p sva=0x%x eva=0x%x",
 2363             pmap, sva, eva);
 2364 
 2365         /*
 2366          * Perform an unsynchronized read.  This is, however, safe.
 2367          */
 2368         if (pmap->pm_stats.resident_count == 0)
 2369                 return;
 2370 
 2371         anyvalid = 0;
 2372 
 2373         vm_page_lock_queues();
 2374         sched_pin();
 2375         PMAP_LOCK(pmap);
 2376 
 2377         /*
 2378          * special handling of removing one page.  a very
 2379          * common operation and easy to short circuit some
 2380          * code.
 2381          */
 2382         if ((sva + PAGE_SIZE == eva) && 
 2383             ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
 2384                 pmap_remove_page(pmap, sva, &free);
 2385                 goto out;
 2386         }
 2387 
 2388         for (; sva < eva; sva = pdnxt) {
 2389                 u_int pdirindex;
 2390 
 2391                 /*
 2392                  * Calculate index for next page table.
 2393                  */
 2394                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 2395                 if (pdnxt < sva)
 2396                         pdnxt = eva;
 2397                 if (pmap->pm_stats.resident_count == 0)
 2398                         break;
 2399 
 2400                 pdirindex = sva >> PDRSHIFT;
 2401                 ptpaddr = pmap->pm_pdir[pdirindex];
 2402 
 2403                 /*
 2404                  * Weed out invalid mappings. Note: we assume that the page
 2405                  * directory table is always allocated, and in kernel virtual.
 2406                  */
 2407                 if (ptpaddr == 0)
 2408                         continue;
 2409 
 2410                 /*
 2411                  * Check for large page.
 2412                  */
 2413                 if ((ptpaddr & PG_PS) != 0) {
 2414                         PD_CLEAR_VA(pmap, pdirindex, TRUE);
 2415                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 2416                         anyvalid = 1;
 2417                         continue;
 2418                 }
 2419 
 2420                 /*
 2421                  * Limit our scan to either the end of the va represented
 2422                  * by the current page table page, or to the end of the
 2423                  * range being removed.
 2424                  */
 2425                 if (pdnxt > eva)
 2426                         pdnxt = eva;
 2427 
 2428                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 2429                     sva += PAGE_SIZE) {
 2430                         if ((*pte & PG_V) == 0)
 2431                                 continue;
 2432 
 2433                         /*
 2434                          * The TLB entry for a PG_G mapping is invalidated
 2435                          * by pmap_remove_pte().
 2436                          */
 2437                         if ((*pte & PG_G) == 0)
 2438                                 anyvalid = 1;
 2439                         if (pmap_remove_pte(pmap, pte, sva, &free))
 2440                                 break;
 2441                 }
 2442         }
 2443         PT_UPDATES_FLUSH();
 2444         if (*PMAP1)
 2445                 PT_SET_VA_MA(PMAP1, 0, TRUE);
 2446 out:
 2447         if (anyvalid)
 2448                 pmap_invalidate_all(pmap);
 2449         sched_unpin();
 2450         vm_page_unlock_queues();
 2451         PMAP_UNLOCK(pmap);
 2452         pmap_free_zero_pages(free);
 2453 }
 2454 
 2455 /*
 2456  *      Routine:        pmap_remove_all
 2457  *      Function:
 2458  *              Removes this physical page from
 2459  *              all physical maps in which it resides.
 2460  *              Reflects back modify bits to the pager.
 2461  *
 2462  *      Notes:
 2463  *              Original versions of this routine were very
 2464  *              inefficient because they iteratively called
 2465  *              pmap_remove (slow...)
 2466  */
 2467 
 2468 void
 2469 pmap_remove_all(vm_page_t m)
 2470 {
 2471         pv_entry_t pv;
 2472         pmap_t pmap;
 2473         pt_entry_t *pte, tpte;
 2474         vm_page_t free;
 2475 
 2476         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 2477             ("pmap_remove_all: page %p is not managed", m));
 2478         free = NULL;
 2479         vm_page_lock_queues();
 2480         sched_pin();
 2481         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 2482                 pmap = PV_PMAP(pv);
 2483                 PMAP_LOCK(pmap);
 2484                 pmap->pm_stats.resident_count--;
 2485                 pte = pmap_pte_quick(pmap, pv->pv_va);
 2486                 tpte = *pte;
 2487                 PT_SET_VA_MA(pte, 0, TRUE);
 2488                 KASSERT(tpte != 0, ("pmap_remove_all: pmap %p va %x zero pte",
 2489                     pmap, pv->pv_va));
 2490                 if (tpte & PG_W)
 2491                         pmap->pm_stats.wired_count--;
 2492                 if (tpte & PG_A)
 2493                         vm_page_aflag_set(m, PGA_REFERENCED);
 2494 
 2495                 /*
 2496                  * Update the vm_page_t clean and reference bits.
 2497                  */
 2498                 if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 2499                         vm_page_dirty(m);
 2500                 pmap_unuse_pt(pmap, pv->pv_va, &free);
 2501                 pmap_invalidate_page(pmap, pv->pv_va);
 2502                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2503                 free_pv_entry(pmap, pv);
 2504                 PMAP_UNLOCK(pmap);
 2505         }
 2506         vm_page_aflag_clear(m, PGA_WRITEABLE);
 2507         PT_UPDATES_FLUSH();
 2508         if (*PMAP1)
 2509                 PT_SET_MA(PADDR1, 0);
 2510         sched_unpin();
 2511         vm_page_unlock_queues();
 2512         pmap_free_zero_pages(free);
 2513 }
 2514 
 2515 /*
 2516  *      Set the physical protection on the
 2517  *      specified range of this map as requested.
 2518  */
 2519 void
 2520 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 2521 {
 2522         vm_offset_t pdnxt;
 2523         pd_entry_t ptpaddr;
 2524         pt_entry_t *pte;
 2525         int anychanged;
 2526 
 2527         CTR4(KTR_PMAP, "pmap_protect: pmap=%p sva=0x%x eva=0x%x prot=0x%x",
 2528             pmap, sva, eva, prot);
 2529         
 2530         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 2531                 pmap_remove(pmap, sva, eva);
 2532                 return;
 2533         }
 2534 
 2535 #ifdef PAE
 2536         if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
 2537             (VM_PROT_WRITE|VM_PROT_EXECUTE))
 2538                 return;
 2539 #else
 2540         if (prot & VM_PROT_WRITE)
 2541                 return;
 2542 #endif
 2543 
 2544         anychanged = 0;
 2545 
 2546         vm_page_lock_queues();
 2547         sched_pin();
 2548         PMAP_LOCK(pmap);
 2549         for (; sva < eva; sva = pdnxt) {
 2550                 pt_entry_t obits, pbits;
 2551                 u_int pdirindex;
 2552 
 2553                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 2554                 if (pdnxt < sva)
 2555                         pdnxt = eva;
 2556 
 2557                 pdirindex = sva >> PDRSHIFT;
 2558                 ptpaddr = pmap->pm_pdir[pdirindex];
 2559 
 2560                 /*
 2561                  * Weed out invalid mappings. Note: we assume that the page
 2562                  * directory table is always allocated, and in kernel virtual.
 2563                  */
 2564                 if (ptpaddr == 0)
 2565                         continue;
 2566 
 2567                 /*
 2568                  * Check for large page.
 2569                  */
 2570                 if ((ptpaddr & PG_PS) != 0) {
 2571                         if ((prot & VM_PROT_WRITE) == 0)
 2572                                 pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW);
 2573 #ifdef PAE
 2574                         if ((prot & VM_PROT_EXECUTE) == 0)
 2575                                 pmap->pm_pdir[pdirindex] |= pg_nx;
 2576 #endif
 2577                         anychanged = 1;
 2578                         continue;
 2579                 }
 2580 
 2581                 if (pdnxt > eva)
 2582                         pdnxt = eva;
 2583 
 2584                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 2585                     sva += PAGE_SIZE) {
 2586                         vm_page_t m;
 2587 
 2588 retry:
 2589                         /*
 2590                          * Regardless of whether a pte is 32 or 64 bits in
 2591                          * size, PG_RW, PG_A, and PG_M are among the least
 2592                          * significant 32 bits.
 2593                          */
 2594                         obits = pbits = *pte;
 2595                         if ((pbits & PG_V) == 0)
 2596                                 continue;
 2597 
 2598                         if ((prot & VM_PROT_WRITE) == 0) {
 2599                                 if ((pbits & (PG_MANAGED | PG_M | PG_RW)) ==
 2600                                     (PG_MANAGED | PG_M | PG_RW)) {
 2601                                         m = PHYS_TO_VM_PAGE(xpmap_mtop(pbits) &
 2602                                             PG_FRAME);
 2603                                         vm_page_dirty(m);
 2604                                 }
 2605                                 pbits &= ~(PG_RW | PG_M);
 2606                         }
 2607 #ifdef PAE
 2608                         if ((prot & VM_PROT_EXECUTE) == 0)
 2609                                 pbits |= pg_nx;
 2610 #endif
 2611 
 2612                         if (pbits != obits) {
 2613                                 obits = *pte;
 2614                                 PT_SET_VA_MA(pte, pbits, TRUE);
 2615                                 if (*pte != pbits)
 2616                                         goto retry;
 2617                                 if (obits & PG_G)
 2618                                         pmap_invalidate_page(pmap, sva);
 2619                                 else
 2620                                         anychanged = 1;
 2621                         }
 2622                 }
 2623         }
 2624         PT_UPDATES_FLUSH();
 2625         if (*PMAP1)
 2626                 PT_SET_VA_MA(PMAP1, 0, TRUE);
 2627         if (anychanged)
 2628                 pmap_invalidate_all(pmap);
 2629         sched_unpin();
 2630         vm_page_unlock_queues();
 2631         PMAP_UNLOCK(pmap);
 2632 }
 2633 
 2634 /*
 2635  *      Insert the given physical page (p) at
 2636  *      the specified virtual address (v) in the
 2637  *      target physical map with the protection requested.
 2638  *
 2639  *      If specified, the page will be wired down, meaning
 2640  *      that the related pte can not be reclaimed.
 2641  *
 2642  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 2643  *      or lose information.  That is, this routine must actually
 2644  *      insert this page into the given map NOW.
 2645  */
 2646 void
 2647 pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
 2648     vm_prot_t prot, boolean_t wired)
 2649 {
 2650         pd_entry_t *pde;
 2651         pt_entry_t *pte;
 2652         pt_entry_t newpte, origpte;
 2653         pv_entry_t pv;
 2654         vm_paddr_t opa, pa;
 2655         vm_page_t mpte, om;
 2656         boolean_t invlva;
 2657 
 2658         CTR6(KTR_PMAP, "pmap_enter: pmap=%08p va=0x%08x access=0x%x ma=0x%08x prot=0x%x wired=%d",
 2659             pmap, va, access, VM_PAGE_TO_MACH(m), prot, wired);
 2660         va = trunc_page(va);
 2661         KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
 2662         KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS,
 2663             ("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)",
 2664             va));
 2665         KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 ||
 2666             VM_OBJECT_LOCKED(m->object),
 2667             ("pmap_enter: page %p is not busy", m));
 2668 
 2669         mpte = NULL;
 2670 
 2671         vm_page_lock_queues();
 2672         PMAP_LOCK(pmap);
 2673         sched_pin();
 2674 
 2675         /*
 2676          * In the case that a page table page is not
 2677          * resident, we are creating it here.
 2678          */
 2679         if (va < VM_MAXUSER_ADDRESS) {
 2680                 mpte = pmap_allocpte(pmap, va, M_WAITOK);
 2681         }
 2682 
 2683         pde = pmap_pde(pmap, va);
 2684         if ((*pde & PG_PS) != 0)
 2685                 panic("pmap_enter: attempted pmap_enter on 4MB page");
 2686         pte = pmap_pte_quick(pmap, va);
 2687 
 2688         /*
 2689          * Page Directory table entry not valid, we need a new PT page
 2690          */
 2691         if (pte == NULL) {
 2692                 panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x",
 2693                         (uintmax_t)pmap->pm_pdir[va >> PDRSHIFT], va);
 2694         }
 2695 
 2696         pa = VM_PAGE_TO_PHYS(m);
 2697         om = NULL;
 2698         opa = origpte = 0;
 2699 
 2700 #if 0
 2701         KASSERT((*pte & PG_V) || (*pte == 0), ("address set but not valid pte=%p *pte=0x%016jx",
 2702                 pte, *pte));
 2703 #endif
 2704         origpte = *pte;
 2705         if (origpte)
 2706                 origpte = xpmap_mtop(origpte);
 2707         opa = origpte & PG_FRAME;
 2708 
 2709         /*
 2710          * Mapping has not changed, must be protection or wiring change.
 2711          */
 2712         if (origpte && (opa == pa)) {
 2713                 /*
 2714                  * Wiring change, just update stats. We don't worry about
 2715                  * wiring PT pages as they remain resident as long as there
 2716                  * are valid mappings in them. Hence, if a user page is wired,
 2717                  * the PT page will be also.
 2718                  */
 2719                 if (wired && ((origpte & PG_W) == 0))
 2720                         pmap->pm_stats.wired_count++;
 2721                 else if (!wired && (origpte & PG_W))
 2722                         pmap->pm_stats.wired_count--;
 2723 
 2724                 /*
 2725                  * Remove extra pte reference
 2726                  */
 2727                 if (mpte)
 2728                         mpte->wire_count--;
 2729 
 2730                 if (origpte & PG_MANAGED) {
 2731                         om = m;
 2732                         pa |= PG_MANAGED;
 2733                 }
 2734                 goto validate;
 2735         } 
 2736 
 2737         pv = NULL;
 2738 
 2739         /*
 2740          * Mapping has changed, invalidate old range and fall through to
 2741          * handle validating new mapping.
 2742          */
 2743         if (opa) {
 2744                 if (origpte & PG_W)
 2745                         pmap->pm_stats.wired_count--;
 2746                 if (origpte & PG_MANAGED) {
 2747                         om = PHYS_TO_VM_PAGE(opa);
 2748                         pv = pmap_pvh_remove(&om->md, pmap, va);
 2749                 } else if (va < VM_MAXUSER_ADDRESS) 
 2750                         printf("va=0x%x is unmanaged :-( \n", va);
 2751                         
 2752                 if (mpte != NULL) {
 2753                         mpte->wire_count--;
 2754                         KASSERT(mpte->wire_count > 0,
 2755                             ("pmap_enter: missing reference to page table page,"
 2756                              " va: 0x%x", va));
 2757                 }
 2758         } else
 2759                 pmap->pm_stats.resident_count++;
 2760 
 2761         /*
 2762          * Enter on the PV list if part of our managed memory.
 2763          */
 2764         if ((m->oflags & VPO_UNMANAGED) == 0) {
 2765                 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva,
 2766                     ("pmap_enter: managed mapping within the clean submap"));
 2767                 if (pv == NULL)
 2768                         pv = get_pv_entry(pmap, FALSE);
 2769                 pv->pv_va = va;
 2770                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2771                 pa |= PG_MANAGED;
 2772         } else if (pv != NULL)
 2773                 free_pv_entry(pmap, pv);
 2774 
 2775         /*
 2776          * Increment counters
 2777          */
 2778         if (wired)
 2779                 pmap->pm_stats.wired_count++;
 2780 
 2781 validate:
 2782         /*
 2783          * Now validate mapping with desired protection/wiring.
 2784          */
 2785         newpte = (pt_entry_t)(pa | PG_V);
 2786         if ((prot & VM_PROT_WRITE) != 0) {
 2787                 newpte |= PG_RW;
 2788                 if ((newpte & PG_MANAGED) != 0)
 2789                         vm_page_aflag_set(m, PGA_WRITEABLE);
 2790         }
 2791 #ifdef PAE
 2792         if ((prot & VM_PROT_EXECUTE) == 0)
 2793                 newpte |= pg_nx;
 2794 #endif
 2795         if (wired)
 2796                 newpte |= PG_W;
 2797         if (va < VM_MAXUSER_ADDRESS)
 2798                 newpte |= PG_U;
 2799         if (pmap == kernel_pmap)
 2800                 newpte |= pgeflag;
 2801 
 2802         critical_enter();
 2803         /*
 2804          * if the mapping or permission bits are different, we need
 2805          * to update the pte.
 2806          */
 2807         if ((origpte & ~(PG_M|PG_A)) != newpte) {
 2808                 if (origpte) {
 2809                         invlva = FALSE;
 2810                         origpte = *pte;
 2811                         PT_SET_VA(pte, newpte | PG_A, FALSE);
 2812                         if (origpte & PG_A) {
 2813                                 if (origpte & PG_MANAGED)
 2814                                         vm_page_aflag_set(om, PGA_REFERENCED);
 2815                                 if (opa != VM_PAGE_TO_PHYS(m))
 2816                                         invlva = TRUE;
 2817 #ifdef PAE
 2818                                 if ((origpte & PG_NX) == 0 &&
 2819                                     (newpte & PG_NX) != 0)
 2820                                         invlva = TRUE;
 2821 #endif
 2822                         }
 2823                         if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 2824                                 if ((origpte & PG_MANAGED) != 0)
 2825                                         vm_page_dirty(om);
 2826                                 if ((prot & VM_PROT_WRITE) == 0)
 2827                                         invlva = TRUE;
 2828                         }
 2829                         if ((origpte & PG_MANAGED) != 0 &&
 2830                             TAILQ_EMPTY(&om->md.pv_list))
 2831                                 vm_page_aflag_clear(om, PGA_WRITEABLE);
 2832                         if (invlva)
 2833                                 pmap_invalidate_page(pmap, va);
 2834                 } else{
 2835                         PT_SET_VA(pte, newpte | PG_A, FALSE);
 2836                 }
 2837                 
 2838         }
 2839         PT_UPDATES_FLUSH();
 2840         critical_exit();
 2841         if (*PMAP1)
 2842                 PT_SET_VA_MA(PMAP1, 0, TRUE);
 2843         sched_unpin();
 2844         vm_page_unlock_queues();
 2845         PMAP_UNLOCK(pmap);
 2846 }
 2847 
 2848 /*
 2849  * Maps a sequence of resident pages belonging to the same object.
 2850  * The sequence begins with the given page m_start.  This page is
 2851  * mapped at the given virtual address start.  Each subsequent page is
 2852  * mapped at a virtual address that is offset from start by the same
 2853  * amount as the page is offset from m_start within the object.  The
 2854  * last page in the sequence is the page with the largest offset from
 2855  * m_start that can be mapped at a virtual address less than the given
 2856  * virtual address end.  Not every virtual page between start and end
 2857  * is mapped; only those for which a resident page exists with the
 2858  * corresponding offset from m_start are mapped.
 2859  */
 2860 void
 2861 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 2862     vm_page_t m_start, vm_prot_t prot)
 2863 {
 2864         vm_page_t m, mpte;
 2865         vm_pindex_t diff, psize;
 2866         multicall_entry_t mcl[16];
 2867         multicall_entry_t *mclp = mcl;
 2868         int error, count = 0;
 2869 
 2870         VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED);
 2871         psize = atop(end - start);
 2872         mpte = NULL;
 2873         m = m_start;
 2874         vm_page_lock_queues();
 2875         PMAP_LOCK(pmap);
 2876         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 2877                 mpte = pmap_enter_quick_locked(&mclp, &count, pmap, start + ptoa(diff), m,
 2878                     prot, mpte);
 2879                 m = TAILQ_NEXT(m, listq);
 2880                 if (count == 16) {
 2881                         error = HYPERVISOR_multicall(mcl, count);
 2882                         KASSERT(error == 0, ("bad multicall %d", error));
 2883                         mclp = mcl;
 2884                         count = 0;
 2885                 }
 2886         }
 2887         if (count) {
 2888                 error = HYPERVISOR_multicall(mcl, count);
 2889                 KASSERT(error == 0, ("bad multicall %d", error));
 2890         }
 2891         vm_page_unlock_queues();
 2892         PMAP_UNLOCK(pmap);
 2893 }
 2894 
 2895 /*
 2896  * this code makes some *MAJOR* assumptions:
 2897  * 1. Current pmap & pmap exists.
 2898  * 2. Not wired.
 2899  * 3. Read access.
 2900  * 4. No page table pages.
 2901  * but is *MUCH* faster than pmap_enter...
 2902  */
 2903 
 2904 void
 2905 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 2906 {
 2907         multicall_entry_t mcl, *mclp;
 2908         int count = 0;
 2909         mclp = &mcl;
 2910 
 2911         CTR4(KTR_PMAP, "pmap_enter_quick: pmap=%p va=0x%x m=%p prot=0x%x",
 2912             pmap, va, m, prot);
 2913         
 2914         vm_page_lock_queues();
 2915         PMAP_LOCK(pmap);
 2916         (void)pmap_enter_quick_locked(&mclp, &count, pmap, va, m, prot, NULL);
 2917         if (count)
 2918                 HYPERVISOR_multicall(&mcl, count);
 2919         vm_page_unlock_queues();
 2920         PMAP_UNLOCK(pmap);
 2921 }
 2922 
 2923 #ifdef notyet
 2924 void
 2925 pmap_enter_quick_range(pmap_t pmap, vm_offset_t *addrs, vm_page_t *pages, vm_prot_t *prots, int count)
 2926 {
 2927         int i, error, index = 0;
 2928         multicall_entry_t mcl[16];
 2929         multicall_entry_t *mclp = mcl;
 2930                 
 2931         PMAP_LOCK(pmap);
 2932         for (i = 0; i < count; i++, addrs++, pages++, prots++) {
 2933                 if (!pmap_is_prefaultable_locked(pmap, *addrs))
 2934                         continue;
 2935 
 2936                 (void) pmap_enter_quick_locked(&mclp, &index, pmap, *addrs, *pages, *prots, NULL);
 2937                 if (index == 16) {
 2938                         error = HYPERVISOR_multicall(mcl, index);
 2939                         mclp = mcl;
 2940                         index = 0;
 2941                         KASSERT(error == 0, ("bad multicall %d", error));
 2942                 }
 2943         }
 2944         if (index) {
 2945                 error = HYPERVISOR_multicall(mcl, index);
 2946                 KASSERT(error == 0, ("bad multicall %d", error));
 2947         }
 2948         
 2949         PMAP_UNLOCK(pmap);
 2950 }
 2951 #endif
 2952 
 2953 static vm_page_t
 2954 pmap_enter_quick_locked(multicall_entry_t **mclpp, int *count, pmap_t pmap, vm_offset_t va, vm_page_t m,
 2955     vm_prot_t prot, vm_page_t mpte)
 2956 {
 2957         pt_entry_t *pte;
 2958         vm_paddr_t pa;
 2959         vm_page_t free;
 2960         multicall_entry_t *mcl = *mclpp;
 2961 
 2962         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 2963             (m->oflags & VPO_UNMANAGED) != 0,
 2964             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 2965         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2966         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2967 
 2968         /*
 2969          * In the case that a page table page is not
 2970          * resident, we are creating it here.
 2971          */
 2972         if (va < VM_MAXUSER_ADDRESS) {
 2973                 u_int ptepindex;
 2974                 pd_entry_t ptema;
 2975 
 2976                 /*
 2977                  * Calculate pagetable page index
 2978                  */
 2979                 ptepindex = va >> PDRSHIFT;
 2980                 if (mpte && (mpte->pindex == ptepindex)) {
 2981                         mpte->wire_count++;
 2982                 } else {
 2983                         /*
 2984                          * Get the page directory entry
 2985                          */
 2986                         ptema = pmap->pm_pdir[ptepindex];
 2987 
 2988                         /*
 2989                          * If the page table page is mapped, we just increment
 2990                          * the hold count, and activate it.
 2991                          */
 2992                         if (ptema & PG_V) {
 2993                                 if (ptema & PG_PS)
 2994                                         panic("pmap_enter_quick: unexpected mapping into 4MB page");
 2995                                 mpte = PHYS_TO_VM_PAGE(xpmap_mtop(ptema) & PG_FRAME);
 2996                                 mpte->wire_count++;
 2997                         } else {
 2998                                 mpte = _pmap_allocpte(pmap, ptepindex,
 2999                                     M_NOWAIT);
 3000                                 if (mpte == NULL)
 3001                                         return (mpte);
 3002                         }
 3003                 }
 3004         } else {
 3005                 mpte = NULL;
 3006         }
 3007 
 3008         /*
 3009          * This call to vtopte makes the assumption that we are
 3010          * entering the page into the current pmap.  In order to support
 3011          * quick entry into any pmap, one would likely use pmap_pte_quick.
 3012          * But that isn't as quick as vtopte.
 3013          */
 3014         KASSERT(pmap_is_current(pmap), ("entering pages in non-current pmap"));
 3015         pte = vtopte(va);
 3016         if (*pte & PG_V) {
 3017                 if (mpte != NULL) {
 3018                         mpte->wire_count--;
 3019                         mpte = NULL;
 3020                 }
 3021                 return (mpte);
 3022         }
 3023 
 3024         /*
 3025          * Enter on the PV list if part of our managed memory.
 3026          */
 3027         if ((m->oflags & VPO_UNMANAGED) == 0 &&
 3028             !pmap_try_insert_pv_entry(pmap, va, m)) {
 3029                 if (mpte != NULL) {
 3030                         free = NULL;
 3031                         if (pmap_unwire_ptp(pmap, mpte, &free)) {
 3032                                 pmap_invalidate_page(pmap, va);
 3033                                 pmap_free_zero_pages(free);
 3034                         }
 3035                         
 3036                         mpte = NULL;
 3037                 }
 3038                 return (mpte);
 3039         }
 3040 
 3041         /*
 3042          * Increment counters
 3043          */
 3044         pmap->pm_stats.resident_count++;
 3045 
 3046         pa = VM_PAGE_TO_PHYS(m);
 3047 #ifdef PAE
 3048         if ((prot & VM_PROT_EXECUTE) == 0)
 3049                 pa |= pg_nx;
 3050 #endif
 3051 
 3052 #if 0
 3053         /*
 3054          * Now validate mapping with RO protection
 3055          */
 3056         if ((m->oflags & VPO_UNMANAGED) != 0)
 3057                 pte_store(pte, pa | PG_V | PG_U);
 3058         else
 3059                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 3060 #else
 3061         /*
 3062          * Now validate mapping with RO protection
 3063          */
 3064         if ((m->oflags & VPO_UNMANAGED) != 0)
 3065                 pa =    xpmap_ptom(pa | PG_V | PG_U);
 3066         else
 3067                 pa = xpmap_ptom(pa | PG_V | PG_U | PG_MANAGED);
 3068 
 3069         mcl->op = __HYPERVISOR_update_va_mapping;
 3070         mcl->args[0] = va;
 3071         mcl->args[1] = (uint32_t)(pa & 0xffffffff);
 3072         mcl->args[2] = (uint32_t)(pa >> 32);
 3073         mcl->args[3] = 0;
 3074         *mclpp = mcl + 1;
 3075         *count = *count + 1;
 3076 #endif  
 3077         return (mpte);
 3078 }
 3079 
 3080 /*
 3081  * Make a temporary mapping for a physical address.  This is only intended
 3082  * to be used for panic dumps.
 3083  */
 3084 void *
 3085 pmap_kenter_temporary(vm_paddr_t pa, int i)
 3086 {
 3087         vm_offset_t va;
 3088         vm_paddr_t ma = xpmap_ptom(pa);
 3089 
 3090         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 3091         PT_SET_MA(va, (ma & ~PAGE_MASK) | PG_V | pgeflag);
 3092         invlpg(va);
 3093         return ((void *)crashdumpmap);
 3094 }
 3095 
 3096 /*
 3097  * This code maps large physical mmap regions into the
 3098  * processor address space.  Note that some shortcuts
 3099  * are taken, but the code works.
 3100  */
 3101 void
 3102 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object,
 3103     vm_pindex_t pindex, vm_size_t size)
 3104 {
 3105         pd_entry_t *pde;
 3106         vm_paddr_t pa, ptepa;
 3107         vm_page_t p;
 3108         int pat_mode;
 3109 
 3110         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 3111         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 3112             ("pmap_object_init_pt: non-device object"));
 3113         if (pseflag && 
 3114             (addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) {
 3115                 if (!vm_object_populate(object, pindex, pindex + atop(size)))
 3116                         return;
 3117                 p = vm_page_lookup(object, pindex);
 3118                 KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3119                     ("pmap_object_init_pt: invalid page %p", p));
 3120                 pat_mode = p->md.pat_mode;
 3121 
 3122                 /*
 3123                  * Abort the mapping if the first page is not physically
 3124                  * aligned to a 2/4MB page boundary.
 3125                  */
 3126                 ptepa = VM_PAGE_TO_PHYS(p);
 3127                 if (ptepa & (NBPDR - 1))
 3128                         return;
 3129 
 3130                 /*
 3131                  * Skip the first page.  Abort the mapping if the rest of
 3132                  * the pages are not physically contiguous or have differing
 3133                  * memory attributes.
 3134                  */
 3135                 p = TAILQ_NEXT(p, listq);
 3136                 for (pa = ptepa + PAGE_SIZE; pa < ptepa + size;
 3137                     pa += PAGE_SIZE) {
 3138                         KASSERT(p->valid == VM_PAGE_BITS_ALL,
 3139                             ("pmap_object_init_pt: invalid page %p", p));
 3140                         if (pa != VM_PAGE_TO_PHYS(p) ||
 3141                             pat_mode != p->md.pat_mode)
 3142                                 return;
 3143                         p = TAILQ_NEXT(p, listq);
 3144                 }
 3145 
 3146                 /*
 3147                  * Map using 2/4MB pages.  Since "ptepa" is 2/4M aligned and
 3148                  * "size" is a multiple of 2/4M, adding the PAT setting to
 3149                  * "pa" will not affect the termination of this loop.
 3150                  */
 3151                 PMAP_LOCK(pmap);
 3152                 for (pa = ptepa | pmap_cache_bits(pat_mode, 1); pa < ptepa +
 3153                     size; pa += NBPDR) {
 3154                         pde = pmap_pde(pmap, addr);
 3155                         if (*pde == 0) {
 3156                                 pde_store(pde, pa | PG_PS | PG_M | PG_A |
 3157                                     PG_U | PG_RW | PG_V);
 3158                                 pmap->pm_stats.resident_count += NBPDR /
 3159                                     PAGE_SIZE;
 3160                                 pmap_pde_mappings++;
 3161                         }
 3162                         /* Else continue on if the PDE is already valid. */
 3163                         addr += NBPDR;
 3164                 }
 3165                 PMAP_UNLOCK(pmap);
 3166         }
 3167 }
 3168 
 3169 /*
 3170  *      Routine:        pmap_change_wiring
 3171  *      Function:       Change the wiring attribute for a map/virtual-address
 3172  *                      pair.
 3173  *      In/out conditions:
 3174  *                      The mapping must already exist in the pmap.
 3175  */
 3176 void
 3177 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 3178 {
 3179         pt_entry_t *pte;
 3180 
 3181         vm_page_lock_queues();
 3182         PMAP_LOCK(pmap);
 3183         pte = pmap_pte(pmap, va);
 3184 
 3185         if (wired && !pmap_pte_w(pte)) {
 3186                 PT_SET_VA_MA((pte), *(pte) | PG_W, TRUE);
 3187                 pmap->pm_stats.wired_count++;
 3188         } else if (!wired && pmap_pte_w(pte)) {
 3189                 PT_SET_VA_MA((pte), *(pte) & ~PG_W, TRUE);
 3190                 pmap->pm_stats.wired_count--;
 3191         }
 3192         
 3193         /*
 3194          * Wiring is not a hardware characteristic so there is no need to
 3195          * invalidate TLB.
 3196          */
 3197         pmap_pte_release(pte);
 3198         PMAP_UNLOCK(pmap);
 3199         vm_page_unlock_queues();
 3200 }
 3201 
 3202 
 3203 
 3204 /*
 3205  *      Copy the range specified by src_addr/len
 3206  *      from the source map to the range dst_addr/len
 3207  *      in the destination map.
 3208  *
 3209  *      This routine is only advisory and need not do anything.
 3210  */
 3211 
 3212 void
 3213 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 3214     vm_offset_t src_addr)
 3215 {
 3216         vm_page_t   free;
 3217         vm_offset_t addr;
 3218         vm_offset_t end_addr = src_addr + len;
 3219         vm_offset_t pdnxt;
 3220 
 3221         if (dst_addr != src_addr)
 3222                 return;
 3223 
 3224         if (!pmap_is_current(src_pmap)) {
 3225                 CTR2(KTR_PMAP,
 3226                     "pmap_copy, skipping: pdir[PTDPTDI]=0x%jx PTDpde[0]=0x%jx",
 3227                     (src_pmap->pm_pdir[PTDPTDI] & PG_FRAME), (PTDpde[0] & PG_FRAME));
 3228                 
 3229                 return;
 3230         }
 3231         CTR5(KTR_PMAP, "pmap_copy:  dst_pmap=%p src_pmap=%p dst_addr=0x%x len=%d src_addr=0x%x",
 3232             dst_pmap, src_pmap, dst_addr, len, src_addr);
 3233         
 3234 #ifdef HAMFISTED_LOCKING
 3235         mtx_lock(&createdelete_lock);
 3236 #endif
 3237 
 3238         vm_page_lock_queues();
 3239         if (dst_pmap < src_pmap) {
 3240                 PMAP_LOCK(dst_pmap);
 3241                 PMAP_LOCK(src_pmap);
 3242         } else {
 3243                 PMAP_LOCK(src_pmap);
 3244                 PMAP_LOCK(dst_pmap);
 3245         }
 3246         sched_pin();
 3247         for (addr = src_addr; addr < end_addr; addr = pdnxt) {
 3248                 pt_entry_t *src_pte, *dst_pte;
 3249                 vm_page_t dstmpte, srcmpte;
 3250                 pd_entry_t srcptepaddr;
 3251                 u_int ptepindex;
 3252 
 3253                 KASSERT(addr < UPT_MIN_ADDRESS,
 3254                     ("pmap_copy: invalid to pmap_copy page tables"));
 3255 
 3256                 pdnxt = (addr + NBPDR) & ~PDRMASK;
 3257                 if (pdnxt < addr)
 3258                         pdnxt = end_addr;
 3259                 ptepindex = addr >> PDRSHIFT;
 3260 
 3261                 srcptepaddr = PT_GET(&src_pmap->pm_pdir[ptepindex]);
 3262                 if (srcptepaddr == 0)
 3263                         continue;
 3264                         
 3265                 if (srcptepaddr & PG_PS) {
 3266                         if (dst_pmap->pm_pdir[ptepindex] == 0) {
 3267                                 PD_SET_VA(dst_pmap, ptepindex, srcptepaddr & ~PG_W, TRUE);
 3268                                 dst_pmap->pm_stats.resident_count +=
 3269                                     NBPDR / PAGE_SIZE;
 3270                         }
 3271                         continue;
 3272                 }
 3273 
 3274                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 3275                 KASSERT(srcmpte->wire_count > 0,
 3276                     ("pmap_copy: source page table page is unused"));
 3277 
 3278                 if (pdnxt > end_addr)
 3279                         pdnxt = end_addr;
 3280 
 3281                 src_pte = vtopte(addr);
 3282                 while (addr < pdnxt) {
 3283                         pt_entry_t ptetemp;
 3284                         ptetemp = *src_pte;
 3285                         /*
 3286                          * we only virtual copy managed pages
 3287                          */
 3288                         if ((ptetemp & PG_MANAGED) != 0) {
 3289                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 3290                                     M_NOWAIT);
 3291                                 if (dstmpte == NULL)
 3292                                         goto out;
 3293                                 dst_pte = pmap_pte_quick(dst_pmap, addr);
 3294                                 if (*dst_pte == 0 &&
 3295                                     pmap_try_insert_pv_entry(dst_pmap, addr,
 3296                                     PHYS_TO_VM_PAGE(xpmap_mtop(ptetemp) & PG_FRAME))) {
 3297                                         /*
 3298                                          * Clear the wired, modified, and
 3299                                          * accessed (referenced) bits
 3300                                          * during the copy.
 3301                                          */
 3302                                         KASSERT(ptetemp != 0, ("src_pte not set"));
 3303                                         PT_SET_VA_MA(dst_pte, ptetemp & ~(PG_W | PG_M | PG_A), TRUE /* XXX debug */);
 3304                                         KASSERT(*dst_pte == (ptetemp & ~(PG_W | PG_M | PG_A)),
 3305                                             ("no pmap copy expected: 0x%jx saw: 0x%jx",
 3306                                                 ptetemp &  ~(PG_W | PG_M | PG_A), *dst_pte));
 3307                                         dst_pmap->pm_stats.resident_count++;
 3308                                 } else {
 3309                                         free = NULL;
 3310                                         if (pmap_unwire_ptp(dst_pmap, dstmpte,
 3311                                             &free)) {
 3312                                                 pmap_invalidate_page(dst_pmap,
 3313                                                     addr);
 3314                                                 pmap_free_zero_pages(free);
 3315                                         }
 3316                                         goto out;
 3317                                 }
 3318                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 3319                                         break;
 3320                         }
 3321                         addr += PAGE_SIZE;
 3322                         src_pte++;
 3323                 }
 3324         }
 3325 out:
 3326         PT_UPDATES_FLUSH();
 3327         sched_unpin();
 3328         vm_page_unlock_queues();
 3329         PMAP_UNLOCK(src_pmap);
 3330         PMAP_UNLOCK(dst_pmap);
 3331 
 3332 #ifdef HAMFISTED_LOCKING
 3333         mtx_unlock(&createdelete_lock);
 3334 #endif
 3335 }       
 3336 
 3337 static __inline void
 3338 pagezero(void *page)
 3339 {
 3340 #if defined(I686_CPU)
 3341         if (cpu_class == CPUCLASS_686) {
 3342 #if defined(CPU_ENABLE_SSE)
 3343                 if (cpu_feature & CPUID_SSE2)
 3344                         sse2_pagezero(page);
 3345                 else
 3346 #endif
 3347                         i686_pagezero(page);
 3348         } else
 3349 #endif
 3350                 bzero(page, PAGE_SIZE);
 3351 }
 3352 
 3353 /*
 3354  *      pmap_zero_page zeros the specified hardware page by mapping 
 3355  *      the page into KVM and using bzero to clear its contents.
 3356  */
 3357 void
 3358 pmap_zero_page(vm_page_t m)
 3359 {
 3360         struct sysmaps *sysmaps;
 3361 
 3362         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 3363         mtx_lock(&sysmaps->lock);
 3364         if (*sysmaps->CMAP2)
 3365                 panic("pmap_zero_page: CMAP2 busy");
 3366         sched_pin();
 3367         PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | VM_PAGE_TO_MACH(m) | PG_A | PG_M);
 3368         pagezero(sysmaps->CADDR2);
 3369         PT_SET_MA(sysmaps->CADDR2, 0);
 3370         sched_unpin();
 3371         mtx_unlock(&sysmaps->lock);
 3372 }
 3373 
 3374 /*
 3375  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 3376  *      the page into KVM and using bzero to clear its contents.
 3377  *
 3378  *      off and size may not cover an area beyond a single hardware page.
 3379  */
 3380 void
 3381 pmap_zero_page_area(vm_page_t m, int off, int size)
 3382 {
 3383         struct sysmaps *sysmaps;
 3384 
 3385         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 3386         mtx_lock(&sysmaps->lock);
 3387         if (*sysmaps->CMAP2)
 3388                 panic("pmap_zero_page_area: CMAP2 busy");
 3389         sched_pin();
 3390         PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | VM_PAGE_TO_MACH(m) | PG_A | PG_M);
 3391 
 3392         if (off == 0 && size == PAGE_SIZE) 
 3393                 pagezero(sysmaps->CADDR2);
 3394         else
 3395                 bzero((char *)sysmaps->CADDR2 + off, size);
 3396         PT_SET_MA(sysmaps->CADDR2, 0);
 3397         sched_unpin();
 3398         mtx_unlock(&sysmaps->lock);
 3399 }
 3400 
 3401 /*
 3402  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 3403  *      the page into KVM and using bzero to clear its contents.  This
 3404  *      is intended to be called from the vm_pagezero process only and
 3405  *      outside of Giant.
 3406  */
 3407 void
 3408 pmap_zero_page_idle(vm_page_t m)
 3409 {
 3410 
 3411         if (*CMAP3)
 3412                 panic("pmap_zero_page_idle: CMAP3 busy");
 3413         sched_pin();
 3414         PT_SET_MA(CADDR3, PG_V | PG_RW | VM_PAGE_TO_MACH(m) | PG_A | PG_M);
 3415         pagezero(CADDR3);
 3416         PT_SET_MA(CADDR3, 0);
 3417         sched_unpin();
 3418 }
 3419 
 3420 /*
 3421  *      pmap_copy_page copies the specified (machine independent)
 3422  *      page by mapping the page into virtual memory and using
 3423  *      bcopy to copy the page, one machine dependent page at a
 3424  *      time.
 3425  */
 3426 void
 3427 pmap_copy_page(vm_page_t src, vm_page_t dst)
 3428 {
 3429         struct sysmaps *sysmaps;
 3430 
 3431         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 3432         mtx_lock(&sysmaps->lock);
 3433         if (*sysmaps->CMAP1)
 3434                 panic("pmap_copy_page: CMAP1 busy");
 3435         if (*sysmaps->CMAP2)
 3436                 panic("pmap_copy_page: CMAP2 busy");
 3437         sched_pin();
 3438         PT_SET_MA(sysmaps->CADDR1, PG_V | VM_PAGE_TO_MACH(src) | PG_A);
 3439         PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | VM_PAGE_TO_MACH(dst) | PG_A | PG_M);
 3440         bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
 3441         PT_SET_MA(sysmaps->CADDR1, 0);
 3442         PT_SET_MA(sysmaps->CADDR2, 0);
 3443         sched_unpin();
 3444         mtx_unlock(&sysmaps->lock);
 3445 }
 3446 
 3447 int unmapped_buf_allowed = 1;
 3448 
 3449 void
 3450 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 3451     vm_offset_t b_offset, int xfersize)
 3452 {
 3453         struct sysmaps *sysmaps;
 3454         vm_page_t a_pg, b_pg;
 3455         char *a_cp, *b_cp;
 3456         vm_offset_t a_pg_offset, b_pg_offset;
 3457         int cnt;
 3458 
 3459         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 3460         mtx_lock(&sysmaps->lock);
 3461         if (*sysmaps->CMAP1 != 0)
 3462                 panic("pmap_copy_pages: CMAP1 busy");
 3463         if (*sysmaps->CMAP2 != 0)
 3464                 panic("pmap_copy_pages: CMAP2 busy");
 3465         sched_pin();
 3466         while (xfersize > 0) {
 3467                 a_pg = ma[a_offset >> PAGE_SHIFT];
 3468                 a_pg_offset = a_offset & PAGE_MASK;
 3469                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
 3470                 b_pg = mb[b_offset >> PAGE_SHIFT];
 3471                 b_pg_offset = b_offset & PAGE_MASK;
 3472                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 3473                 PT_SET_MA(sysmaps->CADDR1, PG_V | VM_PAGE_TO_MACH(a_pg) | PG_A);
 3474                 PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW |
 3475                     VM_PAGE_TO_MACH(b_pg) | PG_A | PG_M);
 3476                 a_cp = sysmaps->CADDR1 + a_pg_offset;
 3477                 b_cp = sysmaps->CADDR2 + b_pg_offset;
 3478                 bcopy(a_cp, b_cp, cnt);
 3479                 a_offset += cnt;
 3480                 b_offset += cnt;
 3481                 xfersize -= cnt;
 3482         }
 3483         PT_SET_MA(sysmaps->CADDR1, 0);
 3484         PT_SET_MA(sysmaps->CADDR2, 0);
 3485         sched_unpin();
 3486         mtx_unlock(&sysmaps->lock);
 3487 }
 3488 
 3489 /*
 3490  * Returns true if the pmap's pv is one of the first
 3491  * 16 pvs linked to from this page.  This count may
 3492  * be changed upwards or downwards in the future; it
 3493  * is only necessary that true be returned for a small
 3494  * subset of pmaps for proper page aging.
 3495  */
 3496 boolean_t
 3497 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 3498 {
 3499         pv_entry_t pv;
 3500         int loops = 0;
 3501         boolean_t rv;
 3502 
 3503         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3504             ("pmap_page_exists_quick: page %p is not managed", m));
 3505         rv = FALSE;
 3506         vm_page_lock_queues();
 3507         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3508                 if (PV_PMAP(pv) == pmap) {
 3509                         rv = TRUE;
 3510                         break;
 3511                 }
 3512                 loops++;
 3513                 if (loops >= 16)
 3514                         break;
 3515         }
 3516         vm_page_unlock_queues();
 3517         return (rv);
 3518 }
 3519 
 3520 /*
 3521  *      pmap_page_wired_mappings:
 3522  *
 3523  *      Return the number of managed mappings to the given physical page
 3524  *      that are wired.
 3525  */
 3526 int
 3527 pmap_page_wired_mappings(vm_page_t m)
 3528 {
 3529         pv_entry_t pv;
 3530         pt_entry_t *pte;
 3531         pmap_t pmap;
 3532         int count;
 3533 
 3534         count = 0;
 3535         if ((m->oflags & VPO_UNMANAGED) != 0)
 3536                 return (count);
 3537         vm_page_lock_queues();
 3538         sched_pin();
 3539         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3540                 pmap = PV_PMAP(pv);
 3541                 PMAP_LOCK(pmap);
 3542                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3543                 if ((*pte & PG_W) != 0)
 3544                         count++;
 3545                 PMAP_UNLOCK(pmap);
 3546         }
 3547         sched_unpin();
 3548         vm_page_unlock_queues();
 3549         return (count);
 3550 }
 3551 
 3552 /*
 3553  * Returns TRUE if the given page is mapped.  Otherwise, returns FALSE.
 3554  */
 3555 boolean_t
 3556 pmap_page_is_mapped(vm_page_t m)
 3557 {
 3558 
 3559         if ((m->oflags & VPO_UNMANAGED) != 0)
 3560                 return (FALSE);
 3561         return (!TAILQ_EMPTY(&m->md.pv_list));
 3562 }
 3563 
 3564 /*
 3565  * Remove all pages from specified address space
 3566  * this aids process exit speeds.  Also, this code
 3567  * is special cased for current process only, but
 3568  * can have the more generic (and slightly slower)
 3569  * mode enabled.  This is much faster than pmap_remove
 3570  * in the case of running down an entire address space.
 3571  */
 3572 void
 3573 pmap_remove_pages(pmap_t pmap)
 3574 {
 3575         pt_entry_t *pte, tpte;
 3576         vm_page_t m, free = NULL;
 3577         pv_entry_t pv;
 3578         struct pv_chunk *pc, *npc;
 3579         int field, idx;
 3580         int32_t bit;
 3581         uint32_t inuse, bitmask;
 3582         int allfree;
 3583 
 3584         CTR1(KTR_PMAP, "pmap_remove_pages: pmap=%p", pmap);
 3585         
 3586         if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
 3587                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 3588                 return;
 3589         }
 3590         vm_page_lock_queues();
 3591         KASSERT(pmap_is_current(pmap), ("removing pages from non-current pmap"));
 3592         PMAP_LOCK(pmap);
 3593         sched_pin();
 3594         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 3595                 KASSERT(pc->pc_pmap == pmap, ("Wrong pmap %p %p", pmap,
 3596                     pc->pc_pmap));
 3597                 allfree = 1;
 3598                 for (field = 0; field < _NPCM; field++) {
 3599                         inuse = ~pc->pc_map[field] & pc_freemask[field];
 3600                         while (inuse != 0) {
 3601                                 bit = bsfl(inuse);
 3602                                 bitmask = 1UL << bit;
 3603                                 idx = field * 32 + bit;
 3604                                 pv = &pc->pc_pventry[idx];
 3605                                 inuse &= ~bitmask;
 3606 
 3607                                 pte = vtopte(pv->pv_va);
 3608                                 tpte = *pte ? xpmap_mtop(*pte) : 0;
 3609 
 3610                                 if (tpte == 0) {
 3611                                         printf(
 3612                                             "TPTE at %p  IS ZERO @ VA %08x\n",
 3613                                             pte, pv->pv_va);
 3614                                         panic("bad pte");
 3615                                 }
 3616 
 3617 /*
 3618  * We cannot remove wired pages from a process' mapping at this time
 3619  */
 3620                                 if (tpte & PG_W) {
 3621                                         allfree = 0;
 3622                                         continue;
 3623                                 }
 3624 
 3625                                 m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 3626                                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 3627                                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 3628                                     m, (uintmax_t)m->phys_addr,
 3629                                     (uintmax_t)tpte));
 3630 
 3631                                 KASSERT(m < &vm_page_array[vm_page_array_size],
 3632                                         ("pmap_remove_pages: bad tpte %#jx",
 3633                                         (uintmax_t)tpte));
 3634 
 3635 
 3636                                 PT_CLEAR_VA(pte, FALSE);
 3637                                 
 3638                                 /*
 3639                                  * Update the vm_page_t clean/reference bits.
 3640                                  */
 3641                                 if (tpte & PG_M)
 3642                                         vm_page_dirty(m);
 3643 
 3644                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 3645                                 if (TAILQ_EMPTY(&m->md.pv_list))
 3646                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 3647 
 3648                                 pmap_unuse_pt(pmap, pv->pv_va, &free);
 3649 
 3650                                 /* Mark free */
 3651                                 PV_STAT(pv_entry_frees++);
 3652                                 PV_STAT(pv_entry_spare++);
 3653                                 pv_entry_count--;
 3654                                 pc->pc_map[field] |= bitmask;
 3655                                 pmap->pm_stats.resident_count--;                        
 3656                         }
 3657                 }
 3658                 PT_UPDATES_FLUSH();
 3659                 if (allfree) {
 3660                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 3661                         free_pv_chunk(pc);
 3662                 }
 3663         }
 3664         PT_UPDATES_FLUSH();
 3665         if (*PMAP1)
 3666                 PT_SET_MA(PADDR1, 0);
 3667 
 3668         sched_unpin();
 3669         pmap_invalidate_all(pmap);
 3670         vm_page_unlock_queues();
 3671         PMAP_UNLOCK(pmap);
 3672         pmap_free_zero_pages(free);
 3673 }
 3674 
 3675 /*
 3676  *      pmap_is_modified:
 3677  *
 3678  *      Return whether or not the specified physical page was modified
 3679  *      in any physical maps.
 3680  */
 3681 boolean_t
 3682 pmap_is_modified(vm_page_t m)
 3683 {
 3684         pv_entry_t pv;
 3685         pt_entry_t *pte;
 3686         pmap_t pmap;
 3687         boolean_t rv;
 3688 
 3689         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3690             ("pmap_is_modified: page %p is not managed", m));
 3691         rv = FALSE;
 3692 
 3693         /*
 3694          * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be
 3695          * concurrently set while the object is locked.  Thus, if PGA_WRITEABLE
 3696          * is clear, no PTEs can have PG_M set.
 3697          */
 3698         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 3699         if ((m->oflags & VPO_BUSY) == 0 &&
 3700             (m->aflags & PGA_WRITEABLE) == 0)
 3701                 return (rv);
 3702         vm_page_lock_queues();
 3703         sched_pin();
 3704         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3705                 pmap = PV_PMAP(pv);
 3706                 PMAP_LOCK(pmap);
 3707                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3708                 rv = (*pte & PG_M) != 0;
 3709                 PMAP_UNLOCK(pmap);
 3710                 if (rv)
 3711                         break;
 3712         }
 3713         if (*PMAP1)
 3714                 PT_SET_MA(PADDR1, 0);
 3715         sched_unpin();
 3716         vm_page_unlock_queues();
 3717         return (rv);
 3718 }
 3719 
 3720 /*
 3721  *      pmap_is_prefaultable:
 3722  *
 3723  *      Return whether or not the specified virtual address is elgible
 3724  *      for prefault.
 3725  */
 3726 static boolean_t
 3727 pmap_is_prefaultable_locked(pmap_t pmap, vm_offset_t addr)
 3728 {
 3729         pt_entry_t *pte;
 3730         boolean_t rv = FALSE;
 3731 
 3732         return (rv);
 3733         
 3734         if (pmap_is_current(pmap) && *pmap_pde(pmap, addr)) {
 3735                 pte = vtopte(addr);
 3736                 rv = (*pte == 0);
 3737         }
 3738         return (rv);
 3739 }
 3740 
 3741 boolean_t
 3742 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 3743 {
 3744         boolean_t rv;
 3745         
 3746         PMAP_LOCK(pmap);
 3747         rv = pmap_is_prefaultable_locked(pmap, addr);
 3748         PMAP_UNLOCK(pmap);
 3749         return (rv);
 3750 }
 3751 
 3752 boolean_t
 3753 pmap_is_referenced(vm_page_t m)
 3754 {
 3755         pv_entry_t pv;
 3756         pt_entry_t *pte;
 3757         pmap_t pmap;
 3758         boolean_t rv;
 3759 
 3760         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3761             ("pmap_is_referenced: page %p is not managed", m));
 3762         rv = FALSE;
 3763         vm_page_lock_queues();
 3764         sched_pin();
 3765         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3766                 pmap = PV_PMAP(pv);
 3767                 PMAP_LOCK(pmap);
 3768                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3769                 rv = (*pte & (PG_A | PG_V)) == (PG_A | PG_V);
 3770                 PMAP_UNLOCK(pmap);
 3771                 if (rv)
 3772                         break;
 3773         }
 3774         if (*PMAP1)
 3775                 PT_SET_MA(PADDR1, 0);
 3776         sched_unpin();
 3777         vm_page_unlock_queues();
 3778         return (rv);
 3779 }
 3780 
 3781 void
 3782 pmap_map_readonly(pmap_t pmap, vm_offset_t va, int len)
 3783 {
 3784         int i, npages = round_page(len) >> PAGE_SHIFT;
 3785         for (i = 0; i < npages; i++) {
 3786                 pt_entry_t *pte;
 3787                 pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE));
 3788                 vm_page_lock_queues();
 3789                 pte_store(pte, xpmap_mtop(*pte & ~(PG_RW|PG_M)));
 3790                 vm_page_unlock_queues();
 3791                 PMAP_MARK_PRIV(xpmap_mtop(*pte));
 3792                 pmap_pte_release(pte);
 3793         }
 3794 }
 3795 
 3796 void
 3797 pmap_map_readwrite(pmap_t pmap, vm_offset_t va, int len)
 3798 {
 3799         int i, npages = round_page(len) >> PAGE_SHIFT;
 3800         for (i = 0; i < npages; i++) {
 3801                 pt_entry_t *pte;
 3802                 pte = pmap_pte(pmap, (vm_offset_t)(va + i*PAGE_SIZE));
 3803                 PMAP_MARK_UNPRIV(xpmap_mtop(*pte));
 3804                 vm_page_lock_queues();
 3805                 pte_store(pte, xpmap_mtop(*pte) | (PG_RW|PG_M));
 3806                 vm_page_unlock_queues();
 3807                 pmap_pte_release(pte);
 3808         }
 3809 }
 3810 
 3811 /*
 3812  * Clear the write and modified bits in each of the given page's mappings.
 3813  */
 3814 void
 3815 pmap_remove_write(vm_page_t m)
 3816 {
 3817         pv_entry_t pv;
 3818         pmap_t pmap;
 3819         pt_entry_t oldpte, *pte;
 3820 
 3821         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3822             ("pmap_remove_write: page %p is not managed", m));
 3823 
 3824         /*
 3825          * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by
 3826          * another thread while the object is locked.  Thus, if PGA_WRITEABLE
 3827          * is clear, no page table entries need updating.
 3828          */
 3829         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 3830         if ((m->oflags & VPO_BUSY) == 0 &&
 3831             (m->aflags & PGA_WRITEABLE) == 0)
 3832                 return;
 3833         vm_page_lock_queues();
 3834         sched_pin();
 3835         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3836                 pmap = PV_PMAP(pv);
 3837                 PMAP_LOCK(pmap);
 3838                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3839 retry:
 3840                 oldpte = *pte;
 3841                 if ((oldpte & PG_RW) != 0) {
 3842                         vm_paddr_t newpte = oldpte & ~(PG_RW | PG_M);
 3843                         
 3844                         /*
 3845                          * Regardless of whether a pte is 32 or 64 bits
 3846                          * in size, PG_RW and PG_M are among the least
 3847                          * significant 32 bits.
 3848                          */
 3849                         PT_SET_VA_MA(pte, newpte, TRUE);
 3850                         if (*pte != newpte)
 3851                                 goto retry;
 3852                         
 3853                         if ((oldpte & PG_M) != 0)
 3854                                 vm_page_dirty(m);
 3855                         pmap_invalidate_page(pmap, pv->pv_va);
 3856                 }
 3857                 PMAP_UNLOCK(pmap);
 3858         }
 3859         vm_page_aflag_clear(m, PGA_WRITEABLE);
 3860         PT_UPDATES_FLUSH();
 3861         if (*PMAP1)
 3862                 PT_SET_MA(PADDR1, 0);
 3863         sched_unpin();
 3864         vm_page_unlock_queues();
 3865 }
 3866 
 3867 /*
 3868  *      pmap_ts_referenced:
 3869  *
 3870  *      Return a count of reference bits for a page, clearing those bits.
 3871  *      It is not necessary for every reference bit to be cleared, but it
 3872  *      is necessary that 0 only be returned when there are truly no
 3873  *      reference bits set.
 3874  *
 3875  *      XXX: The exact number of bits to check and clear is a matter that
 3876  *      should be tested and standardized at some point in the future for
 3877  *      optimal aging of shared pages.
 3878  */
 3879 int
 3880 pmap_ts_referenced(vm_page_t m)
 3881 {
 3882         pv_entry_t pv, pvf, pvn;
 3883         pmap_t pmap;
 3884         pt_entry_t *pte;
 3885         int rtval = 0;
 3886 
 3887         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3888             ("pmap_ts_referenced: page %p is not managed", m));
 3889         vm_page_lock_queues();
 3890         sched_pin();
 3891         if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 3892                 pvf = pv;
 3893                 do {
 3894                         pvn = TAILQ_NEXT(pv, pv_list);
 3895                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 3896                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 3897                         pmap = PV_PMAP(pv);
 3898                         PMAP_LOCK(pmap);
 3899                         pte = pmap_pte_quick(pmap, pv->pv_va);
 3900                         if ((*pte & PG_A) != 0) {
 3901                                 PT_SET_VA_MA(pte, *pte & ~PG_A, FALSE);
 3902                                 pmap_invalidate_page(pmap, pv->pv_va);
 3903                                 rtval++;
 3904                                 if (rtval > 4)
 3905                                         pvn = NULL;
 3906                         }
 3907                         PMAP_UNLOCK(pmap);
 3908                 } while ((pv = pvn) != NULL && pv != pvf);
 3909         }
 3910         PT_UPDATES_FLUSH();
 3911         if (*PMAP1)
 3912                 PT_SET_MA(PADDR1, 0);
 3913         sched_unpin();
 3914         vm_page_unlock_queues();
 3915         return (rtval);
 3916 }
 3917 
 3918 /*
 3919  *      Clear the modify bits on the specified physical page.
 3920  */
 3921 void
 3922 pmap_clear_modify(vm_page_t m)
 3923 {
 3924         pv_entry_t pv;
 3925         pmap_t pmap;
 3926         pt_entry_t *pte;
 3927 
 3928         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3929             ("pmap_clear_modify: page %p is not managed", m));
 3930         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 3931         KASSERT((m->oflags & VPO_BUSY) == 0,
 3932             ("pmap_clear_modify: page %p is busy", m));
 3933 
 3934         /*
 3935          * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
 3936          * If the object containing the page is locked and the page is not
 3937          * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set.
 3938          */
 3939         if ((m->aflags & PGA_WRITEABLE) == 0)
 3940                 return;
 3941         vm_page_lock_queues();
 3942         sched_pin();
 3943         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3944                 pmap = PV_PMAP(pv);
 3945                 PMAP_LOCK(pmap);
 3946                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3947                 if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) {
 3948                         /*
 3949                          * Regardless of whether a pte is 32 or 64 bits
 3950                          * in size, PG_M is among the least significant
 3951                          * 32 bits. 
 3952                          */
 3953                         PT_SET_VA_MA(pte, *pte & ~PG_M, FALSE);
 3954                         pmap_invalidate_page(pmap, pv->pv_va);
 3955                 }
 3956                 PMAP_UNLOCK(pmap);
 3957         }
 3958         sched_unpin();
 3959         vm_page_unlock_queues();
 3960 }
 3961 
 3962 /*
 3963  *      pmap_clear_reference:
 3964  *
 3965  *      Clear the reference bit on the specified physical page.
 3966  */
 3967 void
 3968 pmap_clear_reference(vm_page_t m)
 3969 {
 3970         pv_entry_t pv;
 3971         pmap_t pmap;
 3972         pt_entry_t *pte;
 3973 
 3974         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3975             ("pmap_clear_reference: page %p is not managed", m));
 3976         vm_page_lock_queues();
 3977         sched_pin();
 3978         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3979                 pmap = PV_PMAP(pv);
 3980                 PMAP_LOCK(pmap);
 3981                 pte = pmap_pte_quick(pmap, pv->pv_va);
 3982                 if ((*pte & PG_A) != 0) {
 3983                         /*
 3984                          * Regardless of whether a pte is 32 or 64 bits
 3985                          * in size, PG_A is among the least significant
 3986                          * 32 bits. 
 3987                          */
 3988                         PT_SET_VA_MA(pte, *pte & ~PG_A, FALSE);
 3989                         pmap_invalidate_page(pmap, pv->pv_va);
 3990                 }
 3991                 PMAP_UNLOCK(pmap);
 3992         }
 3993         sched_unpin();
 3994         vm_page_unlock_queues();
 3995 }
 3996 
 3997 /*
 3998  * Miscellaneous support routines follow
 3999  */
 4000 
 4001 /*
 4002  * Map a set of physical memory pages into the kernel virtual
 4003  * address space. Return a pointer to where it is mapped. This
 4004  * routine is intended to be used for mapping device memory,
 4005  * NOT real memory.
 4006  */
 4007 void *
 4008 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 4009 {
 4010         vm_offset_t va, offset;
 4011         vm_size_t tmpsize;
 4012 
 4013         offset = pa & PAGE_MASK;
 4014         size = roundup(offset + size, PAGE_SIZE);
 4015         pa = pa & PG_FRAME;
 4016 
 4017         if (pa < KERNLOAD && pa + size <= KERNLOAD)
 4018                 va = KERNBASE + pa;
 4019         else
 4020                 va = kmem_alloc_nofault(kernel_map, size);
 4021         if (!va)
 4022                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 4023 
 4024         for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
 4025                 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
 4026         pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
 4027         pmap_invalidate_cache_range(va, va + size);
 4028         return ((void *)(va + offset));
 4029 }
 4030 
 4031 void *
 4032 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 4033 {
 4034 
 4035         return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));
 4036 }
 4037 
 4038 void *
 4039 pmap_mapbios(vm_paddr_t pa, vm_size_t size)
 4040 {
 4041 
 4042         return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK));
 4043 }
 4044 
 4045 void
 4046 pmap_unmapdev(vm_offset_t va, vm_size_t size)
 4047 {
 4048         vm_offset_t base, offset;
 4049 
 4050         if (va >= KERNBASE && va + size <= KERNBASE + KERNLOAD)
 4051                 return;
 4052         base = trunc_page(va);
 4053         offset = va & PAGE_MASK;
 4054         size = roundup(offset + size, PAGE_SIZE);
 4055         kmem_free(kernel_map, base, size);
 4056 }
 4057 
 4058 /*
 4059  * Sets the memory attribute for the specified page.
 4060  */
 4061 void
 4062 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
 4063 {
 4064 
 4065         m->md.pat_mode = ma;
 4066         if ((m->flags & PG_FICTITIOUS) != 0)
 4067                 return;
 4068 
 4069         /*
 4070          * If "m" is a normal page, flush it from the cache.
 4071          * See pmap_invalidate_cache_range().
 4072          *
 4073          * First, try to find an existing mapping of the page by sf
 4074          * buffer. sf_buf_invalidate_cache() modifies mapping and
 4075          * flushes the cache.
 4076          */    
 4077         if (sf_buf_invalidate_cache(m))
 4078                 return;
 4079 
 4080         /*
 4081          * If page is not mapped by sf buffer, but CPU does not
 4082          * support self snoop, map the page transient and do
 4083          * invalidation. In the worst case, whole cache is flushed by
 4084          * pmap_invalidate_cache_range().
 4085          */
 4086         if ((cpu_feature & CPUID_SS) == 0)
 4087                 pmap_flush_page(m);
 4088 }
 4089 
 4090 static void
 4091 pmap_flush_page(vm_page_t m)
 4092 {
 4093         struct sysmaps *sysmaps;
 4094         vm_offset_t sva, eva;
 4095 
 4096         if ((cpu_feature & CPUID_CLFSH) != 0) {
 4097                 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 4098                 mtx_lock(&sysmaps->lock);
 4099                 if (*sysmaps->CMAP2)
 4100                         panic("pmap_flush_page: CMAP2 busy");
 4101                 sched_pin();
 4102                 PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW |
 4103                     VM_PAGE_TO_MACH(m) | PG_A | PG_M |
 4104                     pmap_cache_bits(m->md.pat_mode, 0));
 4105                 invlcaddr(sysmaps->CADDR2);
 4106                 sva = (vm_offset_t)sysmaps->CADDR2;
 4107                 eva = sva + PAGE_SIZE;
 4108 
 4109                 /*
 4110                  * Use mfence despite the ordering implied by
 4111                  * mtx_{un,}lock() because clflush is not guaranteed
 4112                  * to be ordered by any other instruction.
 4113                  */
 4114                 mfence();
 4115                 for (; sva < eva; sva += cpu_clflush_line_size)
 4116                         clflush(sva);
 4117                 mfence();
 4118                 PT_SET_MA(sysmaps->CADDR2, 0);
 4119                 sched_unpin();
 4120                 mtx_unlock(&sysmaps->lock);
 4121         } else
 4122                 pmap_invalidate_cache();
 4123 }
 4124 
 4125 /*
 4126  * Changes the specified virtual address range's memory type to that given by
 4127  * the parameter "mode".  The specified virtual address range must be
 4128  * completely contained within either the kernel map.
 4129  *
 4130  * Returns zero if the change completed successfully, and either EINVAL or
 4131  * ENOMEM if the change failed.  Specifically, EINVAL is returned if some part
 4132  * of the virtual address range was not mapped, and ENOMEM is returned if
 4133  * there was insufficient memory available to complete the change.
 4134  */
 4135 int
 4136 pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
 4137 {
 4138         vm_offset_t base, offset, tmpva;
 4139         pt_entry_t *pte;
 4140         u_int opte, npte;
 4141         pd_entry_t *pde;
 4142         boolean_t changed;
 4143 
 4144         base = trunc_page(va);
 4145         offset = va & PAGE_MASK;
 4146         size = roundup(offset + size, PAGE_SIZE);
 4147 
 4148         /* Only supported on kernel virtual addresses. */
 4149         if (base <= VM_MAXUSER_ADDRESS)
 4150                 return (EINVAL);
 4151 
 4152         /* 4MB pages and pages that aren't mapped aren't supported. */
 4153         for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE) {
 4154                 pde = pmap_pde(kernel_pmap, tmpva);
 4155                 if (*pde & PG_PS)
 4156                         return (EINVAL);
 4157                 if ((*pde & PG_V) == 0)
 4158                         return (EINVAL);
 4159                 pte = vtopte(va);
 4160                 if ((*pte & PG_V) == 0)
 4161                         return (EINVAL);
 4162         }
 4163 
 4164         changed = FALSE;
 4165 
 4166         /*
 4167          * Ok, all the pages exist and are 4k, so run through them updating
 4168          * their cache mode.
 4169          */
 4170         for (tmpva = base; size > 0; ) {
 4171                 pte = vtopte(tmpva);
 4172 
 4173                 /*
 4174                  * The cache mode bits are all in the low 32-bits of the
 4175                  * PTE, so we can just spin on updating the low 32-bits.
 4176                  */
 4177                 do {
 4178                         opte = *(u_int *)pte;
 4179                         npte = opte & ~(PG_PTE_PAT | PG_NC_PCD | PG_NC_PWT);
 4180                         npte |= pmap_cache_bits(mode, 0);
 4181                         PT_SET_VA_MA(pte, npte, TRUE);
 4182                 } while (npte != opte && (*pte != npte));
 4183                 if (npte != opte)
 4184                         changed = TRUE;
 4185                 tmpva += PAGE_SIZE;
 4186                 size -= PAGE_SIZE;
 4187         }
 4188 
 4189         /*
 4190          * Flush CPU caches to make sure any data isn't cached that
 4191          * shouldn't be, etc.
 4192          */
 4193         if (changed) {
 4194                 pmap_invalidate_range(kernel_pmap, base, tmpva);
 4195                 pmap_invalidate_cache_range(base, tmpva);
 4196         }
 4197         return (0);
 4198 }
 4199 
 4200 /*
 4201  * perform the pmap work for mincore
 4202  */
 4203 int
 4204 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
 4205 {
 4206         pt_entry_t *ptep, pte;
 4207         vm_paddr_t pa;
 4208         int val;
 4209 
 4210         PMAP_LOCK(pmap);
 4211 retry:
 4212         ptep = pmap_pte(pmap, addr);
 4213         pte = (ptep != NULL) ? PT_GET(ptep) : 0;
 4214         pmap_pte_release(ptep);
 4215         val = 0;
 4216         if ((pte & PG_V) != 0) {
 4217                 val |= MINCORE_INCORE;
 4218                 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW))
 4219                         val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
 4220                 if ((pte & PG_A) != 0)
 4221                         val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
 4222         }
 4223         if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
 4224             (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
 4225             (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) {
 4226                 pa = pte & PG_FRAME;
 4227                 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */
 4228                 if (vm_page_pa_tryrelock(pmap, pa, locked_pa))
 4229                         goto retry;
 4230         } else
 4231                 PA_UNLOCK_COND(*locked_pa);
 4232         PMAP_UNLOCK(pmap);
 4233         return (val);
 4234 }
 4235 
 4236 void
 4237 pmap_activate(struct thread *td)
 4238 {
 4239         pmap_t  pmap, oldpmap;
 4240         u_int   cpuid;
 4241         u_int32_t  cr3;
 4242 
 4243         critical_enter();
 4244         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 4245         oldpmap = PCPU_GET(curpmap);
 4246         cpuid = PCPU_GET(cpuid);
 4247 #if defined(SMP)
 4248         CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
 4249         CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
 4250 #else
 4251         CPU_CLR(cpuid, &oldpmap->pm_active);
 4252         CPU_SET(cpuid, &pmap->pm_active);
 4253 #endif
 4254 #ifdef PAE
 4255         cr3 = vtophys(pmap->pm_pdpt);
 4256 #else
 4257         cr3 = vtophys(pmap->pm_pdir);
 4258 #endif
 4259         /*
 4260          * pmap_activate is for the current thread on the current cpu
 4261          */
 4262         td->td_pcb->pcb_cr3 = cr3;
 4263         PT_UPDATES_FLUSH();
 4264         load_cr3(cr3);
 4265         PCPU_SET(curpmap, pmap);
 4266         critical_exit();
 4267 }
 4268 
 4269 void
 4270 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
 4271 {
 4272 }
 4273 
 4274 /*
 4275  *      Increase the starting virtual address of the given mapping if a
 4276  *      different alignment might result in more superpage mappings.
 4277  */
 4278 void
 4279 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
 4280     vm_offset_t *addr, vm_size_t size)
 4281 {
 4282         vm_offset_t superpage_offset;
 4283 
 4284         if (size < NBPDR)
 4285                 return;
 4286         if (object != NULL && (object->flags & OBJ_COLORED) != 0)
 4287                 offset += ptoa(object->pg_color);
 4288         superpage_offset = offset & PDRMASK;
 4289         if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR ||
 4290             (*addr & PDRMASK) == superpage_offset)
 4291                 return;
 4292         if ((*addr & PDRMASK) < superpage_offset)
 4293                 *addr = (*addr & ~PDRMASK) + superpage_offset;
 4294         else
 4295                 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
 4296 }
 4297 
 4298 void
 4299 pmap_suspend()
 4300 {
 4301         pmap_t pmap;
 4302         int i, pdir, offset;
 4303         vm_paddr_t pdirma;
 4304         mmu_update_t mu[4];
 4305 
 4306         /*
 4307          * We need to remove the recursive mapping structure from all
 4308          * our pmaps so that Xen doesn't get confused when it restores
 4309          * the page tables. The recursive map lives at page directory
 4310          * index PTDPTDI. We assume that the suspend code has stopped
 4311          * the other vcpus (if any).
 4312          */
 4313         LIST_FOREACH(pmap, &allpmaps, pm_list) {
 4314                 for (i = 0; i < 4; i++) {
 4315                         /*
 4316                          * Figure out which page directory (L2) page
 4317                          * contains this bit of the recursive map and
 4318                          * the offset within that page of the map
 4319                          * entry
 4320                          */
 4321                         pdir = (PTDPTDI + i) / NPDEPG;
 4322                         offset = (PTDPTDI + i) % NPDEPG;
 4323                         pdirma = pmap->pm_pdpt[pdir] & PG_FRAME;
 4324                         mu[i].ptr = pdirma + offset * sizeof(pd_entry_t);
 4325                         mu[i].val = 0;
 4326                 }
 4327                 HYPERVISOR_mmu_update(mu, 4, NULL, DOMID_SELF);
 4328         }
 4329 }
 4330 
 4331 void
 4332 pmap_resume()
 4333 {
 4334         pmap_t pmap;
 4335         int i, pdir, offset;
 4336         vm_paddr_t pdirma;
 4337         mmu_update_t mu[4];
 4338 
 4339         /*
 4340          * Restore the recursive map that we removed on suspend.
 4341          */
 4342         LIST_FOREACH(pmap, &allpmaps, pm_list) {
 4343                 for (i = 0; i < 4; i++) {
 4344                         /*
 4345                          * Figure out which page directory (L2) page
 4346                          * contains this bit of the recursive map and
 4347                          * the offset within that page of the map
 4348                          * entry
 4349                          */
 4350                         pdir = (PTDPTDI + i) / NPDEPG;
 4351                         offset = (PTDPTDI + i) % NPDEPG;
 4352                         pdirma = pmap->pm_pdpt[pdir] & PG_FRAME;
 4353                         mu[i].ptr = pdirma + offset * sizeof(pd_entry_t);
 4354                         mu[i].val = (pmap->pm_pdpt[i] & PG_FRAME) | PG_V;
 4355                 }
 4356                 HYPERVISOR_mmu_update(mu, 4, NULL, DOMID_SELF);
 4357         }
 4358 }
 4359 
 4360 #if defined(PMAP_DEBUG)
 4361 pmap_pid_dump(int pid)
 4362 {
 4363         pmap_t pmap;
 4364         struct proc *p;
 4365         int npte = 0;
 4366         int index;
 4367 
 4368         sx_slock(&allproc_lock);
 4369         FOREACH_PROC_IN_SYSTEM(p) {
 4370                 if (p->p_pid != pid)
 4371                         continue;
 4372 
 4373                 if (p->p_vmspace) {
 4374                         int i,j;
 4375                         index = 0;
 4376                         pmap = vmspace_pmap(p->p_vmspace);
 4377                         for (i = 0; i < NPDEPTD; i++) {
 4378                                 pd_entry_t *pde;
 4379                                 pt_entry_t *pte;
 4380                                 vm_offset_t base = i << PDRSHIFT;
 4381                                 
 4382                                 pde = &pmap->pm_pdir[i];
 4383                                 if (pde && pmap_pde_v(pde)) {
 4384                                         for (j = 0; j < NPTEPG; j++) {
 4385                                                 vm_offset_t va = base + (j << PAGE_SHIFT);
 4386                                                 if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) {
 4387                                                         if (index) {
 4388                                                                 index = 0;
 4389                                                                 printf("\n");
 4390                                                         }
 4391                                                         sx_sunlock(&allproc_lock);
 4392                                                         return (npte);
 4393                                                 }
 4394                                                 pte = pmap_pte(pmap, va);
 4395                                                 if (pte && pmap_pte_v(pte)) {
 4396                                                         pt_entry_t pa;
 4397                                                         vm_page_t m;
 4398                                                         pa = PT_GET(pte);
 4399                                                         m = PHYS_TO_VM_PAGE(pa & PG_FRAME);
 4400                                                         printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x",
 4401                                                                 va, pa, m->hold_count, m->wire_count, m->flags);
 4402                                                         npte++;
 4403                                                         index++;
 4404                                                         if (index >= 2) {
 4405                                                                 index = 0;
 4406                                                                 printf("\n");
 4407                                                         } else {
 4408                                                                 printf(" ");
 4409                                                         }
 4410                                                 }
 4411                                         }
 4412                                 }
 4413                         }
 4414                 }
 4415         }
 4416         sx_sunlock(&allproc_lock);
 4417         return (npte);
 4418 }
 4419 #endif
 4420 
 4421 #if defined(DEBUG)
 4422 
 4423 static void     pads(pmap_t pm);
 4424 void            pmap_pvdump(vm_paddr_t pa);
 4425 
 4426 /* print address space of pmap*/
 4427 static void
 4428 pads(pmap_t pm)
 4429 {
 4430         int i, j;
 4431         vm_paddr_t va;
 4432         pt_entry_t *ptep;
 4433 
 4434         if (pm == kernel_pmap)
 4435                 return;
 4436         for (i = 0; i < NPDEPTD; i++)
 4437                 if (pm->pm_pdir[i])
 4438                         for (j = 0; j < NPTEPG; j++) {
 4439                                 va = (i << PDRSHIFT) + (j << PAGE_SHIFT);
 4440                                 if (pm == kernel_pmap && va < KERNBASE)
 4441                                         continue;
 4442                                 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS)
 4443                                         continue;
 4444                                 ptep = pmap_pte(pm, va);
 4445                                 if (pmap_pte_v(ptep))
 4446                                         printf("%x:%x ", va, *ptep);
 4447                         };
 4448 
 4449 }
 4450 
 4451 void
 4452 pmap_pvdump(vm_paddr_t pa)
 4453 {
 4454         pv_entry_t pv;
 4455         pmap_t pmap;
 4456         vm_page_t m;
 4457 
 4458         printf("pa %x", pa);
 4459         m = PHYS_TO_VM_PAGE(pa);
 4460         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 4461                 pmap = PV_PMAP(pv);
 4462                 printf(" -> pmap %p, va %x", (void *)pmap, pv->pv_va);
 4463                 pads(pmap);
 4464         }
 4465         printf(" ");
 4466 }
 4467 #endif

Cache object: cd8f739552b8372144b44e504bbfdd33


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.