The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/pmap.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1991 Regents of the University of California.
    3  * All rights reserved.
    4  * Copyright (c) 1994 John S. Dyson
    5  * All rights reserved.
    6  * Copyright (c) 1994 David Greenman
    7  * All rights reserved.
    8  * Copyright (c) 2005 Alan L. Cox <alc@cs.rice.edu>
    9  * All rights reserved.
   10  *
   11  * This code is derived from software contributed to Berkeley by
   12  * the Systems Programming Group of the University of Utah Computer
   13  * Science Department and William Jolitz of UUNET Technologies Inc.
   14  *
   15  * Redistribution and use in source and binary forms, with or without
   16  * modification, are permitted provided that the following conditions
   17  * are met:
   18  * 1. Redistributions of source code must retain the above copyright
   19  *    notice, this list of conditions and the following disclaimer.
   20  * 2. Redistributions in binary form must reproduce the above copyright
   21  *    notice, this list of conditions and the following disclaimer in the
   22  *    documentation and/or other materials provided with the distribution.
   23  * 3. All advertising materials mentioning features or use of this software
   24  *    must display the following acknowledgement:
   25  *      This product includes software developed by the University of
   26  *      California, Berkeley and its contributors.
   27  * 4. Neither the name of the University nor the names of its contributors
   28  *    may be used to endorse or promote products derived from this software
   29  *    without specific prior written permission.
   30  *
   31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   41  * SUCH DAMAGE.
   42  *
   43  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   44  */
   45 /*-
   46  * Copyright (c) 2003 Networks Associates Technology, Inc.
   47  * All rights reserved.
   48  *
   49  * This software was developed for the FreeBSD Project by Jake Burkholder,
   50  * Safeport Network Services, and Network Associates Laboratories, the
   51  * Security Research Division of Network Associates, Inc. under
   52  * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA
   53  * CHATS research program.
   54  *
   55  * Redistribution and use in source and binary forms, with or without
   56  * modification, are permitted provided that the following conditions
   57  * are met:
   58  * 1. Redistributions of source code must retain the above copyright
   59  *    notice, this list of conditions and the following disclaimer.
   60  * 2. Redistributions in binary form must reproduce the above copyright
   61  *    notice, this list of conditions and the following disclaimer in the
   62  *    documentation and/or other materials provided with the distribution.
   63  *
   64  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   65  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   66  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   67  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   68  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   69  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   70  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   71  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   72  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   73  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   74  * SUCH DAMAGE.
   75  */
   76 
   77 #include <sys/cdefs.h>
   78 __FBSDID("$FreeBSD: releng/6.2/sys/i386/i386/pmap.c 162707 2006-09-27 18:10:16Z alc $");
   79 
   80 /*
   81  *      Manages physical address maps.
   82  *
   83  *      In addition to hardware address maps, this
   84  *      module is called upon to provide software-use-only
   85  *      maps which may or may not be stored in the same
   86  *      form as hardware maps.  These pseudo-maps are
   87  *      used to store intermediate results from copy
   88  *      operations to and from address spaces.
   89  *
   90  *      Since the information managed by this module is
   91  *      also stored by the logical address mapping module,
   92  *      this module may throw away valid virtual-to-physical
   93  *      mappings at almost any time.  However, invalidations
   94  *      of virtual-to-physical mappings must be done as
   95  *      requested.
   96  *
   97  *      In order to cope with hardware architectures which
   98  *      make virtual-to-physical map invalidates expensive,
   99  *      this module may delay invalidate or reduced protection
  100  *      operations until such time as they are actually
  101  *      necessary.  This module is given full information as
  102  *      to which processors are currently using which maps,
  103  *      and to when physical maps must be made correct.
  104  */
  105 
  106 #include "opt_cpu.h"
  107 #include "opt_pmap.h"
  108 #include "opt_msgbuf.h"
  109 #include "opt_xbox.h"
  110 
  111 #include <sys/param.h>
  112 #include <sys/systm.h>
  113 #include <sys/kernel.h>
  114 #include <sys/lock.h>
  115 #include <sys/malloc.h>
  116 #include <sys/mman.h>
  117 #include <sys/msgbuf.h>
  118 #include <sys/mutex.h>
  119 #include <sys/proc.h>
  120 #include <sys/sx.h>
  121 #include <sys/vmmeter.h>
  122 #include <sys/sched.h>
  123 #include <sys/sysctl.h>
  124 #ifdef SMP
  125 #include <sys/smp.h>
  126 #endif
  127 
  128 #ifdef XBOX
  129 #include <machine/xbox.h>
  130 #endif
  131 
  132 #include <vm/vm.h>
  133 #include <vm/vm_param.h>
  134 #include <vm/vm_kern.h>
  135 #include <vm/vm_page.h>
  136 #include <vm/vm_map.h>
  137 #include <vm/vm_object.h>
  138 #include <vm/vm_extern.h>
  139 #include <vm/vm_pageout.h>
  140 #include <vm/vm_pager.h>
  141 #include <vm/uma.h>
  142 
  143 #include <machine/cpu.h>
  144 #include <machine/cputypes.h>
  145 #include <machine/md_var.h>
  146 #include <machine/pcb.h>
  147 #include <machine/specialreg.h>
  148 #ifdef SMP
  149 #include <machine/smp.h>
  150 #endif
  151 
  152 #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
  153 #define CPU_ENABLE_SSE
  154 #endif
  155 
  156 #ifndef PMAP_SHPGPERPROC
  157 #define PMAP_SHPGPERPROC 200
  158 #endif
  159 
  160 #if defined(DIAGNOSTIC)
  161 #define PMAP_DIAGNOSTIC
  162 #endif
  163 
  164 #if !defined(PMAP_DIAGNOSTIC)
  165 #define PMAP_INLINE __inline
  166 #else
  167 #define PMAP_INLINE
  168 #endif
  169 
  170 /*
  171  * Get PDEs and PTEs for user/kernel address space
  172  */
  173 #define pmap_pde(m, v)  (&((m)->pm_pdir[(vm_offset_t)(v) >> PDRSHIFT]))
  174 #define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDRSHIFT])
  175 
  176 #define pmap_pde_v(pte)         ((*(int *)pte & PG_V) != 0)
  177 #define pmap_pte_w(pte)         ((*(int *)pte & PG_W) != 0)
  178 #define pmap_pte_m(pte)         ((*(int *)pte & PG_M) != 0)
  179 #define pmap_pte_u(pte)         ((*(int *)pte & PG_A) != 0)
  180 #define pmap_pte_v(pte)         ((*(int *)pte & PG_V) != 0)
  181 
  182 #define pmap_pte_set_w(pte, v)  ((v) ? atomic_set_int((u_int *)(pte), PG_W) : \
  183     atomic_clear_int((u_int *)(pte), PG_W))
  184 #define pmap_pte_set_prot(pte, v) ((*(int *)pte &= ~PG_PROT), (*(int *)pte |= (v)))
  185 
  186 struct pmap kernel_pmap_store;
  187 LIST_HEAD(pmaplist, pmap);
  188 static struct pmaplist allpmaps;
  189 static struct mtx allpmaps_lock;
  190 
  191 vm_paddr_t avail_end;   /* PA of last available physical page */
  192 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  193 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  194 int pgeflag = 0;                /* PG_G or-in */
  195 int pseflag = 0;                /* PG_PS or-in */
  196 
  197 static int nkpt;
  198 vm_offset_t kernel_vm_end;
  199 extern u_int32_t KERNend;
  200 
  201 #ifdef PAE
  202 static uma_zone_t pdptzone;
  203 #endif
  204 
  205 /*
  206  * Data for the pv entry allocation mechanism
  207  */
  208 static uma_zone_t pvzone;
  209 static struct vm_object pvzone_obj;
  210 static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0;
  211 int pmap_pagedaemon_waken;
  212 
  213 /*
  214  * All those kernel PT submaps that BSD is so fond of
  215  */
  216 struct sysmaps {
  217         struct  mtx lock;
  218         pt_entry_t *CMAP1;
  219         pt_entry_t *CMAP2;
  220         caddr_t CADDR1;
  221         caddr_t CADDR2;
  222 };
  223 static struct sysmaps sysmaps_pcpu[MAXCPU];
  224 pt_entry_t *CMAP1 = 0;
  225 static pt_entry_t *CMAP3;
  226 caddr_t CADDR1 = 0, ptvmmap = 0;
  227 static caddr_t CADDR3;
  228 struct msgbuf *msgbufp = 0;
  229 
  230 /*
  231  * Crashdump maps.
  232  */
  233 static caddr_t crashdumpmap;
  234 
  235 #ifdef SMP
  236 extern pt_entry_t *SMPpt;
  237 #endif
  238 static pt_entry_t *PMAP1 = 0, *PMAP2;
  239 static pt_entry_t *PADDR1 = 0, *PADDR2;
  240 #ifdef SMP
  241 static int PMAP1cpu;
  242 static int PMAP1changedcpu;
  243 SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, 
  244            &PMAP1changedcpu, 0,
  245            "Number of times pmap_pte_quick changed CPU with same PMAP1");
  246 #endif
  247 static int PMAP1changed;
  248 SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, 
  249            &PMAP1changed, 0,
  250            "Number of times pmap_pte_quick changed PMAP1");
  251 static int PMAP1unchanged;
  252 SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, 
  253            &PMAP1unchanged, 0,
  254            "Number of times pmap_pte_quick didn't change PMAP1");
  255 static struct mtx PMAP2mutex;
  256 
  257 static PMAP_INLINE void free_pv_entry(pv_entry_t pv);
  258 static pv_entry_t get_pv_entry(void);
  259 static void     pmap_clear_ptes(vm_page_t m, int bit);
  260 
  261 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva);
  262 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va);
  263 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
  264                                         vm_offset_t va);
  265 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
  266 
  267 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
  268 
  269 static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags);
  270 static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m);
  271 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
  272 static void pmap_pte_release(pt_entry_t *pte);
  273 static int pmap_unuse_pt(pmap_t, vm_offset_t);
  274 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
  275 #ifdef PAE
  276 static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
  277 #endif
  278 
  279 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
  280 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
  281 
  282 /*
  283  * Move the kernel virtual free pointer to the next
  284  * 4MB.  This is used to help improve performance
  285  * by using a large (4MB) page for much of the kernel
  286  * (.text, .data, .bss)
  287  */
  288 static vm_offset_t
  289 pmap_kmem_choose(vm_offset_t addr)
  290 {
  291         vm_offset_t newaddr = addr;
  292 
  293 #ifndef DISABLE_PSE
  294         if (cpu_feature & CPUID_PSE)
  295                 newaddr = (addr + PDRMASK) & ~PDRMASK;
  296 #endif
  297         return newaddr;
  298 }
  299 
  300 /*
  301  *      Bootstrap the system enough to run with virtual memory.
  302  *
  303  *      On the i386 this is called after mapping has already been enabled
  304  *      and just syncs the pmap module with what has already been done.
  305  *      [We can't call it easily with mapping off since the kernel is not
  306  *      mapped with PA == VA, hence we would have to relocate every address
  307  *      from the linked base (virtual) address "KERNBASE" to the actual
  308  *      (physical) address starting relative to 0]
  309  */
  310 void
  311 pmap_bootstrap(firstaddr, loadaddr)
  312         vm_paddr_t firstaddr;
  313         vm_paddr_t loadaddr;
  314 {
  315         vm_offset_t va;
  316         pt_entry_t *pte, *unused;
  317         struct sysmaps *sysmaps;
  318         int i;
  319 
  320         /*
  321          * XXX The calculation of virtual_avail is wrong. It's NKPT*PAGE_SIZE too
  322          * large. It should instead be correctly calculated in locore.s and
  323          * not based on 'first' (which is a physical address, not a virtual
  324          * address, for the start of unused physical memory). The kernel
  325          * page tables are NOT double mapped and thus should not be included
  326          * in this calculation.
  327          */
  328         virtual_avail = (vm_offset_t) KERNBASE + firstaddr;
  329         virtual_avail = pmap_kmem_choose(virtual_avail);
  330 
  331         virtual_end = VM_MAX_KERNEL_ADDRESS;
  332 
  333         /*
  334          * Initialize the kernel pmap (which is statically allocated).
  335          */
  336         PMAP_LOCK_INIT(kernel_pmap);
  337         kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD);
  338 #ifdef PAE
  339         kernel_pmap->pm_pdpt = (pdpt_entry_t *) (KERNBASE + (u_int)IdlePDPT);
  340 #endif
  341         kernel_pmap->pm_active = -1;    /* don't allow deactivation */
  342         TAILQ_INIT(&kernel_pmap->pm_pvlist);
  343         LIST_INIT(&allpmaps);
  344         mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN);
  345         mtx_lock_spin(&allpmaps_lock);
  346         LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list);
  347         mtx_unlock_spin(&allpmaps_lock);
  348         nkpt = NKPT;
  349 
  350         /*
  351          * Reserve some special page table entries/VA space for temporary
  352          * mapping of pages.
  353          */
  354 #define SYSMAP(c, p, v, n)      \
  355         v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n);
  356 
  357         va = virtual_avail;
  358         pte = vtopte(va);
  359 
  360         /*
  361          * CMAP1/CMAP2 are used for zeroing and copying pages.
  362          * CMAP3 is used for the idle process page zeroing.
  363          */
  364         for (i = 0; i < MAXCPU; i++) {
  365                 sysmaps = &sysmaps_pcpu[i];
  366                 mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF);
  367                 SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1)
  368                 SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1)
  369         }
  370         SYSMAP(caddr_t, CMAP1, CADDR1, 1)
  371         SYSMAP(caddr_t, CMAP3, CADDR3, 1)
  372         *CMAP3 = 0;
  373 
  374         /*
  375          * Crashdump maps.
  376          */
  377         SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS)
  378 
  379         /*
  380          * ptvmmap is used for reading arbitrary physical pages via /dev/mem.
  381          */
  382         SYSMAP(caddr_t, unused, ptvmmap, 1)
  383 
  384         /*
  385          * msgbufp is used to map the system message buffer.
  386          */
  387         SYSMAP(struct msgbuf *, unused, msgbufp, atop(round_page(MSGBUF_SIZE)))
  388 
  389         /*
  390          * ptemap is used for pmap_pte_quick
  391          */
  392         SYSMAP(pt_entry_t *, PMAP1, PADDR1, 1);
  393         SYSMAP(pt_entry_t *, PMAP2, PADDR2, 1);
  394 
  395         mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF);
  396 
  397         virtual_avail = va;
  398 
  399         *CMAP1 = 0;
  400 
  401 #ifdef XBOX
  402         /* FIXME: This is gross, but needed for the XBOX. Since we are in such
  403          * an early stadium, we cannot yet neatly map video memory ... :-(
  404          * Better fixes are very welcome!
  405          */
  406         if (!arch_i386_is_xbox)
  407 #endif
  408         for (i = 0; i < NKPT; i++)
  409                 PTD[i] = 0;
  410 
  411         /* Turn on PG_G on kernel page(s) */
  412         pmap_set_pg();
  413 }
  414 
  415 /*
  416  * Set PG_G on kernel pages.  Only the BSP calls this when SMP is turned on.
  417  */
  418 void
  419 pmap_set_pg(void)
  420 {
  421         pd_entry_t pdir;
  422         pt_entry_t *pte;
  423         vm_offset_t va, endva;
  424         int i; 
  425 
  426         if (pgeflag == 0)
  427                 return;
  428 
  429         i = KERNLOAD/NBPDR;
  430         endva = KERNBASE + KERNend;
  431 
  432         if (pseflag) {
  433                 va = KERNBASE + KERNLOAD;
  434                 while (va  < endva) {
  435                         pdir = kernel_pmap->pm_pdir[KPTDI+i];
  436                         pdir |= pgeflag;
  437                         kernel_pmap->pm_pdir[KPTDI+i] = PTD[KPTDI+i] = pdir;
  438                         invltlb();      /* Play it safe, invltlb() every time */
  439                         i++;
  440                         va += NBPDR;
  441                 }
  442         } else {
  443                 va = (vm_offset_t)btext;
  444                 while (va < endva) {
  445                         pte = vtopte(va);
  446                         if (*pte)
  447                                 *pte |= pgeflag;
  448                         invltlb();      /* Play it safe, invltlb() every time */
  449                         va += PAGE_SIZE;
  450                 }
  451         }
  452 }
  453 
  454 /*
  455  * Initialize a vm_page's machine-dependent fields.
  456  */
  457 void
  458 pmap_page_init(vm_page_t m)
  459 {
  460 
  461         TAILQ_INIT(&m->md.pv_list);
  462         m->md.pv_list_count = 0;
  463 }
  464 
  465 #ifdef PAE
  466 
  467 static MALLOC_DEFINE(M_PMAPPDPT, "pmap", "pmap pdpt");
  468 
  469 static void *
  470 pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
  471 {
  472         *flags = UMA_SLAB_PRIV;
  473         return (contigmalloc(PAGE_SIZE, M_PMAPPDPT, 0, 0x0ULL, 0xffffffffULL,
  474             1, 0));
  475 }
  476 #endif
  477 
  478 /*
  479  *      Initialize the pmap module.
  480  *      Called by vm_init, to initialize any structures that the pmap
  481  *      system needs to map virtual memory.
  482  */
  483 void
  484 pmap_init(void)
  485 {
  486         int shpgperproc = PMAP_SHPGPERPROC;
  487 
  488         /*
  489          * Initialize the address space (zone) for the pv entries.  Set a
  490          * high water mark so that the system can recover from excessive
  491          * numbers of pv entries.
  492          */
  493         pvzone = uma_zcreate("PV ENTRY", sizeof(struct pv_entry), NULL, NULL, 
  494             NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
  495         TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc);
  496         pv_entry_max = shpgperproc * maxproc + cnt.v_page_count;
  497         TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max);
  498         pv_entry_high_water = 9 * (pv_entry_max / 10);
  499         uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max);
  500 
  501 #ifdef PAE
  502         pdptzone = uma_zcreate("PDPT", NPGPTD * sizeof(pdpt_entry_t), NULL,
  503             NULL, NULL, NULL, (NPGPTD * sizeof(pdpt_entry_t)) - 1,
  504             UMA_ZONE_VM | UMA_ZONE_NOFREE);
  505         uma_zone_set_allocf(pdptzone, pmap_pdpt_allocf);
  506 #endif
  507 }
  508 
  509 void
  510 pmap_init2()
  511 {
  512 }
  513 
  514 
  515 /***************************************************
  516  * Low level helper routines.....
  517  ***************************************************/
  518 
  519 #if defined(PMAP_DIAGNOSTIC)
  520 
  521 /*
  522  * This code checks for non-writeable/modified pages.
  523  * This should be an invalid condition.
  524  */
  525 static int
  526 pmap_nw_modified(pt_entry_t ptea)
  527 {
  528         int pte;
  529 
  530         pte = (int) ptea;
  531 
  532         if ((pte & (PG_M|PG_RW)) == PG_M)
  533                 return 1;
  534         else
  535                 return 0;
  536 }
  537 #endif
  538 
  539 
  540 /*
  541  * this routine defines the region(s) of memory that should
  542  * not be tested for the modified bit.
  543  */
  544 static PMAP_INLINE int
  545 pmap_track_modified(vm_offset_t va)
  546 {
  547         if ((va < kmi.clean_sva) || (va >= kmi.clean_eva)) 
  548                 return 1;
  549         else
  550                 return 0;
  551 }
  552 
  553 #ifdef SMP
  554 /*
  555  * For SMP, these functions have to use the IPI mechanism for coherence.
  556  */
  557 void
  558 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  559 {
  560         u_int cpumask;
  561         u_int other_cpus;
  562 
  563         if (smp_started) {
  564                 if (!(read_eflags() & PSL_I))
  565                         panic("%s: interrupts disabled", __func__);
  566                 mtx_lock_spin(&smp_ipi_mtx);
  567         } else
  568                 critical_enter();
  569         /*
  570          * We need to disable interrupt preemption but MUST NOT have
  571          * interrupts disabled here.
  572          * XXX we may need to hold schedlock to get a coherent pm_active
  573          * XXX critical sections disable interrupts again
  574          */
  575         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  576                 invlpg(va);
  577                 smp_invlpg(va);
  578         } else {
  579                 cpumask = PCPU_GET(cpumask);
  580                 other_cpus = PCPU_GET(other_cpus);
  581                 if (pmap->pm_active & cpumask)
  582                         invlpg(va);
  583                 if (pmap->pm_active & other_cpus)
  584                         smp_masked_invlpg(pmap->pm_active & other_cpus, va);
  585         }
  586         if (smp_started)
  587                 mtx_unlock_spin(&smp_ipi_mtx);
  588         else
  589                 critical_exit();
  590 }
  591 
  592 void
  593 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  594 {
  595         u_int cpumask;
  596         u_int other_cpus;
  597         vm_offset_t addr;
  598 
  599         if (smp_started) {
  600                 if (!(read_eflags() & PSL_I))
  601                         panic("%s: interrupts disabled", __func__);
  602                 mtx_lock_spin(&smp_ipi_mtx);
  603         } else
  604                 critical_enter();
  605         /*
  606          * We need to disable interrupt preemption but MUST NOT have
  607          * interrupts disabled here.
  608          * XXX we may need to hold schedlock to get a coherent pm_active
  609          * XXX critical sections disable interrupts again
  610          */
  611         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  612                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  613                         invlpg(addr);
  614                 smp_invlpg_range(sva, eva);
  615         } else {
  616                 cpumask = PCPU_GET(cpumask);
  617                 other_cpus = PCPU_GET(other_cpus);
  618                 if (pmap->pm_active & cpumask)
  619                         for (addr = sva; addr < eva; addr += PAGE_SIZE)
  620                                 invlpg(addr);
  621                 if (pmap->pm_active & other_cpus)
  622                         smp_masked_invlpg_range(pmap->pm_active & other_cpus,
  623                             sva, eva);
  624         }
  625         if (smp_started)
  626                 mtx_unlock_spin(&smp_ipi_mtx);
  627         else
  628                 critical_exit();
  629 }
  630 
  631 void
  632 pmap_invalidate_all(pmap_t pmap)
  633 {
  634         u_int cpumask;
  635         u_int other_cpus;
  636 
  637         if (smp_started) {
  638                 if (!(read_eflags() & PSL_I))
  639                         panic("%s: interrupts disabled", __func__);
  640                 mtx_lock_spin(&smp_ipi_mtx);
  641         } else
  642                 critical_enter();
  643         /*
  644          * We need to disable interrupt preemption but MUST NOT have
  645          * interrupts disabled here.
  646          * XXX we may need to hold schedlock to get a coherent pm_active
  647          * XXX critical sections disable interrupts again
  648          */
  649         if (pmap == kernel_pmap || pmap->pm_active == all_cpus) {
  650                 invltlb();
  651                 smp_invltlb();
  652         } else {
  653                 cpumask = PCPU_GET(cpumask);
  654                 other_cpus = PCPU_GET(other_cpus);
  655                 if (pmap->pm_active & cpumask)
  656                         invltlb();
  657                 if (pmap->pm_active & other_cpus)
  658                         smp_masked_invltlb(pmap->pm_active & other_cpus);
  659         }
  660         if (smp_started)
  661                 mtx_unlock_spin(&smp_ipi_mtx);
  662         else
  663                 critical_exit();
  664 }
  665 #else /* !SMP */
  666 /*
  667  * Normal, non-SMP, 486+ invalidation functions.
  668  * We inline these within pmap.c for speed.
  669  */
  670 PMAP_INLINE void
  671 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  672 {
  673 
  674         if (pmap == kernel_pmap || pmap->pm_active)
  675                 invlpg(va);
  676 }
  677 
  678 PMAP_INLINE void
  679 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  680 {
  681         vm_offset_t addr;
  682 
  683         if (pmap == kernel_pmap || pmap->pm_active)
  684                 for (addr = sva; addr < eva; addr += PAGE_SIZE)
  685                         invlpg(addr);
  686 }
  687 
  688 PMAP_INLINE void
  689 pmap_invalidate_all(pmap_t pmap)
  690 {
  691 
  692         if (pmap == kernel_pmap || pmap->pm_active)
  693                 invltlb();
  694 }
  695 #endif /* !SMP */
  696 
  697 /*
  698  * Are we current address space or kernel?  N.B. We return FALSE when
  699  * a pmap's page table is in use because a kernel thread is borrowing
  700  * it.  The borrowed page table can change spontaneously, making any
  701  * dependence on its continued use subject to a race condition.
  702  */
  703 static __inline int
  704 pmap_is_current(pmap_t pmap)
  705 {
  706 
  707         return (pmap == kernel_pmap ||
  708                 (pmap == vmspace_pmap(curthread->td_proc->p_vmspace) &&
  709             (pmap->pm_pdir[PTDPTDI] & PG_FRAME) == (PTDpde[0] & PG_FRAME)));
  710 }
  711 
  712 /*
  713  * If the given pmap is not the current or kernel pmap, the returned pte must
  714  * be released by passing it to pmap_pte_release().
  715  */
  716 pt_entry_t *
  717 pmap_pte(pmap_t pmap, vm_offset_t va)
  718 {
  719         pd_entry_t newpf;
  720         pd_entry_t *pde;
  721 
  722         pde = pmap_pde(pmap, va);
  723         if (*pde & PG_PS)
  724                 return (pde);
  725         if (*pde != 0) {
  726                 /* are we current address space or kernel? */
  727                 if (pmap_is_current(pmap))
  728                         return (vtopte(va));
  729                 mtx_lock(&PMAP2mutex);
  730                 newpf = *pde & PG_FRAME;
  731                 if ((*PMAP2 & PG_FRAME) != newpf) {
  732                         *PMAP2 = newpf | PG_RW | PG_V | PG_A | PG_M;
  733                         pmap_invalidate_page(kernel_pmap, (vm_offset_t)PADDR2);
  734                 }
  735                 return (PADDR2 + (i386_btop(va) & (NPTEPG - 1)));
  736         }
  737         return (0);
  738 }
  739 
  740 /*
  741  * Releases a pte that was obtained from pmap_pte().  Be prepared for the pte
  742  * being NULL.
  743  */
  744 static __inline void
  745 pmap_pte_release(pt_entry_t *pte)
  746 {
  747 
  748         if ((pt_entry_t *)((vm_offset_t)pte & ~PAGE_MASK) == PADDR2)
  749                 mtx_unlock(&PMAP2mutex);
  750 }
  751 
  752 static __inline void
  753 invlcaddr(void *caddr)
  754 {
  755 
  756         invlpg((u_int)caddr);
  757 }
  758 
  759 /*
  760  * Super fast pmap_pte routine best used when scanning
  761  * the pv lists.  This eliminates many coarse-grained
  762  * invltlb calls.  Note that many of the pv list
  763  * scans are across different pmaps.  It is very wasteful
  764  * to do an entire invltlb for checking a single mapping.
  765  *
  766  * If the given pmap is not the current pmap, vm_page_queue_mtx
  767  * must be held and curthread pinned to a CPU.
  768  */
  769 static pt_entry_t *
  770 pmap_pte_quick(pmap_t pmap, vm_offset_t va)
  771 {
  772         pd_entry_t newpf;
  773         pd_entry_t *pde;
  774 
  775         pde = pmap_pde(pmap, va);
  776         if (*pde & PG_PS)
  777                 return (pde);
  778         if (*pde != 0) {
  779                 /* are we current address space or kernel? */
  780                 if (pmap_is_current(pmap))
  781                         return (vtopte(va));
  782                 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
  783                 KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
  784                 newpf = *pde & PG_FRAME;
  785                 if ((*PMAP1 & PG_FRAME) != newpf) {
  786                         *PMAP1 = newpf | PG_RW | PG_V | PG_A | PG_M;
  787 #ifdef SMP
  788                         PMAP1cpu = PCPU_GET(cpuid);
  789 #endif
  790                         invlcaddr(PADDR1);
  791                         PMAP1changed++;
  792                 } else
  793 #ifdef SMP
  794                 if (PMAP1cpu != PCPU_GET(cpuid)) {
  795                         PMAP1cpu = PCPU_GET(cpuid);
  796                         invlcaddr(PADDR1);
  797                         PMAP1changedcpu++;
  798                 } else
  799 #endif
  800                         PMAP1unchanged++;
  801                 return (PADDR1 + (i386_btop(va) & (NPTEPG - 1)));
  802         }
  803         return (0);
  804 }
  805 
  806 /*
  807  *      Routine:        pmap_extract
  808  *      Function:
  809  *              Extract the physical page address associated
  810  *              with the given map/virtual_address pair.
  811  */
  812 vm_paddr_t 
  813 pmap_extract(pmap_t pmap, vm_offset_t va)
  814 {
  815         vm_paddr_t rtval;
  816         pt_entry_t *pte;
  817         pd_entry_t pde;
  818 
  819         rtval = 0;
  820         PMAP_LOCK(pmap);
  821         pde = pmap->pm_pdir[va >> PDRSHIFT];
  822         if (pde != 0) {
  823                 if ((pde & PG_PS) != 0) {
  824                         rtval = (pde & ~PDRMASK) | (va & PDRMASK);
  825                         PMAP_UNLOCK(pmap);
  826                         return rtval;
  827                 }
  828                 pte = pmap_pte(pmap, va);
  829                 rtval = (*pte & PG_FRAME) | (va & PAGE_MASK);
  830                 pmap_pte_release(pte);
  831         }
  832         PMAP_UNLOCK(pmap);
  833         return (rtval);
  834 }
  835 
  836 /*
  837  *      Routine:        pmap_extract_and_hold
  838  *      Function:
  839  *              Atomically extract and hold the physical page
  840  *              with the given pmap and virtual address pair
  841  *              if that mapping permits the given protection.
  842  */
  843 vm_page_t
  844 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
  845 {
  846         pd_entry_t pde;
  847         pt_entry_t pte;
  848         vm_page_t m;
  849 
  850         m = NULL;
  851         vm_page_lock_queues();
  852         PMAP_LOCK(pmap);
  853         pde = *pmap_pde(pmap, va);
  854         if (pde != 0) {
  855                 if (pde & PG_PS) {
  856                         if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
  857                                 m = PHYS_TO_VM_PAGE((pde & ~PDRMASK) |
  858                                     (va & PDRMASK));
  859                                 vm_page_hold(m);
  860                         }
  861                 } else {
  862                         sched_pin();
  863                         pte = *pmap_pte_quick(pmap, va);
  864                         if (pte != 0 &&
  865                             ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) {
  866                                 m = PHYS_TO_VM_PAGE(pte & PG_FRAME);
  867                                 vm_page_hold(m);
  868                         }
  869                         sched_unpin();
  870                 }
  871         }
  872         vm_page_unlock_queues();
  873         PMAP_UNLOCK(pmap);
  874         return (m);
  875 }
  876 
  877 /***************************************************
  878  * Low level mapping routines.....
  879  ***************************************************/
  880 
  881 /*
  882  * Add a wired page to the kva.
  883  * Note: not SMP coherent.
  884  */
  885 PMAP_INLINE void 
  886 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
  887 {
  888         pt_entry_t *pte;
  889 
  890         pte = vtopte(va);
  891         pte_store(pte, pa | PG_RW | PG_V | pgeflag);
  892 }
  893 
  894 /*
  895  * Remove a page from the kernel pagetables.
  896  * Note: not SMP coherent.
  897  */
  898 PMAP_INLINE void
  899 pmap_kremove(vm_offset_t va)
  900 {
  901         pt_entry_t *pte;
  902 
  903         pte = vtopte(va);
  904         pte_clear(pte);
  905 }
  906 
  907 /*
  908  *      Used to map a range of physical addresses into kernel
  909  *      virtual address space.
  910  *
  911  *      The value passed in '*virt' is a suggested virtual address for
  912  *      the mapping. Architectures which can support a direct-mapped
  913  *      physical to virtual region can return the appropriate address
  914  *      within that region, leaving '*virt' unchanged. Other
  915  *      architectures should map the pages starting at '*virt' and
  916  *      update '*virt' with the first usable address after the mapped
  917  *      region.
  918  */
  919 vm_offset_t
  920 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
  921 {
  922         vm_offset_t va, sva;
  923 
  924         va = sva = *virt;
  925         while (start < end) {
  926                 pmap_kenter(va, start);
  927                 va += PAGE_SIZE;
  928                 start += PAGE_SIZE;
  929         }
  930         pmap_invalidate_range(kernel_pmap, sva, va);
  931         *virt = va;
  932         return (sva);
  933 }
  934 
  935 
  936 /*
  937  * Add a list of wired pages to the kva
  938  * this routine is only used for temporary
  939  * kernel mappings that do not need to have
  940  * page modification or references recorded.
  941  * Note that old mappings are simply written
  942  * over.  The page *must* be wired.
  943  * Note: SMP coherent.  Uses a ranged shootdown IPI.
  944  */
  945 void
  946 pmap_qenter(vm_offset_t sva, vm_page_t *m, int count)
  947 {
  948         vm_offset_t va;
  949 
  950         va = sva;
  951         while (count-- > 0) {
  952                 pmap_kenter(va, VM_PAGE_TO_PHYS(*m));
  953                 va += PAGE_SIZE;
  954                 m++;
  955         }
  956         pmap_invalidate_range(kernel_pmap, sva, va);
  957 }
  958 
  959 /*
  960  * This routine tears out page mappings from the
  961  * kernel -- it is meant only for temporary mappings.
  962  * Note: SMP coherent.  Uses a ranged shootdown IPI.
  963  */
  964 void
  965 pmap_qremove(vm_offset_t sva, int count)
  966 {
  967         vm_offset_t va;
  968 
  969         va = sva;
  970         while (count-- > 0) {
  971                 pmap_kremove(va);
  972                 va += PAGE_SIZE;
  973         }
  974         pmap_invalidate_range(kernel_pmap, sva, va);
  975 }
  976 
  977 /***************************************************
  978  * Page table page management routines.....
  979  ***************************************************/
  980 
  981 /*
  982  * This routine unholds page table pages, and if the hold count
  983  * drops to zero, then it decrements the wire count.
  984  */
  985 static PMAP_INLINE int
  986 pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
  987 {
  988 
  989         --m->wire_count;
  990         if (m->wire_count == 0)
  991                 return _pmap_unwire_pte_hold(pmap, m);
  992         else
  993                 return 0;
  994 }
  995 
  996 static int 
  997 _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
  998 {
  999         vm_offset_t pteva;
 1000 
 1001         /*
 1002          * unmap the page table page
 1003          */
 1004         pmap->pm_pdir[m->pindex] = 0;
 1005         --pmap->pm_stats.resident_count;
 1006 
 1007         /*
 1008          * Do an invltlb to make the invalidated mapping
 1009          * take effect immediately.
 1010          */
 1011         pteva = VM_MAXUSER_ADDRESS + i386_ptob(m->pindex);
 1012         pmap_invalidate_page(pmap, pteva);
 1013 
 1014         vm_page_free_zero(m);
 1015         atomic_subtract_int(&cnt.v_wire_count, 1);
 1016         return 1;
 1017 }
 1018 
 1019 /*
 1020  * After removing a page table entry, this routine is used to
 1021  * conditionally free the page, and manage the hold/wire counts.
 1022  */
 1023 static int
 1024 pmap_unuse_pt(pmap_t pmap, vm_offset_t va)
 1025 {
 1026         pd_entry_t ptepde;
 1027         vm_page_t mpte;
 1028 
 1029         if (va >= VM_MAXUSER_ADDRESS)
 1030                 return 0;
 1031         ptepde = *pmap_pde(pmap, va);
 1032         mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
 1033         return pmap_unwire_pte_hold(pmap, mpte);
 1034 }
 1035 
 1036 void
 1037 pmap_pinit0(pmap)
 1038         struct pmap *pmap;
 1039 {
 1040 
 1041         PMAP_LOCK_INIT(pmap);
 1042         pmap->pm_pdir = (pd_entry_t *)(KERNBASE + (vm_offset_t)IdlePTD);
 1043 #ifdef PAE
 1044         pmap->pm_pdpt = (pdpt_entry_t *)(KERNBASE + (vm_offset_t)IdlePDPT);
 1045 #endif
 1046         pmap->pm_active = 0;
 1047         PCPU_SET(curpmap, pmap);
 1048         TAILQ_INIT(&pmap->pm_pvlist);
 1049         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1050         mtx_lock_spin(&allpmaps_lock);
 1051         LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);
 1052         mtx_unlock_spin(&allpmaps_lock);
 1053 }
 1054 
 1055 /*
 1056  * Initialize a preallocated and zeroed pmap structure,
 1057  * such as one in a vmspace structure.
 1058  */
 1059 void
 1060 pmap_pinit(pmap)
 1061         register struct pmap *pmap;
 1062 {
 1063         vm_page_t m, ptdpg[NPGPTD];
 1064         vm_paddr_t pa;
 1065         static int color;
 1066         int i;
 1067 
 1068         PMAP_LOCK_INIT(pmap);
 1069 
 1070         /*
 1071          * No need to allocate page table space yet but we do need a valid
 1072          * page directory table.
 1073          */
 1074         if (pmap->pm_pdir == NULL) {
 1075                 pmap->pm_pdir = (pd_entry_t *)kmem_alloc_nofault(kernel_map,
 1076                     NBPTD);
 1077 #ifdef PAE
 1078                 pmap->pm_pdpt = uma_zalloc(pdptzone, M_WAITOK | M_ZERO);
 1079                 KASSERT(((vm_offset_t)pmap->pm_pdpt &
 1080                     ((NPGPTD * sizeof(pdpt_entry_t)) - 1)) == 0,
 1081                     ("pmap_pinit: pdpt misaligned"));
 1082                 KASSERT(pmap_kextract((vm_offset_t)pmap->pm_pdpt) < (4ULL<<30),
 1083                     ("pmap_pinit: pdpt above 4g"));
 1084 #endif
 1085         }
 1086 
 1087         /*
 1088          * allocate the page directory page(s)
 1089          */
 1090         for (i = 0; i < NPGPTD;) {
 1091                 m = vm_page_alloc(NULL, color++,
 1092                     VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
 1093                     VM_ALLOC_ZERO);
 1094                 if (m == NULL)
 1095                         VM_WAIT;
 1096                 else {
 1097                         ptdpg[i++] = m;
 1098                 }
 1099         }
 1100 
 1101         pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD);
 1102 
 1103         for (i = 0; i < NPGPTD; i++) {
 1104                 if ((ptdpg[i]->flags & PG_ZERO) == 0)
 1105                         bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE);
 1106         }
 1107 
 1108         mtx_lock_spin(&allpmaps_lock);
 1109         LIST_INSERT_HEAD(&allpmaps, pmap, pm_list);
 1110         mtx_unlock_spin(&allpmaps_lock);
 1111         /* Wire in kernel global address entries. */
 1112         /* XXX copies current process, does not fill in MPPTDI */
 1113         bcopy(PTD + KPTDI, pmap->pm_pdir + KPTDI, nkpt * sizeof(pd_entry_t));
 1114 #ifdef SMP
 1115         pmap->pm_pdir[MPPTDI] = PTD[MPPTDI];
 1116 #endif
 1117 
 1118         /* install self-referential address mapping entry(s) */
 1119         for (i = 0; i < NPGPTD; i++) {
 1120                 pa = VM_PAGE_TO_PHYS(ptdpg[i]);
 1121                 pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M;
 1122 #ifdef PAE
 1123                 pmap->pm_pdpt[i] = pa | PG_V;
 1124 #endif
 1125         }
 1126 
 1127         pmap->pm_active = 0;
 1128         TAILQ_INIT(&pmap->pm_pvlist);
 1129         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1130 }
 1131 
 1132 /*
 1133  * this routine is called if the page table page is not
 1134  * mapped correctly.
 1135  */
 1136 static vm_page_t
 1137 _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags)
 1138 {
 1139         vm_paddr_t ptepa;
 1140         vm_page_t m;
 1141 
 1142         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1143             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1144             ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1145 
 1146         /*
 1147          * Allocate a page table page.
 1148          */
 1149         if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 1150             VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
 1151                 if (flags & M_WAITOK) {
 1152                         PMAP_UNLOCK(pmap);
 1153                         vm_page_unlock_queues();
 1154                         VM_WAIT;
 1155                         vm_page_lock_queues();
 1156                         PMAP_LOCK(pmap);
 1157                 }
 1158 
 1159                 /*
 1160                  * Indicate the need to retry.  While waiting, the page table
 1161                  * page may have been allocated.
 1162                  */
 1163                 return (NULL);
 1164         }
 1165         if ((m->flags & PG_ZERO) == 0)
 1166                 pmap_zero_page(m);
 1167 
 1168         /*
 1169          * Map the pagetable page into the process address space, if
 1170          * it isn't already there.
 1171          */
 1172 
 1173         pmap->pm_stats.resident_count++;
 1174 
 1175         ptepa = VM_PAGE_TO_PHYS(m);
 1176         pmap->pm_pdir[ptepindex] =
 1177                 (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_A | PG_M);
 1178 
 1179         return m;
 1180 }
 1181 
 1182 static vm_page_t
 1183 pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
 1184 {
 1185         unsigned ptepindex;
 1186         pd_entry_t ptepa;
 1187         vm_page_t m;
 1188 
 1189         KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 1190             (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
 1191             ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
 1192 
 1193         /*
 1194          * Calculate pagetable page index
 1195          */
 1196         ptepindex = va >> PDRSHIFT;
 1197 retry:
 1198         /*
 1199          * Get the page directory entry
 1200          */
 1201         ptepa = pmap->pm_pdir[ptepindex];
 1202 
 1203         /*
 1204          * This supports switching from a 4MB page to a
 1205          * normal 4K page.
 1206          */
 1207         if (ptepa & PG_PS) {
 1208                 pmap->pm_pdir[ptepindex] = 0;
 1209                 ptepa = 0;
 1210                 pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1211                 pmap_invalidate_all(kernel_pmap);
 1212         }
 1213 
 1214         /*
 1215          * If the page table page is mapped, we just increment the
 1216          * hold count, and activate it.
 1217          */
 1218         if (ptepa) {
 1219                 m = PHYS_TO_VM_PAGE(ptepa);
 1220                 m->wire_count++;
 1221         } else {
 1222                 /*
 1223                  * Here if the pte page isn't mapped, or if it has
 1224                  * been deallocated. 
 1225                  */
 1226                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1227                 if (m == NULL && (flags & M_WAITOK))
 1228                         goto retry;
 1229         }
 1230         return (m);
 1231 }
 1232 
 1233 
 1234 /***************************************************
 1235 * Pmap allocation/deallocation routines.
 1236  ***************************************************/
 1237 
 1238 #ifdef SMP
 1239 /*
 1240  * Deal with a SMP shootdown of other users of the pmap that we are
 1241  * trying to dispose of.  This can be a bit hairy.
 1242  */
 1243 static u_int *lazymask;
 1244 static u_int lazyptd;
 1245 static volatile u_int lazywait;
 1246 
 1247 void pmap_lazyfix_action(void);
 1248 
 1249 void
 1250 pmap_lazyfix_action(void)
 1251 {
 1252         u_int mymask = PCPU_GET(cpumask);
 1253 
 1254         if (rcr3() == lazyptd)
 1255                 load_cr3(PCPU_GET(curpcb)->pcb_cr3);
 1256         atomic_clear_int(lazymask, mymask);
 1257         atomic_store_rel_int(&lazywait, 1);
 1258 }
 1259 
 1260 static void
 1261 pmap_lazyfix_self(u_int mymask)
 1262 {
 1263 
 1264         if (rcr3() == lazyptd)
 1265                 load_cr3(PCPU_GET(curpcb)->pcb_cr3);
 1266         atomic_clear_int(lazymask, mymask);
 1267 }
 1268 
 1269 
 1270 static void
 1271 pmap_lazyfix(pmap_t pmap)
 1272 {
 1273         u_int mymask;
 1274         u_int mask;
 1275         register u_int spins;
 1276 
 1277         while ((mask = pmap->pm_active) != 0) {
 1278                 spins = 50000000;
 1279                 mask = mask & -mask;    /* Find least significant set bit */
 1280                 mtx_lock_spin(&smp_ipi_mtx);
 1281 #ifdef PAE
 1282                 lazyptd = vtophys(pmap->pm_pdpt);
 1283 #else
 1284                 lazyptd = vtophys(pmap->pm_pdir);
 1285 #endif
 1286                 mymask = PCPU_GET(cpumask);
 1287                 if (mask == mymask) {
 1288                         lazymask = &pmap->pm_active;
 1289                         pmap_lazyfix_self(mymask);
 1290                 } else {
 1291                         atomic_store_rel_int((u_int *)&lazymask,
 1292                             (u_int)&pmap->pm_active);
 1293                         atomic_store_rel_int(&lazywait, 0);
 1294                         ipi_selected(mask, IPI_LAZYPMAP);
 1295                         while (lazywait == 0) {
 1296                                 ia32_pause();
 1297                                 if (--spins == 0)
 1298                                         break;
 1299                         }
 1300                 }
 1301                 mtx_unlock_spin(&smp_ipi_mtx);
 1302                 if (spins == 0)
 1303                         printf("pmap_lazyfix: spun for 50000000\n");
 1304         }
 1305 }
 1306 
 1307 #else   /* SMP */
 1308 
 1309 /*
 1310  * Cleaning up on uniprocessor is easy.  For various reasons, we're
 1311  * unlikely to have to even execute this code, including the fact
 1312  * that the cleanup is deferred until the parent does a wait(2), which
 1313  * means that another userland process has run.
 1314  */
 1315 static void
 1316 pmap_lazyfix(pmap_t pmap)
 1317 {
 1318         u_int cr3;
 1319 
 1320         cr3 = vtophys(pmap->pm_pdir);
 1321         if (cr3 == rcr3()) {
 1322                 load_cr3(PCPU_GET(curpcb)->pcb_cr3);
 1323                 pmap->pm_active &= ~(PCPU_GET(cpumask));
 1324         }
 1325 }
 1326 #endif  /* SMP */
 1327 
 1328 /*
 1329  * Release any resources held by the given physical map.
 1330  * Called when a pmap initialized by pmap_pinit is being released.
 1331  * Should only be called if the map contains no valid mappings.
 1332  */
 1333 void
 1334 pmap_release(pmap_t pmap)
 1335 {
 1336         vm_page_t m, ptdpg[NPGPTD];
 1337         int i;
 1338 
 1339         KASSERT(pmap->pm_stats.resident_count == 0,
 1340             ("pmap_release: pmap resident count %ld != 0",
 1341             pmap->pm_stats.resident_count));
 1342 
 1343         pmap_lazyfix(pmap);
 1344         mtx_lock_spin(&allpmaps_lock);
 1345         LIST_REMOVE(pmap, pm_list);
 1346         mtx_unlock_spin(&allpmaps_lock);
 1347 
 1348         for (i = 0; i < NPGPTD; i++)
 1349                 ptdpg[i] = PHYS_TO_VM_PAGE(pmap->pm_pdir[PTDPTDI + i]);
 1350 
 1351         bzero(pmap->pm_pdir + PTDPTDI, (nkpt + NPGPTD) *
 1352             sizeof(*pmap->pm_pdir));
 1353 #ifdef SMP
 1354         pmap->pm_pdir[MPPTDI] = 0;
 1355 #endif
 1356 
 1357         pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD);
 1358 
 1359         vm_page_lock_queues();
 1360         for (i = 0; i < NPGPTD; i++) {
 1361                 m = ptdpg[i];
 1362 #ifdef PAE
 1363                 KASSERT(VM_PAGE_TO_PHYS(m) == (pmap->pm_pdpt[i] & PG_FRAME),
 1364                     ("pmap_release: got wrong ptd page"));
 1365 #endif
 1366                 m->wire_count--;
 1367                 atomic_subtract_int(&cnt.v_wire_count, 1);
 1368                 vm_page_free_zero(m);
 1369         }
 1370         vm_page_unlock_queues();
 1371         PMAP_LOCK_DESTROY(pmap);
 1372 }
 1373 
 1374 static int
 1375 kvm_size(SYSCTL_HANDLER_ARGS)
 1376 {
 1377         unsigned long ksize = VM_MAX_KERNEL_ADDRESS - KERNBASE;
 1378 
 1379         return sysctl_handle_long(oidp, &ksize, 0, req);
 1380 }
 1381 SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 
 1382     0, 0, kvm_size, "IU", "Size of KVM");
 1383 
 1384 static int
 1385 kvm_free(SYSCTL_HANDLER_ARGS)
 1386 {
 1387         unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end;
 1388 
 1389         return sysctl_handle_long(oidp, &kfree, 0, req);
 1390 }
 1391 SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 
 1392     0, 0, kvm_free, "IU", "Amount of KVM free");
 1393 
 1394 /*
 1395  * grow the number of kernel page table entries, if needed
 1396  */
 1397 void
 1398 pmap_growkernel(vm_offset_t addr)
 1399 {
 1400         struct pmap *pmap;
 1401         vm_paddr_t ptppaddr;
 1402         vm_page_t nkpg;
 1403         pd_entry_t newpdir;
 1404         pt_entry_t *pde;
 1405 
 1406         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1407         if (kernel_vm_end == 0) {
 1408                 kernel_vm_end = KERNBASE;
 1409                 nkpt = 0;
 1410                 while (pdir_pde(PTD, kernel_vm_end)) {
 1411                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1412                         nkpt++;
 1413                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1414                                 kernel_vm_end = kernel_map->max_offset;
 1415                                 break;
 1416                         }
 1417                 }
 1418         }
 1419         addr = roundup2(addr, PAGE_SIZE * NPTEPG);
 1420         if (addr - 1 >= kernel_map->max_offset)
 1421                 addr = kernel_map->max_offset;
 1422         while (kernel_vm_end < addr) {
 1423                 if (pdir_pde(PTD, kernel_vm_end)) {
 1424                         kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1425                         if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1426                                 kernel_vm_end = kernel_map->max_offset;
 1427                                 break;
 1428                         }
 1429                         continue;
 1430                 }
 1431 
 1432                 /*
 1433                  * This index is bogus, but out of the way
 1434                  */
 1435                 nkpg = vm_page_alloc(NULL, nkpt,
 1436                     VM_ALLOC_NOOBJ | VM_ALLOC_SYSTEM | VM_ALLOC_WIRED);
 1437                 if (!nkpg)
 1438                         panic("pmap_growkernel: no memory to grow kernel");
 1439 
 1440                 nkpt++;
 1441 
 1442                 pmap_zero_page(nkpg);
 1443                 ptppaddr = VM_PAGE_TO_PHYS(nkpg);
 1444                 newpdir = (pd_entry_t) (ptppaddr | PG_V | PG_RW | PG_A | PG_M);
 1445                 pdir_pde(PTD, kernel_vm_end) = newpdir;
 1446 
 1447                 mtx_lock_spin(&allpmaps_lock);
 1448                 LIST_FOREACH(pmap, &allpmaps, pm_list) {
 1449                         pde = pmap_pde(pmap, kernel_vm_end);
 1450                         pde_store(pde, newpdir);
 1451                 }
 1452                 mtx_unlock_spin(&allpmaps_lock);
 1453                 kernel_vm_end = (kernel_vm_end + PAGE_SIZE * NPTEPG) & ~(PAGE_SIZE * NPTEPG - 1);
 1454                 if (kernel_vm_end - 1 >= kernel_map->max_offset) {
 1455                         kernel_vm_end = kernel_map->max_offset;
 1456                         break;
 1457                 }
 1458         }
 1459 }
 1460 
 1461 
 1462 /***************************************************
 1463  * page management routines.
 1464  ***************************************************/
 1465 
 1466 /*
 1467  * free the pv_entry back to the free list
 1468  */
 1469 static PMAP_INLINE void
 1470 free_pv_entry(pv_entry_t pv)
 1471 {
 1472         pv_entry_count--;
 1473         uma_zfree(pvzone, pv);
 1474 }
 1475 
 1476 /*
 1477  * get a new pv_entry, allocating a block from the system
 1478  * when needed.
 1479  * the memory allocation is performed bypassing the malloc code
 1480  * because of the possibility of allocations at interrupt time.
 1481  */
 1482 static pv_entry_t
 1483 get_pv_entry(void)
 1484 {
 1485         pv_entry_count++;
 1486         if ((pv_entry_count > pv_entry_high_water) &&
 1487                 (pmap_pagedaemon_waken == 0)) {
 1488                 pmap_pagedaemon_waken = 1;
 1489                 wakeup (&vm_pages_needed);
 1490         }
 1491         return uma_zalloc(pvzone, M_NOWAIT);
 1492 }
 1493 
 1494 
 1495 static void
 1496 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 1497 {
 1498         pv_entry_t pv;
 1499 
 1500         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1501         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1502         if (m->md.pv_list_count < pmap->pm_stats.resident_count) {
 1503                 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 1504                         if (pmap == pv->pv_pmap && va == pv->pv_va) 
 1505                                 break;
 1506                 }
 1507         } else {
 1508                 TAILQ_FOREACH(pv, &pmap->pm_pvlist, pv_plist) {
 1509                         if (va == pv->pv_va) 
 1510                                 break;
 1511                 }
 1512         }
 1513         KASSERT(pv != NULL, ("pmap_remove_entry: pv not found"));
 1514         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1515         m->md.pv_list_count--;
 1516         if (TAILQ_EMPTY(&m->md.pv_list))
 1517                 vm_page_flag_clear(m, PG_WRITEABLE);
 1518         TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
 1519         free_pv_entry(pv);
 1520 }
 1521 
 1522 /*
 1523  * Create a pv entry for page at pa for
 1524  * (pmap, va).
 1525  */
 1526 static void
 1527 pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1528 {
 1529         pv_entry_t pv;
 1530 
 1531         pv = get_pv_entry();
 1532         if (pv == NULL)
 1533                 panic("no pv entries: increase vm.pmap.shpgperproc");
 1534         pv->pv_va = va;
 1535         pv->pv_pmap = pmap;
 1536 
 1537         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1538         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1539         TAILQ_INSERT_TAIL(&pmap->pm_pvlist, pv, pv_plist);
 1540         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 1541         m->md.pv_list_count++;
 1542 }
 1543 
 1544 /*
 1545  * pmap_remove_pte: do the things to unmap a page in a process
 1546  */
 1547 static int
 1548 pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va)
 1549 {
 1550         pt_entry_t oldpte;
 1551         vm_page_t m;
 1552 
 1553         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1554         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1555         oldpte = pte_load_clear(ptq);
 1556         if (oldpte & PG_W)
 1557                 pmap->pm_stats.wired_count -= 1;
 1558         /*
 1559          * Machines that don't support invlpg, also don't support
 1560          * PG_G.
 1561          */
 1562         if (oldpte & PG_G)
 1563                 pmap_invalidate_page(kernel_pmap, va);
 1564         pmap->pm_stats.resident_count -= 1;
 1565         if (oldpte & PG_MANAGED) {
 1566                 m = PHYS_TO_VM_PAGE(oldpte);
 1567                 if (oldpte & PG_M) {
 1568 #if defined(PMAP_DIAGNOSTIC)
 1569                         if (pmap_nw_modified((pt_entry_t) oldpte)) {
 1570                                 printf(
 1571         "pmap_remove: modified page not writable: va: 0x%x, pte: 0x%x\n",
 1572                                     va, oldpte);
 1573                         }
 1574 #endif
 1575                         if (pmap_track_modified(va))
 1576                                 vm_page_dirty(m);
 1577                 }
 1578                 if (oldpte & PG_A)
 1579                         vm_page_flag_set(m, PG_REFERENCED);
 1580                 pmap_remove_entry(pmap, m, va);
 1581         }
 1582         return (pmap_unuse_pt(pmap, va));
 1583 }
 1584 
 1585 /*
 1586  * Remove a single page from a process address space
 1587  */
 1588 static void
 1589 pmap_remove_page(pmap_t pmap, vm_offset_t va)
 1590 {
 1591         pt_entry_t *pte;
 1592 
 1593         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1594         KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 1595         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1596         if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
 1597                 return;
 1598         pmap_remove_pte(pmap, pte, va);
 1599         pmap_invalidate_page(pmap, va);
 1600 }
 1601 
 1602 /*
 1603  *      Remove the given range of addresses from the specified map.
 1604  *
 1605  *      It is assumed that the start and end are properly
 1606  *      rounded to the page size.
 1607  */
 1608 void
 1609 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1610 {
 1611         vm_offset_t pdnxt;
 1612         pd_entry_t ptpaddr;
 1613         pt_entry_t *pte;
 1614         int anyvalid;
 1615 
 1616         /*
 1617          * Perform an unsynchronized read.  This is, however, safe.
 1618          */
 1619         if (pmap->pm_stats.resident_count == 0)
 1620                 return;
 1621 
 1622         anyvalid = 0;
 1623 
 1624         vm_page_lock_queues();
 1625         sched_pin();
 1626         PMAP_LOCK(pmap);
 1627 
 1628         /*
 1629          * special handling of removing one page.  a very
 1630          * common operation and easy to short circuit some
 1631          * code.
 1632          */
 1633         if ((sva + PAGE_SIZE == eva) && 
 1634             ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
 1635                 pmap_remove_page(pmap, sva);
 1636                 goto out;
 1637         }
 1638 
 1639         for (; sva < eva; sva = pdnxt) {
 1640                 unsigned pdirindex;
 1641 
 1642                 /*
 1643                  * Calculate index for next page table.
 1644                  */
 1645                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 1646                 if (pmap->pm_stats.resident_count == 0)
 1647                         break;
 1648 
 1649                 pdirindex = sva >> PDRSHIFT;
 1650                 ptpaddr = pmap->pm_pdir[pdirindex];
 1651 
 1652                 /*
 1653                  * Weed out invalid mappings. Note: we assume that the page
 1654                  * directory table is always allocated, and in kernel virtual.
 1655                  */
 1656                 if (ptpaddr == 0)
 1657                         continue;
 1658 
 1659                 /*
 1660                  * Check for large page.
 1661                  */
 1662                 if ((ptpaddr & PG_PS) != 0) {
 1663                         pmap->pm_pdir[pdirindex] = 0;
 1664                         pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
 1665                         anyvalid = 1;
 1666                         continue;
 1667                 }
 1668 
 1669                 /*
 1670                  * Limit our scan to either the end of the va represented
 1671                  * by the current page table page, or to the end of the
 1672                  * range being removed.
 1673                  */
 1674                 if (pdnxt > eva)
 1675                         pdnxt = eva;
 1676 
 1677                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 1678                     sva += PAGE_SIZE) {
 1679                         if (*pte == 0)
 1680                                 continue;
 1681                         anyvalid = 1;
 1682                         if (pmap_remove_pte(pmap, pte, sva))
 1683                                 break;
 1684                 }
 1685         }
 1686 out:
 1687         sched_unpin();
 1688         vm_page_unlock_queues();
 1689         if (anyvalid)
 1690                 pmap_invalidate_all(pmap);
 1691         PMAP_UNLOCK(pmap);
 1692 }
 1693 
 1694 /*
 1695  *      Routine:        pmap_remove_all
 1696  *      Function:
 1697  *              Removes this physical page from
 1698  *              all physical maps in which it resides.
 1699  *              Reflects back modify bits to the pager.
 1700  *
 1701  *      Notes:
 1702  *              Original versions of this routine were very
 1703  *              inefficient because they iteratively called
 1704  *              pmap_remove (slow...)
 1705  */
 1706 
 1707 void
 1708 pmap_remove_all(vm_page_t m)
 1709 {
 1710         register pv_entry_t pv;
 1711         pt_entry_t *pte, tpte;
 1712 
 1713 #if defined(PMAP_DIAGNOSTIC)
 1714         /*
 1715          * XXX This makes pmap_remove_all() illegal for non-managed pages!
 1716          */
 1717         if (m->flags & PG_FICTITIOUS) {
 1718                 panic("pmap_remove_all: illegal for unmanaged page, va: 0x%x",
 1719                     VM_PAGE_TO_PHYS(m));
 1720         }
 1721 #endif
 1722         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 1723         sched_pin();
 1724         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 1725                 PMAP_LOCK(pv->pv_pmap);
 1726                 pv->pv_pmap->pm_stats.resident_count--;
 1727                 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
 1728                 tpte = pte_load_clear(pte);
 1729                 if (tpte & PG_W)
 1730                         pv->pv_pmap->pm_stats.wired_count--;
 1731                 if (tpte & PG_A)
 1732                         vm_page_flag_set(m, PG_REFERENCED);
 1733 
 1734                 /*
 1735                  * Update the vm_page_t clean and reference bits.
 1736                  */
 1737                 if (tpte & PG_M) {
 1738 #if defined(PMAP_DIAGNOSTIC)
 1739                         if (pmap_nw_modified((pt_entry_t) tpte)) {
 1740                                 printf(
 1741         "pmap_remove_all: modified page not writable: va: 0x%x, pte: 0x%x\n",
 1742                                     pv->pv_va, tpte);
 1743                         }
 1744 #endif
 1745                         if (pmap_track_modified(pv->pv_va))
 1746                                 vm_page_dirty(m);
 1747                 }
 1748                 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 1749                 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist);
 1750                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1751                 m->md.pv_list_count--;
 1752                 pmap_unuse_pt(pv->pv_pmap, pv->pv_va);
 1753                 PMAP_UNLOCK(pv->pv_pmap);
 1754                 free_pv_entry(pv);
 1755         }
 1756         vm_page_flag_clear(m, PG_WRITEABLE);
 1757         sched_unpin();
 1758 }
 1759 
 1760 /*
 1761  *      Set the physical protection on the
 1762  *      specified range of this map as requested.
 1763  */
 1764 void
 1765 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 1766 {
 1767         vm_offset_t pdnxt;
 1768         pd_entry_t ptpaddr;
 1769         pt_entry_t *pte;
 1770         int anychanged;
 1771 
 1772         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 1773                 pmap_remove(pmap, sva, eva);
 1774                 return;
 1775         }
 1776 
 1777         if (prot & VM_PROT_WRITE)
 1778                 return;
 1779 
 1780         anychanged = 0;
 1781 
 1782         vm_page_lock_queues();
 1783         sched_pin();
 1784         PMAP_LOCK(pmap);
 1785         for (; sva < eva; sva = pdnxt) {
 1786                 unsigned obits, pbits, pdirindex;
 1787 
 1788                 pdnxt = (sva + NBPDR) & ~PDRMASK;
 1789 
 1790                 pdirindex = sva >> PDRSHIFT;
 1791                 ptpaddr = pmap->pm_pdir[pdirindex];
 1792 
 1793                 /*
 1794                  * Weed out invalid mappings. Note: we assume that the page
 1795                  * directory table is always allocated, and in kernel virtual.
 1796                  */
 1797                 if (ptpaddr == 0)
 1798                         continue;
 1799 
 1800                 /*
 1801                  * Check for large page.
 1802                  */
 1803                 if ((ptpaddr & PG_PS) != 0) {
 1804                         pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW);
 1805                         anychanged = 1;
 1806                         continue;
 1807                 }
 1808 
 1809                 if (pdnxt > eva)
 1810                         pdnxt = eva;
 1811 
 1812                 for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++,
 1813                     sva += PAGE_SIZE) {
 1814                         vm_page_t m;
 1815 
 1816 retry:
 1817                         /*
 1818                          * Regardless of whether a pte is 32 or 64 bits in
 1819                          * size, PG_RW, PG_A, and PG_M are among the least
 1820                          * significant 32 bits.
 1821                          */
 1822                         obits = pbits = *(u_int *)pte;
 1823                         if (pbits & PG_MANAGED) {
 1824                                 m = NULL;
 1825                                 if (pbits & PG_A) {
 1826                                         m = PHYS_TO_VM_PAGE(*pte);
 1827                                         vm_page_flag_set(m, PG_REFERENCED);
 1828                                         pbits &= ~PG_A;
 1829                                 }
 1830                                 if ((pbits & PG_M) != 0 &&
 1831                                     pmap_track_modified(sva)) {
 1832                                         if (m == NULL)
 1833                                                 m = PHYS_TO_VM_PAGE(*pte);
 1834                                         vm_page_dirty(m);
 1835                                 }
 1836                         }
 1837 
 1838                         pbits &= ~(PG_RW | PG_M);
 1839 
 1840                         if (pbits != obits) {
 1841                                 if (!atomic_cmpset_int((u_int *)pte, obits,
 1842                                     pbits))
 1843                                         goto retry;
 1844                                 if (obits & PG_G)
 1845                                         pmap_invalidate_page(pmap, sva);
 1846                                 else
 1847                                         anychanged = 1;
 1848                         }
 1849                 }
 1850         }
 1851         sched_unpin();
 1852         vm_page_unlock_queues();
 1853         if (anychanged)
 1854                 pmap_invalidate_all(pmap);
 1855         PMAP_UNLOCK(pmap);
 1856 }
 1857 
 1858 /*
 1859  *      Insert the given physical page (p) at
 1860  *      the specified virtual address (v) in the
 1861  *      target physical map with the protection requested.
 1862  *
 1863  *      If specified, the page will be wired down, meaning
 1864  *      that the related pte can not be reclaimed.
 1865  *
 1866  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 1867  *      or lose information.  That is, this routine must actually
 1868  *      insert this page into the given map NOW.
 1869  */
 1870 void
 1871 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 1872            boolean_t wired)
 1873 {
 1874         vm_paddr_t pa;
 1875         register pt_entry_t *pte;
 1876         vm_paddr_t opa;
 1877         pt_entry_t origpte, newpte;
 1878         vm_page_t mpte, om;
 1879         boolean_t invlva;
 1880 
 1881         va &= PG_FRAME;
 1882 #ifdef PMAP_DIAGNOSTIC
 1883         if (va > VM_MAX_KERNEL_ADDRESS)
 1884                 panic("pmap_enter: toobig");
 1885         if ((va >= UPT_MIN_ADDRESS) && (va < UPT_MAX_ADDRESS))
 1886                 panic("pmap_enter: invalid to pmap_enter page table pages (va: 0x%x)", va);
 1887 #endif
 1888 
 1889         mpte = NULL;
 1890 
 1891         vm_page_lock_queues();
 1892         PMAP_LOCK(pmap);
 1893         sched_pin();
 1894 
 1895         /*
 1896          * In the case that a page table page is not
 1897          * resident, we are creating it here.
 1898          */
 1899         if (va < VM_MAXUSER_ADDRESS) {
 1900                 mpte = pmap_allocpte(pmap, va, M_WAITOK);
 1901         }
 1902 #if 0 && defined(PMAP_DIAGNOSTIC)
 1903         else {
 1904                 pd_entry_t *pdeaddr = pmap_pde(pmap, va);
 1905                 origpte = *pdeaddr;
 1906                 if ((origpte & PG_V) == 0) { 
 1907                         panic("pmap_enter: invalid kernel page table page, pdir=%p, pde=%p, va=%p\n",
 1908                                 pmap->pm_pdir[PTDPTDI], origpte, va);
 1909                 }
 1910         }
 1911 #endif
 1912 
 1913         pte = pmap_pte_quick(pmap, va);
 1914 
 1915         /*
 1916          * Page Directory table entry not valid, we need a new PT page
 1917          */
 1918         if (pte == NULL) {
 1919                 panic("pmap_enter: invalid page directory pdir=%#jx, va=%#x\n",
 1920                         (uintmax_t)pmap->pm_pdir[PTDPTDI], va);
 1921         }
 1922 
 1923         pa = VM_PAGE_TO_PHYS(m);
 1924         om = NULL;
 1925         origpte = *pte;
 1926         opa = origpte & PG_FRAME;
 1927 
 1928         if (origpte & PG_PS) {
 1929                 /*
 1930                  * Yes, I know this will truncate upper address bits for PAE,
 1931                  * but I'm actually more interested in the lower bits
 1932                  */
 1933                 printf("pmap_enter: va %p, pte %p, origpte %p\n",
 1934                     (void *)va, (void *)pte, (void *)(uintptr_t)origpte);
 1935                 panic("pmap_enter: attempted pmap_enter on 4MB page");
 1936         }
 1937 
 1938         /*
 1939          * Mapping has not changed, must be protection or wiring change.
 1940          */
 1941         if (origpte && (opa == pa)) {
 1942                 /*
 1943                  * Wiring change, just update stats. We don't worry about
 1944                  * wiring PT pages as they remain resident as long as there
 1945                  * are valid mappings in them. Hence, if a user page is wired,
 1946                  * the PT page will be also.
 1947                  */
 1948                 if (wired && ((origpte & PG_W) == 0))
 1949                         pmap->pm_stats.wired_count++;
 1950                 else if (!wired && (origpte & PG_W))
 1951                         pmap->pm_stats.wired_count--;
 1952 
 1953                 /*
 1954                  * Remove extra pte reference
 1955                  */
 1956                 if (mpte)
 1957                         mpte->wire_count--;
 1958 
 1959                 /*
 1960                  * We might be turning off write access to the page,
 1961                  * so we go ahead and sense modify status.
 1962                  */
 1963                 if (origpte & PG_MANAGED) {
 1964                         om = m;
 1965                         pa |= PG_MANAGED;
 1966                 }
 1967                 goto validate;
 1968         } 
 1969         /*
 1970          * Mapping has changed, invalidate old range and fall through to
 1971          * handle validating new mapping.
 1972          */
 1973         if (opa) {
 1974                 if (origpte & PG_W)
 1975                         pmap->pm_stats.wired_count--;
 1976                 if (origpte & PG_MANAGED) {
 1977                         om = PHYS_TO_VM_PAGE(opa);
 1978                         pmap_remove_entry(pmap, om, va);
 1979                 }
 1980                 if (mpte != NULL) {
 1981                         mpte->wire_count--;
 1982                         KASSERT(mpte->wire_count > 0,
 1983                             ("pmap_enter: missing reference to page table page,"
 1984                              " va: 0x%x", va));
 1985                 }
 1986         } else
 1987                 pmap->pm_stats.resident_count++;
 1988 
 1989         /*
 1990          * Enter on the PV list if part of our managed memory.
 1991          */
 1992         if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) {
 1993                 pmap_insert_entry(pmap, va, m);
 1994                 pa |= PG_MANAGED;
 1995         }
 1996 
 1997         /*
 1998          * Increment counters
 1999          */
 2000         if (wired)
 2001                 pmap->pm_stats.wired_count++;
 2002 
 2003 validate:
 2004         /*
 2005          * Now validate mapping with desired protection/wiring.
 2006          */
 2007         newpte = (pt_entry_t)(pa | PG_V);
 2008         if ((prot & VM_PROT_WRITE) != 0)
 2009                 newpte |= PG_RW;
 2010         if (wired)
 2011                 newpte |= PG_W;
 2012         if (va < VM_MAXUSER_ADDRESS)
 2013                 newpte |= PG_U;
 2014         if (pmap == kernel_pmap)
 2015                 newpte |= pgeflag;
 2016 
 2017         /*
 2018          * if the mapping or permission bits are different, we need
 2019          * to update the pte.
 2020          */
 2021         if ((origpte & ~(PG_M|PG_A)) != newpte) {
 2022                 if (origpte & PG_V) {
 2023                         invlva = FALSE;
 2024                         origpte = pte_load_store(pte, newpte | PG_A);
 2025                         if (origpte & PG_A) {
 2026                                 if (origpte & PG_MANAGED)
 2027                                         vm_page_flag_set(om, PG_REFERENCED);
 2028                                 if (opa != VM_PAGE_TO_PHYS(m))
 2029                                         invlva = TRUE;
 2030                         }
 2031                         if (origpte & PG_M) {
 2032                                 KASSERT((origpte & PG_RW),
 2033                                     ("pmap_enter: modified page not writable:"
 2034                                      " va: 0x%x, pte: 0x%x", va, origpte));
 2035                                 if ((origpte & PG_MANAGED) &&
 2036                                     pmap_track_modified(va))
 2037                                         vm_page_dirty(om);
 2038                                 if ((prot & VM_PROT_WRITE) == 0)
 2039                                         invlva = TRUE;
 2040                         }
 2041                         if (invlva)
 2042                                 pmap_invalidate_page(pmap, va);
 2043                 } else
 2044                         pte_store(pte, newpte | PG_A);
 2045         }
 2046         sched_unpin();
 2047         vm_page_unlock_queues();
 2048         PMAP_UNLOCK(pmap);
 2049 }
 2050 
 2051 /*
 2052  * this code makes some *MAJOR* assumptions:
 2053  * 1. Current pmap & pmap exists.
 2054  * 2. Not wired.
 2055  * 3. Read access.
 2056  * 4. No page table pages.
 2057  * but is *MUCH* faster than pmap_enter...
 2058  */
 2059 
 2060 vm_page_t
 2061 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 2062     vm_page_t mpte)
 2063 {
 2064         pt_entry_t *pte;
 2065         vm_paddr_t pa;
 2066 
 2067         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2068         VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 2069         PMAP_LOCK(pmap);
 2070 
 2071         /*
 2072          * In the case that a page table page is not
 2073          * resident, we are creating it here.
 2074          */
 2075         if (va < VM_MAXUSER_ADDRESS) {
 2076                 unsigned ptepindex;
 2077                 pd_entry_t ptepa;
 2078 
 2079                 /*
 2080                  * Calculate pagetable page index
 2081                  */
 2082                 ptepindex = va >> PDRSHIFT;
 2083                 if (mpte && (mpte->pindex == ptepindex)) {
 2084                         mpte->wire_count++;
 2085                 } else {
 2086 retry:
 2087                         /*
 2088                          * Get the page directory entry
 2089                          */
 2090                         ptepa = pmap->pm_pdir[ptepindex];
 2091 
 2092                         /*
 2093                          * If the page table page is mapped, we just increment
 2094                          * the hold count, and activate it.
 2095                          */
 2096                         if (ptepa) {
 2097                                 if (ptepa & PG_PS)
 2098                                         panic("pmap_enter_quick: unexpected mapping into 4MB page");
 2099                                 mpte = PHYS_TO_VM_PAGE(ptepa);
 2100                                 mpte->wire_count++;
 2101                         } else {
 2102                                 mpte = _pmap_allocpte(pmap, ptepindex,
 2103                                     M_NOWAIT);
 2104                                 if (mpte == NULL) {
 2105                                         PMAP_UNLOCK(pmap);
 2106                                         vm_page_busy(m);
 2107                                         vm_page_unlock_queues();
 2108                                         VM_OBJECT_UNLOCK(m->object);
 2109                                         VM_WAIT;
 2110                                         VM_OBJECT_LOCK(m->object);
 2111                                         vm_page_lock_queues();
 2112                                         vm_page_wakeup(m);
 2113                                         PMAP_LOCK(pmap);
 2114                                         goto retry;
 2115                                 }
 2116                         }
 2117                 }
 2118         } else {
 2119                 mpte = NULL;
 2120         }
 2121 
 2122         /*
 2123          * This call to vtopte makes the assumption that we are
 2124          * entering the page into the current pmap.  In order to support
 2125          * quick entry into any pmap, one would likely use pmap_pte_quick.
 2126          * But that isn't as quick as vtopte.
 2127          */
 2128         pte = vtopte(va);
 2129         if (*pte) {
 2130                 if (mpte != NULL) {
 2131                         pmap_unwire_pte_hold(pmap, mpte);
 2132                         mpte = NULL;
 2133                 }
 2134                 goto out;
 2135         }
 2136 
 2137         /*
 2138          * Enter on the PV list if part of our managed memory. Note that we
 2139          * raise IPL while manipulating pv_table since pmap_enter can be
 2140          * called at interrupt time.
 2141          */
 2142         if ((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0)
 2143                 pmap_insert_entry(pmap, va, m);
 2144 
 2145         /*
 2146          * Increment counters
 2147          */
 2148         pmap->pm_stats.resident_count++;
 2149 
 2150         pa = VM_PAGE_TO_PHYS(m);
 2151 
 2152         /*
 2153          * Now validate mapping with RO protection
 2154          */
 2155         if (m->flags & (PG_FICTITIOUS|PG_UNMANAGED))
 2156                 pte_store(pte, pa | PG_V | PG_U);
 2157         else
 2158                 pte_store(pte, pa | PG_V | PG_U | PG_MANAGED);
 2159 out:
 2160         PMAP_UNLOCK(pmap);
 2161         return mpte;
 2162 }
 2163 
 2164 /*
 2165  * Make a temporary mapping for a physical address.  This is only intended
 2166  * to be used for panic dumps.
 2167  */
 2168 void *
 2169 pmap_kenter_temporary(vm_paddr_t pa, int i)
 2170 {
 2171         vm_offset_t va;
 2172 
 2173         va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE);
 2174         pmap_kenter(va, pa);
 2175         invlpg(va);
 2176         return ((void *)crashdumpmap);
 2177 }
 2178 
 2179 /*
 2180  * This code maps large physical mmap regions into the
 2181  * processor address space.  Note that some shortcuts
 2182  * are taken, but the code works.
 2183  */
 2184 void
 2185 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
 2186                     vm_object_t object, vm_pindex_t pindex,
 2187                     vm_size_t size)
 2188 {
 2189         vm_page_t p;
 2190 
 2191         VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 2192         KASSERT(object->type == OBJT_DEVICE,
 2193             ("pmap_object_init_pt: non-device object"));
 2194         if (pseflag && 
 2195             ((addr & (NBPDR - 1)) == 0) && ((size & (NBPDR - 1)) == 0)) {
 2196                 int i;
 2197                 vm_page_t m[1];
 2198                 unsigned int ptepindex;
 2199                 int npdes;
 2200                 pd_entry_t ptepa;
 2201 
 2202                 PMAP_LOCK(pmap);
 2203                 if (pmap->pm_pdir[ptepindex = (addr >> PDRSHIFT)])
 2204                         goto out;
 2205                 PMAP_UNLOCK(pmap);
 2206 retry:
 2207                 p = vm_page_lookup(object, pindex);
 2208                 if (p != NULL) {
 2209                         vm_page_lock_queues();
 2210                         if (vm_page_sleep_if_busy(p, FALSE, "init4p"))
 2211                                 goto retry;
 2212                 } else {
 2213                         p = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
 2214                         if (p == NULL)
 2215                                 return;
 2216                         m[0] = p;
 2217 
 2218                         if (vm_pager_get_pages(object, m, 1, 0) != VM_PAGER_OK) {
 2219                                 vm_page_lock_queues();
 2220                                 vm_page_free(p);
 2221                                 vm_page_unlock_queues();
 2222                                 return;
 2223                         }
 2224 
 2225                         p = vm_page_lookup(object, pindex);
 2226                         vm_page_lock_queues();
 2227                         vm_page_wakeup(p);
 2228                 }
 2229                 vm_page_unlock_queues();
 2230 
 2231                 ptepa = VM_PAGE_TO_PHYS(p);
 2232                 if (ptepa & (NBPDR - 1))
 2233                         return;
 2234 
 2235                 p->valid = VM_PAGE_BITS_ALL;
 2236 
 2237                 PMAP_LOCK(pmap);
 2238                 pmap->pm_stats.resident_count += size >> PAGE_SHIFT;
 2239                 npdes = size >> PDRSHIFT;
 2240                 for(i = 0; i < npdes; i++) {
 2241                         pde_store(&pmap->pm_pdir[ptepindex],
 2242                             ptepa | PG_U | PG_RW | PG_V | PG_PS);
 2243                         ptepa += NBPDR;
 2244                         ptepindex += 1;
 2245                 }
 2246                 pmap_invalidate_all(pmap);
 2247 out:
 2248                 PMAP_UNLOCK(pmap);
 2249         }
 2250 }
 2251 
 2252 /*
 2253  *      Routine:        pmap_change_wiring
 2254  *      Function:       Change the wiring attribute for a map/virtual-address
 2255  *                      pair.
 2256  *      In/out conditions:
 2257  *                      The mapping must already exist in the pmap.
 2258  */
 2259 void
 2260 pmap_change_wiring(pmap, va, wired)
 2261         register pmap_t pmap;
 2262         vm_offset_t va;
 2263         boolean_t wired;
 2264 {
 2265         register pt_entry_t *pte;
 2266 
 2267         PMAP_LOCK(pmap);
 2268         pte = pmap_pte(pmap, va);
 2269 
 2270         if (wired && !pmap_pte_w(pte))
 2271                 pmap->pm_stats.wired_count++;
 2272         else if (!wired && pmap_pte_w(pte))
 2273                 pmap->pm_stats.wired_count--;
 2274 
 2275         /*
 2276          * Wiring is not a hardware characteristic so there is no need to
 2277          * invalidate TLB.
 2278          */
 2279         pmap_pte_set_w(pte, wired);
 2280         pmap_pte_release(pte);
 2281         PMAP_UNLOCK(pmap);
 2282 }
 2283 
 2284 
 2285 
 2286 /*
 2287  *      Copy the range specified by src_addr/len
 2288  *      from the source map to the range dst_addr/len
 2289  *      in the destination map.
 2290  *
 2291  *      This routine is only advisory and need not do anything.
 2292  */
 2293 
 2294 void
 2295 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 2296           vm_offset_t src_addr)
 2297 {
 2298         vm_offset_t addr;
 2299         vm_offset_t end_addr = src_addr + len;
 2300         vm_offset_t pdnxt;
 2301         vm_page_t m;
 2302 
 2303         if (dst_addr != src_addr)
 2304                 return;
 2305 
 2306         if (!pmap_is_current(src_pmap))
 2307                 return;
 2308 
 2309         vm_page_lock_queues();
 2310         if (dst_pmap < src_pmap) {
 2311                 PMAP_LOCK(dst_pmap);
 2312                 PMAP_LOCK(src_pmap);
 2313         } else {
 2314                 PMAP_LOCK(src_pmap);
 2315                 PMAP_LOCK(dst_pmap);
 2316         }
 2317         sched_pin();
 2318         for (addr = src_addr; addr < end_addr; addr = pdnxt) {
 2319                 pt_entry_t *src_pte, *dst_pte;
 2320                 vm_page_t dstmpte, srcmpte;
 2321                 pd_entry_t srcptepaddr;
 2322                 unsigned ptepindex;
 2323 
 2324                 if (addr >= UPT_MIN_ADDRESS)
 2325                         panic("pmap_copy: invalid to pmap_copy page tables");
 2326 
 2327                 /*
 2328                  * Don't let optional prefaulting of pages make us go
 2329                  * way below the low water mark of free pages or way
 2330                  * above high water mark of used pv entries.
 2331                  */
 2332                 if (cnt.v_free_count < cnt.v_free_reserved ||
 2333                     pv_entry_count > pv_entry_high_water)
 2334                         break;
 2335                 
 2336                 pdnxt = (addr + NBPDR) & ~PDRMASK;
 2337                 ptepindex = addr >> PDRSHIFT;
 2338 
 2339                 srcptepaddr = src_pmap->pm_pdir[ptepindex];
 2340                 if (srcptepaddr == 0)
 2341                         continue;
 2342                         
 2343                 if (srcptepaddr & PG_PS) {
 2344                         if (dst_pmap->pm_pdir[ptepindex] == 0) {
 2345                                 dst_pmap->pm_pdir[ptepindex] = srcptepaddr &
 2346                                     ~PG_W;
 2347                                 dst_pmap->pm_stats.resident_count +=
 2348                                     NBPDR / PAGE_SIZE;
 2349                         }
 2350                         continue;
 2351                 }
 2352 
 2353                 srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
 2354                 if (srcmpte->wire_count == 0)
 2355                         panic("pmap_copy: source page table page is unused");
 2356 
 2357                 if (pdnxt > end_addr)
 2358                         pdnxt = end_addr;
 2359 
 2360                 src_pte = vtopte(addr);
 2361                 while (addr < pdnxt) {
 2362                         pt_entry_t ptetemp;
 2363                         ptetemp = *src_pte;
 2364                         /*
 2365                          * we only virtual copy managed pages
 2366                          */
 2367                         if ((ptetemp & PG_MANAGED) != 0) {
 2368                                 /*
 2369                                  * We have to check after allocpte for the
 2370                                  * pte still being around...  allocpte can
 2371                                  * block.
 2372                                  */
 2373                                 dstmpte = pmap_allocpte(dst_pmap, addr,
 2374                                     M_NOWAIT);
 2375                                 if (dstmpte == NULL)
 2376                                         break;
 2377                                 dst_pte = pmap_pte_quick(dst_pmap, addr);
 2378                                 if (*dst_pte == 0) {
 2379                                         /*
 2380                                          * Clear the wired, modified, and
 2381                                          * accessed (referenced) bits
 2382                                          * during the copy.
 2383                                          */
 2384                                         m = PHYS_TO_VM_PAGE(ptetemp);
 2385                                         *dst_pte = ptetemp & ~(PG_W | PG_M |
 2386                                             PG_A);
 2387                                         dst_pmap->pm_stats.resident_count++;
 2388                                         pmap_insert_entry(dst_pmap, addr, m);
 2389                                 } else
 2390                                         pmap_unwire_pte_hold(dst_pmap, dstmpte);
 2391                                 if (dstmpte->wire_count >= srcmpte->wire_count)
 2392                                         break;
 2393                         }
 2394                         addr += PAGE_SIZE;
 2395                         src_pte++;
 2396                 }
 2397         }
 2398         sched_unpin();
 2399         vm_page_unlock_queues();
 2400         PMAP_UNLOCK(src_pmap);
 2401         PMAP_UNLOCK(dst_pmap);
 2402 }       
 2403 
 2404 static __inline void
 2405 pagezero(void *page)
 2406 {
 2407 #if defined(I686_CPU)
 2408         if (cpu_class == CPUCLASS_686) {
 2409 #if defined(CPU_ENABLE_SSE)
 2410                 if (cpu_feature & CPUID_SSE2)
 2411                         sse2_pagezero(page);
 2412                 else
 2413 #endif
 2414                         i686_pagezero(page);
 2415         } else
 2416 #endif
 2417                 bzero(page, PAGE_SIZE);
 2418 }
 2419 
 2420 /*
 2421  *      pmap_zero_page zeros the specified hardware page by mapping 
 2422  *      the page into KVM and using bzero to clear its contents.
 2423  */
 2424 void
 2425 pmap_zero_page(vm_page_t m)
 2426 {
 2427         struct sysmaps *sysmaps;
 2428 
 2429         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 2430         mtx_lock(&sysmaps->lock);
 2431         if (*sysmaps->CMAP2)
 2432                 panic("pmap_zero_page: CMAP2 busy");
 2433         sched_pin();
 2434         *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
 2435         invlcaddr(sysmaps->CADDR2);
 2436         pagezero(sysmaps->CADDR2);
 2437         *sysmaps->CMAP2 = 0;
 2438         sched_unpin();
 2439         mtx_unlock(&sysmaps->lock);
 2440 }
 2441 
 2442 /*
 2443  *      pmap_zero_page_area zeros the specified hardware page by mapping 
 2444  *      the page into KVM and using bzero to clear its contents.
 2445  *
 2446  *      off and size may not cover an area beyond a single hardware page.
 2447  */
 2448 void
 2449 pmap_zero_page_area(vm_page_t m, int off, int size)
 2450 {
 2451         struct sysmaps *sysmaps;
 2452 
 2453         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 2454         mtx_lock(&sysmaps->lock);
 2455         if (*sysmaps->CMAP2)
 2456                 panic("pmap_zero_page: CMAP2 busy");
 2457         sched_pin();
 2458         *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
 2459         invlcaddr(sysmaps->CADDR2);
 2460         if (off == 0 && size == PAGE_SIZE) 
 2461                 pagezero(sysmaps->CADDR2);
 2462         else
 2463                 bzero((char *)sysmaps->CADDR2 + off, size);
 2464         *sysmaps->CMAP2 = 0;
 2465         sched_unpin();
 2466         mtx_unlock(&sysmaps->lock);
 2467 }
 2468 
 2469 /*
 2470  *      pmap_zero_page_idle zeros the specified hardware page by mapping 
 2471  *      the page into KVM and using bzero to clear its contents.  This
 2472  *      is intended to be called from the vm_pagezero process only and
 2473  *      outside of Giant.
 2474  */
 2475 void
 2476 pmap_zero_page_idle(vm_page_t m)
 2477 {
 2478 
 2479         if (*CMAP3)
 2480                 panic("pmap_zero_page: CMAP3 busy");
 2481         sched_pin();
 2482         *CMAP3 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | PG_A | PG_M;
 2483         invlcaddr(CADDR3);
 2484         pagezero(CADDR3);
 2485         *CMAP3 = 0;
 2486         sched_unpin();
 2487 }
 2488 
 2489 /*
 2490  *      pmap_copy_page copies the specified (machine independent)
 2491  *      page by mapping the page into virtual memory and using
 2492  *      bcopy to copy the page, one machine dependent page at a
 2493  *      time.
 2494  */
 2495 void
 2496 pmap_copy_page(vm_page_t src, vm_page_t dst)
 2497 {
 2498         struct sysmaps *sysmaps;
 2499 
 2500         sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
 2501         mtx_lock(&sysmaps->lock);
 2502         if (*sysmaps->CMAP1)
 2503                 panic("pmap_copy_page: CMAP1 busy");
 2504         if (*sysmaps->CMAP2)
 2505                 panic("pmap_copy_page: CMAP2 busy");
 2506         sched_pin();
 2507         invlpg((u_int)sysmaps->CADDR1);
 2508         invlpg((u_int)sysmaps->CADDR2);
 2509         *sysmaps->CMAP1 = PG_V | VM_PAGE_TO_PHYS(src) | PG_A;
 2510         *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(dst) | PG_A | PG_M;
 2511         bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE);
 2512         *sysmaps->CMAP1 = 0;
 2513         *sysmaps->CMAP2 = 0;
 2514         sched_unpin();
 2515         mtx_unlock(&sysmaps->lock);
 2516 }
 2517 
 2518 /*
 2519  * Returns true if the pmap's pv is one of the first
 2520  * 16 pvs linked to from this page.  This count may
 2521  * be changed upwards or downwards in the future; it
 2522  * is only necessary that true be returned for a small
 2523  * subset of pmaps for proper page aging.
 2524  */
 2525 boolean_t
 2526 pmap_page_exists_quick(pmap, m)
 2527         pmap_t pmap;
 2528         vm_page_t m;
 2529 {
 2530         pv_entry_t pv;
 2531         int loops = 0;
 2532 
 2533         if (m->flags & PG_FICTITIOUS)
 2534                 return FALSE;
 2535 
 2536         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2537         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2538                 if (pv->pv_pmap == pmap) {
 2539                         return TRUE;
 2540                 }
 2541                 loops++;
 2542                 if (loops >= 16)
 2543                         break;
 2544         }
 2545         return (FALSE);
 2546 }
 2547 
 2548 #define PMAP_REMOVE_PAGES_CURPROC_ONLY
 2549 /*
 2550  * Remove all pages from specified address space
 2551  * this aids process exit speeds.  Also, this code
 2552  * is special cased for current process only, but
 2553  * can have the more generic (and slightly slower)
 2554  * mode enabled.  This is much faster than pmap_remove
 2555  * in the case of running down an entire address space.
 2556  */
 2557 void
 2558 pmap_remove_pages(pmap, sva, eva)
 2559         pmap_t pmap;
 2560         vm_offset_t sva, eva;
 2561 {
 2562         pt_entry_t *pte, tpte;
 2563         vm_page_t m;
 2564         pv_entry_t pv, npv;
 2565 
 2566 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
 2567         if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
 2568                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 2569                 return;
 2570         }
 2571 #endif
 2572         vm_page_lock_queues();
 2573         PMAP_LOCK(pmap);
 2574         sched_pin();
 2575         for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
 2576 
 2577                 if (pv->pv_va >= eva || pv->pv_va < sva) {
 2578                         npv = TAILQ_NEXT(pv, pv_plist);
 2579                         continue;
 2580                 }
 2581 
 2582 #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY
 2583                 pte = vtopte(pv->pv_va);
 2584 #else
 2585                 pte = pmap_pte_quick(pmap, pv->pv_va);
 2586 #endif
 2587                 tpte = *pte;
 2588 
 2589                 if (tpte == 0) {
 2590                         printf("TPTE at %p  IS ZERO @ VA %08x\n",
 2591                                                         pte, pv->pv_va);
 2592                         panic("bad pte");
 2593                 }
 2594 
 2595 /*
 2596  * We cannot remove wired pages from a process' mapping at this time
 2597  */
 2598                 if (tpte & PG_W) {
 2599                         npv = TAILQ_NEXT(pv, pv_plist);
 2600                         continue;
 2601                 }
 2602 
 2603                 m = PHYS_TO_VM_PAGE(tpte);
 2604                 KASSERT(m->phys_addr == (tpte & PG_FRAME),
 2605                     ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 2606                     m, (uintmax_t)m->phys_addr, (uintmax_t)tpte));
 2607 
 2608                 KASSERT(m < &vm_page_array[vm_page_array_size],
 2609                         ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte));
 2610 
 2611                 pmap->pm_stats.resident_count--;
 2612 
 2613                 pte_clear(pte);
 2614 
 2615                 /*
 2616                  * Update the vm_page_t clean and reference bits.
 2617                  */
 2618                 if (tpte & PG_M) {
 2619                         vm_page_dirty(m);
 2620                 }
 2621 
 2622                 npv = TAILQ_NEXT(pv, pv_plist);
 2623                 TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist);
 2624 
 2625                 m->md.pv_list_count--;
 2626                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2627                 if (TAILQ_EMPTY(&m->md.pv_list))
 2628                         vm_page_flag_clear(m, PG_WRITEABLE);
 2629 
 2630                 pmap_unuse_pt(pmap, pv->pv_va);
 2631                 free_pv_entry(pv);
 2632         }
 2633         sched_unpin();
 2634         pmap_invalidate_all(pmap);
 2635         PMAP_UNLOCK(pmap);
 2636         vm_page_unlock_queues();
 2637 }
 2638 
 2639 /*
 2640  *      pmap_is_modified:
 2641  *
 2642  *      Return whether or not the specified physical page was modified
 2643  *      in any physical maps.
 2644  */
 2645 boolean_t
 2646 pmap_is_modified(vm_page_t m)
 2647 {
 2648         pv_entry_t pv;
 2649         pt_entry_t *pte;
 2650         boolean_t rv;
 2651 
 2652         rv = FALSE;
 2653         if (m->flags & PG_FICTITIOUS)
 2654                 return (rv);
 2655 
 2656         sched_pin();
 2657         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2658         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2659                 /*
 2660                  * if the bit being tested is the modified bit, then
 2661                  * mark clean_map and ptes as never
 2662                  * modified.
 2663                  */
 2664                 if (!pmap_track_modified(pv->pv_va))
 2665                         continue;
 2666                 PMAP_LOCK(pv->pv_pmap);
 2667                 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
 2668                 rv = (*pte & PG_M) != 0;
 2669                 PMAP_UNLOCK(pv->pv_pmap);
 2670                 if (rv)
 2671                         break;
 2672         }
 2673         sched_unpin();
 2674         return (rv);
 2675 }
 2676 
 2677 /*
 2678  *      pmap_is_prefaultable:
 2679  *
 2680  *      Return whether or not the specified virtual address is elgible
 2681  *      for prefault.
 2682  */
 2683 boolean_t
 2684 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 2685 {
 2686         pt_entry_t *pte;
 2687         boolean_t rv;
 2688 
 2689         rv = FALSE;
 2690         PMAP_LOCK(pmap);
 2691         if (*pmap_pde(pmap, addr)) {
 2692                 pte = vtopte(addr);
 2693                 rv = *pte == 0;
 2694         }
 2695         PMAP_UNLOCK(pmap);
 2696         return (rv);
 2697 }
 2698 
 2699 /*
 2700  *      Clear the given bit in each of the given page's ptes.  The bit is
 2701  *      expressed as a 32-bit mask.  Consequently, if the pte is 64 bits in
 2702  *      size, only a bit within the least significant 32 can be cleared.
 2703  */
 2704 static __inline void
 2705 pmap_clear_ptes(vm_page_t m, int bit)
 2706 {
 2707         register pv_entry_t pv;
 2708         pt_entry_t pbits, *pte;
 2709 
 2710         if ((m->flags & PG_FICTITIOUS) ||
 2711             (bit == PG_RW && (m->flags & PG_WRITEABLE) == 0))
 2712                 return;
 2713 
 2714         sched_pin();
 2715         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2716         /*
 2717          * Loop over all current mappings setting/clearing as appropos If
 2718          * setting RO do we need to clear the VAC?
 2719          */
 2720         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2721                 /*
 2722                  * don't write protect pager mappings
 2723                  */
 2724                 if (bit == PG_RW) {
 2725                         if (!pmap_track_modified(pv->pv_va))
 2726                                 continue;
 2727                 }
 2728 
 2729                 PMAP_LOCK(pv->pv_pmap);
 2730                 pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
 2731 retry:
 2732                 pbits = *pte;
 2733                 if (pbits & bit) {
 2734                         if (bit == PG_RW) {
 2735                                 /*
 2736                                  * Regardless of whether a pte is 32 or 64 bits
 2737                                  * in size, PG_RW and PG_M are among the least
 2738                                  * significant 32 bits.
 2739                                  */
 2740                                 if (!atomic_cmpset_int((u_int *)pte, pbits,
 2741                                     pbits & ~(PG_RW | PG_M)))
 2742                                         goto retry;
 2743                                 if (pbits & PG_M) {
 2744                                         vm_page_dirty(m);
 2745                                 }
 2746                         } else {
 2747                                 atomic_clear_int((u_int *)pte, bit);
 2748                         }
 2749                         pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 2750                 }
 2751                 PMAP_UNLOCK(pv->pv_pmap);
 2752         }
 2753         if (bit == PG_RW)
 2754                 vm_page_flag_clear(m, PG_WRITEABLE);
 2755         sched_unpin();
 2756 }
 2757 
 2758 /*
 2759  *      pmap_page_protect:
 2760  *
 2761  *      Lower the permission for all mappings to a given page.
 2762  */
 2763 void
 2764 pmap_page_protect(vm_page_t m, vm_prot_t prot)
 2765 {
 2766         if ((prot & VM_PROT_WRITE) == 0) {
 2767                 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) {
 2768                         pmap_clear_ptes(m, PG_RW);
 2769                 } else {
 2770                         pmap_remove_all(m);
 2771                 }
 2772         }
 2773 }
 2774 
 2775 /*
 2776  *      pmap_ts_referenced:
 2777  *
 2778  *      Return a count of reference bits for a page, clearing those bits.
 2779  *      It is not necessary for every reference bit to be cleared, but it
 2780  *      is necessary that 0 only be returned when there are truly no
 2781  *      reference bits set.
 2782  *
 2783  *      XXX: The exact number of bits to check and clear is a matter that
 2784  *      should be tested and standardized at some point in the future for
 2785  *      optimal aging of shared pages.
 2786  */
 2787 int
 2788 pmap_ts_referenced(vm_page_t m)
 2789 {
 2790         register pv_entry_t pv, pvf, pvn;
 2791         pt_entry_t *pte;
 2792         pt_entry_t v;
 2793         int rtval = 0;
 2794 
 2795         if (m->flags & PG_FICTITIOUS)
 2796                 return (rtval);
 2797 
 2798         sched_pin();
 2799         mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 2800         if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 2801 
 2802                 pvf = pv;
 2803 
 2804                 do {
 2805                         pvn = TAILQ_NEXT(pv, pv_list);
 2806 
 2807                         TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2808 
 2809                         TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2810 
 2811                         if (!pmap_track_modified(pv->pv_va))
 2812                                 continue;
 2813 
 2814                         PMAP_LOCK(pv->pv_pmap);
 2815                         pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va);
 2816 
 2817                         if (pte && ((v = pte_load(pte)) & PG_A) != 0) {
 2818                                 atomic_clear_int((u_int *)pte, PG_A);
 2819                                 pmap_invalidate_page(pv->pv_pmap, pv->pv_va);
 2820 
 2821                                 rtval++;
 2822                                 if (rtval > 4) {
 2823                                         PMAP_UNLOCK(pv->pv_pmap);
 2824                                         break;
 2825                                 }
 2826                         }
 2827                         PMAP_UNLOCK(pv->pv_pmap);
 2828                 } while ((pv = pvn) != NULL && pv != pvf);
 2829         }
 2830         sched_unpin();
 2831 
 2832         return (rtval);
 2833 }
 2834 
 2835 /*
 2836  *      Clear the modify bits on the specified physical page.
 2837  */
 2838 void
 2839 pmap_clear_modify(vm_page_t m)
 2840 {
 2841         pmap_clear_ptes(m, PG_M);
 2842 }
 2843 
 2844 /*
 2845  *      pmap_clear_reference:
 2846  *
 2847  *      Clear the reference bit on the specified physical page.
 2848  */
 2849 void
 2850 pmap_clear_reference(vm_page_t m)
 2851 {
 2852         pmap_clear_ptes(m, PG_A);
 2853 }
 2854 
 2855 /*
 2856  * Miscellaneous support routines follow
 2857  */
 2858 
 2859 /*
 2860  * Map a set of physical memory pages into the kernel virtual
 2861  * address space. Return a pointer to where it is mapped. This
 2862  * routine is intended to be used for mapping device memory,
 2863  * NOT real memory.
 2864  */
 2865 void *
 2866 pmap_mapdev(pa, size)
 2867         vm_paddr_t pa;
 2868         vm_size_t size;
 2869 {
 2870         vm_offset_t va, tmpva, offset;
 2871 
 2872         offset = pa & PAGE_MASK;
 2873         size = roundup(offset + size, PAGE_SIZE);
 2874         pa = pa & PG_FRAME;
 2875 
 2876         if (pa < KERNLOAD && pa + size <= KERNLOAD)
 2877                 va = KERNBASE + pa;
 2878         else
 2879                 va = kmem_alloc_nofault(kernel_map, size);
 2880         if (!va)
 2881                 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 2882 
 2883         for (tmpva = va; size > 0; ) {
 2884                 pmap_kenter(tmpva, pa);
 2885                 size -= PAGE_SIZE;
 2886                 tmpva += PAGE_SIZE;
 2887                 pa += PAGE_SIZE;
 2888         }
 2889         pmap_invalidate_range(kernel_pmap, va, tmpva);
 2890         return ((void *)(va + offset));
 2891 }
 2892 
 2893 void
 2894 pmap_unmapdev(va, size)
 2895         vm_offset_t va;
 2896         vm_size_t size;
 2897 {
 2898         vm_offset_t base, offset, tmpva;
 2899 
 2900         if (va >= KERNBASE && va + size <= KERNBASE + KERNLOAD)
 2901                 return;
 2902         base = va & PG_FRAME;
 2903         offset = va & PAGE_MASK;
 2904         size = roundup(offset + size, PAGE_SIZE);
 2905         for (tmpva = base; tmpva < (base + size); tmpva += PAGE_SIZE)
 2906                 pmap_kremove(tmpva);
 2907         pmap_invalidate_range(kernel_pmap, va, tmpva);
 2908         kmem_free(kernel_map, base, size);
 2909 }
 2910 
 2911 /*
 2912  * perform the pmap work for mincore
 2913  */
 2914 int
 2915 pmap_mincore(pmap, addr)
 2916         pmap_t pmap;
 2917         vm_offset_t addr;
 2918 {
 2919         pt_entry_t *ptep, pte;
 2920         vm_page_t m;
 2921         int val = 0;
 2922         
 2923         PMAP_LOCK(pmap);
 2924         ptep = pmap_pte(pmap, addr);
 2925         pte = (ptep != NULL) ? *ptep : 0;
 2926         pmap_pte_release(ptep);
 2927         PMAP_UNLOCK(pmap);
 2928 
 2929         if (pte != 0) {
 2930                 vm_paddr_t pa;
 2931 
 2932                 val = MINCORE_INCORE;
 2933                 if ((pte & PG_MANAGED) == 0)
 2934                         return val;
 2935 
 2936                 pa = pte & PG_FRAME;
 2937 
 2938                 m = PHYS_TO_VM_PAGE(pa);
 2939 
 2940                 /*
 2941                  * Modified by us
 2942                  */
 2943                 if (pte & PG_M)
 2944                         val |= MINCORE_MODIFIED|MINCORE_MODIFIED_OTHER;
 2945                 else {
 2946                         /*
 2947                          * Modified by someone else
 2948                          */
 2949                         vm_page_lock_queues();
 2950                         if (m->dirty || pmap_is_modified(m))
 2951                                 val |= MINCORE_MODIFIED_OTHER;
 2952                         vm_page_unlock_queues();
 2953                 }
 2954                 /*
 2955                  * Referenced by us
 2956                  */
 2957                 if (pte & PG_A)
 2958                         val |= MINCORE_REFERENCED|MINCORE_REFERENCED_OTHER;
 2959                 else {
 2960                         /*
 2961                          * Referenced by someone else
 2962                          */
 2963                         vm_page_lock_queues();
 2964                         if ((m->flags & PG_REFERENCED) ||
 2965                             pmap_ts_referenced(m)) {
 2966                                 val |= MINCORE_REFERENCED_OTHER;
 2967                                 vm_page_flag_set(m, PG_REFERENCED);
 2968                         }
 2969                         vm_page_unlock_queues();
 2970                 }
 2971         } 
 2972         return val;
 2973 }
 2974 
 2975 void
 2976 pmap_activate(struct thread *td)
 2977 {
 2978         struct proc *p = td->td_proc;
 2979         pmap_t  pmap, oldpmap;
 2980         u_int32_t  cr3;
 2981 
 2982         critical_enter();
 2983         pmap = vmspace_pmap(td->td_proc->p_vmspace);
 2984         oldpmap = PCPU_GET(curpmap);
 2985 #if defined(SMP)
 2986         atomic_clear_int(&oldpmap->pm_active, PCPU_GET(cpumask));
 2987         atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask));
 2988 #else
 2989         oldpmap->pm_active &= ~1;
 2990         pmap->pm_active |= 1;
 2991 #endif
 2992 #ifdef PAE
 2993         cr3 = vtophys(pmap->pm_pdpt);
 2994 #else
 2995         cr3 = vtophys(pmap->pm_pdir);
 2996 #endif
 2997         /* XXXKSE this is wrong.
 2998          * pmap_activate is for the current thread on the current cpu
 2999          */
 3000         if (p->p_flag & P_SA) {
 3001                 /* Make sure all other cr3 entries are updated. */
 3002                 /* what if they are running?  XXXKSE (maybe abort them) */
 3003                 FOREACH_THREAD_IN_PROC(p, td) {
 3004                         td->td_pcb->pcb_cr3 = cr3;
 3005                 }
 3006         } else {
 3007                 td->td_pcb->pcb_cr3 = cr3;
 3008         }
 3009         load_cr3(cr3);
 3010         PCPU_SET(curpmap, pmap);
 3011         critical_exit();
 3012 }
 3013 
 3014 vm_offset_t
 3015 pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
 3016 {
 3017 
 3018         if ((obj == NULL) || (size < NBPDR) || (obj->type != OBJT_DEVICE)) {
 3019                 return addr;
 3020         }
 3021 
 3022         addr = (addr + PDRMASK) & ~PDRMASK;
 3023         return addr;
 3024 }
 3025 
 3026 
 3027 #if defined(PMAP_DEBUG)
 3028 pmap_pid_dump(int pid)
 3029 {
 3030         pmap_t pmap;
 3031         struct proc *p;
 3032         int npte = 0;
 3033         int index;
 3034 
 3035         sx_slock(&allproc_lock);
 3036         LIST_FOREACH(p, &allproc, p_list) {
 3037                 if (p->p_pid != pid)
 3038                         continue;
 3039 
 3040                 if (p->p_vmspace) {
 3041                         int i,j;
 3042                         index = 0;
 3043                         pmap = vmspace_pmap(p->p_vmspace);
 3044                         for (i = 0; i < NPDEPTD; i++) {
 3045                                 pd_entry_t *pde;
 3046                                 pt_entry_t *pte;
 3047                                 vm_offset_t base = i << PDRSHIFT;
 3048                                 
 3049                                 pde = &pmap->pm_pdir[i];
 3050                                 if (pde && pmap_pde_v(pde)) {
 3051                                         for (j = 0; j < NPTEPG; j++) {
 3052                                                 vm_offset_t va = base + (j << PAGE_SHIFT);
 3053                                                 if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) {
 3054                                                         if (index) {
 3055                                                                 index = 0;
 3056                                                                 printf("\n");
 3057                                                         }
 3058                                                         sx_sunlock(&allproc_lock);
 3059                                                         return npte;
 3060                                                 }
 3061                                                 pte = pmap_pte(pmap, va);
 3062                                                 if (pte && pmap_pte_v(pte)) {
 3063                                                         pt_entry_t pa;
 3064                                                         vm_page_t m;
 3065                                                         pa = *pte;
 3066                                                         m = PHYS_TO_VM_PAGE(pa);
 3067                                                         printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x",
 3068                                                                 va, pa, m->hold_count, m->wire_count, m->flags);
 3069                                                         npte++;
 3070                                                         index++;
 3071                                                         if (index >= 2) {
 3072                                                                 index = 0;
 3073                                                                 printf("\n");
 3074                                                         } else {
 3075                                                                 printf(" ");
 3076                                                         }
 3077                                                 }
 3078                                         }
 3079                                 }
 3080                         }
 3081                 }
 3082         }
 3083         sx_sunlock(&allproc_lock);
 3084         return npte;
 3085 }
 3086 #endif
 3087 
 3088 #if defined(DEBUG)
 3089 
 3090 static void     pads(pmap_t pm);
 3091 void            pmap_pvdump(vm_offset_t pa);
 3092 
 3093 /* print address space of pmap*/
 3094 static void
 3095 pads(pm)
 3096         pmap_t pm;
 3097 {
 3098         int i, j;
 3099         vm_paddr_t va;
 3100         pt_entry_t *ptep;
 3101 
 3102         if (pm == kernel_pmap)
 3103                 return;
 3104         for (i = 0; i < NPDEPTD; i++)
 3105                 if (pm->pm_pdir[i])
 3106                         for (j = 0; j < NPTEPG; j++) {
 3107                                 va = (i << PDRSHIFT) + (j << PAGE_SHIFT);
 3108                                 if (pm == kernel_pmap && va < KERNBASE)
 3109                                         continue;
 3110                                 if (pm != kernel_pmap && va > UPT_MAX_ADDRESS)
 3111                                         continue;
 3112                                 ptep = pmap_pte(pm, va);
 3113                                 if (pmap_pte_v(ptep))
 3114                                         printf("%x:%x ", va, *ptep);
 3115                         };
 3116 
 3117 }
 3118 
 3119 void
 3120 pmap_pvdump(pa)
 3121         vm_paddr_t pa;
 3122 {
 3123         pv_entry_t pv;
 3124         vm_page_t m;
 3125 
 3126         printf("pa %x", pa);
 3127         m = PHYS_TO_VM_PAGE(pa);
 3128         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3129                 printf(" -> pmap %p, va %x", (void *)pv->pv_pmap, pv->pv_va);
 3130                 pads(pv->pv_pmap);
 3131         }
 3132         printf(" ");
 3133 }
 3134 #endif

Cache object: 0c0760ade3535cbeb804898203ad2053


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.