The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/mips/mips/pmap.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-3-Clause
    3  *
    4  * Copyright (c) 1991 Regents of the University of California.
    5  * All rights reserved.
    6  * Copyright (c) 1994 John S. Dyson
    7  * All rights reserved.
    8  * Copyright (c) 1994 David Greenman
    9  * All rights reserved.
   10  *
   11  * This code is derived from software contributed to Berkeley by
   12  * the Systems Programming Group of the University of Utah Computer
   13  * Science Department and William Jolitz of UUNET Technologies Inc.
   14  *
   15  * Redistribution and use in source and binary forms, with or without
   16  * modification, are permitted provided that the following conditions
   17  * are met:
   18  * 1. Redistributions of source code must retain the above copyright
   19  *    notice, this list of conditions and the following disclaimer.
   20  * 2. Redistributions in binary form must reproduce the above copyright
   21  *    notice, this list of conditions and the following disclaimer in the
   22  *    documentation and/or other materials provided with the distribution.
   23  * 3. Neither the name of the University nor the names of its contributors
   24  *    may be used to endorse or promote products derived from this software
   25  *    without specific prior written permission.
   26  *
   27  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   28  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   29  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   30  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   31  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   32  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   33  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   34  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   35  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   36  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   37  * SUCH DAMAGE.
   38  *
   39  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
   40  *      from: src/sys/i386/i386/pmap.c,v 1.250.2.8 2000/11/21 00:09:14 ps
   41  *      JNPR: pmap.c,v 1.11.2.1 2007/08/16 11:51:06 girish
   42  */
   43 
   44 /*
   45  *      Manages physical address maps.
   46  *
   47  *      Since the information managed by this module is
   48  *      also stored by the logical address mapping module,
   49  *      this module may throw away valid virtual-to-physical
   50  *      mappings at almost any time.  However, invalidations
   51  *      of virtual-to-physical mappings must be done as
   52  *      requested.
   53  *
   54  *      In order to cope with hardware architectures which
   55  *      make virtual-to-physical map invalidates expensive,
   56  *      this module may delay invalidate or reduced protection
   57  *      operations until such time as they are actually
   58  *      necessary.  This module is given full information as
   59  *      to which processors are currently using which maps,
   60  *      and to when physical maps must be made correct.
   61  */
   62 
   63 #include <sys/cdefs.h>
   64 __FBSDID("$FreeBSD: stable/12/sys/mips/mips/pmap.c 370164 2021-07-21 13:25:33Z mjg $");
   65 
   66 #include "opt_ddb.h"
   67 #include "opt_pmap.h"
   68 
   69 #include <sys/param.h>
   70 #include <sys/systm.h>
   71 #include <sys/lock.h>
   72 #include <sys/mman.h>
   73 #include <sys/msgbuf.h>
   74 #include <sys/mutex.h>
   75 #include <sys/pcpu.h>
   76 #include <sys/proc.h>
   77 #include <sys/rwlock.h>
   78 #include <sys/sched.h>
   79 #include <sys/smp.h>
   80 #include <sys/sysctl.h>
   81 #include <sys/vmmeter.h>
   82 
   83 #ifdef DDB
   84 #include <ddb/ddb.h>
   85 #endif
   86 
   87 #include <vm/vm.h>
   88 #include <vm/vm_param.h>
   89 #include <vm/vm_kern.h>
   90 #include <vm/vm_page.h>
   91 #include <vm/vm_map.h>
   92 #include <vm/vm_object.h>
   93 #include <vm/vm_extern.h>
   94 #include <vm/vm_pageout.h>
   95 #include <vm/vm_pager.h>
   96 #include <vm/uma.h>
   97 
   98 #include <machine/cache.h>
   99 #include <machine/md_var.h>
  100 #include <machine/tlb.h>
  101 
  102 #undef PMAP_DEBUG
  103 
  104 #if !defined(DIAGNOSTIC)
  105 #define PMAP_INLINE __inline
  106 #else
  107 #define PMAP_INLINE
  108 #endif
  109 
  110 #ifdef PV_STATS
  111 #define PV_STAT(x)      do { x ; } while (0)
  112 #else
  113 #define PV_STAT(x)      do { } while (0)
  114 #endif
  115 
  116 /*
  117  * Get PDEs and PTEs for user/kernel address space
  118  */
  119 #define pmap_seg_index(v)       (((v) >> SEGSHIFT) & (NPDEPG - 1))
  120 #define pmap_pde_index(v)       (((v) >> PDRSHIFT) & (NPDEPG - 1))
  121 #define pmap_pte_index(v)       (((v) >> PAGE_SHIFT) & (NPTEPG - 1))
  122 #define pmap_pde_pindex(v)      ((v) >> PDRSHIFT)
  123 
  124 #ifdef __mips_n64
  125 #define NUPDE                   (NPDEPG * NPDEPG)
  126 #define NUSERPGTBLS             (NUPDE + NPDEPG)
  127 #else
  128 #define NUPDE                   (NPDEPG)
  129 #define NUSERPGTBLS             (NUPDE)
  130 #endif
  131 
  132 #define is_kernel_pmap(x)       ((x) == kernel_pmap)
  133 
  134 struct pmap kernel_pmap_store;
  135 pd_entry_t *kernel_segmap;
  136 
  137 vm_offset_t virtual_avail;      /* VA of first avail page (after kernel bss) */
  138 vm_offset_t virtual_end;        /* VA of last avail page (end of kernel AS) */
  139 
  140 static int nkpt;
  141 unsigned pmap_max_asid;         /* max ASID supported by the system */
  142 
  143 #define PMAP_ASID_RESERVED      0
  144 
  145 vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS;
  146 
  147 static void pmap_asid_alloc(pmap_t pmap);
  148 
  149 static struct rwlock_padalign pvh_global_lock;
  150 
  151 /*
  152  * Data for the pv entry allocation mechanism
  153  */
  154 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks);
  155 static int pv_entry_count;
  156 
  157 static void free_pv_chunk(struct pv_chunk *pc);
  158 static void free_pv_entry(pmap_t pmap, pv_entry_t pv);
  159 static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try);
  160 static vm_page_t pmap_pv_reclaim(pmap_t locked_pmap);
  161 static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
  162 static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
  163     vm_offset_t va);
  164 static vm_page_t pmap_alloc_direct_page(unsigned int index, int req);
  165 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
  166     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
  167 static void pmap_grow_direct_page(int req);
  168 static int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
  169     pd_entry_t pde);
  170 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va);
  171 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va);
  172 static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte,
  173     vm_offset_t va, vm_page_t m);
  174 static void pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte);
  175 static void pmap_invalidate_all(pmap_t pmap);
  176 static void pmap_invalidate_page(pmap_t pmap, vm_offset_t va);
  177 static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m);
  178 
  179 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags);
  180 static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, u_int flags);
  181 static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t);
  182 static pt_entry_t init_pte_prot(vm_page_t m, vm_prot_t access, vm_prot_t prot);
  183 
  184 static void pmap_invalidate_page_action(void *arg);
  185 static void pmap_invalidate_range_action(void *arg);
  186 static void pmap_update_page_action(void *arg);
  187 
  188 #ifndef __mips_n64
  189 /*
  190  * This structure is for high memory (memory above 512Meg in 32 bit) support.
  191  * The highmem area does not have a KSEG0 mapping, and we need a mechanism to
  192  * do temporary per-CPU mappings for pmap_zero_page, pmap_copy_page etc.
  193  *
  194  * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To
  195  * access a highmem physical address on a CPU, we map the physical address to
  196  * the reserved virtual address for the CPU in the kernel pagetable.  This is
  197  * done with interrupts disabled(although a spinlock and sched_pin would be
  198  * sufficient).
  199  */
  200 struct local_sysmaps {
  201         vm_offset_t     base;
  202         uint32_t        saved_intr;
  203         uint16_t        valid1, valid2;
  204 };
  205 static struct local_sysmaps sysmap_lmem[MAXCPU];
  206 
  207 static __inline void
  208 pmap_alloc_lmem_map(void)
  209 {
  210         int i;
  211 
  212         for (i = 0; i < MAXCPU; i++) {
  213                 sysmap_lmem[i].base = virtual_avail;
  214                 virtual_avail += PAGE_SIZE * 2;
  215                 sysmap_lmem[i].valid1 = sysmap_lmem[i].valid2 = 0;
  216         }
  217 }
  218 
  219 static __inline vm_offset_t
  220 pmap_lmem_map1(vm_paddr_t phys)
  221 {
  222         struct local_sysmaps *sysm;
  223         pt_entry_t *pte, npte;
  224         vm_offset_t va;
  225         uint32_t intr;
  226         int cpu;
  227 
  228         intr = intr_disable();
  229         cpu = PCPU_GET(cpuid);
  230         sysm = &sysmap_lmem[cpu];
  231         sysm->saved_intr = intr;
  232         va = sysm->base;
  233         npte = TLBLO_PA_TO_PFN(phys) | PTE_C_CACHE | PTE_D | PTE_V | PTE_G;
  234         pte = pmap_pte(kernel_pmap, va);
  235         *pte = npte;
  236         sysm->valid1 = 1;
  237         return (va);
  238 }
  239 
  240 static __inline vm_offset_t
  241 pmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2)
  242 {
  243         struct local_sysmaps *sysm;
  244         pt_entry_t *pte, npte;
  245         vm_offset_t va1, va2;
  246         uint32_t intr;
  247         int cpu;
  248 
  249         intr = intr_disable();
  250         cpu = PCPU_GET(cpuid);
  251         sysm = &sysmap_lmem[cpu];
  252         sysm->saved_intr = intr;
  253         va1 = sysm->base;
  254         va2 = sysm->base + PAGE_SIZE;
  255         npte = TLBLO_PA_TO_PFN(phys1) | PTE_C_CACHE | PTE_D | PTE_V | PTE_G;
  256         pte = pmap_pte(kernel_pmap, va1);
  257         *pte = npte;
  258         npte = TLBLO_PA_TO_PFN(phys2) | PTE_C_CACHE | PTE_D | PTE_V | PTE_G;
  259         pte = pmap_pte(kernel_pmap, va2);
  260         *pte = npte;
  261         sysm->valid1 = 1;
  262         sysm->valid2 = 1;
  263         return (va1);
  264 }
  265 
  266 static __inline void
  267 pmap_lmem_unmap(void)
  268 {
  269         struct local_sysmaps *sysm;
  270         pt_entry_t *pte;
  271         int cpu;
  272 
  273         cpu = PCPU_GET(cpuid);
  274         sysm = &sysmap_lmem[cpu];
  275         pte = pmap_pte(kernel_pmap, sysm->base);
  276         *pte = PTE_G;
  277         tlb_invalidate_address(kernel_pmap, sysm->base);
  278         sysm->valid1 = 0;
  279         if (sysm->valid2) {
  280                 pte = pmap_pte(kernel_pmap, sysm->base + PAGE_SIZE);
  281                 *pte = PTE_G;
  282                 tlb_invalidate_address(kernel_pmap, sysm->base + PAGE_SIZE);
  283                 sysm->valid2 = 0;
  284         }
  285         intr_restore(sysm->saved_intr);
  286 }
  287 #else  /* __mips_n64 */
  288 
  289 static __inline void
  290 pmap_alloc_lmem_map(void)
  291 {
  292 }
  293 
  294 static __inline vm_offset_t
  295 pmap_lmem_map1(vm_paddr_t phys)
  296 {
  297 
  298         return (0);
  299 }
  300 
  301 static __inline vm_offset_t
  302 pmap_lmem_map2(vm_paddr_t phys1, vm_paddr_t phys2)
  303 {
  304 
  305         return (0);
  306 }
  307 
  308 static __inline vm_offset_t
  309 pmap_lmem_unmap(void)
  310 {
  311 
  312         return (0);
  313 }
  314 #endif /* !__mips_n64 */
  315 
  316 static __inline int
  317 pmap_pte_cache_bits(vm_paddr_t pa, vm_page_t m)
  318 {
  319         vm_memattr_t ma;
  320 
  321         ma = pmap_page_get_memattr(m);
  322         if (ma == VM_MEMATTR_WRITE_BACK && !is_cacheable_mem(pa))
  323                 ma = VM_MEMATTR_UNCACHEABLE;
  324         return PTE_C(ma);
  325 }
  326 #define PMAP_PTE_SET_CACHE_BITS(pte, pa, m) {   \
  327         pte &= ~PTE_C_MASK;                     \
  328         pte |= pmap_pte_cache_bits(pa, m);      \
  329 }
  330 
  331 /*
  332  * Page table entry lookup routines.
  333  */
  334 static __inline pd_entry_t *
  335 pmap_segmap(pmap_t pmap, vm_offset_t va)
  336 {
  337 
  338         return (&pmap->pm_segtab[pmap_seg_index(va)]);
  339 }
  340 
  341 #ifdef __mips_n64
  342 static __inline pd_entry_t *
  343 pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va)
  344 {
  345         pd_entry_t *pde;
  346 
  347         pde = (pd_entry_t *)*pdpe;
  348         return (&pde[pmap_pde_index(va)]);
  349 }
  350 
  351 static __inline pd_entry_t *
  352 pmap_pde(pmap_t pmap, vm_offset_t va)
  353 {
  354         pd_entry_t *pdpe;
  355 
  356         pdpe = pmap_segmap(pmap, va);
  357         if (*pdpe == NULL)
  358                 return (NULL);
  359 
  360         return (pmap_pdpe_to_pde(pdpe, va));
  361 }
  362 #else
  363 static __inline pd_entry_t *
  364 pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_offset_t va)
  365 {
  366 
  367         return (pdpe);
  368 }
  369 
  370 static __inline
  371 pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va)
  372 {
  373 
  374         return (pmap_segmap(pmap, va));
  375 }
  376 #endif
  377 
  378 static __inline pt_entry_t *
  379 pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va)
  380 {
  381         pt_entry_t *pte;
  382 
  383         pte = (pt_entry_t *)*pde;
  384         return (&pte[pmap_pte_index(va)]);
  385 }
  386 
  387 pt_entry_t *
  388 pmap_pte(pmap_t pmap, vm_offset_t va)
  389 {
  390         pd_entry_t *pde;
  391 
  392         pde = pmap_pde(pmap, va);
  393         if (pde == NULL || *pde == NULL)
  394                 return (NULL);
  395 
  396         return (pmap_pde_to_pte(pde, va));
  397 }
  398 
  399 vm_offset_t
  400 pmap_steal_memory(vm_size_t size)
  401 {
  402         vm_paddr_t bank_size, pa;
  403         vm_offset_t va;
  404 
  405         size = round_page(size);
  406         bank_size = phys_avail[1] - phys_avail[0];
  407         while (size > bank_size) {
  408                 int i;
  409 
  410                 for (i = 0; phys_avail[i + 2]; i += 2) {
  411                         phys_avail[i] = phys_avail[i + 2];
  412                         phys_avail[i + 1] = phys_avail[i + 3];
  413                 }
  414                 phys_avail[i] = 0;
  415                 phys_avail[i + 1] = 0;
  416                 if (!phys_avail[0])
  417                         panic("pmap_steal_memory: out of memory");
  418                 bank_size = phys_avail[1] - phys_avail[0];
  419         }
  420 
  421         pa = phys_avail[0];
  422         phys_avail[0] += size;
  423         if (MIPS_DIRECT_MAPPABLE(pa) == 0)
  424                 panic("Out of memory below 512Meg?");
  425         va = MIPS_PHYS_TO_DIRECT(pa);
  426         bzero((caddr_t)va, size);
  427         return (va);
  428 }
  429 
  430 /*
  431  * Bootstrap the system enough to run with virtual memory.  This
  432  * assumes that the phys_avail array has been initialized.
  433  */
  434 static void
  435 pmap_create_kernel_pagetable(void)
  436 {
  437         int i, j;
  438         vm_offset_t ptaddr;
  439         pt_entry_t *pte;
  440 #ifdef __mips_n64
  441         pd_entry_t *pde;
  442         vm_offset_t pdaddr;
  443         int npt, npde;
  444 #endif
  445 
  446         /*
  447          * Allocate segment table for the kernel
  448          */
  449         kernel_segmap = (pd_entry_t *)pmap_steal_memory(PAGE_SIZE);
  450 
  451         /*
  452          * Allocate second level page tables for the kernel
  453          */
  454 #ifdef __mips_n64
  455         npde = howmany(NKPT, NPDEPG);
  456         pdaddr = pmap_steal_memory(PAGE_SIZE * npde);
  457 #endif
  458         nkpt = NKPT;
  459         ptaddr = pmap_steal_memory(PAGE_SIZE * nkpt);
  460 
  461         /*
  462          * The R[4-7]?00 stores only one copy of the Global bit in the
  463          * translation lookaside buffer for each 2 page entry. Thus invalid
  464          * entrys must have the Global bit set so when Entry LO and Entry HI
  465          * G bits are anded together they will produce a global bit to store
  466          * in the tlb.
  467          */
  468         for (i = 0, pte = (pt_entry_t *)ptaddr; i < (nkpt * NPTEPG); i++, pte++)
  469                 *pte = PTE_G;
  470 
  471 #ifdef __mips_n64
  472         for (i = 0,  npt = nkpt; npt > 0; i++) {
  473                 kernel_segmap[i] = (pd_entry_t)(pdaddr + i * PAGE_SIZE);
  474                 pde = (pd_entry_t *)kernel_segmap[i];
  475 
  476                 for (j = 0; j < NPDEPG && npt > 0; j++, npt--)
  477                         pde[j] = (pd_entry_t)(ptaddr + (i * NPDEPG + j) * PAGE_SIZE);
  478         }
  479 #else
  480         for (i = 0, j = pmap_seg_index(VM_MIN_KERNEL_ADDRESS); i < nkpt; i++, j++)
  481                 kernel_segmap[j] = (pd_entry_t)(ptaddr + (i * PAGE_SIZE));
  482 #endif
  483 
  484         PMAP_LOCK_INIT(kernel_pmap);
  485         kernel_pmap->pm_segtab = kernel_segmap;
  486         CPU_FILL(&kernel_pmap->pm_active);
  487         TAILQ_INIT(&kernel_pmap->pm_pvchunk);
  488         kernel_pmap->pm_asid[0].asid = PMAP_ASID_RESERVED;
  489         kernel_pmap->pm_asid[0].gen = 0;
  490         kernel_vm_end += nkpt * NPTEPG * PAGE_SIZE;
  491 }
  492 
  493 void
  494 pmap_bootstrap(void)
  495 {
  496         int i;
  497         int need_local_mappings = 0;
  498 
  499         /* Sort. */
  500 again:
  501         for (i = 0; phys_avail[i + 1] != 0; i += 2) {
  502                 /*
  503                  * Keep the memory aligned on page boundary.
  504                  */
  505                 phys_avail[i] = round_page(phys_avail[i]);
  506                 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
  507 
  508                 if (i < 2)
  509                         continue;
  510                 if (phys_avail[i - 2] > phys_avail[i]) {
  511                         vm_paddr_t ptemp[2];
  512 
  513                         ptemp[0] = phys_avail[i + 0];
  514                         ptemp[1] = phys_avail[i + 1];
  515 
  516                         phys_avail[i + 0] = phys_avail[i - 2];
  517                         phys_avail[i + 1] = phys_avail[i - 1];
  518 
  519                         phys_avail[i - 2] = ptemp[0];
  520                         phys_avail[i - 1] = ptemp[1];
  521                         goto again;
  522                 }
  523         }
  524 
  525         /*
  526          * In 32 bit, we may have memory which cannot be mapped directly.
  527          * This memory will need temporary mapping before it can be
  528          * accessed.
  529          */
  530         if (!MIPS_DIRECT_MAPPABLE(phys_avail[i - 1] - 1))
  531                 need_local_mappings = 1;
  532 
  533         /*
  534          * Copy the phys_avail[] array before we start stealing memory from it.
  535          */
  536         for (i = 0; phys_avail[i + 1] != 0; i += 2) {
  537                 physmem_desc[i] = phys_avail[i];
  538                 physmem_desc[i + 1] = phys_avail[i + 1];
  539         }
  540 
  541         Maxmem = atop(phys_avail[i - 1]);
  542 
  543         if (bootverbose) {
  544                 printf("Physical memory chunk(s):\n");
  545                 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
  546                         vm_paddr_t size;
  547 
  548                         size = phys_avail[i + 1] - phys_avail[i];
  549                         printf("%#08jx - %#08jx, %ju bytes (%ju pages)\n",
  550                             (uintmax_t) phys_avail[i],
  551                             (uintmax_t) phys_avail[i + 1] - 1,
  552                             (uintmax_t) size, (uintmax_t) size / PAGE_SIZE);
  553                 }
  554                 printf("Maxmem is 0x%0jx\n", ptoa((uintmax_t)Maxmem));
  555         }
  556         /*
  557          * Steal the message buffer from the beginning of memory.
  558          */
  559         msgbufp = (struct msgbuf *)pmap_steal_memory(msgbufsize);
  560         msgbufinit(msgbufp, msgbufsize);
  561 
  562         /*
  563          * Steal thread0 kstack.
  564          */
  565         kstack0 = pmap_steal_memory(KSTACK_PAGES << PAGE_SHIFT);
  566 
  567         virtual_avail = VM_MIN_KERNEL_ADDRESS;
  568         virtual_end = VM_MAX_KERNEL_ADDRESS;
  569 
  570 #ifdef SMP
  571         /*
  572          * Steal some virtual address space to map the pcpu area.
  573          */
  574         virtual_avail = roundup2(virtual_avail, PAGE_SIZE * 2);
  575         pcpup = (struct pcpu *)virtual_avail;
  576         virtual_avail += PAGE_SIZE * 2;
  577 
  578         /*
  579          * Initialize the wired TLB entry mapping the pcpu region for
  580          * the BSP at 'pcpup'. Up until this point we were operating
  581          * with the 'pcpup' for the BSP pointing to a virtual address
  582          * in KSEG0 so there was no need for a TLB mapping.
  583          */
  584         mips_pcpu_tlb_init(PCPU_ADDR(0));
  585 
  586         if (bootverbose)
  587                 printf("pcpu is available at virtual address %p.\n", pcpup);
  588 #endif
  589 
  590         if (need_local_mappings)
  591                 pmap_alloc_lmem_map();
  592         pmap_create_kernel_pagetable();
  593         pmap_max_asid = VMNUM_PIDS;
  594         mips_wr_entryhi(0);
  595         mips_wr_pagemask(0);
  596 
  597         /*
  598          * Initialize the global pv list lock.
  599          */
  600         rw_init(&pvh_global_lock, "pmap pv global");
  601 }
  602 
  603 /*
  604  * Initialize a vm_page's machine-dependent fields.
  605  */
  606 void
  607 pmap_page_init(vm_page_t m)
  608 {
  609 
  610         TAILQ_INIT(&m->md.pv_list);
  611         m->md.pv_flags = VM_MEMATTR_DEFAULT << PV_MEMATTR_SHIFT;
  612 }
  613 
  614 /*
  615  *      Initialize the pmap module.
  616  *      Called by vm_init, to initialize any structures that the pmap
  617  *      system needs to map virtual memory.
  618  */
  619 void
  620 pmap_init(void)
  621 {
  622 }
  623 
  624 /***************************************************
  625  * Low level helper routines.....
  626  ***************************************************/
  627 
  628 #ifdef  SMP
  629 static __inline void
  630 pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg)
  631 {
  632         int     cpuid, cpu, self;
  633         cpuset_t active_cpus;
  634 
  635         sched_pin();
  636         if (is_kernel_pmap(pmap)) {
  637                 smp_rendezvous(NULL, fn, NULL, arg);
  638                 goto out;
  639         }
  640         /* Force ASID update on inactive CPUs */
  641         CPU_FOREACH(cpu) {
  642                 if (!CPU_ISSET(cpu, &pmap->pm_active))
  643                         pmap->pm_asid[cpu].gen = 0;
  644         }
  645         cpuid = PCPU_GET(cpuid);
  646         /*
  647          * XXX: barrier/locking for active?
  648          *
  649          * Take a snapshot of active here, any further changes are ignored.
  650          * tlb update/invalidate should be harmless on inactive CPUs
  651          */
  652         active_cpus = pmap->pm_active;
  653         self = CPU_ISSET(cpuid, &active_cpus);
  654         CPU_CLR(cpuid, &active_cpus);
  655         /* Optimize for the case where this cpu is the only active one */
  656         if (CPU_EMPTY(&active_cpus)) {
  657                 if (self)
  658                         fn(arg);
  659         } else {
  660                 if (self)
  661                         CPU_SET(cpuid, &active_cpus);
  662                 smp_rendezvous_cpus(active_cpus, NULL, fn, NULL, arg);
  663         }
  664 out:
  665         sched_unpin();
  666 }
  667 #else /* !SMP */
  668 static __inline void
  669 pmap_call_on_active_cpus(pmap_t pmap, void (*fn)(void *), void *arg)
  670 {
  671         int     cpuid;
  672 
  673         if (is_kernel_pmap(pmap)) {
  674                 fn(arg);
  675                 return;
  676         }
  677         cpuid = PCPU_GET(cpuid);
  678         if (!CPU_ISSET(cpuid, &pmap->pm_active))
  679                 pmap->pm_asid[cpuid].gen = 0;
  680         else
  681                 fn(arg);
  682 }
  683 #endif /* SMP */
  684 
  685 static void
  686 pmap_invalidate_all(pmap_t pmap)
  687 {
  688 
  689         pmap_call_on_active_cpus(pmap,
  690             (void (*)(void *))tlb_invalidate_all_user, pmap);
  691 }
  692 
  693 struct pmap_invalidate_page_arg {
  694         pmap_t pmap;
  695         vm_offset_t va;
  696 };
  697 
  698 static void
  699 pmap_invalidate_page_action(void *arg)
  700 {
  701         struct pmap_invalidate_page_arg *p = arg;
  702 
  703         tlb_invalidate_address(p->pmap, p->va);
  704 }
  705 
  706 static void
  707 pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
  708 {
  709         struct pmap_invalidate_page_arg arg;
  710 
  711         arg.pmap = pmap;
  712         arg.va = va;
  713         pmap_call_on_active_cpus(pmap, pmap_invalidate_page_action, &arg);
  714 }
  715 
  716 struct pmap_invalidate_range_arg {
  717         pmap_t pmap;
  718         vm_offset_t sva;
  719         vm_offset_t eva;
  720 };
  721 
  722 static void
  723 pmap_invalidate_range_action(void *arg)
  724 {
  725         struct pmap_invalidate_range_arg *p = arg;
  726 
  727         tlb_invalidate_range(p->pmap, p->sva, p->eva);
  728 }
  729 
  730 static void
  731 pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
  732 {
  733         struct pmap_invalidate_range_arg arg;
  734 
  735         arg.pmap = pmap;
  736         arg.sva = sva;
  737         arg.eva = eva;
  738         pmap_call_on_active_cpus(pmap, pmap_invalidate_range_action, &arg);
  739 }
  740 
  741 struct pmap_update_page_arg {
  742         pmap_t pmap;
  743         vm_offset_t va;
  744         pt_entry_t pte;
  745 };
  746 
  747 static void
  748 pmap_update_page_action(void *arg)
  749 {
  750         struct pmap_update_page_arg *p = arg;
  751 
  752         tlb_update(p->pmap, p->va, p->pte);
  753 }
  754 
  755 static void
  756 pmap_update_page(pmap_t pmap, vm_offset_t va, pt_entry_t pte)
  757 {
  758         struct pmap_update_page_arg arg;
  759 
  760         arg.pmap = pmap;
  761         arg.va = va;
  762         arg.pte = pte;
  763         pmap_call_on_active_cpus(pmap, pmap_update_page_action, &arg);
  764 }
  765 
  766 /*
  767  *      Routine:        pmap_extract
  768  *      Function:
  769  *              Extract the physical page address associated
  770  *              with the given map/virtual_address pair.
  771  */
  772 vm_paddr_t
  773 pmap_extract(pmap_t pmap, vm_offset_t va)
  774 {
  775         pt_entry_t *pte;
  776         vm_offset_t retval = 0;
  777 
  778         PMAP_LOCK(pmap);
  779         pte = pmap_pte(pmap, va);
  780         if (pte) {
  781                 retval = TLBLO_PTE_TO_PA(*pte) | (va & PAGE_MASK);
  782         }
  783         PMAP_UNLOCK(pmap);
  784         return (retval);
  785 }
  786 
  787 /*
  788  *      Routine:        pmap_extract_and_hold
  789  *      Function:
  790  *              Atomically extract and hold the physical page
  791  *              with the given pmap and virtual address pair
  792  *              if that mapping permits the given protection.
  793  */
  794 vm_page_t
  795 pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
  796 {
  797         pt_entry_t pte, *ptep;
  798         vm_paddr_t pa, pte_pa;
  799         vm_page_t m;
  800 
  801         m = NULL;
  802         pa = 0;
  803         PMAP_LOCK(pmap);
  804 retry:
  805         ptep = pmap_pte(pmap, va);
  806         if (ptep != NULL) {
  807                 pte = *ptep;
  808                 if (pte_test(&pte, PTE_V) && (!pte_test(&pte, PTE_RO) ||
  809                     (prot & VM_PROT_WRITE) == 0)) {
  810                         pte_pa = TLBLO_PTE_TO_PA(pte);
  811                         if (vm_page_pa_tryrelock(pmap, pte_pa, &pa))
  812                                 goto retry;
  813                         m = PHYS_TO_VM_PAGE(pte_pa);
  814                         vm_page_hold(m);
  815                 }
  816         }
  817         PA_UNLOCK_COND(pa);
  818         PMAP_UNLOCK(pmap);
  819         return (m);
  820 }
  821 
  822 /***************************************************
  823  * Low level mapping routines.....
  824  ***************************************************/
  825 
  826 /*
  827  * add a wired page to the kva
  828  */
  829 void
  830 pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
  831 {
  832         pt_entry_t *pte;
  833         pt_entry_t opte, npte;
  834 
  835 #ifdef PMAP_DEBUG
  836         printf("pmap_kenter:  va: %p -> pa: %p\n", (void *)va, (void *)pa);
  837 #endif
  838 
  839         pte = pmap_pte(kernel_pmap, va);
  840         opte = *pte;
  841         npte = TLBLO_PA_TO_PFN(pa) | PTE_C(ma) | PTE_D | PTE_V | PTE_G;
  842         *pte = npte;
  843         if (pte_test(&opte, PTE_V) && opte != npte)
  844                 pmap_update_page(kernel_pmap, va, npte);
  845 }
  846 
  847 void
  848 pmap_kenter(vm_offset_t va, vm_paddr_t pa)
  849 {
  850 
  851         KASSERT(is_cacheable_mem(pa),
  852                 ("pmap_kenter: memory at 0x%lx is not cacheable", (u_long)pa));
  853 
  854         pmap_kenter_attr(va, pa, VM_MEMATTR_DEFAULT);
  855 }
  856 
  857 void
  858 pmap_kenter_device(vm_offset_t va, vm_size_t size, vm_paddr_t pa)
  859 {
  860 
  861         KASSERT((size & PAGE_MASK) == 0,
  862             ("%s: device mapping not page-sized", __func__));
  863 
  864         for (; size > 0; size -= PAGE_SIZE) {
  865                 /*
  866                  * XXXCEM: this is somewhat inefficient on SMP systems in that
  867                  * every single page is individually TLB-invalidated via
  868                  * rendezvous (pmap_update_page()), instead of invalidating the
  869                  * entire range via a single rendezvous.
  870                  */
  871                 pmap_kenter_attr(va, pa, VM_MEMATTR_UNCACHEABLE);
  872                 va += PAGE_SIZE;
  873                 pa += PAGE_SIZE;
  874         }
  875 }
  876 
  877 void
  878 pmap_kremove_device(vm_offset_t va, vm_size_t size)
  879 {
  880 
  881         KASSERT((size & PAGE_MASK) == 0,
  882             ("%s: device mapping not page-sized", __func__));
  883 
  884         /*
  885          * XXXCEM: Similar to pmap_kenter_device, this is inefficient on SMP,
  886          * in that pages are invalidated individually instead of a single range
  887          * rendezvous.
  888          */
  889         for (; size > 0; size -= PAGE_SIZE) {
  890                 pmap_kremove(va);
  891                 va += PAGE_SIZE;
  892         }
  893 }
  894 
  895 /*
  896  * remove a page from the kernel pagetables
  897  */
  898  /* PMAP_INLINE */ void
  899 pmap_kremove(vm_offset_t va)
  900 {
  901         pt_entry_t *pte;
  902 
  903         /*
  904          * Write back all caches from the page being destroyed
  905          */
  906         mips_dcache_wbinv_range_index(va, PAGE_SIZE);
  907 
  908         pte = pmap_pte(kernel_pmap, va);
  909         *pte = PTE_G;
  910         pmap_invalidate_page(kernel_pmap, va);
  911 }
  912 
  913 /*
  914  *      Used to map a range of physical addresses into kernel
  915  *      virtual address space.
  916  *
  917  *      The value passed in '*virt' is a suggested virtual address for
  918  *      the mapping. Architectures which can support a direct-mapped
  919  *      physical to virtual region can return the appropriate address
  920  *      within that region, leaving '*virt' unchanged. Other
  921  *      architectures should map the pages starting at '*virt' and
  922  *      update '*virt' with the first usable address after the mapped
  923  *      region.
  924  *
  925  *      Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit.
  926  */
  927 vm_offset_t
  928 pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot)
  929 {
  930         vm_offset_t va, sva;
  931 
  932         if (MIPS_DIRECT_MAPPABLE(end - 1))
  933                 return (MIPS_PHYS_TO_DIRECT(start));
  934 
  935         va = sva = *virt;
  936         while (start < end) {
  937                 pmap_kenter(va, start);
  938                 va += PAGE_SIZE;
  939                 start += PAGE_SIZE;
  940         }
  941         *virt = va;
  942         return (sva);
  943 }
  944 
  945 /*
  946  * Add a list of wired pages to the kva
  947  * this routine is only used for temporary
  948  * kernel mappings that do not need to have
  949  * page modification or references recorded.
  950  * Note that old mappings are simply written
  951  * over.  The page *must* be wired.
  952  */
  953 void
  954 pmap_qenter(vm_offset_t va, vm_page_t *m, int count)
  955 {
  956         int i;
  957         vm_offset_t origva = va;
  958 
  959         for (i = 0; i < count; i++) {
  960                 pmap_flush_pvcache(m[i]);
  961                 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i]));
  962                 va += PAGE_SIZE;
  963         }
  964 
  965         mips_dcache_wbinv_range_index(origva, PAGE_SIZE*count);
  966 }
  967 
  968 /*
  969  * this routine jerks page mappings from the
  970  * kernel -- it is meant only for temporary mappings.
  971  */
  972 void
  973 pmap_qremove(vm_offset_t va, int count)
  974 {
  975         pt_entry_t *pte;
  976         vm_offset_t origva;
  977 
  978         if (count < 1)
  979                 return;
  980         mips_dcache_wbinv_range_index(va, PAGE_SIZE * count);
  981         origva = va;
  982         do {
  983                 pte = pmap_pte(kernel_pmap, va);
  984                 *pte = PTE_G;
  985                 va += PAGE_SIZE;
  986         } while (--count > 0);
  987         pmap_invalidate_range(kernel_pmap, origva, va);
  988 }
  989 
  990 /***************************************************
  991  * Page table page management routines.....
  992  ***************************************************/
  993 
  994 /*
  995  * Decrements a page table page's wire count, which is used to record the
  996  * number of valid page table entries within the page.  If the wire count
  997  * drops to zero, then the page table page is unmapped.  Returns TRUE if the
  998  * page table page was unmapped and FALSE otherwise.
  999  */
 1000 static PMAP_INLINE boolean_t
 1001 pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1002 {
 1003 
 1004         --m->wire_count;
 1005         if (m->wire_count == 0) {
 1006                 _pmap_unwire_ptp(pmap, va, m);
 1007                 return (TRUE);
 1008         } else
 1009                 return (FALSE);
 1010 }
 1011 
 1012 static void
 1013 _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m)
 1014 {
 1015         pd_entry_t *pde;
 1016         vm_offset_t sva, eva;
 1017 
 1018         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1019         /*
 1020          * unmap the page table page
 1021          */
 1022 #ifdef __mips_n64
 1023         if (m->pindex < NUPDE) {
 1024                 pde = pmap_pde(pmap, va);
 1025                 sva = va & ~PDRMASK;
 1026                 eva = sva + NBPDR;
 1027         } else {
 1028                 pde = pmap_segmap(pmap, va);
 1029                 sva = va & ~SEGMASK;
 1030                 eva = sva + NBSEG;
 1031         }
 1032 #else
 1033         pde = pmap_pde(pmap, va);
 1034         sva = va & ~SEGMASK;
 1035         eva = sva + NBSEG;
 1036 #endif
 1037         *pde = 0;
 1038         pmap->pm_stats.resident_count--;
 1039 
 1040 #ifdef __mips_n64
 1041         if (m->pindex < NUPDE) {
 1042                 pd_entry_t *pdp;
 1043                 vm_page_t pdpg;
 1044 
 1045                 /*
 1046                  * Recursively decrement next level pagetable refcount.
 1047                  * Either that shoots down a larger range from TLBs (below)
 1048                  * or we're to shoot down just the page in question.
 1049                  */
 1050                 pdp = (pd_entry_t *)*pmap_segmap(pmap, va);
 1051                 pdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pdp));
 1052                 if (!pmap_unwire_ptp(pmap, va, pdpg)) {
 1053                         pmap_invalidate_range(pmap, sva, eva);
 1054                 }
 1055         } else {
 1056                 /* Segmap entry shootdown */
 1057                 pmap_invalidate_range(pmap, sva, eva);
 1058         }
 1059 #else
 1060         /* Segmap entry shootdown */
 1061         pmap_invalidate_range(pmap, sva, eva);
 1062 #endif
 1063 
 1064         /*
 1065          * If the page is finally unwired, simply free it.
 1066          */
 1067         vm_page_free_zero(m);
 1068         vm_wire_sub(1);
 1069 }
 1070 
 1071 /*
 1072  * After removing a page table entry, this routine is used to
 1073  * conditionally free the page, and manage the hold/wire counts.
 1074  */
 1075 static int
 1076 pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t pde)
 1077 {
 1078         vm_page_t mpte;
 1079 
 1080         if (va >= VM_MAXUSER_ADDRESS)
 1081                 return (0);
 1082         KASSERT(pde != 0, ("pmap_unuse_pt: pde != 0"));
 1083         mpte = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(pde));
 1084         return (pmap_unwire_ptp(pmap, va, mpte));
 1085 }
 1086 
 1087 void
 1088 pmap_pinit0(pmap_t pmap)
 1089 {
 1090         int i;
 1091 
 1092         PMAP_LOCK_INIT(pmap);
 1093         pmap->pm_segtab = kernel_segmap;
 1094         CPU_ZERO(&pmap->pm_active);
 1095         for (i = 0; i < MAXCPU; i++) {
 1096                 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED;
 1097                 pmap->pm_asid[i].gen = 0;
 1098         }
 1099         PCPU_SET(curpmap, pmap);
 1100         TAILQ_INIT(&pmap->pm_pvchunk);
 1101         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1102 }
 1103 
 1104 static void
 1105 pmap_grow_direct_page(int req)
 1106 {
 1107 
 1108 #ifdef __mips_n64
 1109         vm_wait(NULL);
 1110 #else
 1111         if (!vm_page_reclaim_contig(req, 1, 0, MIPS_KSEG0_LARGEST_PHYS,
 1112             PAGE_SIZE, 0))
 1113                 vm_wait(NULL);
 1114 #endif
 1115 }
 1116 
 1117 static vm_page_t
 1118 pmap_alloc_direct_page(unsigned int index, int req)
 1119 {
 1120         vm_page_t m;
 1121 
 1122         m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_WIRED |
 1123             VM_ALLOC_ZERO);
 1124         if (m == NULL)
 1125                 return (NULL);
 1126 
 1127         if ((m->flags & PG_ZERO) == 0)
 1128                 pmap_zero_page(m);
 1129 
 1130         m->pindex = index;
 1131         return (m);
 1132 }
 1133 
 1134 /*
 1135  * Initialize a preallocated and zeroed pmap structure,
 1136  * such as one in a vmspace structure.
 1137  */
 1138 int
 1139 pmap_pinit(pmap_t pmap)
 1140 {
 1141         vm_offset_t ptdva;
 1142         vm_page_t ptdpg;
 1143         int i, req_class;
 1144 
 1145         /*
 1146          * allocate the page directory page
 1147          */
 1148         req_class = VM_ALLOC_NORMAL;
 1149         while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, req_class)) ==
 1150             NULL)
 1151                 pmap_grow_direct_page(req_class);
 1152 
 1153         ptdva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(ptdpg));
 1154         pmap->pm_segtab = (pd_entry_t *)ptdva;
 1155         CPU_ZERO(&pmap->pm_active);
 1156         for (i = 0; i < MAXCPU; i++) {
 1157                 pmap->pm_asid[i].asid = PMAP_ASID_RESERVED;
 1158                 pmap->pm_asid[i].gen = 0;
 1159         }
 1160         TAILQ_INIT(&pmap->pm_pvchunk);
 1161         bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
 1162 
 1163         return (1);
 1164 }
 1165 
 1166 /*
 1167  * this routine is called if the page table page is not
 1168  * mapped correctly.
 1169  */
 1170 static vm_page_t
 1171 _pmap_allocpte(pmap_t pmap, unsigned ptepindex, u_int flags)
 1172 {
 1173         vm_offset_t pageva;
 1174         vm_page_t m;
 1175         int req_class;
 1176 
 1177         /*
 1178          * Find or fabricate a new pagetable page
 1179          */
 1180         req_class = VM_ALLOC_NORMAL;
 1181         if ((m = pmap_alloc_direct_page(ptepindex, req_class)) == NULL) {
 1182                 if ((flags & PMAP_ENTER_NOSLEEP) == 0) {
 1183                         PMAP_UNLOCK(pmap);
 1184                         rw_wunlock(&pvh_global_lock);
 1185                         pmap_grow_direct_page(req_class);
 1186                         rw_wlock(&pvh_global_lock);
 1187                         PMAP_LOCK(pmap);
 1188                 }
 1189 
 1190                 /*
 1191                  * Indicate the need to retry.  While waiting, the page
 1192                  * table page may have been allocated.
 1193                  */
 1194                 return (NULL);
 1195         }
 1196 
 1197         /*
 1198          * Map the pagetable page into the process address space, if it
 1199          * isn't already there.
 1200          */
 1201         pageva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m));
 1202 
 1203 #ifdef __mips_n64
 1204         if (ptepindex >= NUPDE) {
 1205                 pmap->pm_segtab[ptepindex - NUPDE] = (pd_entry_t)pageva;
 1206         } else {
 1207                 pd_entry_t *pdep, *pde;
 1208                 int segindex = ptepindex >> (SEGSHIFT - PDRSHIFT);
 1209                 int pdeindex = ptepindex & (NPDEPG - 1);
 1210                 vm_page_t pg;
 1211 
 1212                 pdep = &pmap->pm_segtab[segindex];
 1213                 if (*pdep == NULL) {
 1214                         /* recurse for allocating page dir */
 1215                         if (_pmap_allocpte(pmap, NUPDE + segindex,
 1216                             flags) == NULL) {
 1217                                 /* alloc failed, release current */
 1218                                 vm_page_unwire_noq(m);
 1219                                 vm_page_free_zero(m);
 1220                                 return (NULL);
 1221                         }
 1222                 } else {
 1223                         pg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pdep));
 1224                         pg->wire_count++;
 1225                 }
 1226                 /* Next level entry */
 1227                 pde = (pd_entry_t *)*pdep;
 1228                 pde[pdeindex] = (pd_entry_t)pageva;
 1229         }
 1230 #else
 1231         pmap->pm_segtab[ptepindex] = (pd_entry_t)pageva;
 1232 #endif
 1233         pmap->pm_stats.resident_count++;
 1234         return (m);
 1235 }
 1236 
 1237 static vm_page_t
 1238 pmap_allocpte(pmap_t pmap, vm_offset_t va, u_int flags)
 1239 {
 1240         unsigned ptepindex;
 1241         pd_entry_t *pde;
 1242         vm_page_t m;
 1243 
 1244         /*
 1245          * Calculate pagetable page index
 1246          */
 1247         ptepindex = pmap_pde_pindex(va);
 1248 retry:
 1249         /*
 1250          * Get the page directory entry
 1251          */
 1252         pde = pmap_pde(pmap, va);
 1253 
 1254         /*
 1255          * If the page table page is mapped, we just increment the hold
 1256          * count, and activate it.
 1257          */
 1258         if (pde != NULL && *pde != NULL) {
 1259                 m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(*pde));
 1260                 m->wire_count++;
 1261         } else {
 1262                 /*
 1263                  * Here if the pte page isn't mapped, or if it has been
 1264                  * deallocated.
 1265                  */
 1266                 m = _pmap_allocpte(pmap, ptepindex, flags);
 1267                 if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0)
 1268                         goto retry;
 1269         }
 1270         return (m);
 1271 }
 1272 
 1273 
 1274 /***************************************************
 1275  * Pmap allocation/deallocation routines.
 1276  ***************************************************/
 1277 
 1278 /*
 1279  * Release any resources held by the given physical map.
 1280  * Called when a pmap initialized by pmap_pinit is being released.
 1281  * Should only be called if the map contains no valid mappings.
 1282  */
 1283 void
 1284 pmap_release(pmap_t pmap)
 1285 {
 1286         vm_offset_t ptdva;
 1287         vm_page_t ptdpg;
 1288 
 1289         KASSERT(pmap->pm_stats.resident_count == 0,
 1290             ("pmap_release: pmap resident count %ld != 0",
 1291             pmap->pm_stats.resident_count));
 1292 
 1293         ptdva = (vm_offset_t)pmap->pm_segtab;
 1294         ptdpg = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(ptdva));
 1295 
 1296         vm_page_unwire_noq(ptdpg);
 1297         vm_page_free_zero(ptdpg);
 1298 }
 1299 
 1300 /*
 1301  * grow the number of kernel page table entries, if needed
 1302  */
 1303 void
 1304 pmap_growkernel(vm_offset_t addr)
 1305 {
 1306         vm_page_t nkpg;
 1307         pd_entry_t *pde, *pdpe;
 1308         pt_entry_t *pte;
 1309         int i, req_class;
 1310 
 1311         mtx_assert(&kernel_map->system_mtx, MA_OWNED);
 1312         req_class = VM_ALLOC_INTERRUPT;
 1313         addr = roundup2(addr, NBSEG);
 1314         if (addr - 1 >= vm_map_max(kernel_map))
 1315                 addr = vm_map_max(kernel_map);
 1316         while (kernel_vm_end < addr) {
 1317                 pdpe = pmap_segmap(kernel_pmap, kernel_vm_end);
 1318 #ifdef __mips_n64
 1319                 if (*pdpe == 0) {
 1320                         /* new intermediate page table entry */
 1321                         nkpg = pmap_alloc_direct_page(nkpt, req_class);
 1322                         if (nkpg == NULL)
 1323                                 panic("pmap_growkernel: no memory to grow kernel");
 1324                         *pdpe = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg));
 1325                         continue; /* try again */
 1326                 }
 1327 #endif
 1328                 pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end);
 1329                 if (*pde != 0) {
 1330                         kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 1331                         if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
 1332                                 kernel_vm_end = vm_map_max(kernel_map);
 1333                                 break;
 1334                         }
 1335                         continue;
 1336                 }
 1337 
 1338                 /*
 1339                  * This index is bogus, but out of the way
 1340                  */
 1341                 nkpg = pmap_alloc_direct_page(nkpt, req_class);
 1342 #ifndef __mips_n64
 1343                 if (nkpg == NULL && vm_page_reclaim_contig(req_class, 1,
 1344                     0, MIPS_KSEG0_LARGEST_PHYS, PAGE_SIZE, 0))
 1345                         nkpg = pmap_alloc_direct_page(nkpt, req_class);
 1346 #endif
 1347                 if (nkpg == NULL)
 1348                         panic("pmap_growkernel: no memory to grow kernel");
 1349                 nkpt++;
 1350                 *pde = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg));
 1351 
 1352                 /*
 1353                  * The R[4-7]?00 stores only one copy of the Global bit in
 1354                  * the translation lookaside buffer for each 2 page entry.
 1355                  * Thus invalid entrys must have the Global bit set so when
 1356                  * Entry LO and Entry HI G bits are anded together they will
 1357                  * produce a global bit to store in the tlb.
 1358                  */
 1359                 pte = (pt_entry_t *)*pde;
 1360                 for (i = 0; i < NPTEPG; i++)
 1361                         pte[i] = PTE_G;
 1362 
 1363                 kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK;
 1364                 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
 1365                         kernel_vm_end = vm_map_max(kernel_map);
 1366                         break;
 1367                 }
 1368         }
 1369 }
 1370 
 1371 /***************************************************
 1372  * page management routines.
 1373  ***************************************************/
 1374 
 1375 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE);
 1376 #ifdef __mips_n64
 1377 CTASSERT(_NPCM == 3);
 1378 CTASSERT(_NPCPV == 168);
 1379 #else
 1380 CTASSERT(_NPCM == 11);
 1381 CTASSERT(_NPCPV == 336);
 1382 #endif
 1383 
 1384 static __inline struct pv_chunk *
 1385 pv_to_chunk(pv_entry_t pv)
 1386 {
 1387 
 1388         return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK));
 1389 }
 1390 
 1391 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
 1392 
 1393 #ifdef __mips_n64
 1394 #define PC_FREE0_1      0xfffffffffffffffful
 1395 #define PC_FREE2        0x000000fffffffffful
 1396 #else
 1397 #define PC_FREE0_9      0xfffffffful    /* Free values for index 0 through 9 */
 1398 #define PC_FREE10       0x0000fffful    /* Free values for index 10 */
 1399 #endif
 1400 
 1401 static const u_long pc_freemask[_NPCM] = {
 1402 #ifdef __mips_n64
 1403         PC_FREE0_1, PC_FREE0_1, PC_FREE2
 1404 #else
 1405         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 1406         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 1407         PC_FREE0_9, PC_FREE0_9, PC_FREE0_9,
 1408         PC_FREE0_9, PC_FREE10
 1409 #endif
 1410 };
 1411 
 1412 static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
 1413 
 1414 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0,
 1415     "Current number of pv entries");
 1416 
 1417 #ifdef PV_STATS
 1418 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail;
 1419 
 1420 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0,
 1421     "Current number of pv entry chunks");
 1422 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0,
 1423     "Current number of pv entry chunks allocated");
 1424 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0,
 1425     "Current number of pv entry chunks frees");
 1426 SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0,
 1427     "Number of times tried to get a chunk page but failed.");
 1428 
 1429 static long pv_entry_frees, pv_entry_allocs;
 1430 static int pv_entry_spare;
 1431 
 1432 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0,
 1433     "Current number of pv entry frees");
 1434 SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0,
 1435     "Current number of pv entry allocs");
 1436 SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0,
 1437     "Current number of spare pv entries");
 1438 #endif
 1439 
 1440 /*
 1441  * We are in a serious low memory condition.  Resort to
 1442  * drastic measures to free some pages so we can allocate
 1443  * another pv entry chunk.
 1444  */
 1445 static vm_page_t
 1446 pmap_pv_reclaim(pmap_t locked_pmap)
 1447 {
 1448         struct pch newtail;
 1449         struct pv_chunk *pc;
 1450         pd_entry_t *pde;
 1451         pmap_t pmap;
 1452         pt_entry_t *pte, oldpte;
 1453         pv_entry_t pv;
 1454         vm_offset_t va;
 1455         vm_page_t m, m_pc;
 1456         u_long inuse;
 1457         int bit, field, freed, idx;
 1458 
 1459         PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED);
 1460         pmap = NULL;
 1461         m_pc = NULL;
 1462         TAILQ_INIT(&newtail);
 1463         while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL) {
 1464                 TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 1465                 if (pmap != pc->pc_pmap) {
 1466                         if (pmap != NULL) {
 1467                                 pmap_invalidate_all(pmap);
 1468                                 if (pmap != locked_pmap)
 1469                                         PMAP_UNLOCK(pmap);
 1470                         }
 1471                         pmap = pc->pc_pmap;
 1472                         /* Avoid deadlock and lock recursion. */
 1473                         if (pmap > locked_pmap)
 1474                                 PMAP_LOCK(pmap);
 1475                         else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) {
 1476                                 pmap = NULL;
 1477                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 1478                                 continue;
 1479                         }
 1480                 }
 1481 
 1482                 /*
 1483                  * Destroy every non-wired, 4 KB page mapping in the chunk.
 1484                  */
 1485                 freed = 0;
 1486                 for (field = 0; field < _NPCM; field++) {
 1487                         for (inuse = ~pc->pc_map[field] & pc_freemask[field];
 1488                             inuse != 0; inuse &= ~(1UL << bit)) {
 1489                                 bit = ffsl(inuse) - 1;
 1490                                 idx = field * sizeof(inuse) * NBBY + bit;
 1491                                 pv = &pc->pc_pventry[idx];
 1492                                 va = pv->pv_va;
 1493                                 pde = pmap_pde(pmap, va);
 1494                                 KASSERT(pde != NULL && *pde != 0,
 1495                                     ("pmap_pv_reclaim: pde"));
 1496                                 pte = pmap_pde_to_pte(pde, va);
 1497                                 oldpte = *pte;
 1498                                 if (pte_test(&oldpte, PTE_W))
 1499                                         continue;
 1500                                 if (is_kernel_pmap(pmap))
 1501                                         *pte = PTE_G;
 1502                                 else
 1503                                         *pte = 0;
 1504                                 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(oldpte));
 1505                                 if (pte_test(&oldpte, PTE_D))
 1506                                         vm_page_dirty(m);
 1507                                 if (m->md.pv_flags & PV_TABLE_REF)
 1508                                         vm_page_aflag_set(m, PGA_REFERENCED);
 1509                                 m->md.pv_flags &= ~PV_TABLE_REF;
 1510                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 1511                                 if (TAILQ_EMPTY(&m->md.pv_list))
 1512                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 1513                                 pc->pc_map[field] |= 1UL << bit;
 1514 
 1515                                 /*
 1516                                  * For simplicity, we will unconditionally shoot
 1517                                  * down TLBs either at the end of this function
 1518                                  * or at the top of the loop above if we switch
 1519                                  * to a different pmap.
 1520                                  */
 1521                                 (void)pmap_unuse_pt(pmap, va, *pde);
 1522 
 1523                                 freed++;
 1524                         }
 1525                 }
 1526                 if (freed == 0) {
 1527                         TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 1528                         continue;
 1529                 }
 1530                 /* Every freed mapping is for a 4 KB page. */
 1531                 pmap->pm_stats.resident_count -= freed;
 1532                 PV_STAT(pv_entry_frees += freed);
 1533                 PV_STAT(pv_entry_spare += freed);
 1534                 pv_entry_count -= freed;
 1535                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 1536                 for (field = 0; field < _NPCM; field++)
 1537                         if (pc->pc_map[field] != pc_freemask[field]) {
 1538                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 1539                                     pc_list);
 1540                                 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru);
 1541 
 1542                                 /*
 1543                                  * One freed pv entry in locked_pmap is
 1544                                  * sufficient.
 1545                                  */
 1546                                 if (pmap == locked_pmap)
 1547                                         goto out;
 1548                                 break;
 1549                         }
 1550                 if (field == _NPCM) {
 1551                         PV_STAT(pv_entry_spare -= _NPCPV);
 1552                         PV_STAT(pc_chunk_count--);
 1553                         PV_STAT(pc_chunk_frees++);
 1554                         /* Entire chunk is free; return it. */
 1555                         m_pc = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS(
 1556                             (vm_offset_t)pc));
 1557                         dump_drop_page(m_pc->phys_addr);
 1558                         break;
 1559                 }
 1560         }
 1561 out:
 1562         TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru);
 1563         if (pmap != NULL) {
 1564                 pmap_invalidate_all(pmap);
 1565                 if (pmap != locked_pmap)
 1566                         PMAP_UNLOCK(pmap);
 1567         }
 1568         return (m_pc);
 1569 }
 1570 
 1571 /*
 1572  * free the pv_entry back to the free list
 1573  */
 1574 static void
 1575 free_pv_entry(pmap_t pmap, pv_entry_t pv)
 1576 {
 1577         struct pv_chunk *pc;
 1578         int bit, field, idx;
 1579 
 1580         rw_assert(&pvh_global_lock, RA_WLOCKED);
 1581         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1582         PV_STAT(pv_entry_frees++);
 1583         PV_STAT(pv_entry_spare++);
 1584         pv_entry_count--;
 1585         pc = pv_to_chunk(pv);
 1586         idx = pv - &pc->pc_pventry[0];
 1587         field = idx / (sizeof(u_long) * NBBY);
 1588         bit = idx % (sizeof(u_long) * NBBY);
 1589         pc->pc_map[field] |= 1ul << bit;
 1590         for (idx = 0; idx < _NPCM; idx++)
 1591                 if (pc->pc_map[idx] != pc_freemask[idx]) {
 1592                         /*
 1593                          * 98% of the time, pc is already at the head of the
 1594                          * list.  If it isn't already, move it to the head.
 1595                          */
 1596                         if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) !=
 1597                             pc)) {
 1598                                 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 1599                                 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc,
 1600                                     pc_list);
 1601                         }
 1602                         return;
 1603                 }
 1604         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 1605         free_pv_chunk(pc);
 1606 }
 1607 
 1608 static void
 1609 free_pv_chunk(struct pv_chunk *pc)
 1610 {
 1611         vm_page_t m;
 1612 
 1613         TAILQ_REMOVE(&pv_chunks, pc, pc_lru);
 1614         PV_STAT(pv_entry_spare -= _NPCPV);
 1615         PV_STAT(pc_chunk_count--);
 1616         PV_STAT(pc_chunk_frees++);
 1617         /* entire chunk is free, return it */
 1618         m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS((vm_offset_t)pc));
 1619         dump_drop_page(m->phys_addr);
 1620         vm_page_unwire_noq(m);
 1621         vm_page_free(m);
 1622 }
 1623 
 1624 /*
 1625  * get a new pv_entry, allocating a block from the system
 1626  * when needed.
 1627  */
 1628 static pv_entry_t
 1629 get_pv_entry(pmap_t pmap, boolean_t try)
 1630 {
 1631         struct pv_chunk *pc;
 1632         pv_entry_t pv;
 1633         vm_page_t m;
 1634         int bit, field, idx;
 1635 
 1636         rw_assert(&pvh_global_lock, RA_WLOCKED);
 1637         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1638         PV_STAT(pv_entry_allocs++);
 1639         pv_entry_count++;
 1640 retry:
 1641         pc = TAILQ_FIRST(&pmap->pm_pvchunk);
 1642         if (pc != NULL) {
 1643                 for (field = 0; field < _NPCM; field++) {
 1644                         if (pc->pc_map[field]) {
 1645                                 bit = ffsl(pc->pc_map[field]) - 1;
 1646                                 break;
 1647                         }
 1648                 }
 1649                 if (field < _NPCM) {
 1650                         idx = field * sizeof(pc->pc_map[field]) * NBBY + bit;
 1651                         pv = &pc->pc_pventry[idx];
 1652                         pc->pc_map[field] &= ~(1ul << bit);
 1653                         /* If this was the last item, move it to tail */
 1654                         for (field = 0; field < _NPCM; field++)
 1655                                 if (pc->pc_map[field] != 0) {
 1656                                         PV_STAT(pv_entry_spare--);
 1657                                         return (pv);    /* not full, return */
 1658                                 }
 1659                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 1660                         TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
 1661                         PV_STAT(pv_entry_spare--);
 1662                         return (pv);
 1663                 }
 1664         }
 1665         /* No free items, allocate another chunk */
 1666         m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, VM_ALLOC_NORMAL |
 1667             VM_ALLOC_WIRED);
 1668         if (m == NULL) {
 1669                 if (try) {
 1670                         pv_entry_count--;
 1671                         PV_STAT(pc_chunk_tryfail++);
 1672                         return (NULL);
 1673                 }
 1674                 m = pmap_pv_reclaim(pmap);
 1675                 if (m == NULL)
 1676                         goto retry;
 1677         }
 1678         PV_STAT(pc_chunk_count++);
 1679         PV_STAT(pc_chunk_allocs++);
 1680         dump_add_page(m->phys_addr);
 1681         pc = (struct pv_chunk *)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m));
 1682         pc->pc_pmap = pmap;
 1683         pc->pc_map[0] = pc_freemask[0] & ~1ul;  /* preallocated bit 0 */
 1684         for (field = 1; field < _NPCM; field++)
 1685                 pc->pc_map[field] = pc_freemask[field];
 1686         TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru);
 1687         pv = &pc->pc_pventry[0];
 1688         TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
 1689         PV_STAT(pv_entry_spare += _NPCPV - 1);
 1690         return (pv);
 1691 }
 1692 
 1693 static pv_entry_t
 1694 pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 1695 {
 1696         pv_entry_t pv;
 1697 
 1698         rw_assert(&pvh_global_lock, RA_WLOCKED);
 1699         TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) {
 1700                 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
 1701                         TAILQ_REMOVE(&pvh->pv_list, pv, pv_list);
 1702                         break;
 1703                 }
 1704         }
 1705         return (pv);
 1706 }
 1707 
 1708 static void
 1709 pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va)
 1710 {
 1711         pv_entry_t pv;
 1712 
 1713         pv = pmap_pvh_remove(pvh, pmap, va);
 1714         KASSERT(pv != NULL, ("pmap_pvh_free: pv not found, pa %lx va %lx",
 1715              (u_long)VM_PAGE_TO_PHYS(__containerof(pvh, struct vm_page, md)),
 1716              (u_long)va));
 1717         free_pv_entry(pmap, pv);
 1718 }
 1719 
 1720 static void
 1721 pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va)
 1722 {
 1723 
 1724         rw_assert(&pvh_global_lock, RA_WLOCKED);
 1725         pmap_pvh_free(&m->md, pmap, va);
 1726         if (TAILQ_EMPTY(&m->md.pv_list))
 1727                 vm_page_aflag_clear(m, PGA_WRITEABLE);
 1728 }
 1729 
 1730 /*
 1731  * Conditionally create a pv entry.
 1732  */
 1733 static boolean_t
 1734 pmap_try_insert_pv_entry(pmap_t pmap, vm_page_t mpte, vm_offset_t va,
 1735     vm_page_t m)
 1736 {
 1737         pv_entry_t pv;
 1738 
 1739         rw_assert(&pvh_global_lock, RA_WLOCKED);
 1740         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1741         if ((pv = get_pv_entry(pmap, TRUE)) != NULL) {
 1742                 pv->pv_va = va;
 1743                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 1744                 return (TRUE);
 1745         } else
 1746                 return (FALSE);
 1747 }
 1748 
 1749 /*
 1750  * pmap_remove_pte: do the things to unmap a page in a process
 1751  *
 1752  * Returns true if this was the last PTE in the PT (and possibly the last PT in
 1753  * the PD, and possibly the last PD in the segmap), in which case...
 1754  *
 1755  *   1) the TLB has been invalidated for the whole PT's span (at least),
 1756  *   already, to ensure that MipsDoTLBMiss does not attempt to follow a
 1757  *   dangling pointer into a freed page.  No additional TLB shootdown is
 1758  *   required.
 1759  *
 1760  *   2) if this removal was part of a sweep to remove PTEs, it is safe to jump
 1761  *   to the PT span boundary and continue.
 1762  *
 1763  *   3) The given pde may now point onto a freed page and must not be
 1764  *   dereferenced
 1765  *
 1766  * If the return value is false, the TLB has not been shot down (and the segmap
 1767  * entry, PD, and PT all remain in place).
 1768  */
 1769 static int
 1770 pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
 1771     pd_entry_t pde)
 1772 {
 1773         pt_entry_t oldpte;
 1774         vm_page_t m;
 1775         vm_paddr_t pa;
 1776 
 1777         rw_assert(&pvh_global_lock, RA_WLOCKED);
 1778         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1779 
 1780         /*
 1781          * Write back all cache lines from the page being unmapped.
 1782          */
 1783         mips_dcache_wbinv_range_index(va, PAGE_SIZE);
 1784 
 1785         oldpte = *ptq;
 1786         if (is_kernel_pmap(pmap))
 1787                 *ptq = PTE_G;
 1788         else
 1789                 *ptq = 0;
 1790 
 1791         if (pte_test(&oldpte, PTE_W))
 1792                 pmap->pm_stats.wired_count -= 1;
 1793 
 1794         pmap->pm_stats.resident_count -= 1;
 1795 
 1796         if (pte_test(&oldpte, PTE_MANAGED)) {
 1797                 pa = TLBLO_PTE_TO_PA(oldpte);
 1798                 m = PHYS_TO_VM_PAGE(pa);
 1799                 if (pte_test(&oldpte, PTE_D)) {
 1800                         KASSERT(!pte_test(&oldpte, PTE_RO),
 1801                             ("%s: modified page not writable: va: %p, pte: %#jx",
 1802                             __func__, (void *)va, (uintmax_t)oldpte));
 1803                         vm_page_dirty(m);
 1804                 }
 1805                 if (m->md.pv_flags & PV_TABLE_REF)
 1806                         vm_page_aflag_set(m, PGA_REFERENCED);
 1807                 m->md.pv_flags &= ~PV_TABLE_REF;
 1808 
 1809                 pmap_remove_entry(pmap, m, va);
 1810         }
 1811         return (pmap_unuse_pt(pmap, va, pde));
 1812 }
 1813 
 1814 /*
 1815  * Remove a single page from a process address space
 1816  */
 1817 static void
 1818 pmap_remove_page(struct pmap *pmap, vm_offset_t va)
 1819 {
 1820         pd_entry_t *pde;
 1821         pt_entry_t *ptq;
 1822 
 1823         rw_assert(&pvh_global_lock, RA_WLOCKED);
 1824         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 1825         pde = pmap_pde(pmap, va);
 1826         if (pde == NULL || *pde == 0)
 1827                 return;
 1828         ptq = pmap_pde_to_pte(pde, va);
 1829 
 1830         /*
 1831          * If there is no pte for this address, just skip it!
 1832          */
 1833         if (!pte_test(ptq, PTE_V))
 1834                 return;
 1835 
 1836         /*
 1837          * Remove this PTE from the PT.  If this is the last one, then
 1838          * the TLB has already been shot down, so don't bother again
 1839          */
 1840         if (!pmap_remove_pte(pmap, ptq, va, *pde))
 1841                 pmap_invalidate_page(pmap, va);
 1842 }
 1843 
 1844 /*
 1845  *      Remove the given range of addresses from the specified map.
 1846  *
 1847  *      It is assumed that the start and end are properly
 1848  *      rounded to the page size.
 1849  */
 1850 void
 1851 pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 1852 {
 1853         pd_entry_t *pde, *pdpe;
 1854         pt_entry_t *pte;
 1855         vm_offset_t va_next;
 1856         vm_offset_t va_init, va_fini;
 1857         bool need_tlb_shootdown;
 1858 
 1859         /*
 1860          * Perform an unsynchronized read.  This is, however, safe.
 1861          */
 1862         if (pmap->pm_stats.resident_count == 0)
 1863                 return;
 1864 
 1865         rw_wlock(&pvh_global_lock);
 1866         PMAP_LOCK(pmap);
 1867 
 1868         /*
 1869          * special handling of removing one page.  a very common operation
 1870          * and easy to short circuit some code.
 1871          */
 1872         if ((sva + PAGE_SIZE) == eva) {
 1873                 pmap_remove_page(pmap, sva);
 1874                 goto out;
 1875         }
 1876         for (; sva < eva; sva = va_next) {
 1877                 pdpe = pmap_segmap(pmap, sva);
 1878 #ifdef __mips_n64
 1879                 if (*pdpe == 0) {
 1880                         va_next = (sva + NBSEG) & ~SEGMASK;
 1881                         if (va_next < sva)
 1882                                 va_next = eva;
 1883                         continue;
 1884                 }
 1885 #endif
 1886 
 1887                 /* Scan up to the end of the page table pointed to by pde */
 1888                 va_next = (sva + NBPDR) & ~PDRMASK;
 1889                 if (va_next < sva)
 1890                         va_next = eva;
 1891 
 1892                 pde = pmap_pdpe_to_pde(pdpe, sva);
 1893                 if (*pde == NULL)
 1894                         continue;
 1895 
 1896                 /*
 1897                  * Limit our scan to either the end of the va represented
 1898                  * by the current page table page, or to the end of the
 1899                  * range being removed.
 1900                  */
 1901                 if (va_next > eva)
 1902                         va_next = eva;
 1903 
 1904                 need_tlb_shootdown = false;
 1905                 va_init = sva;
 1906                 va_fini = va_next;
 1907                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 1908                     sva += PAGE_SIZE) {
 1909 
 1910                         /* Skip over invalid entries; no need to shootdown */
 1911                         if (!pte_test(pte, PTE_V)) {
 1912                                 /*
 1913                                  * If we have not yet found a valid entry, then
 1914                                  * we can move the lower edge of the region to
 1915                                  * invalidate to the next PTE.
 1916                                  */
 1917                                 if (!need_tlb_shootdown)
 1918                                         va_init = sva + PAGE_SIZE;
 1919                                 continue;
 1920                         }
 1921 
 1922                         /*
 1923                          * A valid entry; the range we are shooting down must
 1924                          * include this page.  va_fini is used instead of sva
 1925                          * so that if the range ends with a run of !PTE_V PTEs,
 1926                          * but doesn't clear out so much that pmap_remove_pte
 1927                          * removes the entire PT, we won't include these !PTE_V
 1928                          * entries in the region to be shot down.
 1929                          */
 1930                         va_fini = sva + PAGE_SIZE;
 1931 
 1932                         if (pmap_remove_pte(pmap, pte, sva, *pde)) {
 1933                                 /* Entire PT removed and TLBs shot down. */
 1934                                 need_tlb_shootdown = false;
 1935                                 break;
 1936                         } else {
 1937                                 need_tlb_shootdown = true;
 1938                         }
 1939                 }
 1940                 if (need_tlb_shootdown)
 1941                         pmap_invalidate_range(pmap, va_init, va_fini);
 1942         }
 1943 out:
 1944         rw_wunlock(&pvh_global_lock);
 1945         PMAP_UNLOCK(pmap);
 1946 }
 1947 
 1948 /*
 1949  *      Routine:        pmap_remove_all
 1950  *      Function:
 1951  *              Removes this physical page from
 1952  *              all physical maps in which it resides.
 1953  *              Reflects back modify bits to the pager.
 1954  *
 1955  *      Notes:
 1956  *              Original versions of this routine were very
 1957  *              inefficient because they iteratively called
 1958  *              pmap_remove (slow...)
 1959  */
 1960 
 1961 void
 1962 pmap_remove_all(vm_page_t m)
 1963 {
 1964         pv_entry_t pv;
 1965         pmap_t pmap;
 1966         pd_entry_t *pde;
 1967         pt_entry_t *pte, tpte;
 1968 
 1969         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 1970             ("pmap_remove_all: page %p is not managed", m));
 1971         rw_wlock(&pvh_global_lock);
 1972 
 1973         if (m->md.pv_flags & PV_TABLE_REF)
 1974                 vm_page_aflag_set(m, PGA_REFERENCED);
 1975 
 1976         while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
 1977                 pmap = PV_PMAP(pv);
 1978                 PMAP_LOCK(pmap);
 1979 
 1980                 /*
 1981                  * If it's last mapping writeback all caches from
 1982                  * the page being destroyed
 1983                  */
 1984                 if (TAILQ_NEXT(pv, pv_list) == NULL)
 1985                         mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
 1986 
 1987                 pmap->pm_stats.resident_count--;
 1988 
 1989                 pde = pmap_pde(pmap, pv->pv_va);
 1990                 KASSERT(pde != NULL && *pde != 0, ("pmap_remove_all: pde"));
 1991                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 1992 
 1993                 tpte = *pte;
 1994                 if (is_kernel_pmap(pmap))
 1995                         *pte = PTE_G;
 1996                 else
 1997                         *pte = 0;
 1998 
 1999                 if (pte_test(&tpte, PTE_W))
 2000                         pmap->pm_stats.wired_count--;
 2001 
 2002                 /*
 2003                  * Update the vm_page_t clean and reference bits.
 2004                  */
 2005                 if (pte_test(&tpte, PTE_D)) {
 2006                         KASSERT(!pte_test(&tpte, PTE_RO),
 2007                             ("%s: modified page not writable: va: %p, pte: %#jx",
 2008                             __func__, (void *)pv->pv_va, (uintmax_t)tpte));
 2009                         vm_page_dirty(m);
 2010                 }
 2011 
 2012                 if (!pmap_unuse_pt(pmap, pv->pv_va, *pde))
 2013                         pmap_invalidate_page(pmap, pv->pv_va);
 2014 
 2015                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2016                 free_pv_entry(pmap, pv);
 2017                 PMAP_UNLOCK(pmap);
 2018         }
 2019 
 2020         vm_page_aflag_clear(m, PGA_WRITEABLE);
 2021         m->md.pv_flags &= ~PV_TABLE_REF;
 2022         rw_wunlock(&pvh_global_lock);
 2023 }
 2024 
 2025 /*
 2026  *      Set the physical protection on the
 2027  *      specified range of this map as requested.
 2028  */
 2029 void
 2030 pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 2031 {
 2032         pt_entry_t pbits, *pte;
 2033         pd_entry_t *pde, *pdpe;
 2034         vm_offset_t va, va_next;
 2035         vm_paddr_t pa;
 2036         vm_page_t m;
 2037 
 2038         if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 2039                 pmap_remove(pmap, sva, eva);
 2040                 return;
 2041         }
 2042         if (prot & VM_PROT_WRITE)
 2043                 return;
 2044 
 2045         PMAP_LOCK(pmap);
 2046         for (; sva < eva; sva = va_next) {
 2047                 pdpe = pmap_segmap(pmap, sva);
 2048 #ifdef __mips_n64
 2049                 if (*pdpe == 0) {
 2050                         va_next = (sva + NBSEG) & ~SEGMASK;
 2051                         if (va_next < sva)
 2052                                 va_next = eva;
 2053                         continue;
 2054                 }
 2055 #endif
 2056                 va_next = (sva + NBPDR) & ~PDRMASK;
 2057                 if (va_next < sva)
 2058                         va_next = eva;
 2059 
 2060                 pde = pmap_pdpe_to_pde(pdpe, sva);
 2061                 if (*pde == NULL)
 2062                         continue;
 2063 
 2064                 /*
 2065                  * Limit our scan to either the end of the va represented
 2066                  * by the current page table page, or to the end of the
 2067                  * range being write protected.
 2068                  */
 2069                 if (va_next > eva)
 2070                         va_next = eva;
 2071 
 2072                 va = va_next;
 2073                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 2074                     sva += PAGE_SIZE) {
 2075                         pbits = *pte;
 2076                         if (!pte_test(&pbits, PTE_V) || pte_test(&pbits,
 2077                             PTE_RO)) {
 2078                                 if (va != va_next) {
 2079                                         pmap_invalidate_range(pmap, va, sva);
 2080                                         va = va_next;
 2081                                 }
 2082                                 continue;
 2083                         }
 2084                         pte_set(&pbits, PTE_RO);
 2085                         if (pte_test(&pbits, PTE_D)) {
 2086                                 pte_clear(&pbits, PTE_D);
 2087                                 if (pte_test(&pbits, PTE_MANAGED)) {
 2088                                         pa = TLBLO_PTE_TO_PA(pbits);
 2089                                         m = PHYS_TO_VM_PAGE(pa);
 2090                                         vm_page_dirty(m);
 2091                                 }
 2092                                 if (va == va_next)
 2093                                         va = sva;
 2094                         } else {
 2095                                 /*
 2096                                  * Unless PTE_D is set, any TLB entries
 2097                                  * mapping "sva" don't allow write access, so
 2098                                  * they needn't be invalidated.
 2099                                  */
 2100                                 if (va != va_next) {
 2101                                         pmap_invalidate_range(pmap, va, sva);
 2102                                         va = va_next;
 2103                                 }
 2104                         }
 2105                         *pte = pbits;
 2106                 }
 2107                 if (va != va_next)
 2108                         pmap_invalidate_range(pmap, va, sva);
 2109         }
 2110         PMAP_UNLOCK(pmap);
 2111 }
 2112 
 2113 /*
 2114  *      Insert the given physical page (p) at
 2115  *      the specified virtual address (v) in the
 2116  *      target physical map with the protection requested.
 2117  *
 2118  *      If specified, the page will be wired down, meaning
 2119  *      that the related pte can not be reclaimed.
 2120  *
 2121  *      NB:  This is the only routine which MAY NOT lazy-evaluate
 2122  *      or lose information.  That is, this routine must actually
 2123  *      insert this page into the given map NOW.
 2124  */
 2125 int
 2126 pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 2127     u_int flags, int8_t psind __unused)
 2128 {
 2129         vm_paddr_t pa, opa;
 2130         pt_entry_t *pte;
 2131         pt_entry_t origpte, newpte;
 2132         pv_entry_t pv;
 2133         vm_page_t mpte, om;
 2134 
 2135         va &= ~PAGE_MASK;
 2136         KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig"));
 2137         KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva ||
 2138             va >= kmi.clean_eva,
 2139             ("pmap_enter: managed mapping within the clean submap"));
 2140         if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m))
 2141                 VM_OBJECT_ASSERT_LOCKED(m->object);
 2142         pa = VM_PAGE_TO_PHYS(m);
 2143         newpte = TLBLO_PA_TO_PFN(pa) | init_pte_prot(m, flags, prot);
 2144         if ((flags & PMAP_ENTER_WIRED) != 0)
 2145                 newpte |= PTE_W;
 2146         if (is_kernel_pmap(pmap))
 2147                 newpte |= PTE_G;
 2148         PMAP_PTE_SET_CACHE_BITS(newpte, pa, m);
 2149         if ((m->oflags & VPO_UNMANAGED) == 0)
 2150                 newpte |= PTE_MANAGED;
 2151 
 2152         mpte = NULL;
 2153 
 2154         rw_wlock(&pvh_global_lock);
 2155         PMAP_LOCK(pmap);
 2156 
 2157         /*
 2158          * In the case that a page table page is not resident, we are
 2159          * creating it here.
 2160          */
 2161         if (va < VM_MAXUSER_ADDRESS) {
 2162                 mpte = pmap_allocpte(pmap, va, flags);
 2163                 if (mpte == NULL) {
 2164                         KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0,
 2165                             ("pmap_allocpte failed with sleep allowed"));
 2166                         rw_wunlock(&pvh_global_lock);
 2167                         PMAP_UNLOCK(pmap);
 2168                         return (KERN_RESOURCE_SHORTAGE);
 2169                 }
 2170         }
 2171         pte = pmap_pte(pmap, va);
 2172 
 2173         /*
 2174          * Page Directory table entry not valid, we need a new PT page
 2175          */
 2176         if (pte == NULL) {
 2177                 panic("pmap_enter: invalid page directory, pdir=%p, va=%p",
 2178                     (void *)pmap->pm_segtab, (void *)va);
 2179         }
 2180 
 2181         origpte = *pte;
 2182         KASSERT(!pte_test(&origpte, PTE_D | PTE_RO | PTE_V),
 2183             ("pmap_enter: modified page not writable: va: %p, pte: %#jx",
 2184             (void *)va, (uintmax_t)origpte));
 2185         opa = TLBLO_PTE_TO_PA(origpte);
 2186 
 2187         /*
 2188          * Mapping has not changed, must be protection or wiring change.
 2189          */
 2190         if (pte_test(&origpte, PTE_V) && opa == pa) {
 2191                 /*
 2192                  * Wiring change, just update stats. We don't worry about
 2193                  * wiring PT pages as they remain resident as long as there
 2194                  * are valid mappings in them. Hence, if a user page is
 2195                  * wired, the PT page will be also.
 2196                  */
 2197                 if (pte_test(&newpte, PTE_W) && !pte_test(&origpte, PTE_W))
 2198                         pmap->pm_stats.wired_count++;
 2199                 else if (!pte_test(&newpte, PTE_W) && pte_test(&origpte,
 2200                     PTE_W))
 2201                         pmap->pm_stats.wired_count--;
 2202 
 2203                 /*
 2204                  * Remove extra pte reference
 2205                  */
 2206                 if (mpte)
 2207                         mpte->wire_count--;
 2208 
 2209                 if (pte_test(&origpte, PTE_MANAGED)) {
 2210                         m->md.pv_flags |= PV_TABLE_REF;
 2211                         if (!pte_test(&newpte, PTE_RO))
 2212                                 vm_page_aflag_set(m, PGA_WRITEABLE);
 2213                 }
 2214                 goto validate;
 2215         }
 2216 
 2217         pv = NULL;
 2218 
 2219         /*
 2220          * Mapping has changed, invalidate old range and fall through to
 2221          * handle validating new mapping.
 2222          */
 2223         if (opa) {
 2224                 if (is_kernel_pmap(pmap))
 2225                         *pte = PTE_G;
 2226                 else
 2227                         *pte = 0;
 2228                 if (pte_test(&origpte, PTE_W))
 2229                         pmap->pm_stats.wired_count--;
 2230                 if (pte_test(&origpte, PTE_MANAGED)) {
 2231                         om = PHYS_TO_VM_PAGE(opa);
 2232                         if (pte_test(&origpte, PTE_D))
 2233                                 vm_page_dirty(om);
 2234                         if ((om->md.pv_flags & PV_TABLE_REF) != 0) {
 2235                                 om->md.pv_flags &= ~PV_TABLE_REF;
 2236                                 vm_page_aflag_set(om, PGA_REFERENCED);
 2237                         }
 2238                         pv = pmap_pvh_remove(&om->md, pmap, va);
 2239                         if (!pte_test(&newpte, PTE_MANAGED))
 2240                                 free_pv_entry(pmap, pv);
 2241                         if ((om->aflags & PGA_WRITEABLE) != 0 &&
 2242                             TAILQ_EMPTY(&om->md.pv_list))
 2243                                 vm_page_aflag_clear(om, PGA_WRITEABLE);
 2244                 }
 2245                 pmap_invalidate_page(pmap, va);
 2246                 origpte = 0;
 2247                 if (mpte != NULL) {
 2248                         mpte->wire_count--;
 2249                         KASSERT(mpte->wire_count > 0,
 2250                             ("pmap_enter: missing reference to page table page,"
 2251                             " va: %p", (void *)va));
 2252                 }
 2253         } else
 2254                 pmap->pm_stats.resident_count++;
 2255 
 2256         /*
 2257          * Enter on the PV list if part of our managed memory.
 2258          */
 2259         if (pte_test(&newpte, PTE_MANAGED)) {
 2260                 m->md.pv_flags |= PV_TABLE_REF;
 2261                 if (pv == NULL) {
 2262                         pv = get_pv_entry(pmap, FALSE);
 2263                         pv->pv_va = va;
 2264                 }
 2265                 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
 2266                 if (!pte_test(&newpte, PTE_RO))
 2267                         vm_page_aflag_set(m, PGA_WRITEABLE);
 2268         }
 2269 
 2270         /*
 2271          * Increment counters
 2272          */
 2273         if (pte_test(&newpte, PTE_W))
 2274                 pmap->pm_stats.wired_count++;
 2275 
 2276 validate:
 2277 
 2278 #ifdef PMAP_DEBUG
 2279         printf("pmap_enter:  va: %p -> pa: %p\n", (void *)va, (void *)pa);
 2280 #endif
 2281 
 2282         /*
 2283          * if the mapping or permission bits are different, we need to
 2284          * update the pte.
 2285          */
 2286         if (origpte != newpte) {
 2287                 *pte = newpte;
 2288                 if (pte_test(&origpte, PTE_V)) {
 2289                         KASSERT(opa == pa, ("pmap_enter: invalid update"));
 2290                         if (pte_test(&origpte, PTE_D)) {
 2291                                 if (pte_test(&origpte, PTE_MANAGED))
 2292                                         vm_page_dirty(m);
 2293                         }
 2294                         pmap_update_page(pmap, va, newpte);
 2295                 }
 2296         }
 2297 
 2298         /*
 2299          * Sync I & D caches for executable pages.  Do this only if the
 2300          * target pmap belongs to the current process.  Otherwise, an
 2301          * unresolvable TLB miss may occur.
 2302          */
 2303         if (!is_kernel_pmap(pmap) && (pmap == &curproc->p_vmspace->vm_pmap) &&
 2304             (prot & VM_PROT_EXECUTE)) {
 2305                 mips_icache_sync_range(va, PAGE_SIZE);
 2306                 mips_dcache_wbinv_range(va, PAGE_SIZE);
 2307         }
 2308         rw_wunlock(&pvh_global_lock);
 2309         PMAP_UNLOCK(pmap);
 2310         return (KERN_SUCCESS);
 2311 }
 2312 
 2313 /*
 2314  * this code makes some *MAJOR* assumptions:
 2315  * 1. Current pmap & pmap exists.
 2316  * 2. Not wired.
 2317  * 3. Read access.
 2318  * 4. No page table pages.
 2319  * but is *MUCH* faster than pmap_enter...
 2320  */
 2321 
 2322 void
 2323 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot)
 2324 {
 2325 
 2326         rw_wlock(&pvh_global_lock);
 2327         PMAP_LOCK(pmap);
 2328         (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL);
 2329         rw_wunlock(&pvh_global_lock);
 2330         PMAP_UNLOCK(pmap);
 2331 }
 2332 
 2333 static vm_page_t
 2334 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 2335     vm_prot_t prot, vm_page_t mpte)
 2336 {
 2337         pt_entry_t *pte, npte;
 2338         vm_paddr_t pa;
 2339 
 2340         KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 2341             (m->oflags & VPO_UNMANAGED) != 0,
 2342             ("pmap_enter_quick_locked: managed mapping within the clean submap"));
 2343         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2344         PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 2345 
 2346         /*
 2347          * In the case that a page table page is not resident, we are
 2348          * creating it here.
 2349          */
 2350         if (va < VM_MAXUSER_ADDRESS) {
 2351                 pd_entry_t *pde;
 2352                 unsigned ptepindex;
 2353 
 2354                 /*
 2355                  * Calculate pagetable page index
 2356                  */
 2357                 ptepindex = pmap_pde_pindex(va);
 2358                 if (mpte && (mpte->pindex == ptepindex)) {
 2359                         mpte->wire_count++;
 2360                 } else {
 2361                         /*
 2362                          * Get the page directory entry
 2363                          */
 2364                         pde = pmap_pde(pmap, va);
 2365 
 2366                         /*
 2367                          * If the page table page is mapped, we just
 2368                          * increment the hold count, and activate it.
 2369                          */
 2370                         if (pde && *pde != 0) {
 2371                                 mpte = PHYS_TO_VM_PAGE(
 2372                                     MIPS_DIRECT_TO_PHYS(*pde));
 2373                                 mpte->wire_count++;
 2374                         } else {
 2375                                 mpte = _pmap_allocpte(pmap, ptepindex,
 2376                                     PMAP_ENTER_NOSLEEP);
 2377                                 if (mpte == NULL)
 2378                                         return (mpte);
 2379                         }
 2380                 }
 2381         } else {
 2382                 mpte = NULL;
 2383         }
 2384 
 2385         pte = pmap_pte(pmap, va);
 2386         if (pte_test(pte, PTE_V)) {
 2387                 if (mpte != NULL) {
 2388                         mpte->wire_count--;
 2389                         mpte = NULL;
 2390                 }
 2391                 return (mpte);
 2392         }
 2393 
 2394         /*
 2395          * Enter on the PV list if part of our managed memory.
 2396          */
 2397         if ((m->oflags & VPO_UNMANAGED) == 0 &&
 2398             !pmap_try_insert_pv_entry(pmap, mpte, va, m)) {
 2399                 if (mpte != NULL) {
 2400                         pmap_unwire_ptp(pmap, va, mpte);
 2401                         mpte = NULL;
 2402                 }
 2403                 return (mpte);
 2404         }
 2405 
 2406         /*
 2407          * Increment counters
 2408          */
 2409         pmap->pm_stats.resident_count++;
 2410 
 2411         pa = VM_PAGE_TO_PHYS(m);
 2412 
 2413         /*
 2414          * Now validate mapping with RO protection
 2415          */
 2416         npte = PTE_RO | TLBLO_PA_TO_PFN(pa) | PTE_V;
 2417         if ((m->oflags & VPO_UNMANAGED) == 0)
 2418                 npte |= PTE_MANAGED;
 2419 
 2420         PMAP_PTE_SET_CACHE_BITS(npte, pa, m);
 2421 
 2422         if (is_kernel_pmap(pmap))
 2423                 *pte = npte | PTE_G;
 2424         else {
 2425                 *pte = npte;
 2426                 /*
 2427                  * Sync I & D caches.  Do this only if the target pmap
 2428                  * belongs to the current process.  Otherwise, an
 2429                  * unresolvable TLB miss may occur. */
 2430                 if (pmap == &curproc->p_vmspace->vm_pmap) {
 2431                         va &= ~PAGE_MASK;
 2432                         mips_icache_sync_range(va, PAGE_SIZE);
 2433                         mips_dcache_wbinv_range(va, PAGE_SIZE);
 2434                 }
 2435         }
 2436         return (mpte);
 2437 }
 2438 
 2439 /*
 2440  * Make a temporary mapping for a physical address.  This is only intended
 2441  * to be used for panic dumps.
 2442  *
 2443  * Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit.
 2444  */
 2445 void *
 2446 pmap_kenter_temporary(vm_paddr_t pa, int i)
 2447 {
 2448         vm_offset_t va;
 2449 
 2450         if (i != 0)
 2451                 printf("%s: ERROR!!! More than one page of virtual address mapping not supported\n",
 2452                     __func__);
 2453 
 2454         if (MIPS_DIRECT_MAPPABLE(pa)) {
 2455                 va = MIPS_PHYS_TO_DIRECT(pa);
 2456         } else {
 2457 #ifndef __mips_n64    /* XXX : to be converted to new style */
 2458                 int cpu;
 2459                 register_t intr;
 2460                 struct local_sysmaps *sysm;
 2461                 pt_entry_t *pte, npte;
 2462 
 2463                 /* If this is used other than for dumps, we may need to leave
 2464                  * interrupts disasbled on return. If crash dumps don't work when
 2465                  * we get to this point, we might want to consider this (leaving things
 2466                  * disabled as a starting point ;-)
 2467                  */
 2468                 intr = intr_disable();
 2469                 cpu = PCPU_GET(cpuid);
 2470                 sysm = &sysmap_lmem[cpu];
 2471                 /* Since this is for the debugger, no locks or any other fun */
 2472                 npte = TLBLO_PA_TO_PFN(pa) | PTE_C_CACHE | PTE_D | PTE_V |
 2473                     PTE_G;
 2474                 pte = pmap_pte(kernel_pmap, sysm->base);
 2475                 *pte = npte;
 2476                 sysm->valid1 = 1;
 2477                 pmap_update_page(kernel_pmap, sysm->base, npte);
 2478                 va = sysm->base;
 2479                 intr_restore(intr);
 2480 #endif
 2481         }
 2482         return ((void *)va);
 2483 }
 2484 
 2485 void
 2486 pmap_kenter_temporary_free(vm_paddr_t pa)
 2487 {
 2488 #ifndef __mips_n64    /* XXX : to be converted to new style */
 2489         int cpu;
 2490         register_t intr;
 2491         struct local_sysmaps *sysm;
 2492 #endif
 2493 
 2494         if (MIPS_DIRECT_MAPPABLE(pa)) {
 2495                 /* nothing to do for this case */
 2496                 return;
 2497         }
 2498 #ifndef __mips_n64    /* XXX : to be converted to new style */
 2499         cpu = PCPU_GET(cpuid);
 2500         sysm = &sysmap_lmem[cpu];
 2501         if (sysm->valid1) {
 2502                 pt_entry_t *pte;
 2503 
 2504                 intr = intr_disable();
 2505                 pte = pmap_pte(kernel_pmap, sysm->base);
 2506                 *pte = PTE_G;
 2507                 pmap_invalidate_page(kernel_pmap, sysm->base);
 2508                 intr_restore(intr);
 2509                 sysm->valid1 = 0;
 2510         }
 2511 #endif
 2512 }
 2513 
 2514 /*
 2515  * Maps a sequence of resident pages belonging to the same object.
 2516  * The sequence begins with the given page m_start.  This page is
 2517  * mapped at the given virtual address start.  Each subsequent page is
 2518  * mapped at a virtual address that is offset from start by the same
 2519  * amount as the page is offset from m_start within the object.  The
 2520  * last page in the sequence is the page with the largest offset from
 2521  * m_start that can be mapped at a virtual address less than the given
 2522  * virtual address end.  Not every virtual page between start and end
 2523  * is mapped; only those for which a resident page exists with the
 2524  * corresponding offset from m_start are mapped.
 2525  */
 2526 void
 2527 pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end,
 2528     vm_page_t m_start, vm_prot_t prot)
 2529 {
 2530         vm_page_t m, mpte;
 2531         vm_pindex_t diff, psize;
 2532 
 2533         VM_OBJECT_ASSERT_LOCKED(m_start->object);
 2534 
 2535         psize = atop(end - start);
 2536         mpte = NULL;
 2537         m = m_start;
 2538         rw_wlock(&pvh_global_lock);
 2539         PMAP_LOCK(pmap);
 2540         while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
 2541                 mpte = pmap_enter_quick_locked(pmap, start + ptoa(diff), m,
 2542                     prot, mpte);
 2543                 m = TAILQ_NEXT(m, listq);
 2544         }
 2545         rw_wunlock(&pvh_global_lock);
 2546         PMAP_UNLOCK(pmap);
 2547 }
 2548 
 2549 /*
 2550  * pmap_object_init_pt preloads the ptes for a given object
 2551  * into the specified pmap.  This eliminates the blast of soft
 2552  * faults on process startup and immediately after an mmap.
 2553  */
 2554 void
 2555 pmap_object_init_pt(pmap_t pmap, vm_offset_t addr,
 2556     vm_object_t object, vm_pindex_t pindex, vm_size_t size)
 2557 {
 2558         VM_OBJECT_ASSERT_WLOCKED(object);
 2559         KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
 2560             ("pmap_object_init_pt: non-device object"));
 2561 }
 2562 
 2563 /*
 2564  *      Clear the wired attribute from the mappings for the specified range of
 2565  *      addresses in the given pmap.  Every valid mapping within that range
 2566  *      must have the wired attribute set.  In contrast, invalid mappings
 2567  *      cannot have the wired attribute set, so they are ignored.
 2568  *
 2569  *      The wired attribute of the page table entry is not a hardware feature,
 2570  *      so there is no need to invalidate any TLB entries.
 2571  */
 2572 void
 2573 pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 2574 {
 2575         pd_entry_t *pde, *pdpe;
 2576         pt_entry_t *pte;
 2577         vm_offset_t va_next;
 2578 
 2579         PMAP_LOCK(pmap);
 2580         for (; sva < eva; sva = va_next) {
 2581                 pdpe = pmap_segmap(pmap, sva);
 2582 #ifdef __mips_n64
 2583                 if (*pdpe == NULL) {
 2584                         va_next = (sva + NBSEG) & ~SEGMASK;
 2585                         if (va_next < sva)
 2586                                 va_next = eva;
 2587                         continue;
 2588                 }
 2589 #endif
 2590                 va_next = (sva + NBPDR) & ~PDRMASK;
 2591                 if (va_next < sva)
 2592                         va_next = eva;
 2593                 pde = pmap_pdpe_to_pde(pdpe, sva);
 2594                 if (*pde == NULL)
 2595                         continue;
 2596                 if (va_next > eva)
 2597                         va_next = eva;
 2598                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 2599                     sva += PAGE_SIZE) {
 2600                         if (!pte_test(pte, PTE_V))
 2601                                 continue;
 2602                         if (!pte_test(pte, PTE_W))
 2603                                 panic("pmap_unwire: pte %#jx is missing PG_W",
 2604                                     (uintmax_t)*pte);
 2605                         pte_clear(pte, PTE_W);
 2606                         pmap->pm_stats.wired_count--;
 2607                 }
 2608         }
 2609         PMAP_UNLOCK(pmap);
 2610 }
 2611 
 2612 /*
 2613  *      Copy the range specified by src_addr/len
 2614  *      from the source map to the range dst_addr/len
 2615  *      in the destination map.
 2616  *
 2617  *      This routine is only advisory and need not do anything.
 2618  */
 2619 
 2620 void
 2621 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr,
 2622     vm_size_t len, vm_offset_t src_addr)
 2623 {
 2624 }
 2625 
 2626 /*
 2627  *      pmap_zero_page zeros the specified hardware page by mapping
 2628  *      the page into KVM and using bzero to clear its contents.
 2629  *
 2630  *      Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit.
 2631  */
 2632 void
 2633 pmap_zero_page(vm_page_t m)
 2634 {
 2635         vm_offset_t va;
 2636         vm_paddr_t phys = VM_PAGE_TO_PHYS(m);
 2637 
 2638         if (MIPS_DIRECT_MAPPABLE(phys)) {
 2639                 va = MIPS_PHYS_TO_DIRECT(phys);
 2640                 bzero((caddr_t)va, PAGE_SIZE);
 2641                 mips_dcache_wbinv_range(va, PAGE_SIZE);
 2642         } else {
 2643                 va = pmap_lmem_map1(phys);
 2644                 bzero((caddr_t)va, PAGE_SIZE);
 2645                 mips_dcache_wbinv_range(va, PAGE_SIZE);
 2646                 pmap_lmem_unmap();
 2647         }
 2648 }
 2649 
 2650 /*
 2651  *      pmap_zero_page_area zeros the specified hardware page by mapping
 2652  *      the page into KVM and using bzero to clear its contents.
 2653  *
 2654  *      off and size may not cover an area beyond a single hardware page.
 2655  */
 2656 void
 2657 pmap_zero_page_area(vm_page_t m, int off, int size)
 2658 {
 2659         vm_offset_t va;
 2660         vm_paddr_t phys = VM_PAGE_TO_PHYS(m);
 2661 
 2662         if (MIPS_DIRECT_MAPPABLE(phys)) {
 2663                 va = MIPS_PHYS_TO_DIRECT(phys);
 2664                 bzero((char *)(caddr_t)va + off, size);
 2665                 mips_dcache_wbinv_range(va + off, size);
 2666         } else {
 2667                 va = pmap_lmem_map1(phys);
 2668                 bzero((char *)va + off, size);
 2669                 mips_dcache_wbinv_range(va + off, size);
 2670                 pmap_lmem_unmap();
 2671         }
 2672 }
 2673 
 2674 /*
 2675  *      pmap_copy_page copies the specified (machine independent)
 2676  *      page by mapping the page into virtual memory and using
 2677  *      bcopy to copy the page, one machine dependent page at a
 2678  *      time.
 2679  *
 2680  *      Use XKPHYS for 64 bit, and KSEG0 where possible for 32 bit.
 2681  */
 2682 void
 2683 pmap_copy_page(vm_page_t src, vm_page_t dst)
 2684 {
 2685         vm_offset_t va_src, va_dst;
 2686         vm_paddr_t phys_src = VM_PAGE_TO_PHYS(src);
 2687         vm_paddr_t phys_dst = VM_PAGE_TO_PHYS(dst);
 2688 
 2689         if (MIPS_DIRECT_MAPPABLE(phys_src) && MIPS_DIRECT_MAPPABLE(phys_dst)) {
 2690                 /* easy case, all can be accessed via KSEG0 */
 2691                 /*
 2692                  * Flush all caches for VA that are mapped to this page
 2693                  * to make sure that data in SDRAM is up to date
 2694                  */
 2695                 pmap_flush_pvcache(src);
 2696                 mips_dcache_wbinv_range_index(
 2697                     MIPS_PHYS_TO_DIRECT(phys_dst), PAGE_SIZE);
 2698                 va_src = MIPS_PHYS_TO_DIRECT(phys_src);
 2699                 va_dst = MIPS_PHYS_TO_DIRECT(phys_dst);
 2700                 bcopy((caddr_t)va_src, (caddr_t)va_dst, PAGE_SIZE);
 2701                 mips_dcache_wbinv_range(va_dst, PAGE_SIZE);
 2702         } else {
 2703                 va_src = pmap_lmem_map2(phys_src, phys_dst);
 2704                 va_dst = va_src + PAGE_SIZE;
 2705                 bcopy((void *)va_src, (void *)va_dst, PAGE_SIZE);
 2706                 mips_dcache_wbinv_range(va_dst, PAGE_SIZE);
 2707                 pmap_lmem_unmap();
 2708         }
 2709 }
 2710 
 2711 int unmapped_buf_allowed;
 2712 
 2713 void
 2714 pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
 2715     vm_offset_t b_offset, int xfersize)
 2716 {
 2717         char *a_cp, *b_cp;
 2718         vm_page_t a_m, b_m;
 2719         vm_offset_t a_pg_offset, b_pg_offset;
 2720         vm_paddr_t a_phys, b_phys;
 2721         int cnt;
 2722 
 2723         while (xfersize > 0) {
 2724                 a_pg_offset = a_offset & PAGE_MASK;
 2725                 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
 2726                 a_m = ma[a_offset >> PAGE_SHIFT];
 2727                 a_phys = VM_PAGE_TO_PHYS(a_m);
 2728                 b_pg_offset = b_offset & PAGE_MASK;
 2729                 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
 2730                 b_m = mb[b_offset >> PAGE_SHIFT];
 2731                 b_phys = VM_PAGE_TO_PHYS(b_m);
 2732                 if (MIPS_DIRECT_MAPPABLE(a_phys) &&
 2733                     MIPS_DIRECT_MAPPABLE(b_phys)) {
 2734                         pmap_flush_pvcache(a_m);
 2735                         mips_dcache_wbinv_range_index(
 2736                             MIPS_PHYS_TO_DIRECT(b_phys), PAGE_SIZE);
 2737                         a_cp = (char *)MIPS_PHYS_TO_DIRECT(a_phys) +
 2738                             a_pg_offset;
 2739                         b_cp = (char *)MIPS_PHYS_TO_DIRECT(b_phys) +
 2740                             b_pg_offset;
 2741                         bcopy(a_cp, b_cp, cnt);
 2742                         mips_dcache_wbinv_range((vm_offset_t)b_cp, cnt);
 2743                 } else {
 2744                         a_cp = (char *)pmap_lmem_map2(a_phys, b_phys);
 2745                         b_cp = (char *)a_cp + PAGE_SIZE;
 2746                         a_cp += a_pg_offset;
 2747                         b_cp += b_pg_offset;
 2748                         bcopy(a_cp, b_cp, cnt);
 2749                         mips_dcache_wbinv_range((vm_offset_t)b_cp, cnt);
 2750                         pmap_lmem_unmap();
 2751                 }
 2752                 a_offset += cnt;
 2753                 b_offset += cnt;
 2754                 xfersize -= cnt;
 2755         }
 2756 }
 2757 
 2758 vm_offset_t
 2759 pmap_quick_enter_page(vm_page_t m)
 2760 {
 2761 #if defined(__mips_n64)
 2762         return MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(m));
 2763 #else
 2764         vm_paddr_t pa;
 2765         struct local_sysmaps *sysm;
 2766         pt_entry_t *pte, npte;
 2767 
 2768         pa = VM_PAGE_TO_PHYS(m);
 2769 
 2770         if (MIPS_DIRECT_MAPPABLE(pa)) {
 2771                 if (pmap_page_get_memattr(m) != VM_MEMATTR_WRITE_BACK)
 2772                         return (MIPS_PHYS_TO_DIRECT_UNCACHED(pa));
 2773                 else
 2774                         return (MIPS_PHYS_TO_DIRECT(pa));
 2775         }
 2776         critical_enter();
 2777         sysm = &sysmap_lmem[PCPU_GET(cpuid)];
 2778 
 2779         KASSERT(sysm->valid1 == 0, ("pmap_quick_enter_page: PTE busy"));
 2780 
 2781         pte = pmap_pte(kernel_pmap, sysm->base);
 2782         npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G;
 2783         PMAP_PTE_SET_CACHE_BITS(npte, pa, m);
 2784         *pte = npte;
 2785         sysm->valid1 = 1;
 2786 
 2787         return (sysm->base);
 2788 #endif
 2789 }
 2790 
 2791 void
 2792 pmap_quick_remove_page(vm_offset_t addr)
 2793 {
 2794         mips_dcache_wbinv_range(addr, PAGE_SIZE);
 2795 
 2796 #if !defined(__mips_n64)
 2797         struct local_sysmaps *sysm;
 2798         pt_entry_t *pte;
 2799 
 2800         if (addr >= MIPS_KSEG0_START && addr < MIPS_KSEG0_END)
 2801                 return;
 2802 
 2803         sysm = &sysmap_lmem[PCPU_GET(cpuid)];
 2804 
 2805         KASSERT(sysm->valid1 != 0,
 2806             ("pmap_quick_remove_page: PTE not in use"));
 2807         KASSERT(sysm->base == addr,
 2808             ("pmap_quick_remove_page: invalid address"));
 2809 
 2810         pte = pmap_pte(kernel_pmap, addr);
 2811         *pte = PTE_G;
 2812         tlb_invalidate_address(kernel_pmap, addr);
 2813         sysm->valid1 = 0;
 2814         critical_exit();
 2815 #endif
 2816 }
 2817 
 2818 /*
 2819  * Returns true if the pmap's pv is one of the first
 2820  * 16 pvs linked to from this page.  This count may
 2821  * be changed upwards or downwards in the future; it
 2822  * is only necessary that true be returned for a small
 2823  * subset of pmaps for proper page aging.
 2824  */
 2825 boolean_t
 2826 pmap_page_exists_quick(pmap_t pmap, vm_page_t m)
 2827 {
 2828         pv_entry_t pv;
 2829         int loops = 0;
 2830         boolean_t rv;
 2831 
 2832         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 2833             ("pmap_page_exists_quick: page %p is not managed", m));
 2834         rv = FALSE;
 2835         rw_wlock(&pvh_global_lock);
 2836         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2837                 if (PV_PMAP(pv) == pmap) {
 2838                         rv = TRUE;
 2839                         break;
 2840                 }
 2841                 loops++;
 2842                 if (loops >= 16)
 2843                         break;
 2844         }
 2845         rw_wunlock(&pvh_global_lock);
 2846         return (rv);
 2847 }
 2848 
 2849 /*
 2850  * Remove all pages from specified address space
 2851  * this aids process exit speeds.  Also, this code
 2852  * is special cased for current process only, but
 2853  * can have the more generic (and slightly slower)
 2854  * mode enabled.  This is much faster than pmap_remove
 2855  * in the case of running down an entire address space.
 2856  */
 2857 void
 2858 pmap_remove_pages(pmap_t pmap)
 2859 {
 2860         pd_entry_t *pde;
 2861         pt_entry_t *pte, tpte;
 2862         pv_entry_t pv;
 2863         vm_page_t m;
 2864         struct pv_chunk *pc, *npc;
 2865         u_long inuse, bitmask;
 2866         int allfree, bit, field, idx;
 2867 
 2868         if (pmap != vmspace_pmap(curthread->td_proc->p_vmspace)) {
 2869                 printf("warning: pmap_remove_pages called with non-current pmap\n");
 2870                 return;
 2871         }
 2872         rw_wlock(&pvh_global_lock);
 2873         PMAP_LOCK(pmap);
 2874         TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
 2875                 allfree = 1;
 2876                 for (field = 0; field < _NPCM; field++) {
 2877                         inuse = ~pc->pc_map[field] & pc_freemask[field];
 2878                         while (inuse != 0) {
 2879                                 bit = ffsl(inuse) - 1;
 2880                                 bitmask = 1UL << bit;
 2881                                 idx = field * sizeof(inuse) * NBBY + bit;
 2882                                 pv = &pc->pc_pventry[idx];
 2883                                 inuse &= ~bitmask;
 2884 
 2885                                 pde = pmap_pde(pmap, pv->pv_va);
 2886                                 KASSERT(pde != NULL && *pde != 0,
 2887                                     ("pmap_remove_pages: pde"));
 2888                                 pte = pmap_pde_to_pte(pde, pv->pv_va);
 2889                                 if (!pte_test(pte, PTE_V))
 2890                                         panic("pmap_remove_pages: bad pte");
 2891                                 tpte = *pte;
 2892 
 2893 /*
 2894  * We cannot remove wired pages from a process' mapping at this time
 2895  */
 2896                                 if (pte_test(&tpte, PTE_W)) {
 2897                                         allfree = 0;
 2898                                         continue;
 2899                                 }
 2900                                 *pte = is_kernel_pmap(pmap) ? PTE_G : 0;
 2901 
 2902                                 m = PHYS_TO_VM_PAGE(TLBLO_PTE_TO_PA(tpte));
 2903                                 KASSERT(m != NULL,
 2904                                     ("pmap_remove_pages: bad tpte %#jx",
 2905                                     (uintmax_t)tpte));
 2906 
 2907                                 /*
 2908                                  * Update the vm_page_t clean and reference bits.
 2909                                  */
 2910                                 if (pte_test(&tpte, PTE_D))
 2911                                         vm_page_dirty(m);
 2912 
 2913                                 /* Mark free */
 2914                                 PV_STAT(pv_entry_frees++);
 2915                                 PV_STAT(pv_entry_spare++);
 2916                                 pv_entry_count--;
 2917                                 pc->pc_map[field] |= bitmask;
 2918                                 pmap->pm_stats.resident_count--;
 2919                                 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 2920                                 if (TAILQ_EMPTY(&m->md.pv_list))
 2921                                         vm_page_aflag_clear(m, PGA_WRITEABLE);
 2922 
 2923                                 /*
 2924                                  * For simplicity, unconditionally call
 2925                                  * pmap_invalidate_all(), below.
 2926                                  */
 2927                                 (void)pmap_unuse_pt(pmap, pv->pv_va, *pde);
 2928                         }
 2929                 }
 2930                 if (allfree) {
 2931                         TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
 2932                         free_pv_chunk(pc);
 2933                 }
 2934         }
 2935         pmap_invalidate_all(pmap);
 2936         PMAP_UNLOCK(pmap);
 2937         rw_wunlock(&pvh_global_lock);
 2938 }
 2939 
 2940 /*
 2941  * pmap_testbit tests bits in pte's
 2942  */
 2943 static boolean_t
 2944 pmap_testbit(vm_page_t m, int bit)
 2945 {
 2946         pv_entry_t pv;
 2947         pmap_t pmap;
 2948         pt_entry_t *pte;
 2949         boolean_t rv = FALSE;
 2950 
 2951         if (m->oflags & VPO_UNMANAGED)
 2952                 return (rv);
 2953 
 2954         rw_assert(&pvh_global_lock, RA_WLOCKED);
 2955         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2956                 pmap = PV_PMAP(pv);
 2957                 PMAP_LOCK(pmap);
 2958                 pte = pmap_pte(pmap, pv->pv_va);
 2959                 rv = pte_test(pte, bit);
 2960                 PMAP_UNLOCK(pmap);
 2961                 if (rv)
 2962                         break;
 2963         }
 2964         return (rv);
 2965 }
 2966 
 2967 /*
 2968  *      pmap_page_wired_mappings:
 2969  *
 2970  *      Return the number of managed mappings to the given physical page
 2971  *      that are wired.
 2972  */
 2973 int
 2974 pmap_page_wired_mappings(vm_page_t m)
 2975 {
 2976         pv_entry_t pv;
 2977         pmap_t pmap;
 2978         pt_entry_t *pte;
 2979         int count;
 2980 
 2981         count = 0;
 2982         if ((m->oflags & VPO_UNMANAGED) != 0)
 2983                 return (count);
 2984         rw_wlock(&pvh_global_lock);
 2985         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 2986                 pmap = PV_PMAP(pv);
 2987                 PMAP_LOCK(pmap);
 2988                 pte = pmap_pte(pmap, pv->pv_va);
 2989                 if (pte_test(pte, PTE_W))
 2990                         count++;
 2991                 PMAP_UNLOCK(pmap);
 2992         }
 2993         rw_wunlock(&pvh_global_lock);
 2994         return (count);
 2995 }
 2996 
 2997 /*
 2998  * Clear the write and modified bits in each of the given page's mappings.
 2999  */
 3000 void
 3001 pmap_remove_write(vm_page_t m)
 3002 {
 3003         pmap_t pmap;
 3004         pt_entry_t pbits, *pte;
 3005         pv_entry_t pv;
 3006 
 3007         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3008             ("pmap_remove_write: page %p is not managed", m));
 3009 
 3010         /*
 3011          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
 3012          * set by another thread while the object is locked.  Thus,
 3013          * if PGA_WRITEABLE is clear, no page table entries need updating.
 3014          */
 3015         VM_OBJECT_ASSERT_WLOCKED(m->object);
 3016         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
 3017                 return;
 3018         rw_wlock(&pvh_global_lock);
 3019         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3020                 pmap = PV_PMAP(pv);
 3021                 PMAP_LOCK(pmap);
 3022                 pte = pmap_pte(pmap, pv->pv_va);
 3023                 KASSERT(pte != NULL && pte_test(pte, PTE_V),
 3024                     ("page on pv_list has no pte"));
 3025                 pbits = *pte;
 3026                 if (pte_test(&pbits, PTE_D)) {
 3027                         pte_clear(&pbits, PTE_D);
 3028                         vm_page_dirty(m);
 3029                 }
 3030                 pte_set(&pbits, PTE_RO);
 3031                 if (pbits != *pte) {
 3032                         *pte = pbits;
 3033                         pmap_update_page(pmap, pv->pv_va, pbits);
 3034                 }
 3035                 PMAP_UNLOCK(pmap);
 3036         }
 3037         vm_page_aflag_clear(m, PGA_WRITEABLE);
 3038         rw_wunlock(&pvh_global_lock);
 3039 }
 3040 
 3041 /*
 3042  *      pmap_ts_referenced:
 3043  *
 3044  *      Return the count of reference bits for a page, clearing all of them.
 3045  */
 3046 int
 3047 pmap_ts_referenced(vm_page_t m)
 3048 {
 3049 
 3050         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3051             ("pmap_ts_referenced: page %p is not managed", m));
 3052         if (m->md.pv_flags & PV_TABLE_REF) {
 3053                 rw_wlock(&pvh_global_lock);
 3054                 m->md.pv_flags &= ~PV_TABLE_REF;
 3055                 rw_wunlock(&pvh_global_lock);
 3056                 return (1);
 3057         }
 3058         return (0);
 3059 }
 3060 
 3061 /*
 3062  *      pmap_is_modified:
 3063  *
 3064  *      Return whether or not the specified physical page was modified
 3065  *      in any physical maps.
 3066  */
 3067 boolean_t
 3068 pmap_is_modified(vm_page_t m)
 3069 {
 3070         boolean_t rv;
 3071 
 3072         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3073             ("pmap_is_modified: page %p is not managed", m));
 3074 
 3075         /*
 3076          * If the page is not exclusive busied, then PGA_WRITEABLE cannot be
 3077          * concurrently set while the object is locked.  Thus, if PGA_WRITEABLE
 3078          * is clear, no PTEs can have PTE_D set.
 3079          */
 3080         VM_OBJECT_ASSERT_WLOCKED(m->object);
 3081         if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0)
 3082                 return (FALSE);
 3083         rw_wlock(&pvh_global_lock);
 3084         rv = pmap_testbit(m, PTE_D);
 3085         rw_wunlock(&pvh_global_lock);
 3086         return (rv);
 3087 }
 3088 
 3089 /* N/C */
 3090 
 3091 /*
 3092  *      pmap_is_prefaultable:
 3093  *
 3094  *      Return whether or not the specified virtual address is elgible
 3095  *      for prefault.
 3096  */
 3097 boolean_t
 3098 pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr)
 3099 {
 3100         pd_entry_t *pde;
 3101         pt_entry_t *pte;
 3102         boolean_t rv;
 3103 
 3104         rv = FALSE;
 3105         PMAP_LOCK(pmap);
 3106         pde = pmap_pde(pmap, addr);
 3107         if (pde != NULL && *pde != 0) {
 3108                 pte = pmap_pde_to_pte(pde, addr);
 3109                 rv = (*pte == 0);
 3110         }
 3111         PMAP_UNLOCK(pmap);
 3112         return (rv);
 3113 }
 3114 
 3115 /*
 3116  *      Apply the given advice to the specified range of addresses within the
 3117  *      given pmap.  Depending on the advice, clear the referenced and/or
 3118  *      modified flags in each mapping and set the mapped page's dirty field.
 3119  */
 3120 void
 3121 pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice)
 3122 {
 3123         pd_entry_t *pde, *pdpe;
 3124         pt_entry_t *pte;
 3125         vm_offset_t va, va_next;
 3126         vm_paddr_t pa;
 3127         vm_page_t m;
 3128 
 3129         if (advice != MADV_DONTNEED && advice != MADV_FREE)
 3130                 return;
 3131         rw_wlock(&pvh_global_lock);
 3132         PMAP_LOCK(pmap);
 3133         for (; sva < eva; sva = va_next) {
 3134                 pdpe = pmap_segmap(pmap, sva);
 3135 #ifdef __mips_n64
 3136                 if (*pdpe == 0) {
 3137                         va_next = (sva + NBSEG) & ~SEGMASK;
 3138                         if (va_next < sva)
 3139                                 va_next = eva;
 3140                         continue;
 3141                 }
 3142 #endif
 3143                 va_next = (sva + NBPDR) & ~PDRMASK;
 3144                 if (va_next < sva)
 3145                         va_next = eva;
 3146 
 3147                 pde = pmap_pdpe_to_pde(pdpe, sva);
 3148                 if (*pde == NULL)
 3149                         continue;
 3150 
 3151                 /*
 3152                  * Limit our scan to either the end of the va represented
 3153                  * by the current page table page, or to the end of the
 3154                  * range being write protected.
 3155                  */
 3156                 if (va_next > eva)
 3157                         va_next = eva;
 3158 
 3159                 va = va_next;
 3160                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 3161                     sva += PAGE_SIZE) {
 3162                         if (!pte_test(pte, PTE_MANAGED | PTE_V)) {
 3163                                 if (va != va_next) {
 3164                                         pmap_invalidate_range(pmap, va, sva);
 3165                                         va = va_next;
 3166                                 }
 3167                                 continue;
 3168                         }
 3169                         pa = TLBLO_PTE_TO_PA(*pte);
 3170                         m = PHYS_TO_VM_PAGE(pa);
 3171                         m->md.pv_flags &= ~PV_TABLE_REF;
 3172                         if (pte_test(pte, PTE_D)) {
 3173                                 if (advice == MADV_DONTNEED) {
 3174                                         /*
 3175                                          * Future calls to pmap_is_modified()
 3176                                          * can be avoided by making the page
 3177                                          * dirty now.
 3178                                          */
 3179                                         vm_page_dirty(m);
 3180                                 } else {
 3181                                         pte_clear(pte, PTE_D);
 3182                                         if (va == va_next)
 3183                                                 va = sva;
 3184                                 }
 3185                         } else {
 3186                                 /*
 3187                                  * Unless PTE_D is set, any TLB entries
 3188                                  * mapping "sva" don't allow write access, so
 3189                                  * they needn't be invalidated.
 3190                                  */
 3191                                 if (va != va_next) {
 3192                                         pmap_invalidate_range(pmap, va, sva);
 3193                                         va = va_next;
 3194                                 }
 3195                         }
 3196                 }
 3197                 if (va != va_next)
 3198                         pmap_invalidate_range(pmap, va, sva);
 3199         }
 3200         rw_wunlock(&pvh_global_lock);
 3201         PMAP_UNLOCK(pmap);
 3202 }
 3203 
 3204 /*
 3205  *      Clear the modify bits on the specified physical page.
 3206  */
 3207 void
 3208 pmap_clear_modify(vm_page_t m)
 3209 {
 3210         pmap_t pmap;
 3211         pt_entry_t *pte;
 3212         pv_entry_t pv;
 3213 
 3214         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3215             ("pmap_clear_modify: page %p is not managed", m));
 3216         VM_OBJECT_ASSERT_WLOCKED(m->object);
 3217         KASSERT(!vm_page_xbusied(m),
 3218             ("pmap_clear_modify: page %p is exclusive busied", m));
 3219 
 3220         /*
 3221          * If the page is not PGA_WRITEABLE, then no PTEs can have PTE_D set.
 3222          * If the object containing the page is locked and the page is not
 3223          * write busied, then PGA_WRITEABLE cannot be concurrently set.
 3224          */
 3225         if ((m->aflags & PGA_WRITEABLE) == 0)
 3226                 return;
 3227         rw_wlock(&pvh_global_lock);
 3228         TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) {
 3229                 pmap = PV_PMAP(pv);
 3230                 PMAP_LOCK(pmap);
 3231                 pte = pmap_pte(pmap, pv->pv_va);
 3232                 if (pte_test(pte, PTE_D)) {
 3233                         pte_clear(pte, PTE_D);
 3234                         pmap_update_page(pmap, pv->pv_va, *pte);
 3235                 }
 3236                 PMAP_UNLOCK(pmap);
 3237         }
 3238         rw_wunlock(&pvh_global_lock);
 3239 }
 3240 
 3241 /*
 3242  *      pmap_is_referenced:
 3243  *
 3244  *      Return whether or not the specified physical page was referenced
 3245  *      in any physical maps.
 3246  */
 3247 boolean_t
 3248 pmap_is_referenced(vm_page_t m)
 3249 {
 3250 
 3251         KASSERT((m->oflags & VPO_UNMANAGED) == 0,
 3252             ("pmap_is_referenced: page %p is not managed", m));
 3253         return ((m->md.pv_flags & PV_TABLE_REF) != 0);
 3254 }
 3255 
 3256 /*
 3257  * Miscellaneous support routines follow
 3258  */
 3259 
 3260 /*
 3261  * Map a set of physical memory pages into the kernel virtual
 3262  * address space. Return a pointer to where it is mapped. This
 3263  * routine is intended to be used for mapping device memory,
 3264  * NOT real memory.
 3265  *
 3266  * Use XKPHYS uncached for 64 bit, and KSEG1 where possible for 32 bit.
 3267  */
 3268 void *
 3269 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma)
 3270 {
 3271         vm_offset_t va, tmpva, offset;
 3272 
 3273         /*
 3274          * KSEG1 maps only first 512M of phys address space. For
 3275          * pa > 0x20000000 we should make proper mapping * using pmap_kenter.
 3276          */
 3277         if (MIPS_DIRECT_MAPPABLE(pa + size - 1) && ma == VM_MEMATTR_UNCACHEABLE)
 3278                 return ((void *)MIPS_PHYS_TO_DIRECT_UNCACHED(pa));
 3279         else {
 3280                 offset = pa & PAGE_MASK;
 3281                 size = roundup(size + offset, PAGE_SIZE);
 3282 
 3283                 va = kva_alloc(size);
 3284                 if (!va)
 3285                         panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 3286                 pa = trunc_page(pa);
 3287                 for (tmpva = va; size > 0;) {
 3288                         pmap_kenter_attr(tmpva, pa, ma);
 3289                         size -= PAGE_SIZE;
 3290                         tmpva += PAGE_SIZE;
 3291                         pa += PAGE_SIZE;
 3292                 }
 3293         }
 3294 
 3295         return ((void *)(va + offset));
 3296 }
 3297 
 3298 void *
 3299 pmap_mapdev(vm_paddr_t pa, vm_size_t size)
 3300 {
 3301         return pmap_mapdev_attr(pa, size, VM_MEMATTR_UNCACHEABLE);
 3302 }
 3303 
 3304 void
 3305 pmap_unmapdev(vm_offset_t va, vm_size_t size)
 3306 {
 3307 #ifndef __mips_n64
 3308         vm_offset_t base, offset;
 3309 
 3310         /* If the address is within KSEG1 then there is nothing to do */
 3311         if (va >= MIPS_KSEG1_START && va <= MIPS_KSEG1_END)
 3312                 return;
 3313 
 3314         base = trunc_page(va);
 3315         offset = va & PAGE_MASK;
 3316         size = roundup(size + offset, PAGE_SIZE);
 3317         pmap_qremove(base, atop(size));
 3318         kva_free(base, size);
 3319 #endif
 3320 }
 3321 
 3322 /*
 3323  * perform the pmap work for mincore
 3324  */
 3325 int
 3326 pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa)
 3327 {
 3328         pt_entry_t *ptep, pte;
 3329         vm_paddr_t pa;
 3330         vm_page_t m;
 3331         int val;
 3332 
 3333         PMAP_LOCK(pmap);
 3334 retry:
 3335         ptep = pmap_pte(pmap, addr);
 3336         pte = (ptep != NULL) ? *ptep : 0;
 3337         if (!pte_test(&pte, PTE_V)) {
 3338                 val = 0;
 3339                 goto out;
 3340         }
 3341         val = MINCORE_INCORE;
 3342         if (pte_test(&pte, PTE_D))
 3343                 val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER;
 3344         pa = TLBLO_PTE_TO_PA(pte);
 3345         if (pte_test(&pte, PTE_MANAGED)) {
 3346                 /*
 3347                  * This may falsely report the given address as
 3348                  * MINCORE_REFERENCED.  Unfortunately, due to the lack of
 3349                  * per-PTE reference information, it is impossible to
 3350                  * determine if the address is MINCORE_REFERENCED.
 3351                  */
 3352                 m = PHYS_TO_VM_PAGE(pa);
 3353                 if ((m->aflags & PGA_REFERENCED) != 0)
 3354                         val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER;
 3355         }
 3356         if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) !=
 3357             (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) &&
 3358             pte_test(&pte, PTE_MANAGED)) {
 3359                 /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */
 3360                 if (vm_page_pa_tryrelock(pmap, pa, locked_pa))
 3361                         goto retry;
 3362         } else
 3363 out:
 3364                 PA_UNLOCK_COND(*locked_pa);
 3365         PMAP_UNLOCK(pmap);
 3366         return (val);
 3367 }
 3368 
 3369 void
 3370 pmap_activate(struct thread *td)
 3371 {
 3372         pmap_t pmap, oldpmap;
 3373         struct proc *p = td->td_proc;
 3374         u_int cpuid;
 3375 
 3376         critical_enter();
 3377 
 3378         pmap = vmspace_pmap(p->p_vmspace);
 3379         oldpmap = PCPU_GET(curpmap);
 3380         cpuid = PCPU_GET(cpuid);
 3381 
 3382         if (oldpmap)
 3383                 CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
 3384         CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
 3385         pmap_asid_alloc(pmap);
 3386         if (td == curthread) {
 3387                 PCPU_SET(segbase, pmap->pm_segtab);
 3388                 mips_wr_entryhi(pmap->pm_asid[cpuid].asid);
 3389         }
 3390 
 3391         PCPU_SET(curpmap, pmap);
 3392         critical_exit();
 3393 }
 3394 
 3395 static void
 3396 pmap_sync_icache_one(void *arg __unused)
 3397 {
 3398 
 3399         mips_icache_sync_all();
 3400         mips_dcache_wbinv_all();
 3401 }
 3402 
 3403 void
 3404 pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
 3405 {
 3406 
 3407         smp_rendezvous(NULL, pmap_sync_icache_one, NULL, NULL);
 3408 }
 3409 
 3410 /*
 3411  *      Increase the starting virtual address of the given mapping if a
 3412  *      different alignment might result in more superpage mappings.
 3413  */
 3414 void
 3415 pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
 3416     vm_offset_t *addr, vm_size_t size)
 3417 {
 3418         vm_offset_t superpage_offset;
 3419 
 3420         if (size < PDRSIZE)
 3421                 return;
 3422         if (object != NULL && (object->flags & OBJ_COLORED) != 0)
 3423                 offset += ptoa(object->pg_color);
 3424         superpage_offset = offset & PDRMASK;
 3425         if (size - ((PDRSIZE - superpage_offset) & PDRMASK) < PDRSIZE ||
 3426             (*addr & PDRMASK) == superpage_offset)
 3427                 return;
 3428         if ((*addr & PDRMASK) < superpage_offset)
 3429                 *addr = (*addr & ~PDRMASK) + superpage_offset;
 3430         else
 3431                 *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
 3432 }
 3433 
 3434 #ifdef DDB
 3435 DB_SHOW_COMMAND(ptable, ddb_pid_dump)
 3436 {
 3437         pmap_t pmap;
 3438         struct thread *td = NULL;
 3439         struct proc *p;
 3440         int i, j, k;
 3441         vm_paddr_t pa;
 3442         vm_offset_t va;
 3443 
 3444         if (have_addr) {
 3445                 td = db_lookup_thread(addr, true);
 3446                 if (td == NULL) {
 3447                         db_printf("Invalid pid or tid");
 3448                         return;
 3449                 }
 3450                 p = td->td_proc;
 3451                 if (p->p_vmspace == NULL) {
 3452                         db_printf("No vmspace for process");
 3453                         return;
 3454                 }
 3455                         pmap = vmspace_pmap(p->p_vmspace);
 3456         } else
 3457                 pmap = kernel_pmap;
 3458 
 3459         db_printf("pmap:%p segtab:%p asid:%x generation:%x\n",
 3460             pmap, pmap->pm_segtab, pmap->pm_asid[0].asid,
 3461             pmap->pm_asid[0].gen);
 3462         for (i = 0; i < NPDEPG; i++) {
 3463                 pd_entry_t *pdpe;
 3464                 pt_entry_t *pde;
 3465                 pt_entry_t pte;
 3466 
 3467                 pdpe = (pd_entry_t *)pmap->pm_segtab[i];
 3468                 if (pdpe == NULL)
 3469                         continue;
 3470                 db_printf("[%4d] %p\n", i, pdpe);
 3471 #ifdef __mips_n64
 3472                 for (j = 0; j < NPDEPG; j++) {
 3473                         pde = (pt_entry_t *)pdpe[j];
 3474                         if (pde == NULL)
 3475                                 continue;
 3476                         db_printf("\t[%4d] %p\n", j, pde);
 3477 #else
 3478                 {
 3479                         j = 0;
 3480                         pde =  (pt_entry_t *)pdpe;
 3481 #endif
 3482                         for (k = 0; k < NPTEPG; k++) {
 3483                                 pte = pde[k];
 3484                                 if (pte == 0 || !pte_test(&pte, PTE_V))
 3485                                         continue;
 3486                                 pa = TLBLO_PTE_TO_PA(pte);
 3487                                 va = ((u_long)i << SEGSHIFT) | (j << PDRSHIFT) | (k << PAGE_SHIFT);
 3488                                 db_printf("\t\t[%04d] va: %p pte: %8jx pa:%jx\n",
 3489                                        k, (void *)va, (uintmax_t)pte, (uintmax_t)pa);
 3490                         }
 3491                 }
 3492         }
 3493 }
 3494 #endif
 3495 
 3496 /*
 3497  * Allocate TLB address space tag (called ASID or TLBPID) and return it.
 3498  * It takes almost as much or more time to search the TLB for a
 3499  * specific ASID and flush those entries as it does to flush the entire TLB.
 3500  * Therefore, when we allocate a new ASID, we just take the next number. When
 3501  * we run out of numbers, we flush the TLB, increment the generation count
 3502  * and start over. ASID zero is reserved for kernel use.
 3503  */
 3504 static void
 3505 pmap_asid_alloc(pmap)
 3506         pmap_t pmap;
 3507 {
 3508         if (pmap->pm_asid[PCPU_GET(cpuid)].asid != PMAP_ASID_RESERVED &&
 3509             pmap->pm_asid[PCPU_GET(cpuid)].gen == PCPU_GET(asid_generation));
 3510         else {
 3511                 if (PCPU_GET(next_asid) == pmap_max_asid) {
 3512                         tlb_invalidate_all_user(NULL);
 3513                         PCPU_SET(asid_generation,
 3514                             (PCPU_GET(asid_generation) + 1) & ASIDGEN_MASK);
 3515                         if (PCPU_GET(asid_generation) == 0) {
 3516                                 PCPU_SET(asid_generation, 1);
 3517                         }
 3518                         PCPU_SET(next_asid, 1); /* 0 means invalid */
 3519                 }
 3520                 pmap->pm_asid[PCPU_GET(cpuid)].asid = PCPU_GET(next_asid);
 3521                 pmap->pm_asid[PCPU_GET(cpuid)].gen = PCPU_GET(asid_generation);
 3522                 PCPU_SET(next_asid, PCPU_GET(next_asid) + 1);
 3523         }
 3524 }
 3525 
 3526 static pt_entry_t
 3527 init_pte_prot(vm_page_t m, vm_prot_t access, vm_prot_t prot)
 3528 {
 3529         pt_entry_t rw;
 3530 
 3531         if (!(prot & VM_PROT_WRITE))
 3532                 rw = PTE_V | PTE_RO;
 3533         else if ((m->oflags & VPO_UNMANAGED) == 0) {
 3534                 if ((access & VM_PROT_WRITE) != 0)
 3535                         rw = PTE_V | PTE_D;
 3536                 else
 3537                         rw = PTE_V;
 3538         } else
 3539                 /* Needn't emulate a modified bit for unmanaged pages. */
 3540                 rw = PTE_V | PTE_D;
 3541         return (rw);
 3542 }
 3543 
 3544 /*
 3545  * pmap_emulate_modified : do dirty bit emulation
 3546  *
 3547  * On SMP, update just the local TLB, other CPUs will update their
 3548  * TLBs from PTE lazily, if they get the exception.
 3549  * Returns 0 in case of sucess, 1 if the page is read only and we
 3550  * need to fault.
 3551  */
 3552 int
 3553 pmap_emulate_modified(pmap_t pmap, vm_offset_t va)
 3554 {
 3555         pt_entry_t *pte;
 3556 
 3557         PMAP_LOCK(pmap);
 3558         pte = pmap_pte(pmap, va);
 3559 
 3560         /*
 3561          * It is possible that some other CPU or thread changed the pmap while
 3562          * we weren't looking; in the SMP case, this is readily apparent, but
 3563          * it can even happen in the UP case, because we may have been blocked
 3564          * on PMAP_LOCK(pmap) above while someone changed this out from
 3565          * underneath us.
 3566          */
 3567 
 3568         if (pte == NULL) {
 3569                 /*
 3570                  * This PTE's PTP (or one of its ancestors) has been reclaimed;
 3571                  * trigger a full fault to reconstruct it via pmap_enter.
 3572                  */
 3573                 PMAP_UNLOCK(pmap);
 3574                 return (1);
 3575         }
 3576 
 3577         if (!pte_test(pte, PTE_V)) {
 3578                 /*
 3579                  * This PTE is no longer valid; the other thread or other
 3580                  * processor must have arranged for our TLB to no longer
 3581                  * have this entry, possibly by IPI, so no tlb_update is
 3582                  * required.  Fall out of the fast path and go take a
 3583                  * general fault before retrying the instruction (or taking
 3584                  * a signal).
 3585                  */
 3586                 PMAP_UNLOCK(pmap);
 3587                 return (1);
 3588         }
 3589 
 3590         if (pte_test(pte, PTE_D)) {
 3591                 /*
 3592                  * This PTE is valid and has the PTE_D bit asserted; since
 3593                  * this is an increase in permission, we may have been expected
 3594                  * to update the TLB lazily.  Do so here and return, on the
 3595                  * fast path, to retry the instruction.
 3596                  */
 3597                 tlb_update(pmap, va, *pte);
 3598                 PMAP_UNLOCK(pmap);
 3599                 return (0);
 3600         }
 3601 
 3602         if (pte_test(pte, PTE_RO)) {
 3603                 /*
 3604                  * This PTE is valid, not dirty, and read-only.  Go take a
 3605                  * full fault (most likely to upgrade this part of the address
 3606                  * space to writeable).
 3607                  */
 3608                 PMAP_UNLOCK(pmap);
 3609                 return (1);
 3610         }
 3611 
 3612         if (!pte_test(pte, PTE_MANAGED))
 3613                 panic("pmap_emulate_modified: unmanaged page");
 3614 
 3615         /*
 3616          * PTE is valid, managed, not dirty, and not read-only.  Set PTE_D
 3617          * and eagerly update the local TLB, returning on the fast path.
 3618          */
 3619 
 3620         pte_set(pte, PTE_D);
 3621         tlb_update(pmap, va, *pte);
 3622         PMAP_UNLOCK(pmap);
 3623 
 3624         return (0);
 3625 }
 3626 
 3627 /*
 3628  *      Routine:        pmap_kextract
 3629  *      Function:
 3630  *              Extract the physical page address associated
 3631  *              virtual address.
 3632  */
 3633 vm_paddr_t
 3634 pmap_kextract(vm_offset_t va)
 3635 {
 3636         int mapped;
 3637 
 3638         /*
 3639          * First, the direct-mapped regions.
 3640          */
 3641 #if defined(__mips_n64)
 3642         if (va >= MIPS_XKPHYS_START && va < MIPS_XKPHYS_END)
 3643                 return (MIPS_XKPHYS_TO_PHYS(va));
 3644 #endif
 3645         if (va >= MIPS_KSEG0_START && va < MIPS_KSEG0_END)
 3646                 return (MIPS_KSEG0_TO_PHYS(va));
 3647 
 3648         if (va >= MIPS_KSEG1_START && va < MIPS_KSEG1_END)
 3649                 return (MIPS_KSEG1_TO_PHYS(va));
 3650 
 3651         /*
 3652          * User virtual addresses.
 3653          */
 3654         if (va < VM_MAXUSER_ADDRESS) {
 3655                 pt_entry_t *ptep;
 3656 
 3657                 if (curproc && curproc->p_vmspace) {
 3658                         ptep = pmap_pte(&curproc->p_vmspace->vm_pmap, va);
 3659                         if (ptep) {
 3660                                 return (TLBLO_PTE_TO_PA(*ptep) |
 3661                                     (va & PAGE_MASK));
 3662                         }
 3663                         return (0);
 3664                 }
 3665         }
 3666 
 3667         /*
 3668          * Should be kernel virtual here, otherwise fail
 3669          */
 3670         mapped = (va >= MIPS_KSEG2_START || va < MIPS_KSEG2_END);
 3671 #if defined(__mips_n64)
 3672         mapped = mapped || (va >= MIPS_XKSEG_START || va < MIPS_XKSEG_END);
 3673 #endif
 3674         /*
 3675          * Kernel virtual.
 3676          */
 3677 
 3678         if (mapped) {
 3679                 pt_entry_t *ptep;
 3680 
 3681                 /* Is the kernel pmap initialized? */
 3682                 if (!CPU_EMPTY(&kernel_pmap->pm_active)) {
 3683                         /* It's inside the virtual address range */
 3684                         ptep = pmap_pte(kernel_pmap, va);
 3685                         if (ptep) {
 3686                                 return (TLBLO_PTE_TO_PA(*ptep) |
 3687                                     (va & PAGE_MASK));
 3688                         }
 3689                 }
 3690                 return (0);
 3691         }
 3692 
 3693         panic("%s for unknown address space %p.", __func__, (void *)va);
 3694 }
 3695 
 3696 
 3697 void
 3698 pmap_flush_pvcache(vm_page_t m)
 3699 {
 3700         pv_entry_t pv;
 3701 
 3702         if (m != NULL) {
 3703                 for (pv = TAILQ_FIRST(&m->md.pv_list); pv;
 3704                     pv = TAILQ_NEXT(pv, pv_list)) {
 3705                         mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
 3706                 }
 3707         }
 3708 }
 3709 
 3710 void
 3711 pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
 3712 {
 3713 
 3714         /*
 3715          * It appears that this function can only be called before any mappings
 3716          * for the page are established.  If this ever changes, this code will
 3717          * need to walk the pv_list and make each of the existing mappings
 3718          * uncacheable, being careful to sync caches and PTEs (and maybe
 3719          * invalidate TLB?) for any current mapping it modifies.
 3720          */
 3721         if (TAILQ_FIRST(&m->md.pv_list) != NULL)
 3722                 panic("Can't change memattr on page with existing mappings");
 3723 
 3724         /* Clean memattr portion of pv_flags */
 3725         m->md.pv_flags &= ~PV_MEMATTR_MASK;
 3726         m->md.pv_flags |= (ma << PV_MEMATTR_SHIFT) & PV_MEMATTR_MASK;
 3727 }
 3728 
 3729 static __inline void
 3730 pmap_pte_attr(pt_entry_t *pte, vm_memattr_t ma)
 3731 {
 3732         u_int npte;
 3733 
 3734         npte = *(u_int *)pte;
 3735         npte &= ~PTE_C_MASK;
 3736         npte |= PTE_C(ma);
 3737         *pte = npte;
 3738 }
 3739 
 3740 int
 3741 pmap_change_attr(vm_offset_t sva, vm_size_t size, vm_memattr_t ma)
 3742 {
 3743         pd_entry_t *pde, *pdpe;
 3744         pt_entry_t *pte;
 3745         vm_offset_t ova, eva, va, va_next;
 3746         pmap_t pmap;
 3747 
 3748         ova = sva;
 3749         eva = sva + size;
 3750         if (eva < sva)
 3751                 return (EINVAL);
 3752 
 3753         pmap = kernel_pmap;
 3754         PMAP_LOCK(pmap);
 3755 
 3756         for (; sva < eva; sva = va_next) {
 3757                 pdpe = pmap_segmap(pmap, sva);
 3758 #ifdef __mips_n64
 3759                 if (*pdpe == 0) {
 3760                         va_next = (sva + NBSEG) & ~SEGMASK;
 3761                         if (va_next < sva)
 3762                                 va_next = eva;
 3763                         continue;
 3764                 }
 3765 #endif
 3766                 va_next = (sva + NBPDR) & ~PDRMASK;
 3767                 if (va_next < sva)
 3768                         va_next = eva;
 3769 
 3770                 pde = pmap_pdpe_to_pde(pdpe, sva);
 3771                 if (*pde == NULL)
 3772                         continue;
 3773 
 3774                 /*
 3775                  * Limit our scan to either the end of the va represented
 3776                  * by the current page table page, or to the end of the
 3777                  * range being removed.
 3778                  */
 3779                 if (va_next > eva)
 3780                         va_next = eva;
 3781 
 3782                 va = va_next;
 3783                 for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
 3784                     sva += PAGE_SIZE) {
 3785                         if (!pte_test(pte, PTE_V) || pte_cache_bits(pte) == ma) {
 3786                                 if (va != va_next) {
 3787                                         pmap_invalidate_range(pmap, va, sva);
 3788                                         va = va_next;
 3789                                 }
 3790                                 continue;
 3791                         }
 3792                         if (va == va_next)
 3793                                 va = sva;
 3794 
 3795                         pmap_pte_attr(pte, ma);
 3796                 }
 3797                 if (va != va_next)
 3798                         pmap_invalidate_range(pmap, va, sva);
 3799         }
 3800         PMAP_UNLOCK(pmap);
 3801 
 3802         /* Flush caches to be in the safe side */
 3803         mips_dcache_wbinv_range(ova, size);
 3804         return 0;
 3805 }
 3806 
 3807 boolean_t
 3808 pmap_is_valid_memattr(pmap_t pmap __unused, vm_memattr_t mode)
 3809 {
 3810 
 3811         switch (mode) {
 3812         case VM_MEMATTR_UNCACHEABLE:
 3813         case VM_MEMATTR_WRITE_BACK:
 3814 #ifdef MIPS_CCA_WC
 3815         case VM_MEMATTR_WRITE_COMBINING:
 3816 #endif
 3817                 return (TRUE);
 3818         default:
 3819                 return (FALSE);
 3820         }
 3821 }

Cache object: 46e53d0eba7f578a44cd5c76f5e4bbcc


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.