The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/mips/mips/tlb.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2004-2010 Juli Mallett <jmallett@FreeBSD.org>
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  *
   28  * $FreeBSD$
   29  */
   30 
   31 #include "opt_ddb.h"
   32 
   33 #include <sys/param.h>
   34 #include <sys/kernel.h>
   35 #include <sys/systm.h>
   36 #include <sys/pcpu.h>
   37 #include <sys/smp.h>
   38 
   39 #include <vm/vm.h>
   40 #include <vm/pmap.h>
   41 
   42 #include <machine/pte.h>
   43 #include <machine/tlb.h>
   44 
   45 #if defined(CPU_CNMIPS)
   46 #define MIPS_MAX_TLB_ENTRIES    128
   47 #elif defined(CPU_NLM)
   48 #define MIPS_MAX_TLB_ENTRIES    (2048 + 128)
   49 #else
   50 #define MIPS_MAX_TLB_ENTRIES    64
   51 #endif
   52 
   53 struct tlb_state {
   54         unsigned wired;
   55         struct tlb_entry {
   56                 register_t entryhi;
   57                 register_t entrylo0;
   58                 register_t entrylo1;
   59                 register_t pagemask;
   60         } entry[MIPS_MAX_TLB_ENTRIES];
   61 };
   62 
   63 static struct tlb_state tlb_state[MAXCPU];
   64 
   65 #if 0
   66 /*
   67  * PageMask must increment in steps of 2 bits.
   68  */
   69 COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0);
   70 #endif
   71 
   72 static inline void
   73 tlb_probe(void)
   74 {
   75         __asm __volatile ("tlbp" : : : "memory");
   76         mips_cp0_sync();
   77 }
   78 
   79 static inline void
   80 tlb_read(void)
   81 {
   82         __asm __volatile ("tlbr" : : : "memory");
   83         mips_cp0_sync();
   84 }
   85 
   86 static inline void
   87 tlb_write_indexed(void)
   88 {
   89         __asm __volatile ("tlbwi" : : : "memory");
   90         mips_cp0_sync();
   91 }
   92 
   93 static void tlb_invalidate_one(unsigned);
   94 
   95 void
   96 tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
   97 {
   98         register_t asid;
   99         register_t s;
  100 
  101         va &= ~PAGE_MASK;
  102 
  103         s = intr_disable();
  104         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  105 
  106         mips_wr_index(i);
  107         mips_wr_pagemask(0);
  108         mips_wr_entryhi(TLBHI_ENTRY(va, 0));
  109         mips_wr_entrylo0(pte0);
  110         mips_wr_entrylo1(pte1);
  111         tlb_write_indexed();
  112 
  113         mips_wr_entryhi(asid);
  114         intr_restore(s);
  115 }
  116 
  117 void
  118 tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
  119 {
  120         register_t asid;
  121         register_t s;
  122         int i;
  123 
  124         va &= ~PAGE_MASK;
  125 
  126         s = intr_disable();
  127         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  128 
  129         mips_wr_pagemask(0);
  130         mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
  131         tlb_probe();
  132         i = mips_rd_index();
  133         if (i >= 0)
  134                 tlb_invalidate_one(i);
  135 
  136         mips_wr_entryhi(asid);
  137         intr_restore(s);
  138 }
  139 
  140 void
  141 tlb_invalidate_all(void)
  142 {
  143         register_t asid;
  144         register_t s;
  145         unsigned i;
  146 
  147         s = intr_disable();
  148         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  149 
  150         for (i = mips_rd_wired(); i < num_tlbentries; i++)
  151                 tlb_invalidate_one(i);
  152 
  153         mips_wr_entryhi(asid);
  154         intr_restore(s);
  155 }
  156 
  157 void
  158 tlb_invalidate_all_user(struct pmap *pmap)
  159 {
  160         register_t asid;
  161         register_t s;
  162         unsigned i;
  163 
  164         s = intr_disable();
  165         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  166 
  167         for (i = mips_rd_wired(); i < num_tlbentries; i++) {
  168                 register_t uasid;
  169 
  170                 mips_wr_index(i);
  171                 tlb_read();
  172 
  173                 uasid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  174                 if (pmap == NULL) {
  175                         /*
  176                          * Invalidate all non-kernel entries.
  177                          */
  178                         if (uasid == 0)
  179                                 continue;
  180                 } else {
  181                         /*
  182                          * Invalidate this pmap's entries.
  183                          */
  184                         if (uasid != pmap_asid(pmap))
  185                                 continue;
  186                 }
  187                 tlb_invalidate_one(i);
  188         }
  189 
  190         mips_wr_entryhi(asid);
  191         intr_restore(s);
  192 }
  193 
  194 /*
  195  * Invalidates any TLB entries that map a virtual page from the specified
  196  * address range.  If "end" is zero, then every virtual page is considered to
  197  * be within the address range's upper bound.
  198  */
  199 void
  200 tlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
  201 {
  202         register_t asid, end_hi, hi, hi_pagemask, s, save_asid, start_hi;
  203         int i;
  204 
  205         KASSERT(start < end || (end == 0 && start > 0),
  206             ("tlb_invalidate_range: invalid range"));
  207 
  208         /*
  209          * Truncate the virtual address "start" to an even page frame number,
  210          * and round the virtual address "end" to an even page frame number.
  211          */
  212         start &= ~((1 << TLBMASK_SHIFT) - 1);
  213         end = roundup2(end, 1 << TLBMASK_SHIFT);
  214 
  215         s = intr_disable();
  216         save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  217 
  218         asid = pmap_asid(pmap);
  219         start_hi = TLBHI_ENTRY(start, asid);
  220         end_hi = TLBHI_ENTRY(end, asid);
  221 
  222         /*
  223          * Select the fastest method for invalidating the TLB entries.
  224          */
  225         if (end - start < num_tlbentries << TLBMASK_SHIFT || (end == 0 &&
  226             start >= -(num_tlbentries << TLBMASK_SHIFT))) {
  227                 /*
  228                  * The virtual address range is small compared to the size of
  229                  * the TLB.  Probe the TLB for each even numbered page frame
  230                  * within the virtual address range.
  231                  */
  232                 for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) {
  233                         mips_wr_pagemask(0);
  234                         mips_wr_entryhi(hi);
  235                         tlb_probe();
  236                         i = mips_rd_index();
  237                         if (i >= 0)
  238                                 tlb_invalidate_one(i);
  239                 }
  240         } else {
  241                 /*
  242                  * The virtual address range is large compared to the size of
  243                  * the TLB.  Test every non-wired TLB entry.
  244                  */
  245                 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
  246                         mips_wr_index(i);
  247                         tlb_read();
  248                         hi = mips_rd_entryhi();
  249                         if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi ||
  250                             end == 0)) {
  251                                 /*
  252                                  * If "hi" is a large page that spans
  253                                  * "start_hi", then it must be invalidated.
  254                                  */
  255                                 hi_pagemask = mips_rd_pagemask();
  256                                 if (hi >= (start_hi & ~(hi_pagemask <<
  257                                     TLBMASK_SHIFT)))
  258                                         tlb_invalidate_one(i);
  259                         }
  260                 }
  261         }
  262 
  263         mips_wr_entryhi(save_asid);
  264         intr_restore(s);
  265 }
  266 
  267 /* XXX Only if DDB?  */
  268 void
  269 tlb_save(void)
  270 {
  271         unsigned ntlb, i, cpu;
  272 
  273         cpu = PCPU_GET(cpuid);
  274         if (num_tlbentries > MIPS_MAX_TLB_ENTRIES)
  275                 ntlb = MIPS_MAX_TLB_ENTRIES;
  276         else
  277                 ntlb = num_tlbentries;
  278         tlb_state[cpu].wired = mips_rd_wired();
  279         for (i = 0; i < ntlb; i++) {
  280                 mips_wr_index(i);
  281                 tlb_read();
  282 
  283                 tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi();
  284                 tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask();
  285                 tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0();
  286                 tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1();
  287         }
  288 }
  289 
  290 void
  291 tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
  292 {
  293         register_t asid;
  294         register_t s;
  295         int i;
  296 
  297         va &= ~PAGE_MASK;
  298         pte &= ~TLBLO_SWBITS_MASK;
  299 
  300         s = intr_disable();
  301         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  302 
  303         mips_wr_pagemask(0);
  304         mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
  305         tlb_probe();
  306         i = mips_rd_index();
  307         if (i >= 0) {
  308                 tlb_read();
  309 
  310                 if ((va & PAGE_SIZE) == 0) {
  311                         mips_wr_entrylo0(pte);
  312                 } else {
  313                         mips_wr_entrylo1(pte);
  314                 }
  315                 tlb_write_indexed();
  316         }
  317 
  318         mips_wr_entryhi(asid);
  319         intr_restore(s);
  320 }
  321 
  322 static void
  323 tlb_invalidate_one(unsigned i)
  324 {
  325         /* XXX an invalid ASID? */
  326         mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
  327         mips_wr_entrylo0(0);
  328         mips_wr_entrylo1(0);
  329         mips_wr_pagemask(0);
  330         mips_wr_index(i);
  331         tlb_write_indexed();
  332 }
  333 
  334 #ifdef DDB
  335 #include <ddb/ddb.h>
  336 
  337 DB_SHOW_COMMAND(tlb, ddb_dump_tlb)
  338 {
  339         register_t ehi, elo0, elo1, epagemask;
  340         unsigned i, cpu, ntlb;
  341 
  342         /*
  343          * XXX
  344          * The worst conversion from hex to decimal ever.
  345          */
  346         if (have_addr)
  347                 cpu = ((addr >> 4) % 16) * 10 + (addr % 16);
  348         else
  349                 cpu = PCPU_GET(cpuid);
  350 
  351         if (cpu >= mp_ncpus) {
  352                 db_printf("Invalid CPU %u\n", cpu);
  353                 return;
  354         }
  355         if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) {
  356                 ntlb = MIPS_MAX_TLB_ENTRIES;
  357                 db_printf("Warning: Only %d of %d TLB entries saved!\n",
  358                     ntlb, num_tlbentries);
  359         } else
  360                 ntlb = num_tlbentries;
  361 
  362         if (cpu == PCPU_GET(cpuid))
  363                 tlb_save();
  364 
  365         db_printf("Beginning TLB dump for CPU %u...\n", cpu);
  366         for (i = 0; i < ntlb; i++) {
  367                 if (i == tlb_state[cpu].wired) {
  368                         if (i != 0)
  369                                 db_printf("^^^ WIRED ENTRIES ^^^\n");
  370                         else
  371                                 db_printf("(No wired entries.)\n");
  372                 }
  373 
  374                 /* XXX PageMask.  */
  375                 ehi = tlb_state[cpu].entry[i].entryhi;
  376                 elo0 = tlb_state[cpu].entry[i].entrylo0;
  377                 elo1 = tlb_state[cpu].entry[i].entrylo1;
  378                 epagemask = tlb_state[cpu].entry[i].pagemask;
  379 
  380                 if (elo0 == 0 && elo1 == 0)
  381                         continue;
  382 
  383                 db_printf("#%u\t=> %jx (pagemask %jx)\n", i, (intmax_t)ehi, (intmax_t) epagemask);
  384                 db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0));
  385                 db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1));
  386         }
  387         db_printf("Finished.\n");
  388 }
  389 #endif

Cache object: e734d0108c55f83caf5dbd7fa684cff7


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.