tlb.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2004-2010 Juli Mallett <jmallett@FreeBSD.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  *
   14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   24  * SUCH DAMAGE.
   25  *
   26  * $FreeBSD: releng/10.1/sys/mips/mips/tlb.c 255935 2013-09-29 10:14:16Z adrian $
   27  */
   28 
   29 #include "opt_ddb.h"
   30 
   31 #include <sys/param.h>
   32 #include <sys/kernel.h>
   33 #include <sys/systm.h>
   34 #include <sys/pcpu.h>
   35 #include <sys/smp.h>
   36 
   37 #include <vm/vm.h>
   38 #include <vm/pmap.h>
   39 
   40 #include <machine/pte.h>
   41 #include <machine/tlb.h>
   42 
   43 #if defined(CPU_CNMIPS)
   44 #define MIPS_MAX_TLB_ENTRIES    128
   45 #elif defined(CPU_NLM)
   46 #define MIPS_MAX_TLB_ENTRIES    (2048 + 128)
   47 #else
   48 #define MIPS_MAX_TLB_ENTRIES    64
   49 #endif
   50 
   51 struct tlb_state {
   52         unsigned wired;
   53         struct tlb_entry {
   54                 register_t entryhi;
   55                 register_t entrylo0;
   56                 register_t entrylo1;
   57                 register_t pagemask;
   58         } entry[MIPS_MAX_TLB_ENTRIES];
   59 };
   60 
   61 static struct tlb_state tlb_state[MAXCPU];
   62 
   63 #if 0
   64 /*
   65  * PageMask must increment in steps of 2 bits.
   66  */
   67 COMPILE_TIME_ASSERT(POPCNT(TLBMASK_MASK) % 2 == 0);
   68 #endif
   69 
   70 static inline void
   71 tlb_probe(void)
   72 {
   73         __asm __volatile ("tlbp" : : : "memory");
   74         mips_cp0_sync();
   75 }
   76 
   77 static inline void
   78 tlb_read(void)
   79 {
   80         __asm __volatile ("tlbr" : : : "memory");
   81         mips_cp0_sync();
   82 }
   83 
   84 static inline void
   85 tlb_write_indexed(void)
   86 {
   87         __asm __volatile ("tlbwi" : : : "memory");
   88         mips_cp0_sync();
   89 }
   90 
   91 static inline void
   92 tlb_write_random(void)
   93 {
   94         __asm __volatile ("tlbwr" : : : "memory");
   95         mips_cp0_sync();
   96 }
   97 
   98 static void tlb_invalidate_one(unsigned);
   99 
  100 void
  101 tlb_insert_wired(unsigned i, vm_offset_t va, pt_entry_t pte0, pt_entry_t pte1)
  102 {
  103         register_t asid;
  104         register_t s;
  105 
  106         va &= ~PAGE_MASK;
  107 
  108         s = intr_disable();
  109         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  110 
  111         mips_wr_index(i);
  112         mips_wr_pagemask(0);
  113         mips_wr_entryhi(TLBHI_ENTRY(va, 0));
  114         mips_wr_entrylo0(pte0);
  115         mips_wr_entrylo1(pte1);
  116         tlb_write_indexed();
  117 
  118         mips_wr_entryhi(asid);
  119         intr_restore(s);
  120 }
  121 
  122 void
  123 tlb_invalidate_address(struct pmap *pmap, vm_offset_t va)
  124 {
  125         register_t asid;
  126         register_t s;
  127         int i;
  128 
  129         va &= ~PAGE_MASK;
  130 
  131         s = intr_disable();
  132         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  133 
  134         mips_wr_pagemask(0);
  135         mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
  136         tlb_probe();
  137         i = mips_rd_index();
  138         if (i >= 0)
  139                 tlb_invalidate_one(i);
  140 
  141         mips_wr_entryhi(asid);
  142         intr_restore(s);
  143 }
  144 
  145 void
  146 tlb_invalidate_all(void)
  147 {
  148         register_t asid;
  149         register_t s;
  150         unsigned i;
  151 
  152         s = intr_disable();
  153         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  154 
  155         for (i = mips_rd_wired(); i < num_tlbentries; i++)
  156                 tlb_invalidate_one(i);
  157 
  158         mips_wr_entryhi(asid);
  159         intr_restore(s);
  160 }
  161 
  162 void
  163 tlb_invalidate_all_user(struct pmap *pmap)
  164 {
  165         register_t asid;
  166         register_t s;
  167         unsigned i;
  168 
  169         s = intr_disable();
  170         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  171 
  172         for (i = mips_rd_wired(); i < num_tlbentries; i++) {
  173                 register_t uasid;
  174 
  175                 mips_wr_index(i);
  176                 tlb_read();
  177 
  178                 uasid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  179                 if (pmap == NULL) {
  180                         /*
  181                          * Invalidate all non-kernel entries.
  182                          */
  183                         if (uasid == 0)
  184                                 continue;
  185                 } else {
  186                         /*
  187                          * Invalidate this pmap's entries.
  188                          */
  189                         if (uasid != pmap_asid(pmap))
  190                                 continue;
  191                 }
  192                 tlb_invalidate_one(i);
  193         }
  194 
  195         mips_wr_entryhi(asid);
  196         intr_restore(s);
  197 }
  198 
  199 /*
  200  * Invalidates any TLB entries that map a virtual page from the specified
  201  * address range.  If "end" is zero, then every virtual page is considered to
  202  * be within the address range's upper bound.
  203  */
  204 void
  205 tlb_invalidate_range(pmap_t pmap, vm_offset_t start, vm_offset_t end)
  206 {
  207         register_t asid, end_hi, hi, hi_pagemask, s, save_asid, start_hi;
  208         int i;
  209 
  210         KASSERT(start < end || (end == 0 && start > 0),
  211             ("tlb_invalidate_range: invalid range"));
  212 
  213         /*
  214          * Truncate the virtual address "start" to an even page frame number,
  215          * and round the virtual address "end" to an even page frame number.
  216          */
  217         start &= ~((1 << TLBMASK_SHIFT) - 1);
  218         end = (end + (1 << TLBMASK_SHIFT) - 1) & ~((1 << TLBMASK_SHIFT) - 1);
  219 
  220         s = intr_disable();
  221         save_asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  222 
  223         asid = pmap_asid(pmap);
  224         start_hi = TLBHI_ENTRY(start, asid);
  225         end_hi = TLBHI_ENTRY(end, asid);
  226 
  227         /*
  228          * Select the fastest method for invalidating the TLB entries.
  229          */
  230         if (end - start < num_tlbentries << TLBMASK_SHIFT || (end == 0 &&
  231             start >= -(num_tlbentries << TLBMASK_SHIFT))) {
  232                 /*
  233                  * The virtual address range is small compared to the size of
  234                  * the TLB.  Probe the TLB for each even numbered page frame
  235                  * within the virtual address range.
  236                  */
  237                 for (hi = start_hi; hi != end_hi; hi += 1 << TLBMASK_SHIFT) {
  238                         mips_wr_pagemask(0);
  239                         mips_wr_entryhi(hi);
  240                         tlb_probe();
  241                         i = mips_rd_index();
  242                         if (i >= 0)
  243                                 tlb_invalidate_one(i);
  244                 }
  245         } else {
  246                 /*
  247                  * The virtual address range is large compared to the size of
  248                  * the TLB.  Test every non-wired TLB entry.
  249                  */
  250                 for (i = mips_rd_wired(); i < num_tlbentries; i++) {
  251                         mips_wr_index(i);
  252                         tlb_read();
  253                         hi = mips_rd_entryhi();
  254                         if ((hi & TLBHI_ASID_MASK) == asid && (hi < end_hi ||
  255                             end == 0)) {
  256                                 /*
  257                                  * If "hi" is a large page that spans
  258                                  * "start_hi", then it must be invalidated.
  259                                  */
  260                                 hi_pagemask = mips_rd_pagemask();
  261                                 if (hi >= (start_hi & ~(hi_pagemask <<
  262                                     TLBMASK_SHIFT)))
  263                                         tlb_invalidate_one(i);
  264                         }
  265                 }
  266         }
  267 
  268         mips_wr_entryhi(save_asid);
  269         intr_restore(s);
  270 }
  271 
  272 /* XXX Only if DDB?  */
  273 void
  274 tlb_save(void)
  275 {
  276         unsigned ntlb, i, cpu;
  277 
  278         cpu = PCPU_GET(cpuid);
  279         if (num_tlbentries > MIPS_MAX_TLB_ENTRIES)
  280                 ntlb = MIPS_MAX_TLB_ENTRIES;
  281         else
  282                 ntlb = num_tlbentries;
  283         tlb_state[cpu].wired = mips_rd_wired();
  284         for (i = 0; i < ntlb; i++) {
  285                 mips_wr_index(i);
  286                 tlb_read();
  287 
  288                 tlb_state[cpu].entry[i].entryhi = mips_rd_entryhi();
  289                 tlb_state[cpu].entry[i].pagemask = mips_rd_pagemask();
  290                 tlb_state[cpu].entry[i].entrylo0 = mips_rd_entrylo0();
  291                 tlb_state[cpu].entry[i].entrylo1 = mips_rd_entrylo1();
  292         }
  293 }
  294 
  295 void
  296 tlb_update(struct pmap *pmap, vm_offset_t va, pt_entry_t pte)
  297 {
  298         register_t asid;
  299         register_t s;
  300         int i;
  301 
  302         va &= ~PAGE_MASK;
  303         pte &= ~TLBLO_SWBITS_MASK;
  304 
  305         s = intr_disable();
  306         asid = mips_rd_entryhi() & TLBHI_ASID_MASK;
  307 
  308         mips_wr_pagemask(0);
  309         mips_wr_entryhi(TLBHI_ENTRY(va, pmap_asid(pmap)));
  310         tlb_probe();
  311         i = mips_rd_index();
  312         if (i >= 0) {
  313                 tlb_read();
  314 
  315                 if ((va & PAGE_SIZE) == 0) {
  316                         mips_wr_entrylo0(pte);
  317                 } else {
  318                         mips_wr_entrylo1(pte);
  319                 }
  320                 tlb_write_indexed();
  321         }
  322 
  323         mips_wr_entryhi(asid);
  324         intr_restore(s);
  325 }
  326 
  327 static void
  328 tlb_invalidate_one(unsigned i)
  329 {
  330         /* XXX an invalid ASID? */
  331         mips_wr_entryhi(TLBHI_ENTRY(MIPS_KSEG0_START + (2 * i * PAGE_SIZE), 0));
  332         mips_wr_entrylo0(0);
  333         mips_wr_entrylo1(0);
  334         mips_wr_pagemask(0);
  335         mips_wr_index(i);
  336         tlb_write_indexed();
  337 }
  338 
  339 #ifdef DDB
  340 #include <ddb/ddb.h>
  341 
  342 DB_SHOW_COMMAND(tlb, ddb_dump_tlb)
  343 {
  344         register_t ehi, elo0, elo1, epagemask;
  345         unsigned i, cpu, ntlb;
  346 
  347         /*
  348          * XXX
  349          * The worst conversion from hex to decimal ever.
  350          */
  351         if (have_addr)
  352                 cpu = ((addr >> 4) % 16) * 10 + (addr % 16);
  353         else
  354                 cpu = PCPU_GET(cpuid);
  355 
  356         if (cpu < 0 || cpu >= mp_ncpus) {
  357                 db_printf("Invalid CPU %u\n", cpu);
  358                 return;
  359         }
  360         if (num_tlbentries > MIPS_MAX_TLB_ENTRIES) {
  361                 ntlb = MIPS_MAX_TLB_ENTRIES;
  362                 db_printf("Warning: Only %d of %d TLB entries saved!\n",
  363                     ntlb, num_tlbentries);
  364         } else
  365                 ntlb = num_tlbentries;
  366 
  367         if (cpu == PCPU_GET(cpuid))
  368                 tlb_save();
  369 
  370         db_printf("Beginning TLB dump for CPU %u...\n", cpu);
  371         for (i = 0; i < ntlb; i++) {
  372                 if (i == tlb_state[cpu].wired) {
  373                         if (i != 0)
  374                                 db_printf("^^^ WIRED ENTRIES ^^^\n");
  375                         else
  376                                 db_printf("(No wired entries.)\n");
  377                 }
  378 
  379                 /* XXX PageMask.  */
  380                 ehi = tlb_state[cpu].entry[i].entryhi;
  381                 elo0 = tlb_state[cpu].entry[i].entrylo0;
  382                 elo1 = tlb_state[cpu].entry[i].entrylo1;
  383                 epagemask = tlb_state[cpu].entry[i].pagemask;
  384 
  385                 if (elo0 == 0 && elo1 == 0)
  386                         continue;
  387 
  388                 db_printf("#%u\t=> %jx (pagemask %jx)\n", i, (intmax_t)ehi, (intmax_t) epagemask);
  389                 db_printf(" Lo0\t%jx\t(%#jx)\n", (intmax_t)elo0, (intmax_t)TLBLO_PTE_TO_PA(elo0));
  390                 db_printf(" Lo1\t%jx\t(%#jx)\n", (intmax_t)elo1, (intmax_t)TLBLO_PTE_TO_PA(elo1));
  391         }
  392         db_printf("Finished.\n");
  393 }
  394 #endif
Cache object: 3bf7b991b6230dfeba870cfec4655b8e
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/mips/mips/tlb.c

FreeBSD/Linux Kernel Cross Reference
sys/mips/mips/tlb.c
