The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/powerpc/booke/trap_subr.S

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (C) 2006-2009 Semihalf, Rafal Jaworowski <raj@semihalf.com>
    3  * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
    4  * Copyright (C) 2006 Juniper Networks, Inc.
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  * 3. The name of the author may not be used to endorse or promote products
   16  *    derived from this software without specific prior written permission.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
   19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
   21  * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   22  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
   23  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
   24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
   25  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
   26  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
   27  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   28  *
   29  * $FreeBSD: releng/11.0/sys/powerpc/booke/trap_subr.S 295520 2016-02-11 13:15:37Z jhibbits $
   30  */
   31 /*-
   32  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
   33  * Copyright (C) 1995, 1996 TooLs GmbH.
   34  * All rights reserved.
   35  *
   36  * Redistribution and use in source and binary forms, with or without
   37  * modification, are permitted provided that the following conditions
   38  * are met:
   39  * 1. Redistributions of source code must retain the above copyright
   40  *    notice, this list of conditions and the following disclaimer.
   41  * 2. Redistributions in binary form must reproduce the above copyright
   42  *    notice, this list of conditions and the following disclaimer in the
   43  *    documentation and/or other materials provided with the distribution.
   44  * 3. All advertising materials mentioning features or use of this software
   45  *    must display the following acknowledgement:
   46  *      This product includes software developed by TooLs GmbH.
   47  * 4. The name of TooLs GmbH may not be used to endorse or promote products
   48  *    derived from this software without specific prior written permission.
   49  *
   50  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
   51  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   52  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
   53  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   54  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
   55  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
   56  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
   57  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
   58  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
   59  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   60  *
   61  *      from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
   62  */
   63 
   64 /*
   65  * NOTICE: This is not a standalone file.  to use it, #include it in
   66  * your port's locore.S, like so:
   67  *
   68  *      #include <powerpc/booke/trap_subr.S>
   69  */
   70 
   71 /*
   72  * SPRG usage notes
   73  *
   74  * SPRG0 - pcpu pointer
   75  * SPRG1 - all interrupts except TLB miss, critical, machine check
   76  * SPRG2 - critical
   77  * SPRG3 - machine check
   78  * SPRG4-6 - scratch
   79  *
   80  */
   81 
   82 /* Get the per-CPU data structure */
   83 #define GET_CPUINFO(r) mfsprg0 r
   84 
   85 #define RES_GRANULE     32
   86 #define RES_LOCK        0       /* offset to the 'lock' word */
   87 #define RES_RECURSE     4       /* offset to the 'recurse' word */
   88 
   89 /*
   90  * Standard interrupt prolog
   91  *
   92  * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
   93  * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
   94  * isrr0-1 - save restore registers with CPU state at interrupt time (may be
   95  *           SRR0-1, CSRR0-1, MCSRR0-1
   96  *
   97  * 1. saves in the given savearea:
   98  *   - R30-31
   99  *   - DEAR, ESR
  100  *   - xSRR0-1
  101  *
  102  * 2. saves CR -> R30
  103  *
  104  * 3. switches to kstack if needed
  105  *
  106  * 4. notes:
  107  *   - R31 can be used as scratch register until a new frame is layed on
  108  *     the stack with FRAME_SETUP
  109  *
  110  *   - potential TLB miss: NO. Saveareas are always acessible via TLB1 
  111  *     permanent entries, and within this prolog we do not dereference any
  112  *     locations potentially not in the TLB
  113  */
  114 #define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)                \
  115         mtspr   sprg_sp, %r1;           /* Save SP */                   \
  116         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
  117         stw     %r30, (savearea+CPUSAVE_R30)(%r1);                      \
  118         stw     %r31, (savearea+CPUSAVE_R31)(%r1);                      \
  119         mfdear  %r30;                                                   \
  120         mfesr   %r31;                                                   \
  121         stw     %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
  122         stw     %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
  123         mfspr   %r30, isrr0;                                            \
  124         mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
  125         stw     %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
  126         stw     %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
  127         isync;                                                          \
  128         mfspr   %r1, sprg_sp;           /* Restore SP */                \
  129         mfcr    %r30;                   /* Save CR */                   \
  130         /* switch to per-thread kstack if intr taken in user mode */    \
  131         mtcr    %r31;                   /* MSR at interrupt time  */    \
  132         bf      17, 1f;                                                 \
  133         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
  134         lwz     %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
  135 1:
  136 
  137 #define STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)           \
  138         mtspr   sprg_sp, %r1;           /* Save SP */                   \
  139         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
  140         stw     %r30, (savearea+CPUSAVE_R30)(%r1);                      \
  141         stw     %r31, (savearea+CPUSAVE_R31)(%r1);                      \
  142         mfdear  %r30;                                                   \
  143         mfesr   %r31;                                                   \
  144         stw     %r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);               \
  145         stw     %r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);                \
  146         mfspr   %r30, isrr0;                                            \
  147         mfspr   %r31, isrr1;            /* MSR at interrupt time */     \
  148         stw     %r30, (savearea+CPUSAVE_SRR0)(%r1);                     \
  149         stw     %r31, (savearea+CPUSAVE_SRR1)(%r1);                     \
  150         mfspr   %r30, SPR_SRR0;                                         \
  151         mfspr   %r31, SPR_SRR1;         /* MSR at interrupt time */     \
  152         stw     %r30, (savearea+CPUSAVE_SRR0+8)(%r1);                   \
  153         stw     %r31, (savearea+CPUSAVE_SRR1+8)(%r1);                   \
  154         isync;                                                          \
  155         mfspr   %r1, sprg_sp;           /* Restore SP */                \
  156         mfcr    %r30;                   /* Save CR */                   \
  157         /* switch to per-thread kstack if intr taken in user mode */    \
  158         mtcr    %r31;                   /* MSR at interrupt time  */    \
  159         bf      17, 1f;                                                 \
  160         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
  161         lwz     %r1, PC_CURPCB(%r1);    /* Per-thread kernel stack */   \
  162 1:
  163 
  164 /*
  165  * FRAME_SETUP assumes:
  166  *      SPRG{1-3}       SP at the time interrupt occured
  167  *      savearea        r30-r31, DEAR, ESR, xSRR0-1
  168  *      r30             CR
  169  *      r31             scratch
  170  *      r1              kernel stack
  171  *
  172  * sprg_sp - SPRG reg containing SP at the time interrupt occured
  173  * savearea - temp save
  174  * exc - exception number (EXC_xxx)
  175  *
  176  * 1. sets a new frame
  177  * 2. saves in the frame:
  178  *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
  179  *   - R3-31 (R30-31 first restored from savearea)
  180  *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
  181  *
  182  * Notes:
  183  * - potential TLB miss: YES, since we make dereferences to kstack, which
  184  *   can happen not covered (we can have up to two DTLB misses if fortunate
  185  *   enough i.e. when kstack crosses page boundary and both pages are
  186  *   untranslated)
  187  */
  188 #define FRAME_SETUP(sprg_sp, savearea, exc)                             \
  189         mfspr   %r31, sprg_sp;          /* get saved SP */              \
  190         /* establish a new stack frame and put everything on it */      \
  191         stwu    %r31, -FRAMELEN(%r1);                                   \
  192         stw     %r0, FRAME_0+8(%r1);    /* save r0 in the trapframe */  \
  193         stw     %r31, FRAME_1+8(%r1);   /* save SP   "      "       */  \
  194         stw     %r2, FRAME_2+8(%r1);    /* save r2   "      "       */  \
  195         mflr    %r31;                                                   \
  196         stw     %r31, FRAME_LR+8(%r1);  /* save LR   "      "       */  \
  197         stw     %r30, FRAME_CR+8(%r1);  /* save CR   "      "       */  \
  198         GET_CPUINFO(%r2);                                               \
  199         lwz     %r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */  \
  200         lwz     %r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */  \
  201         /* save R3-31 */                                                \
  202         stmw    %r3,  FRAME_3+8(%r1) ;                                  \
  203         /* save DEAR, ESR */                                            \
  204         lwz     %r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);               \
  205         lwz     %r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);                \
  206         stw     %r28, FRAME_BOOKE_DEAR+8(%r1);                          \
  207         stw     %r29, FRAME_BOOKE_ESR+8(%r1);                           \
  208         /* save XER, CTR, exc number */                                 \
  209         mfxer   %r3;                                                    \
  210         mfctr   %r4;                                                    \
  211         stw     %r3, FRAME_XER+8(%r1);                                  \
  212         stw     %r4, FRAME_CTR+8(%r1);                                  \
  213         li      %r5, exc;                                               \
  214         stw     %r5, FRAME_EXC+8(%r1);                                  \
  215         /* save DBCR0 */                                                \
  216         mfspr   %r3, SPR_DBCR0;                                         \
  217         stw     %r3, FRAME_BOOKE_DBCR0+8(%r1);                          \
  218         /* save xSSR0-1 */                                              \
  219         lwz     %r30, (savearea+CPUSAVE_SRR0)(%r2);                     \
  220         lwz     %r31, (savearea+CPUSAVE_SRR1)(%r2);                     \
  221         stw     %r30, FRAME_SRR0+8(%r1);                                \
  222         stw     %r31, FRAME_SRR1+8(%r1);                                \
  223         lwz     %r2,PC_CURTHREAD(%r2)   /* set curthread pointer */
  224 
  225 /*
  226  *
  227  * isrr0-1 - save restore registers to restore CPU state to (may be
  228  *           SRR0-1, CSRR0-1, MCSRR0-1
  229  *
  230  * Notes:
  231  *  - potential TLB miss: YES. The deref'd kstack may be not covered
  232  */
  233 #define FRAME_LEAVE(isrr0, isrr1)                                       \
  234         /* restore CTR, XER, LR, CR */                                  \
  235         lwz     %r4, FRAME_CTR+8(%r1);                                  \
  236         lwz     %r5, FRAME_XER+8(%r1);                                  \
  237         lwz     %r6, FRAME_LR+8(%r1);                                   \
  238         lwz     %r7, FRAME_CR+8(%r1);                                   \
  239         mtctr   %r4;                                                    \
  240         mtxer   %r5;                                                    \
  241         mtlr    %r6;                                                    \
  242         mtcr    %r7;                                                    \
  243         /* restore DBCR0 */                                             \
  244         lwz     %r4, FRAME_BOOKE_DBCR0+8(%r1);                          \
  245         mtspr   SPR_DBCR0, %r4;                                         \
  246         /* restore xSRR0-1 */                                           \
  247         lwz     %r30, FRAME_SRR0+8(%r1);                                \
  248         lwz     %r31, FRAME_SRR1+8(%r1);                                \
  249         mtspr   isrr0, %r30;                                            \
  250         mtspr   isrr1, %r31;                                            \
  251         /* restore R2-31, SP */                                         \
  252         lmw     %r2, FRAME_2+8(%r1) ;                                   \
  253         lwz     %r0, FRAME_0+8(%r1);                                    \
  254         lwz     %r1, FRAME_1+8(%r1);                                    \
  255         isync
  256 
  257 /*
  258  * TLB miss prolog
  259  *
  260  * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
  261  *
  262  * Notes:
  263  *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
  264  *    miss within the TLB prolog itself!
  265  *  - TLBSAVE is always translated
  266  */
  267 #define TLB_PROLOG                                                      \
  268         mtsprg4 %r1;                    /* Save SP */                   \
  269         mtsprg5 %r28;                                                   \
  270         mtsprg6 %r29;                                                   \
  271         /* calculate TLB nesting level and TLBSAVE instance address */  \
  272         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
  273         lwz     %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
  274         rlwinm  %r29, %r28, 6, 23, 25;  /* 4 x TLBSAVE_LEN */           \
  275         addi    %r28, %r28, 1;                                          \
  276         stw     %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
  277         addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
  278         add     %r1, %r1, %r29;         /* current TLBSAVE ptr */       \
  279                                                                         \
  280         /* save R20-31 */                                               \
  281         mfsprg5 %r28;                                                   \
  282         mfsprg6 %r29;                                                   \
  283         stmw    %r20, (TLBSAVE_BOOKE_R20)(%r1);                         \
  284         /* save LR, CR */                                               \
  285         mflr    %r30;                                                   \
  286         mfcr    %r31;                                                   \
  287         stw     %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
  288         stw     %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
  289         /* save SRR0-1 */                                               \
  290         mfsrr0  %r30;           /* execution addr at interrupt time */  \
  291         mfsrr1  %r31;           /* MSR at interrupt time*/              \
  292         stw     %r30, (TLBSAVE_BOOKE_SRR0)(%r1);        /* save SRR0 */ \
  293         stw     %r31, (TLBSAVE_BOOKE_SRR1)(%r1);        /* save SRR1 */ \
  294         isync;                                                          \
  295         mfsprg4 %r1
  296 
  297 /*
  298  * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
  299  *
  300  * same notes as for the TLB_PROLOG
  301  */
  302 #define TLB_RESTORE                                                     \
  303         mtsprg4 %r1;                    /* Save SP */                   \
  304         GET_CPUINFO(%r1);               /* Per-cpu structure */         \
  305         /* calculate TLB nesting level and TLBSAVE instance addr */     \
  306         lwz     %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
  307         subi    %r28, %r28, 1;                                          \
  308         stw     %r28, PC_BOOKE_TLB_LEVEL(%r1);                          \
  309         rlwinm  %r29, %r28, 6, 23, 25; /* 4 x TLBSAVE_LEN */            \
  310         addi    %r29, %r29, PC_BOOKE_TLBSAVE@l;                         \
  311         add     %r1, %r1, %r29;                                         \
  312                                                                         \
  313         /* restore LR, CR */                                            \
  314         lwz     %r30, (TLBSAVE_BOOKE_LR)(%r1);                          \
  315         lwz     %r31, (TLBSAVE_BOOKE_CR)(%r1);                          \
  316         mtlr    %r30;                                                   \
  317         mtcr    %r31;                                                   \
  318         /* restore SRR0-1 */                                            \
  319         lwz     %r30, (TLBSAVE_BOOKE_SRR0)(%r1);                        \
  320         lwz     %r31, (TLBSAVE_BOOKE_SRR1)(%r1);                        \
  321         mtsrr0  %r30;                                                   \
  322         mtsrr1  %r31;                                                   \
  323         /* restore R20-31 */                                            \
  324         lmw     %r20, (TLBSAVE_BOOKE_R20)(%r1);                         \
  325         mfsprg4 %r1
  326 
  327 #ifdef SMP
  328 #define TLB_LOCK                                                        \
  329         GET_CPUINFO(%r20);                                              \
  330         lwz     %r21, PC_CURTHREAD(%r20);                               \
  331         lwz     %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
  332                                                                         \
  333 1:      lwarx   %r23, 0, %r22;                                          \
  334         cmpwi   %r23, TLB_UNLOCKED;                                     \
  335         beq     2f;                                                     \
  336                                                                         \
  337         /* check if this is recursion */                                \
  338         cmplw   cr0, %r21, %r23;                                        \
  339         bne-    1b;                                                     \
  340                                                                         \
  341 2:      /* try to acquire lock */                                       \
  342         stwcx.  %r21, 0, %r22;                                          \
  343         bne-    1b;                                                     \
  344                                                                         \
  345         /* got it, update recursion counter */                          \
  346         lwz     %r21, RES_RECURSE(%r22);                                \
  347         addi    %r21, %r21, 1;                                          \
  348         stw     %r21, RES_RECURSE(%r22);                                \
  349         isync;                                                          \
  350         msync
  351 
  352 #define TLB_UNLOCK                                                      \
  353         GET_CPUINFO(%r20);                                              \
  354         lwz     %r21, PC_CURTHREAD(%r20);                               \
  355         lwz     %r22, PC_BOOKE_TLB_LOCK(%r20);                          \
  356                                                                         \
  357         /* update recursion counter */                                  \
  358         lwz     %r23, RES_RECURSE(%r22);                                \
  359         subi    %r23, %r23, 1;                                          \
  360         stw     %r23, RES_RECURSE(%r22);                                \
  361                                                                         \
  362         cmpwi   %r23, 0;                                                \
  363         bne     1f;                                                     \
  364         isync;                                                          \
  365         msync;                                                          \
  366                                                                         \
  367         /* release the lock */                                          \
  368         li      %r23, TLB_UNLOCKED;                                     \
  369         stw     %r23, 0(%r22);                                          \
  370 1:      isync;                                                          \
  371         msync
  372 #else
  373 #define TLB_LOCK
  374 #define TLB_UNLOCK
  375 #endif  /* SMP */
  376 
  377 #define INTERRUPT(label)                                                \
  378         .globl  label;                                                  \
  379         .align  5;                                                      \
  380         CNAME(label):
  381 
  382 /*
  383  * Interrupt handling routines in BookE can be flexibly placed and do not have
  384  * to live in pre-defined vectors location. Note they need to be TLB-mapped at
  385  * all times in order to be able to handle exceptions. We thus arrange for
  386  * them to be part of kernel text which is always TLB-accessible.
  387  *
  388  * The interrupt handling routines have to be 16 bytes aligned: we align them
  389  * to 32 bytes (cache line length) which supposedly performs better.
  390  *
  391  */
  392         .text
  393         .globl CNAME(interrupt_vector_base)
  394         .align 5
  395 interrupt_vector_base:
  396 /*****************************************************************************
  397  * Catch-all handler to handle uninstalled IVORs
  398  ****************************************************************************/
  399 INTERRUPT(int_unknown)
  400         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  401         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_RSVD)
  402         b       trap_common
  403 
  404 /*****************************************************************************
  405  * Critical input interrupt
  406  ****************************************************************************/
  407 INTERRUPT(int_critical_input)
  408         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
  409         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
  410         addi    %r3, %r1, 8
  411         bl      CNAME(powerpc_interrupt)
  412         FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
  413         rfci
  414 
  415 
  416 /*****************************************************************************
  417  * Machine check interrupt
  418  ****************************************************************************/
  419 INTERRUPT(int_machine_check)
  420         STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
  421         FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
  422         addi    %r3, %r1, 8
  423         bl      CNAME(powerpc_interrupt)
  424         FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
  425         rfmci
  426 
  427 
  428 /*****************************************************************************
  429  * Data storage interrupt
  430  ****************************************************************************/
  431 INTERRUPT(int_data_storage)
  432         STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
  433         FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
  434         b       trap_common
  435 
  436 
  437 /*****************************************************************************
  438  * Instruction storage interrupt
  439  ****************************************************************************/
  440 INTERRUPT(int_instr_storage)
  441         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  442         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
  443         b       trap_common
  444 
  445 
  446 /*****************************************************************************
  447  * External input interrupt
  448  ****************************************************************************/
  449 INTERRUPT(int_external_input)
  450         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  451         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
  452         addi    %r3, %r1, 8
  453         bl      CNAME(powerpc_interrupt)
  454         b       clear_we
  455 
  456 
  457 INTERRUPT(int_alignment)
  458         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  459         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
  460         b       trap_common
  461 
  462 
  463 INTERRUPT(int_program)
  464         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  465         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
  466         b       trap_common
  467 
  468 
  469 INTERRUPT(int_fpu)
  470         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  471         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FPU)
  472         b       trap_common
  473 
  474 
  475 /*****************************************************************************
  476  * System call
  477  ****************************************************************************/
  478 INTERRUPT(int_syscall)
  479         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  480         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
  481         b       trap_common
  482 
  483 
  484 /*****************************************************************************
  485  * Decrementer interrupt
  486  ****************************************************************************/
  487 INTERRUPT(int_decrementer)
  488         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  489         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
  490         addi    %r3, %r1, 8
  491         bl      CNAME(powerpc_interrupt)
  492         b       clear_we
  493 
  494 
  495 /*****************************************************************************
  496  * Fixed interval timer
  497  ****************************************************************************/
  498 INTERRUPT(int_fixed_interval_timer)
  499         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  500         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
  501         b       trap_common
  502 
  503 
  504 /*****************************************************************************
  505  * Watchdog interrupt
  506  ****************************************************************************/
  507 INTERRUPT(int_watchdog)
  508         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  509         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
  510         b       trap_common
  511 
  512 
  513 /*****************************************************************************
  514  * Altivec Unavailable interrupt
  515  ****************************************************************************/
  516 INTERRUPT(int_vec)
  517         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  518         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VEC)
  519         b       trap_common
  520 
  521 
  522 /*****************************************************************************
  523  * Watchdog interrupt
  524  ****************************************************************************/
  525 INTERRUPT(int_vecast)
  526         STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  527         FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_VECAST_E)
  528         b       trap_common
  529 
  530 
  531 #ifdef HWPMC_HOOKS
  532 /*****************************************************************************
  533  * PMC Interrupt
  534  ****************************************************************************/
  535 INTERRUPT(int_performance_counter)
  536         STANDARD_PROLOG(SPR_SPRG3, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
  537         FRAME_SETUP(SPR_SPRG3, PC_TEMPSAVE, EXC_PERF)
  538         addi    %r3, %r1, 8
  539         bl      CNAME(powerpc_interrupt)
  540         b       trapexit
  541 #endif
  542 
  543 
  544 /*****************************************************************************
  545  * Data TLB miss interrupt
  546  *
  547  * There can be nested TLB misses - while handling a TLB miss we reference
  548  * data structures that may be not covered by translations. We support up to
  549  * TLB_NESTED_MAX-1 nested misses.
  550  * 
  551  * Registers use:
  552  *      r31 - dear
  553  *      r30 - unused
  554  *      r29 - saved mas0
  555  *      r28 - saved mas1
  556  *      r27 - saved mas2
  557  *      r26 - pmap address
  558  *      r25 - pte address
  559  *
  560  *      r20:r23 - scratch registers
  561  ****************************************************************************/
  562 INTERRUPT(int_data_tlb_error)
  563         TLB_PROLOG
  564         TLB_LOCK
  565 
  566         mfdear  %r31
  567 
  568         /*
  569          * Save MAS0-MAS2 registers. There might be another tlb miss during
  570          * pte lookup overwriting current contents (which was hw filled).
  571          */
  572         mfspr   %r29, SPR_MAS0
  573         mfspr   %r28, SPR_MAS1
  574         mfspr   %r27, SPR_MAS2
  575 
  576         /* Check faulting address. */
  577         lis     %r21, VM_MAXUSER_ADDRESS@h
  578         ori     %r21, %r21, VM_MAXUSER_ADDRESS@l
  579         cmplw   cr0, %r31, %r21
  580         blt     search_user_pmap
  581         
  582         /* If it's kernel address, allow only supervisor mode misses. */
  583         mfsrr1  %r21
  584         mtcr    %r21
  585         bt      17, search_failed       /* check MSR[PR] */
  586 
  587 search_kernel_pmap:
  588         /* Load r26 with kernel_pmap address */
  589         bl      1f
  590         .long kernel_pmap_store-.
  591 1:      mflr    %r21
  592         lwz     %r26, 0(%r21)
  593         add     %r26, %r21, %r26        /* kernel_pmap_store in r26 */
  594 
  595         /* Force kernel tid, set TID to 0 in MAS1. */
  596         li      %r21, 0
  597         rlwimi  %r28, %r21, 0, 8, 15    /* clear TID bits */
  598 
  599 tlb_miss_handle:
  600         /* This may result in nested tlb miss. */
  601         bl      pte_lookup              /* returns PTE address in R25 */
  602 
  603         cmpwi   %r25, 0                 /* pte found? */
  604         beq     search_failed
  605 
  606         /* Finish up, write TLB entry. */
  607         bl      tlb_fill_entry
  608 
  609 tlb_miss_return:
  610         TLB_UNLOCK
  611         TLB_RESTORE
  612         rfi
  613 
  614 search_user_pmap:
  615         /* Load r26 with current user space process pmap */
  616         GET_CPUINFO(%r26)
  617         lwz     %r26, PC_CURPMAP(%r26)
  618 
  619         b       tlb_miss_handle
  620 
  621 search_failed:
  622         /*
  623          * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
  624          * the faulting virtual address anyway, but put a fake RPN and no
  625          * access rights. This should cause a following {D,I}SI exception.
  626          */
  627         lis     %r23, 0xffff0000@h      /* revoke all permissions */
  628 
  629         /* Load MAS registers. */
  630         mtspr   SPR_MAS0, %r29
  631         isync
  632         mtspr   SPR_MAS1, %r28
  633         isync
  634         mtspr   SPR_MAS2, %r27
  635         isync
  636         mtspr   SPR_MAS3, %r23
  637         isync
  638 
  639         bl      zero_mas7
  640         bl      zero_mas8
  641 
  642         tlbwe
  643         msync
  644         isync
  645         b       tlb_miss_return
  646 
  647 /*****************************************************************************
  648  *
  649  * Return pte address that corresponds to given pmap/va.  If there is no valid
  650  * entry return 0.
  651  *
  652  * input: r26 - pmap
  653  * input: r31 - dear
  654  * output: r25 - pte address
  655  *
  656  * scratch regs used: r21
  657  *
  658  ****************************************************************************/
  659 pte_lookup:
  660         cmpwi   %r26, 0
  661         beq     1f                      /* fail quickly if pmap is invalid */
  662 
  663         srwi    %r21, %r31, PDIR_SHIFT          /* pdir offset */
  664         slwi    %r21, %r21, PDIR_ENTRY_SHIFT    /* multiply by pdir entry size */
  665 
  666         addi    %r25, %r26, PM_PDIR     /* pmap pm_dir[] address */
  667         add     %r25, %r25, %r21        /* offset within pm_pdir[] table */
  668         /*
  669          * Get ptbl address, i.e. pmap->pm_pdir[pdir_idx]
  670          * This load may cause a Data TLB miss for non-kernel pmap!
  671          */
  672         lwz     %r25, 0(%r25)
  673         cmpwi   %r25, 0
  674         beq     2f
  675 
  676         lis     %r21, PTBL_MASK@h
  677         ori     %r21, %r21, PTBL_MASK@l
  678         and     %r21, %r21, %r31
  679 
  680         /* ptbl offset, multiply by ptbl entry size */
  681         srwi    %r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
  682 
  683         add     %r25, %r25, %r21                /* address of pte entry */
  684         /*
  685          * Get pte->flags
  686          * This load may cause a Data TLB miss for non-kernel pmap!
  687          */
  688         lwz     %r21, PTE_FLAGS(%r25)
  689         andi.   %r21, %r21, PTE_VALID@l
  690         bne     2f
  691 1:
  692         li      %r25, 0
  693 2:
  694         blr
  695 
  696 /*****************************************************************************
  697  *
  698  * Load MAS1-MAS3 registers with data, write TLB entry
  699  *
  700  * input:
  701  * r29 - mas0
  702  * r28 - mas1
  703  * r27 - mas2
  704  * r25 - pte
  705  *
  706  * output: none
  707  *
  708  * scratch regs: r21-r23
  709  *
  710  ****************************************************************************/
  711 tlb_fill_entry:
  712         /*
  713          * Update PTE flags: we have to do it atomically, as pmap_protect()
  714          * running on other CPUs could attempt to update the flags at the same
  715          * time.
  716          */
  717         li      %r23, PTE_FLAGS
  718 1:
  719         lwarx   %r21, %r23, %r25                /* get pte->flags */
  720         oris    %r21, %r21, PTE_REFERENCED@h    /* set referenced bit */
  721 
  722         andi.   %r22, %r21, (PTE_SW | PTE_UW)@l /* check if writable */
  723         beq     2f
  724         ori     %r21, %r21, PTE_MODIFIED@l      /* set modified bit */
  725 2:
  726         stwcx.  %r21, %r23, %r25                /* write it back */
  727         bne-    1b
  728 
  729         /* Update MAS2. */
  730         rlwimi  %r27, %r21, 13, 27, 30          /* insert WIMG bits from pte */
  731 
  732         /* Setup MAS3 value in r23. */
  733         lwz     %r23, PTE_RPN(%r25)             /* get pte->rpn */
  734         rlwinm  %r22, %r23, 20, 0, 11           /* extract MAS3 portion of RPN */
  735 
  736         rlwimi  %r22, %r21, 30, 26, 31          /* insert protection bits from pte */
  737         rlwimi  %r22, %r21, 20, 12, 19          /* insert lower 8 RPN bits to MAS3 */
  738         rlwinm  %r23, %r23, 20, 24, 31          /* MAS7 portion of RPN */
  739 
  740         /* Load MAS registers. */
  741         mtspr   SPR_MAS0, %r29
  742         isync
  743         mtspr   SPR_MAS1, %r28
  744         isync
  745         mtspr   SPR_MAS2, %r27
  746         isync
  747         mtspr   SPR_MAS3, %r22
  748         isync
  749         mtspr   SPR_MAS7, %r23
  750         isync
  751 
  752         mflr    %r21
  753         bl      zero_mas8
  754         mtlr    %r21
  755 
  756         tlbwe
  757         isync
  758         msync
  759         blr
  760 
  761 /*****************************************************************************
  762  * Instruction TLB miss interrupt
  763  *
  764  * Same notes as for the Data TLB miss
  765  ****************************************************************************/
  766 INTERRUPT(int_inst_tlb_error)
  767         TLB_PROLOG
  768         TLB_LOCK
  769 
  770         mfsrr0  %r31                    /* faulting address */
  771 
  772         /*
  773          * Save MAS0-MAS2 registers. There might be another tlb miss during pte
  774          * lookup overwriting current contents (which was hw filled).
  775          */
  776         mfspr   %r29, SPR_MAS0
  777         mfspr   %r28, SPR_MAS1
  778         mfspr   %r27, SPR_MAS2
  779 
  780         mfsrr1  %r21
  781         mtcr    %r21
  782 
  783         /* check MSR[PR] */
  784         bt      17, search_user_pmap
  785         b       search_kernel_pmap
  786 
  787 
  788         .globl  interrupt_vector_top
  789 interrupt_vector_top:
  790 
  791 /*****************************************************************************
  792  * Debug interrupt
  793  ****************************************************************************/
  794 INTERRUPT(int_debug)
  795         STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
  796         FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
  797         GET_CPUINFO(%r3)
  798         lwz     %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
  799         bl      0f
  800         .long   interrupt_vector_base-.
  801         .long   interrupt_vector_top-.
  802 0:      mflr    %r5
  803         lwz     %r4,0(%r5)      /* interrupt_vector_base in r4 */
  804         add     %r4,%r4,%r5
  805         cmplw   cr0, %r3, %r4
  806         blt     1f
  807         lwz     %r4,4(%r5)      /* interrupt_vector_top in r4 */
  808         add     %r4,%r4,%r5
  809         addi    %r4,%r4,4
  810         cmplw   cr0, %r3, %r4
  811         bge     1f
  812         /* Disable single-stepping for the interrupt handlers. */
  813         lwz     %r3, FRAME_SRR1+8(%r1);
  814         rlwinm  %r3, %r3, 0, 23, 21
  815         stw     %r3, FRAME_SRR1+8(%r1);
  816         /* Restore srr0 and srr1 as they could have been clobbered. */
  817         GET_CPUINFO(%r4)
  818         lwz     %r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r4);
  819         mtspr   SPR_SRR0, %r3
  820         lwz     %r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r4);
  821         mtspr   SPR_SRR1, %r4
  822         b       9f
  823 1:
  824         addi    %r3, %r1, 8
  825         bl      CNAME(trap)
  826         /*
  827          * Handle ASTs, needed for proper support of single-stepping.
  828          * We actually need to return to the process with an rfi.
  829          */
  830         b       trapexit
  831 9:
  832         FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
  833         rfci
  834 
  835 
  836 /*****************************************************************************
  837  * Common trap code
  838  ****************************************************************************/
  839 trap_common:
  840         /* Call C trap dispatcher */
  841         addi    %r3, %r1, 8
  842         bl      CNAME(trap)
  843 
  844         .globl  CNAME(trapexit)         /* exported for db_backtrace use */
  845 CNAME(trapexit):
  846         /* disable interrupts */
  847         wrteei  0
  848 
  849         /* Test AST pending - makes sense for user process only */
  850         lwz     %r5, FRAME_SRR1+8(%r1)
  851         mtcr    %r5
  852         bf      17, 1f
  853 
  854         GET_CPUINFO(%r3)
  855         lwz     %r4, PC_CURTHREAD(%r3)
  856         lwz     %r4, TD_FLAGS(%r4)
  857         lis     %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@h
  858         ori     %r5, %r5, (TDF_ASTPENDING | TDF_NEEDRESCHED)@l
  859         and.    %r4, %r4, %r5
  860         beq     1f
  861 
  862         /* re-enable interrupts before calling ast() */
  863         wrteei  1
  864 
  865         addi    %r3, %r1, 8
  866         bl      CNAME(ast)
  867         .globl  CNAME(asttrapexit)      /* db_backtrace code sentinel #2 */
  868 CNAME(asttrapexit):
  869         b       trapexit                /* test ast ret value ? */
  870 1:
  871         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
  872         rfi
  873 
  874 
  875 #if defined(KDB)
  876 /*
  877  * Deliberate entry to dbtrap
  878  */
  879         .globl  CNAME(breakpoint)
  880 CNAME(breakpoint):
  881         mtsprg1 %r1
  882         mfmsr   %r3
  883         mtsrr1  %r3
  884         andi.   %r3, %r3, ~(PSL_EE | PSL_ME)@l
  885         mtmsr   %r3                     /* disable interrupts */
  886         isync
  887         GET_CPUINFO(%r3)
  888         stw     %r30, (PC_DBSAVE+CPUSAVE_R30)(%r3)
  889         stw     %r31, (PC_DBSAVE+CPUSAVE_R31)(%r3)
  890 
  891         mflr    %r31
  892         mtsrr0  %r31
  893 
  894         mfdear  %r30
  895         mfesr   %r31
  896         stw     %r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
  897         stw     %r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
  898 
  899         mfsrr0  %r30
  900         mfsrr1  %r31
  901         stw     %r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
  902         stw     %r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
  903         isync
  904 
  905         mfcr    %r30
  906 
  907 /*
  908  * Now the kdb trap catching code.
  909  */
  910 dbtrap:
  911         FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
  912 /* Call C trap code: */
  913         addi    %r3, %r1, 8
  914         bl      CNAME(db_trap_glue)
  915         or.     %r3, %r3, %r3
  916         bne     dbleave
  917 /* This wasn't for KDB, so switch to real trap: */
  918         b       trap_common
  919 
  920 dbleave:
  921         FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
  922         rfi
  923 #endif /* KDB */
  924 
  925 clear_we:
  926         lwz     %r3, (FRAME_SRR1+8)(%r1)
  927         rlwinm  %r3, %r3, 0, 14, 12
  928         stw     %r3, (FRAME_SRR1+8)(%r1)
  929         b       trapexit
  930 
  931 #ifdef SMP
  932 ENTRY(tlb_lock)
  933         GET_CPUINFO(%r5)
  934         lwz     %r5, PC_CURTHREAD(%r5)
  935 1:      lwarx   %r4, 0, %r3
  936         cmpwi   %r4, TLB_UNLOCKED
  937         bne     1b
  938         stwcx.  %r5, 0, %r3
  939         bne-    1b
  940         isync
  941         msync
  942         blr
  943 
  944 ENTRY(tlb_unlock)
  945         isync
  946         msync
  947         li      %r4, TLB_UNLOCKED
  948         stw     %r4, 0(%r3)
  949         isync
  950         msync
  951         blr
  952 
  953 /*
  954  * TLB miss spin locks. For each CPU we have a reservation granule (32 bytes);
  955  * only a single word from this granule will actually be used as a spin lock
  956  * for mutual exclusion between TLB miss handler and pmap layer that
  957  * manipulates page table contents.
  958  */
  959         .data
  960         .align  5
  961 GLOBAL(tlb0_miss_locks)
  962         .space  RES_GRANULE * MAXCPU
  963 #endif

Cache object: 9e029344c781f5af0f98aab19a44e625


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.