FreeBSD/Linux Kernel Cross Reference
sys/sparc64/sparc64/exception.S

     /*-
      * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
      *
      * Redistribution and use in source and binary forms, with or without
      * modification, are permitted provided that the following conditions
      * are met:
      * 1. Redistributions of source code must retain the above copyright
      *    notice, this list of conditions and the following disclaimer.
      * 2. Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in the
     *    documentation and/or other materials provided with the distribution.
     * 3. Berkeley Software Design Inc's name may not be used to endorse or
     *    promote products derived from this software without specific prior
     *    written permission.
     *
     * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
     * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
     * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     * SUCH DAMAGE.
     *
     *      BSDI $Id: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp $
     */
    /*-
     * Copyright (c) 2001 Jake Burkholder.
     * All rights reserved.
     *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions
     * are met:
     * 1. Redistributions of source code must retain the above copyright
     *    notice, this list of conditions and the following disclaimer.
     * 2. Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in the
     *    documentation and/or other materials provided with the distribution.
     *
     * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     * SUCH DAMAGE.
     */
    
    #include <machine/asm.h>
    __FBSDID("$FreeBSD$");
    
    #include "opt_compat.h"
    #include "opt_ddb.h"
    
    #include <machine/asi.h>
    #include <machine/asmacros.h>
    #include <machine/frame.h>
    #include <machine/fsr.h>
    #include <machine/intr_machdep.h>
    #include <machine/ktr.h>
    #include <machine/pcb.h>
    #include <machine/pstate.h>
    #include <machine/trap.h>
    #include <machine/tsb.h>
    #include <machine/tstate.h>
    #include <machine/utrap.h>
    #include <machine/wstate.h>
    
    #include "assym.s"
    
    #define TSB_ASI                 0x0
    #define TSB_KERNEL              0x0
    #define TSB_KERNEL_MASK         0x0
    #define TSB_KERNEL_PHYS         0x0
    #define TSB_KERNEL_PHYS_END     0x0
    #define TSB_QUAD_LDD            0x0
    
            .register %g2,#ignore
            .register %g3,#ignore
            .register %g6,#ignore
            .register %g7,#ignore
    
    /*
     * Atomically set a bit in a TTE.
     */
    #define TTE_SET_BIT(r1, r2, r3, bit, a, asi) \
            add     r1, TTE_DATA, r1 ; \
            LD(x, a) [r1] asi, r2 ; \
    9:      or      r2, bit, r3 ; \
            CAS(x, a) [r1] asi, r2, r3 ; \
            cmp     r2, r3 ; \
            bne,pn  %xcc, 9b ; \
            mov    r3, r2
   
   #define TTE_SET_REF(r1, r2, r3, a, asi) TTE_SET_BIT(r1, r2, r3, TD_REF, a, asi)
   #define TTE_SET_W(r1, r2, r3, a, asi)   TTE_SET_BIT(r1, r2, r3, TD_W, a, asi)
   
   /*
    * Macros for spilling and filling live windows.
    *
    * NOTE: These macros use exactly 16 instructions, and it is assumed that the
    * handler will not use more than 24 instructions total, to leave room for
    * resume vectors which occupy the last 8 instructions.
    */
   
   #define SPILL(storer, base, size, asi) \
           storer  %l0, [base + (0 * size)] asi ; \
           storer  %l1, [base + (1 * size)] asi ; \
           storer  %l2, [base + (2 * size)] asi ; \
           storer  %l3, [base + (3 * size)] asi ; \
           storer  %l4, [base + (4 * size)] asi ; \
           storer  %l5, [base + (5 * size)] asi ; \
           storer  %l6, [base + (6 * size)] asi ; \
           storer  %l7, [base + (7 * size)] asi ; \
           storer  %i0, [base + (8 * size)] asi ; \
           storer  %i1, [base + (9 * size)] asi ; \
           storer  %i2, [base + (10 * size)] asi ; \
           storer  %i3, [base + (11 * size)] asi ; \
           storer  %i4, [base + (12 * size)] asi ; \
           storer  %i5, [base + (13 * size)] asi ; \
           storer  %i6, [base + (14 * size)] asi ; \
           storer  %i7, [base + (15 * size)] asi
   
   #define FILL(loader, base, size, asi) \
           loader  [base + (0 * size)] asi, %l0 ; \
           loader  [base + (1 * size)] asi, %l1 ; \
           loader  [base + (2 * size)] asi, %l2 ; \
           loader  [base + (3 * size)] asi, %l3 ; \
           loader  [base + (4 * size)] asi, %l4 ; \
           loader  [base + (5 * size)] asi, %l5 ; \
           loader  [base + (6 * size)] asi, %l6 ; \
           loader  [base + (7 * size)] asi, %l7 ; \
           loader  [base + (8 * size)] asi, %i0 ; \
           loader  [base + (9 * size)] asi, %i1 ; \
           loader  [base + (10 * size)] asi, %i2 ; \
           loader  [base + (11 * size)] asi, %i3 ; \
           loader  [base + (12 * size)] asi, %i4 ; \
           loader  [base + (13 * size)] asi, %i5 ; \
           loader  [base + (14 * size)] asi, %i6 ; \
           loader  [base + (15 * size)] asi, %i7
   
   #define ERRATUM50(reg)  mov reg, reg
   
   #define KSTACK_SLOP     1024
   
   /*
    * Sanity check the kernel stack and bail out if it's wrong.
    * XXX: doesn't handle being on the panic stack.
    */
   #define KSTACK_CHECK \
           dec     16, ASP_REG ; \
           stx     %g1, [ASP_REG + 0] ; \
           stx     %g2, [ASP_REG + 8] ; \
           add     %sp, SPOFF, %g1 ; \
           andcc   %g1, (1 << PTR_SHIFT) - 1, %g0 ; \
           bnz,a   %xcc, tl1_kstack_fault ; \
            inc    16, ASP_REG ; \
           ldx     [PCPU(CURTHREAD)], %g2 ; \
           ldx     [%g2 + TD_KSTACK], %g2 ; \
           add     %g2, KSTACK_SLOP, %g2 ; \
           subcc   %g1, %g2, %g1 ; \
           ble,a   %xcc, tl1_kstack_fault ; \
            inc    16, ASP_REG ; \
           set     KSTACK_PAGES * PAGE_SIZE, %g2 ; \
           cmp     %g1, %g2 ; \
           bgt,a   %xcc, tl1_kstack_fault ; \
            inc    16, ASP_REG ; \
           ldx     [ASP_REG + 8], %g2 ; \
           ldx     [ASP_REG + 0], %g1 ; \
           inc     16, ASP_REG
   
           .globl  tl_text_begin
   tl_text_begin:
           nop
   
   ENTRY(tl1_kstack_fault)
           rdpr    %tl, %g1
   1:      cmp     %g1, 2
           be,a    2f
            nop
   
   #if KTR_COMPILE & KTR_TRAP
           CATR(KTR_TRAP, "tl1_kstack_fault: tl=%#lx tpc=%#lx tnpc=%#lx"
               , %g2, %g3, %g4, 7, 8, 9)
           rdpr    %tl, %g3
           stx     %g3, [%g2 + KTR_PARM1]
           rdpr    %tpc, %g3
           stx     %g3, [%g2 + KTR_PARM1]
           rdpr    %tnpc, %g3
           stx     %g3, [%g2 + KTR_PARM1]
   9:
   #endif
   
           sub     %g1, 1, %g1
           wrpr    %g1, 0, %tl
           ba,a    %xcc, 1b
            nop
   
   2:
   #if KTR_COMPILE & KTR_TRAP
           CATR(KTR_TRAP,
               "tl1_kstack_fault: sp=%#lx ks=%#lx cr=%#lx cs=%#lx ow=%#lx ws=%#lx"
               , %g1, %g2, %g3, 7, 8, 9)
           add     %sp, SPOFF, %g2
           stx     %g2, [%g1 + KTR_PARM1]
           ldx     [PCPU(CURTHREAD)], %g2
           ldx     [%g2 + TD_KSTACK], %g2
           stx     %g2, [%g1 + KTR_PARM2]
           rdpr    %canrestore, %g2
           stx     %g2, [%g1 + KTR_PARM3]
           rdpr    %cansave, %g2
           stx     %g2, [%g1 + KTR_PARM4]
           rdpr    %otherwin, %g2
           stx     %g2, [%g1 + KTR_PARM5]
           rdpr    %wstate, %g2
           stx     %g2, [%g1 + KTR_PARM6]
   9:
   #endif
   
           wrpr    %g0, 0, %canrestore
           wrpr    %g0, 6, %cansave
           wrpr    %g0, 0, %otherwin
           wrpr    %g0, WSTATE_KERNEL, %wstate
   
           sub     ASP_REG, SPOFF + CCFSZ, %sp
           clr     %fp
   
           set     trap, %o2
           ba      %xcc, tl1_trap
            mov    T_KSTACK_FAULT | T_KERNEL, %o0
   END(tl1_kstack_fault)
   
   /*
    * Magic to resume from a spill or fill trap.  If we get an alignment or an
    * MMU fault during a spill or a fill, this macro will detect the fault and
    * resume at a set instruction offset in the trap handler.
    *
    * To check if the previous trap was a spill/fill we convert the trapped pc
    * to a trap type and verify that it is in the range of spill/fill vectors.
    * The spill/fill vectors are types 0x80-0xff and 0x280-0x2ff, masking off the
    * tl bit allows us to detect both ranges with one test.
    *
    * This is:
    *      0x80 <= (((%tpc - %tba) >> 5) & ~0x200) < 0x100
    *
    * To calculate the new pc we take advantage of the xor feature of wrpr.
    * Forcing all the low bits of the trapped pc on we can produce any offset
    * into the spill/fill vector.  The size of a spill/fill trap vector is 0x80.
    *
    *      0x7f ^ 0x1f == 0x60
    *      0x1f == (0x80 - 0x60) - 1
    *
    * Which are the offset and xor value used to resume from alignment faults.
    */
   
   /*
    * Determine if we have trapped inside of a spill/fill vector, and if so resume
    * at a fixed instruction offset in the trap vector.  Must be called on
    * alternate globals.
    */
   #define RESUME_SPILLFILL_MAGIC(stxa_g0_sfsr, xor) \
           dec     16, ASP_REG ; \
           stx     %g1, [ASP_REG + 0] ; \
           stx     %g2, [ASP_REG + 8] ; \
           rdpr    %tpc, %g1 ; \
           ERRATUM50(%g1) ; \
           rdpr    %tba, %g2 ; \
           sub     %g1, %g2, %g2 ; \
           srlx    %g2, 5, %g2 ; \
           andn    %g2, 0x200, %g2 ; \
           cmp     %g2, 0x80 ; \
           blu,pt  %xcc, 9f ; \
            cmp    %g2, 0x100 ; \
           bgeu,pt %xcc, 9f ; \
            or     %g1, 0x7f, %g1 ; \
           wrpr    %g1, xor, %tnpc ; \
           stxa_g0_sfsr ; \
           ldx     [ASP_REG + 8], %g2 ; \
           ldx     [ASP_REG + 0], %g1 ; \
           inc     16, ASP_REG ; \
           done ; \
   9:      ldx     [ASP_REG + 8], %g2 ; \
           ldx     [ASP_REG + 0], %g1 ; \
           inc     16, ASP_REG
   
   /*
    * For certain faults we need to clear the SFSR MMU register before returning.
    */
   #define RSF_CLR_SFSR \
           wr      %g0, ASI_DMMU, %asi ; \
           stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
   
   #define RSF_XOR(off)    ((0x80 - off) - 1)
   
   /*
    * Instruction offsets in spill and fill trap handlers for handling certain
    * nested traps, and corresponding xor constants for wrpr.
    */
   #define RSF_OFF_ALIGN   0x60
   #define RSF_OFF_MMU     0x70
   
   #define RESUME_SPILLFILL_ALIGN \
           RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_ALIGN))
   #define RESUME_SPILLFILL_MMU \
           RESUME_SPILLFILL_MAGIC(EMPTY, RSF_XOR(RSF_OFF_MMU))
   #define RESUME_SPILLFILL_MMU_CLR_SFSR \
           RESUME_SPILLFILL_MAGIC(RSF_CLR_SFSR, RSF_XOR(RSF_OFF_MMU))
   
   /*
    * Constant to add to %tnpc when taking a fill trap just before returning to
    * user mode.
    */
   #define RSF_FILL_INC    tl0_ret_fill_end - tl0_ret_fill
   
   /*
    * Generate a T_SPILL or T_FILL trap if the window operation fails.
    */
   #define RSF_TRAP(type) \
           ba      %xcc, tl0_sftrap ; \
            mov    type, %g2 ; \
           .align  16
   
   /*
    * Game over if the window operation fails.
    */
   #define RSF_FATAL(type) \
           ba      %xcc, rsf_fatal ; \
            mov    type, %g2 ; \
           .align  16
   
   /*
    * Magic to resume from a failed fill a few instructions after the corrsponding
    * restore.  This is used on return from the kernel to usermode.
    */
   #define RSF_FILL_MAGIC \
           rdpr    %tnpc, %g1 ; \
           add     %g1, RSF_FILL_INC, %g1 ; \
           wrpr    %g1, 0, %tnpc ; \
           done ; \
           .align  16
   
   /*
    * Spill to the pcb if a spill to the user stack in kernel mode fails.
    */
   #define RSF_SPILL_TOPCB \
           ba,a    %xcc, tl1_spill_topcb ; \
            nop ; \
           .align  16
   
   ENTRY(rsf_fatal)
   #if KTR_COMPILE & KTR_TRAP
           CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx"
               , %g1, %g3, %g4, 7, 8, 9)
           rdpr    %tt, %g3
           stx     %g3, [%g1 + KTR_PARM1]
           stx     %g2, [%g1 + KTR_PARM2]
   9:
   #endif
   
           KSTACK_CHECK
   
           sir
   END(rsf_fatal)
   
           .data
           _ALIGN_DATA
           .globl  intrnames, eintrnames
   intrnames:
           .space  (IV_MAX + PIL_MAX) * (MAXCOMLEN + 1)
   eintrnames:
           .globl  intrcnt, eintrcnt
   intrcnt:
           .space  (IV_MAX + PIL_MAX) * 8
   eintrcnt:
   
           .text
   
   /*
    * Trap table and associated macros
    *
    * Due to its size a trap table is an inherently hard thing to represent in
    * code in a clean way.  There are approximately 1024 vectors, of 8 or 32
    * instructions each, many of which are identical.  The way that this is
    * laid out is the instructions (8 or 32) for the actual trap vector appear
    * as an AS macro.  In general this code branches to tl0_trap or tl1_trap,
    * but if not supporting code can be placed just after the definition of the
    * macro.  The macros are then instantiated in a different section (.trap),
    * which is setup to be placed by the linker at the beginning of .text, and the
    * code around the macros is moved to the end of trap table.  In this way the
    * code that must be sequential in memory can be split up, and located near
    * its supporting code so that it is easier to follow.
    */
   
           /*
            * Clean window traps occur when %cleanwin is zero to ensure that data
            * is not leaked between address spaces in registers.
            */
           .macro  clean_window
           clr     %o0
           clr     %o1
           clr     %o2
           clr     %o3
           clr     %o4
           clr     %o5
           clr     %o6
           clr     %o7
           clr     %l0
           clr     %l1
           clr     %l2
           clr     %l3
           clr     %l4
           clr     %l5
           clr     %l6
           rdpr    %cleanwin, %l7
           inc     %l7
           wrpr    %l7, 0, %cleanwin
           clr     %l7
           retry
           .align  128
           .endm
   
           /*
            * Stack fixups for entry from user mode.  We are still running on the
            * user stack, and with its live registers, so we must save soon.  We
            * are on alternate globals so we do have some registers.  Set the
            * transitional window state, and do the save.  If this traps we
            * attempt to spill a window to the user stack.  If this fails, we
            * spill the window to the pcb and continue.  Spilling to the pcb
            * must not fail.
            *
            * NOTE: Must be called with alternate globals and clobbers %g1.
            */
   
           .macro  tl0_split
           rdpr    %wstate, %g1
           wrpr    %g1, WSTATE_TRANSITION, %wstate
           save
           .endm
   
           .macro  tl0_setup       type
           tl0_split
           clr     %o1
           set     trap, %o2
           ba      %xcc, tl0_utrap
            mov    \type, %o0
           .endm
   
           /*
            * Generic trap type.  Call trap() with the specified type.
            */
           .macro  tl0_gen         type
           tl0_setup \type
           .align  32
           .endm
   
           /*
            * This is used to suck up the massive swaths of reserved trap types.
            * Generates count "reserved" trap vectors.
            */
           .macro  tl0_reserved    count
           .rept   \count
           tl0_gen T_RESERVED
           .endr
           .endm
   
           .macro  tl1_split
           rdpr    %wstate, %g1
           wrpr    %g1, WSTATE_NESTED, %wstate
           save    %sp, -(CCFSZ + TF_SIZEOF), %sp
           .endm
   
           .macro  tl1_setup       type
           tl1_split
           clr     %o1
           set     trap, %o2
           ba      %xcc, tl1_trap
            mov    \type | T_KERNEL, %o0
           .endm
   
           .macro  tl1_gen         type
           tl1_setup \type
           .align  32
           .endm
   
           .macro  tl1_reserved    count
           .rept   \count
           tl1_gen T_RESERVED
           .endr
           .endm
   
           .macro  tl0_insn_excptn
           wrpr    %g0, PSTATE_ALT, %pstate
           wr      %g0, ASI_IMMU, %asi
           rdpr    %tpc, %g3
           ldxa    [%g0 + AA_IMMU_SFSR] %asi, %g4
           /*
            * XXX in theory, a store to AA_IMMU_SFSR must be immediately
            * followed by a DONE, FLUSH or RETRY for USIII.  In practice,
            * this triggers a RED state exception though.
            */
           stxa    %g0, [%g0 + AA_IMMU_SFSR] %asi
           membar  #Sync
           ba      %xcc, tl0_sfsr_trap
            mov    T_INSTRUCTION_EXCEPTION, %g2
           .align  32
           .endm
   
           .macro  tl0_data_excptn
           wrpr    %g0, PSTATE_ALT, %pstate
           wr      %g0, ASI_DMMU, %asi
           ldxa    [%g0 + AA_DMMU_SFAR] %asi, %g3
           ldxa    [%g0 + AA_DMMU_SFSR] %asi, %g4
           stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
           membar  #Sync
           ba      %xcc, tl0_sfsr_trap
            mov    T_DATA_EXCEPTION, %g2
           .align  32
           .endm
   
           .macro  tl0_align
           wr      %g0, ASI_DMMU, %asi
           ldxa    [%g0 + AA_DMMU_SFAR] %asi, %g3
           ldxa    [%g0 + AA_DMMU_SFSR] %asi, %g4
           stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
           membar  #Sync
           ba      %xcc, tl0_sfsr_trap
            mov    T_MEM_ADDRESS_NOT_ALIGNED, %g2
           .align  32
           .endm
   
   ENTRY(tl0_sfsr_trap)
           tl0_split
           clr     %o1
           set     trap, %o2
           mov     %g3, %o4
           mov     %g4, %o5
           ba      %xcc, tl0_utrap
            mov    %g2, %o0
   END(tl0_sfsr_trap)
   
           .macro  tl0_intr level, mask
           tl0_split
           set     \mask, %o1
           ba      %xcc, tl0_intr
            mov    \level, %o0
           .align  32
           .endm
   
   #define INTR(level, traplvl)                                            \
           tl ## traplvl ## _intr  level, 1 << level
   
   #define TICK(traplvl) \
           tl ## traplvl ## _intr  PIL_TICK, 0x10001
   
   #define INTR_LEVEL(tl)                                                  \
           INTR(1, tl) ;                                                   \
           INTR(2, tl) ;                                                   \
           INTR(3, tl) ;                                                   \
           INTR(4, tl) ;                                                   \
           INTR(5, tl) ;                                                   \
           INTR(6, tl) ;                                                   \
           INTR(7, tl) ;                                                   \
           INTR(8, tl) ;                                                   \
           INTR(9, tl) ;                                                   \
           INTR(10, tl) ;                                                  \
           INTR(11, tl) ;                                                  \
           INTR(12, tl) ;                                                  \
           INTR(13, tl) ;                                                  \
           TICK(tl) ;                                                      \
           INTR(15, tl) ;
   
           .macro  tl0_intr_level
           INTR_LEVEL(0)
           .endm
   
           .macro  intr_vector
           ldxa    [%g0] ASI_INTR_RECEIVE, %g1
           andcc   %g1, IRSR_BUSY, %g0
           bnz,a,pt %xcc, intr_vector
            nop
           ba,a,pt %xcc, intr_vector_stray
            nop
           .align  32
           .endm
   
           .macro  tl0_immu_miss
           /*
            * Load the context and the virtual page number from the tag access
            * register.  We ignore the context.
            */
           wr      %g0, ASI_IMMU, %asi
           ldxa    [%g0 + AA_IMMU_TAR] %asi, %g1
   
           /*
            * Initialize the page size walker.
            */
           mov     TS_MIN, %g2
   
           /*
            * Loop over all supported page sizes.
            */
   
           /*
            * Compute the page shift for the page size we are currently looking
            * for.
            */
   1:      add     %g2, %g2, %g3
           add     %g3, %g2, %g3
           add     %g3, PAGE_SHIFT, %g3
   
           /*
            * Extract the virtual page number from the contents of the tag
            * access register.
            */
           srlx    %g1, %g3, %g3
   
           /*
            * Compute the TTE bucket address.
            */
           ldxa    [%g0 + AA_IMMU_TSB] %asi, %g5
           and     %g3, TSB_BUCKET_MASK, %g4
           sllx    %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4
           add     %g4, %g5, %g4
   
           /*
            * Compute the TTE tag target.
            */
           sllx    %g3, TV_SIZE_BITS, %g3
           or      %g3, %g2, %g3
   
           /*
            * Loop over the TTEs in this bucket.
            */
   
           /*
            * Load the TTE.  Note that this instruction may fault, clobbering
            * the contents of the tag access register, %g5, %g6, and %g7.  We
            * do not use %g5, and %g6 and %g7 are not used until this instruction
            * completes successfully.
            */
   2:      ldda    [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */
   
           /*
            * Check that it's valid and executable and that the TTE tags match.
            */
           brgez,pn %g7, 3f
            andcc  %g7, TD_EXEC, %g0
           bz,pn   %xcc, 3f
            cmp    %g3, %g6
           bne,pn  %xcc, 3f
            EMPTY
   
           /*
            * We matched a TTE, load the TLB.
            */
   
           /*
            * Set the reference bit, if it's currently clear.
            */
            andcc  %g7, TD_REF, %g0
           bz,a,pn %xcc, tl0_immu_miss_set_ref
            nop
   
           /*
            * Load the TTE tag and data into the TLB and retry the instruction.
            */
           stxa    %g1, [%g0 + AA_IMMU_TAR] %asi
           stxa    %g7, [%g0] ASI_ITLB_DATA_IN_REG
           retry
   
           /*
            * Advance to the next TTE in this bucket, and check the low bits
            * of the bucket pointer to see if we've finished the bucket.
            */
   3:      add     %g4, 1 << TTE_SHIFT, %g4
           andcc   %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0
           bnz,pt  %xcc, 2b
            EMPTY
   
           /*
            * See if we just checked the largest page size, and advance to the
            * next one if not.
            */
            cmp    %g2, TS_MAX
           bne,pt  %xcc, 1b
            add    %g2, 1, %g2
   
           /*
            * Not in user TSB, call C code.
            */
           ba,a    %xcc, tl0_immu_miss_trap
           .align  128
           .endm
   
   ENTRY(tl0_immu_miss_set_ref)
           /*
            * Set the reference bit.
            */
           TTE_SET_REF(%g4, %g2, %g3, a, ASI_N)
   
           /*
            * May have become invalid during casxa, in which case start over.
            */
           brgez,pn %g2, 1f
            nop
   
           /*
            * Load the TTE tag and data into the TLB and retry the instruction.
            */
           stxa    %g1, [%g0 + AA_IMMU_TAR] %asi
           stxa    %g2, [%g0] ASI_ITLB_DATA_IN_REG
   1:      retry
   END(tl0_immu_miss_set_ref)
   
   ENTRY(tl0_immu_miss_trap)
           /*
            * Put back the contents of the tag access register, in case we
            * faulted.
            */
           sethi   %hi(KERNBASE), %g2
           stxa    %g1, [%g0 + AA_IMMU_TAR] %asi
           flush   %g2
   
           /*
            * Switch to alternate globals.
            */
           wrpr    %g0, PSTATE_ALT, %pstate
   
           /*
            * Reload the tag access register.
            */
           ldxa    [%g0 + AA_IMMU_TAR] %asi, %g2
   
           /*
            * Save the tag access register, and call common trap code.
            */
           tl0_split
           clr     %o1
           set     trap, %o2
           mov     %g2, %o3
           ba      %xcc, tl0_utrap
            mov    T_INSTRUCTION_MISS, %o0
   END(tl0_immu_miss_trap)
   
           .macro  tl0_dmmu_miss
           /*
            * Load the context and the virtual page number from the tag access
            * register.  We ignore the context.
            */
           wr      %g0, ASI_DMMU, %asi
           ldxa    [%g0 + AA_DMMU_TAR] %asi, %g1
   
           /*
            * Initialize the page size walker.
            */
   tl1_dmmu_miss_user:
           mov     TS_MIN, %g2
   
           /*
            * Loop over all supported page sizes.
            */
   
           /*
            * Compute the page shift for the page size we are currently looking
            * for.
            */
   1:      add     %g2, %g2, %g3
           add     %g3, %g2, %g3
           add     %g3, PAGE_SHIFT, %g3
   
           /*
            * Extract the virtual page number from the contents of the tag
            * access register.
            */
           srlx    %g1, %g3, %g3
   
           /*
            * Compute the TTE bucket address.
            */
           ldxa    [%g0 + AA_DMMU_TSB] %asi, %g5
           and     %g3, TSB_BUCKET_MASK, %g4
           sllx    %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4
           add     %g4, %g5, %g4
   
           /*
            * Compute the TTE tag target.
            */
           sllx    %g3, TV_SIZE_BITS, %g3
           or      %g3, %g2, %g3
   
           /*
            * Loop over the TTEs in this bucket.
            */
   
           /*
            * Load the TTE.  Note that this instruction may fault, clobbering
            * the contents of the tag access register, %g5, %g6, and %g7.  We
            * do not use %g5, and %g6 and %g7 are not used until this instruction
            * completes successfully.
            */
   2:      ldda    [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */
   
           /*
            * Check that it's valid and that the virtual page numbers match.
            */
           brgez,pn %g7, 3f
            cmp    %g3, %g6
           bne,pn  %xcc, 3f
            EMPTY
   
           /*
            * We matched a TTE, load the TLB.
            */
   
           /*
            * Set the reference bit, if it's currently clear.
            */
            andcc  %g7, TD_REF, %g0
           bz,a,pn %xcc, tl0_dmmu_miss_set_ref
            nop
   
           /*
            * Load the TTE tag and data into the TLB and retry the instruction.
            */
           stxa    %g1, [%g0 + AA_DMMU_TAR] %asi
           stxa    %g7, [%g0] ASI_DTLB_DATA_IN_REG
           retry
   
           /*
            * Advance to the next TTE in this bucket, and check the low bits
            * of the bucket pointer to see if we've finished the bucket.
            */
   3:      add     %g4, 1 << TTE_SHIFT, %g4
           andcc   %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0
           bnz,pt  %xcc, 2b
            EMPTY
   
           /*
            * See if we just checked the largest page size, and advance to the
            * next one if not.
            */
            cmp    %g2, TS_MAX
           bne,pt  %xcc, 1b
            add    %g2, 1, %g2
   
           /*
            * Not in user TSB, call C code.
            */
           ba,a    %xcc, tl0_dmmu_miss_trap
           .align  128
           .endm
   
   ENTRY(tl0_dmmu_miss_set_ref)
           /*
            * Set the reference bit.
            */
           TTE_SET_REF(%g4, %g2, %g3, a, ASI_N)
   
           /*
            * May have become invalid during casxa, in which case start over.
            */
           brgez,pn %g2, 1f
            nop
   
           /*
            * Load the TTE tag and data into the TLB and retry the instruction.
            */
           stxa    %g1, [%g0 + AA_DMMU_TAR] %asi
           stxa    %g2, [%g0] ASI_DTLB_DATA_IN_REG
   1:      retry
   END(tl0_dmmu_miss_set_ref)
   
   ENTRY(tl0_dmmu_miss_trap)
           /*
            * Put back the contents of the tag access register, in case we
            * faulted.
            */
           stxa    %g1, [%g0 + AA_DMMU_TAR] %asi
           membar  #Sync
   
           /*
            * Switch to alternate globals.
            */
           wrpr    %g0, PSTATE_ALT, %pstate
   
           /*
            * Check if we actually came from the kernel.
            */
           rdpr    %tl, %g1
           cmp     %g1, 1
           bgt,a,pn %xcc, 1f
            nop
   
           /*
            * Reload the tag access register.
            */
           ldxa    [%g0 + AA_DMMU_TAR] %asi, %g2
   
           /*
            * Save the tag access register and call common trap code.
            */
           tl0_split
           clr     %o1
           set     trap, %o2
           mov     %g2, %o3
           ba      %xcc, tl0_utrap
            mov    T_DATA_MISS, %o0
   
           /*
            * Handle faults during window spill/fill.
            */
   1:      RESUME_SPILLFILL_MMU
   
           /*
            * Reload the tag access register.
            */
           ldxa    [%g0 + AA_DMMU_TAR] %asi, %g2
   
           tl1_split
           clr     %o1
           set     trap, %o2
           mov     %g2, %o3
           ba      %xcc, tl1_trap
            mov    T_DATA_MISS | T_KERNEL, %o0
   END(tl0_dmmu_miss_trap)
   
           .macro  tl0_dmmu_prot
           ba,a    %xcc, tl0_dmmu_prot_1
            nop
           .align  128
           .endm
   
   ENTRY(tl0_dmmu_prot_1)
           /*
            * Load the context and the virtual page number from the tag access
            * register.  We ignore the context.
            */
           wr      %g0, ASI_DMMU, %asi
           ldxa    [%g0 + AA_DMMU_TAR] %asi, %g1
   
           /*
            * Initialize the page size walker.
            */
   tl1_dmmu_prot_user:
           mov     TS_MIN, %g2
   
           /*
            * Loop over all supported page sizes.
            */
   
           /*
            * Compute the page shift for the page size we are currently looking
            * for.
            */
   1:      add     %g2, %g2, %g3
           add     %g3, %g2, %g3
           add     %g3, PAGE_SHIFT, %g3
   
           /*
            * Extract the virtual page number from the contents of the tag
            * access register.
            */
           srlx    %g1, %g3, %g3
   
           /*
            * Compute the TTE bucket address.
            */
           ldxa    [%g0 + AA_DMMU_TSB] %asi, %g5
           and     %g3, TSB_BUCKET_MASK, %g4
           sllx    %g4, TSB_BUCKET_SHIFT + TTE_SHIFT, %g4
           add     %g4, %g5, %g4
   
           /*
            * Compute the TTE tag target.
            */
           sllx    %g3, TV_SIZE_BITS, %g3
           or      %g3, %g2, %g3
   
           /*
            * Loop over the TTEs in this bucket.
            */
   
           /*
            * Load the TTE.  Note that this instruction may fault, clobbering
            * the contents of the tag access register, %g5, %g6, and %g7.  We
            * do not use %g5, and %g6 and %g7 are not used until this instruction
            * completes successfully.
            */
   2:      ldda    [%g4] ASI_NUCLEUS_QUAD_LDD, %g6 /*, %g7 */
   
           /*
            * Check that it's valid and writable and that the virtual page
           * numbers match.
           */
          brgez,pn %g7, 4f
           andcc  %g7, TD_SW, %g0
          bz,pn   %xcc, 4f
           cmp    %g3, %g6
          bne,pn  %xcc, 4f
           nop
  
          /*
           * Set the hardware write bit.
           */
          TTE_SET_W(%g4, %g2, %g3, a, ASI_N)
  
          /*
           * Delete the old TLB entry and clear the SFSR.
           */
          srlx    %g1, PAGE_SHIFT, %g3
          sllx    %g3, PAGE_SHIFT, %g3
          stxa    %g0, [%g3] ASI_DMMU_DEMAP
          stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
          membar  #Sync
  
          /*
           * May have become invalid during casxa, in which case start over.
           */
          brgez,pn %g2, 3f
           or     %g2, TD_W, %g2
  
          /*
           * Load the TTE data into the TLB and retry the instruction.
           */
          stxa    %g1, [%g0 + AA_DMMU_TAR] %asi
          stxa    %g2, [%g0] ASI_DTLB_DATA_IN_REG
  3:      retry
  
          /*
           * Check the low bits to see if we've finished the bucket.
           */
  4:      add     %g4, 1 << TTE_SHIFT, %g4
          andcc   %g4, (1 << (TSB_BUCKET_SHIFT + TTE_SHIFT)) - 1, %g0
          bnz,pt  %xcc, 2b
           EMPTY
  
          /*
           * See if we just checked the largest page size, and advance to the
           * next one if not.
           */
           cmp    %g2, TS_MAX
          bne,pt  %xcc, 1b
           add    %g2, 1, %g2
  
          /*
           * Not in user TSB, call C code.
           */
          ba,a    %xcc, tl0_dmmu_prot_trap
           nop
  END(tl0_dmmu_prot_1)
  
  ENTRY(tl0_dmmu_prot_trap)
          /*
           * Put back the contents of the tag access register, in case we
           * faulted.
           */
          stxa    %g1, [%g0 + AA_DMMU_TAR] %asi
          membar  #Sync
  
          /*
           * Switch to alternate globals.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
  
          /*
           * Check if we actually came from the kernel.
           */
          rdpr    %tl, %g1
          cmp     %g1, 1
          bgt,a,pn %xcc, 1f
           nop
  
          /*
           * Load the SFAR, SFSR and TAR.
           */
          ldxa    [%g0 + AA_DMMU_TAR] %asi, %g2
          ldxa    [%g0 + AA_DMMU_SFAR] %asi, %g3
          ldxa    [%g0 + AA_DMMU_SFSR] %asi, %g4
          stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
          membar  #Sync
  
          /*
           * Save the MMU registers and call common trap code.
           */
          tl0_split
          clr     %o1
          set     trap, %o2
          mov     %g2, %o3
          mov     %g3, %o4
          mov     %g4, %o5
          ba      %xcc, tl0_utrap
           mov    T_DATA_PROTECTION, %o0
  
          /*
           * Handle faults during window spill/fill.
           */
  1:      RESUME_SPILLFILL_MMU_CLR_SFSR
  
          /*
           * Load the SFAR, SFSR and TAR.  Clear the SFSR.
           */
          ldxa    [%g0 + AA_DMMU_TAR] %asi, %g2
          ldxa    [%g0 + AA_DMMU_SFAR] %asi, %g3
          ldxa    [%g0 + AA_DMMU_SFSR] %asi, %g4
          stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
          membar  #Sync
  
          tl1_split
          clr     %o1
          set     trap, %o2
          mov     %g2, %o3
          mov     %g3, %o4
          mov     %g4, %o5
          ba      %xcc, tl1_trap
           mov    T_DATA_PROTECTION | T_KERNEL, %o0
  END(tl0_dmmu_prot_trap)
  
          .macro  tl0_spill_0_n
          wr      %g0, ASI_AIUP, %asi
          SPILL(stxa, %sp + SPOFF, 8, %asi)
          saved
          retry
          .align  32
          RSF_TRAP(T_SPILL)
          RSF_TRAP(T_SPILL)
          .endm
  
          .macro  tl0_spill_1_n
          wr      %g0, ASI_AIUP, %asi
          SPILL(stwa, %sp, 4, %asi)
          saved
          retry
          .align  32
          RSF_TRAP(T_SPILL)
          RSF_TRAP(T_SPILL)
          .endm
  
          .macro  tl0_fill_0_n
          wr      %g0, ASI_AIUP, %asi
          FILL(ldxa, %sp + SPOFF, 8, %asi)
          restored
          retry
          .align  32
          RSF_TRAP(T_FILL)
          RSF_TRAP(T_FILL)
          .endm
  
          .macro  tl0_fill_1_n
          wr      %g0, ASI_AIUP, %asi
          FILL(lduwa, %sp, 4, %asi)
          restored
          retry
          .align  32
          RSF_TRAP(T_FILL)
          RSF_TRAP(T_FILL)
          .endm
  
  ENTRY(tl0_sftrap)
          rdpr    %tstate, %g1
          and     %g1, TSTATE_CWP_MASK, %g1
          wrpr    %g1, 0, %cwp
          tl0_split
          clr     %o1
          set     trap, %o2
          ba      %xcc, tl0_trap
           mov    %g2, %o0
  END(tl0_sftrap)
  
          .macro  tl0_spill_bad   count
          .rept   \count
          sir
          .align  128
          .endr
          .endm
  
          .macro  tl0_fill_bad    count
          .rept   \count
          sir
          .align  128
          .endr
          .endm
  
          .macro  tl0_syscall
          tl0_split
          clr     %o1
          set     syscall, %o2
          ba      %xcc, tl0_trap
           mov    T_SYSCALL, %o0
          .align  32
          .endm
  
          .macro  tl0_fp_restore
          ba,a    %xcc, tl0_fp_restore
           nop
          .align  32
          .endm
  
  ENTRY(tl0_fp_restore)
          ldx     [PCB_REG + PCB_FLAGS], %g1
          andn    %g1, PCB_FEF, %g1
          stx     %g1, [PCB_REG + PCB_FLAGS]
  
          wr      %g0, FPRS_FEF, %fprs
          wr      %g0, ASI_BLK_S, %asi
          ldda    [PCB_REG + PCB_UFP + (0 * 64)] %asi, %f0
          ldda    [PCB_REG + PCB_UFP + (1 * 64)] %asi, %f16
          ldda    [PCB_REG + PCB_UFP + (2 * 64)] %asi, %f32
          ldda    [PCB_REG + PCB_UFP + (3 * 64)] %asi, %f48
          membar  #Sync
          done
  END(tl0_fp_restore)
  
          .macro  tl1_insn_excptn
          wrpr    %g0, PSTATE_ALT, %pstate
          wr      %g0, ASI_IMMU, %asi
          rdpr    %tpc, %g3
          ldxa    [%g0 + AA_IMMU_SFSR] %asi, %g4
          /*
           * XXX in theory, a store to AA_IMMU_SFSR must be immediately
           * followed by a DONE, FLUSH or RETRY for USIII.  In practice,
           * this triggers a RED state exception though.
           */
          stxa    %g0, [%g0 + AA_IMMU_SFSR] %asi
          membar  #Sync
          ba      %xcc, tl1_insn_exceptn_trap
           mov    T_INSTRUCTION_EXCEPTION | T_KERNEL, %g2
          .align  32
          .endm
  
  ENTRY(tl1_insn_exceptn_trap)
          tl1_split
          clr     %o1
          set     trap, %o2
          mov     %g3, %o4
          mov     %g4, %o5
          ba      %xcc, tl1_trap
           mov    %g2, %o0
  END(tl1_insn_exceptn_trap)
  
          .macro  tl1_fp_disabled
          ba,a    %xcc, tl1_fp_disabled_1
           nop
          .align  32
          .endm
  
  ENTRY(tl1_fp_disabled_1)
          rdpr    %tpc, %g1
          set     fpu_fault_begin, %g2
          sub     %g1, %g2, %g1
          cmp     %g1, fpu_fault_size
          bgeu,a,pn %xcc, 1f
           nop
  
          wr      %g0, FPRS_FEF, %fprs
          wr      %g0, ASI_BLK_S, %asi
          ldda    [PCB_REG + PCB_KFP + (0 * 64)] %asi, %f0
          ldda    [PCB_REG + PCB_KFP + (1 * 64)] %asi, %f16
          ldda    [PCB_REG + PCB_KFP + (2 * 64)] %asi, %f32
          ldda    [PCB_REG + PCB_KFP + (3 * 64)] %asi, %f48
          membar  #Sync
          retry
  
  1:      tl1_split
          clr     %o1
          set     trap, %o2
          ba      %xcc, tl1_trap
           mov    T_FP_DISABLED | T_KERNEL, %o0
  END(tl1_fp_disabled_1)
  
          .macro  tl1_data_excptn
          wrpr    %g0, PSTATE_ALT, %pstate
          ba,a    %xcc, tl1_data_excptn_trap
           nop
          .align  32
          .endm
  
  ENTRY(tl1_data_excptn_trap)
          RESUME_SPILLFILL_MMU_CLR_SFSR
          ba      %xcc, tl1_sfsr_trap
           mov    T_DATA_EXCEPTION | T_KERNEL, %g2
  END(tl1_data_excptn_trap)
  
          .macro  tl1_align
          wrpr    %g0, PSTATE_ALT, %pstate
          ba,a    %xcc, tl1_align_trap
           nop
          .align  32
          .endm
  
  ENTRY(tl1_align_trap)
          RESUME_SPILLFILL_ALIGN
          ba      %xcc, tl1_sfsr_trap
           mov    T_MEM_ADDRESS_NOT_ALIGNED | T_KERNEL, %g2
  END(tl1_align_trap)
  
  ENTRY(tl1_sfsr_trap)
          wr      %g0, ASI_DMMU, %asi
          ldxa    [%g0 + AA_DMMU_SFAR] %asi, %g3
          ldxa    [%g0 + AA_DMMU_SFSR] %asi, %g4
          stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
          membar  #Sync
  
          tl1_split
          clr     %o1
          set     trap, %o2
          mov     %g3, %o4
          mov     %g4, %o5
          ba      %xcc, tl1_trap
           mov    %g2, %o0
  END(tl1_sfsr_trap)
  
          .macro  tl1_intr level, mask
          tl1_split
          set     \mask, %o1
          ba      %xcc, tl1_intr
           mov    \level, %o0
          .align  32
          .endm
  
          .macro  tl1_intr_level
          INTR_LEVEL(1)
          .endm
  
          .macro  tl1_immu_miss
          /*
           * Load the context and the virtual page number from the tag access
           * register.  We ignore the context.
           */
          wr      %g0, ASI_IMMU, %asi
          ldxa    [%g0 + AA_IMMU_TAR] %asi, %g5
  
          /*
           * Compute the address of the TTE.  The TSB mask and address of the
           * TSB are patched at startup.
           */
          .globl  tl1_immu_miss_patch_tsb_1
  tl1_immu_miss_patch_tsb_1:
          sethi   %uhi(TSB_KERNEL), %g6
          or      %g6, %ulo(TSB_KERNEL), %g6
          sllx    %g6, 32, %g6
          sethi   %hi(TSB_KERNEL), %g7
          or      %g7, %g6, %g7
          .globl  tl1_immu_miss_patch_tsb_mask_1
  tl1_immu_miss_patch_tsb_mask_1:
          sethi   %hi(TSB_KERNEL_MASK), %g6
          or      %g6, %lo(TSB_KERNEL_MASK), %g6
  
          srlx    %g5, TAR_VPN_SHIFT, %g5
          and     %g5, %g6, %g6
          sllx    %g6, TTE_SHIFT, %g6
          add     %g6, %g7, %g6
  
          /*
           * Load the TTE.
           */
          .globl  tl1_immu_miss_patch_quad_ldd_1
  tl1_immu_miss_patch_quad_ldd_1:
          ldda    [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */
  
          /*
           * Check that it's valid and executable and that the virtual page
           * numbers match.
           */
          brgez,pn %g7, tl1_immu_miss_trap
           andcc  %g7, TD_EXEC, %g0
          bz,pn   %xcc, tl1_immu_miss_trap
           srlx   %g6, TV_SIZE_BITS, %g6
          cmp     %g5, %g6
          bne,pn  %xcc, tl1_immu_miss_trap
           EMPTY
  
          /*
           * Set the reference bit if it's currently clear.
           */
           andcc  %g7, TD_REF, %g0
          bz,a,pn %xcc, tl1_immu_miss_set_ref
           nop
  
          /*
           * Load the TTE data into the TLB and retry the instruction.
           */
          stxa    %g7, [%g0] ASI_ITLB_DATA_IN_REG
          retry
          .align  128
          .endm
  
  ENTRY(tl1_immu_miss_set_ref)
          /*
           * Recompute the TTE address, which we clobbered loading the TTE.
           * The TSB mask and address of the TSB are patched at startup.
           */
          .globl  tl1_immu_miss_patch_tsb_2
  tl1_immu_miss_patch_tsb_2:
          sethi   %uhi(TSB_KERNEL), %g6
          or      %g6, %ulo(TSB_KERNEL), %g6
          sllx    %g6, 32, %g6
          sethi   %hi(TSB_KERNEL), %g7
          or      %g7, %g6, %g7
          .globl  tl1_immu_miss_patch_tsb_mask_2
  tl1_immu_miss_patch_tsb_mask_2:
          sethi   %hi(TSB_KERNEL_MASK), %g6
          or      %g6, %lo(TSB_KERNEL_MASK), %g6
  
          and     %g5, %g6, %g5
          sllx    %g5, TTE_SHIFT, %g5
          add     %g5, %g7, %g5
  
          /*
           * Set the reference bit.
           */
          .globl  tl1_immu_miss_patch_asi_1
  tl1_immu_miss_patch_asi_1:
          wr      %g0, TSB_ASI, %asi
          TTE_SET_REF(%g5, %g6, %g7, a, %asi)
  
          /*
           * May have become invalid during casxa, in which case start over.
           */
          brgez,pn %g6, 1f
           nop
  
          /*
           * Load the TTE data into the TLB and retry the instruction.
           */
          stxa    %g6, [%g0] ASI_ITLB_DATA_IN_REG
  1:      retry
  END(tl1_immu_miss_set_ref)
  
  ENTRY(tl1_immu_miss_trap)
          /*
           * Switch to alternate globals.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
  
          ldxa    [%g0 + AA_IMMU_TAR] %asi, %g2
  
          tl1_split
          clr     %o1
          set     trap, %o2
          mov     %g2, %o3
          ba      %xcc, tl1_trap
           mov    T_INSTRUCTION_MISS | T_KERNEL, %o0
  END(tl1_immu_miss_trap)
  
          .macro  tl1_dmmu_miss
          /*
           * Load the context and the virtual page number from the tag access
           * register.
           */
          wr      %g0, ASI_DMMU, %asi
          ldxa    [%g0 + AA_DMMU_TAR] %asi, %g5
  
          /*
           * Extract the context from the contents of the tag access register.
           * If it's non-zero this is a fault on a user address.  Note that the
           * faulting address is passed in %g1.
           */
          sllx    %g5, 64 - TAR_VPN_SHIFT, %g6
          brnz,a,pn %g6, tl1_dmmu_miss_user
           mov    %g5, %g1
  
          /*
           * Check for the direct mapped physical region.  These addresses have
           * the high bit set so they are negative.
           */
          brlz,pn %g5, tl1_dmmu_miss_direct
           EMPTY
  
          /*
           * Compute the address of the TTE.  The TSB mask and address of the
           * TSB are patched at startup.
           */
          .globl  tl1_dmmu_miss_patch_tsb_1
  tl1_dmmu_miss_patch_tsb_1:
          sethi   %uhi(TSB_KERNEL), %g6
          or      %g6, %ulo(TSB_KERNEL), %g6
          sllx    %g6, 32, %g6
          sethi   %hi(TSB_KERNEL), %g7
          or      %g7, %g6, %g7
          .globl  tl1_dmmu_miss_patch_tsb_mask_1
  tl1_dmmu_miss_patch_tsb_mask_1:
          sethi   %hi(TSB_KERNEL_MASK), %g6
          or      %g6, %lo(TSB_KERNEL_MASK), %g6
  
          srlx    %g5, TAR_VPN_SHIFT, %g5
          and     %g5, %g6, %g6
          sllx    %g6, TTE_SHIFT, %g6
          add     %g6, %g7, %g6
  
          /*
           * Load the TTE.
           */
          .globl  tl1_dmmu_miss_patch_quad_ldd_1
  tl1_dmmu_miss_patch_quad_ldd_1:
          ldda    [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */
  
          /*
           * Check that it's valid and that the virtual page numbers match.
           */
          brgez,pn %g7, tl1_dmmu_miss_trap
           srlx   %g6, TV_SIZE_BITS, %g6
          cmp     %g5, %g6
          bne,pn %xcc, tl1_dmmu_miss_trap
           EMPTY
  
          /*
           * Set the reference bit if it's currently clear.
           */
           andcc  %g7, TD_REF, %g0
          bz,a,pt %xcc, tl1_dmmu_miss_set_ref
           nop
  
          /*
           * Load the TTE data into the TLB and retry the instruction.
           */
          stxa    %g7, [%g0] ASI_DTLB_DATA_IN_REG
          retry
          .align  128
          .endm
  
  ENTRY(tl1_dmmu_miss_set_ref)
          /*
           * Recompute the TTE address, which we clobbered loading the TTE.
           * The TSB mask and address of the TSB are patched at startup.
           */
          .globl  tl1_dmmu_miss_patch_tsb_mask_2
  tl1_dmmu_miss_patch_tsb_2:
          sethi   %uhi(TSB_KERNEL), %g6
          or      %g6, %ulo(TSB_KERNEL), %g6
          sllx    %g6, 32, %g6
          sethi   %hi(TSB_KERNEL), %g7
          or      %g7, %g6, %g7
          .globl  tl1_dmmu_miss_patch_tsb_2
  tl1_dmmu_miss_patch_tsb_mask_2:
          sethi   %hi(TSB_KERNEL_MASK), %g6
          or      %g6, %lo(TSB_KERNEL_MASK), %g6
  
          and     %g5, %g6, %g5
          sllx    %g5, TTE_SHIFT, %g5
          add     %g5, %g7, %g5
  
          /*
           * Set the reference bit.
           */
          .globl  tl1_dmmu_miss_patch_asi_1
  tl1_dmmu_miss_patch_asi_1:
          wr      %g0, TSB_ASI, %asi
          TTE_SET_REF(%g5, %g6, %g7, a, %asi)
  
          /*
           * May have become invalid during casxa, in which case start over.
           */
          brgez,pn %g6, 1f
           nop
  
          /*
           * Load the TTE data into the TLB and retry the instruction.
           */
          stxa    %g6, [%g0] ASI_DTLB_DATA_IN_REG
  1:      retry
  END(tl1_dmmu_miss_set_ref)
  
  ENTRY(tl1_dmmu_miss_trap)
          /*
           * Switch to alternate globals.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
  
          ldxa    [%g0 + AA_DMMU_TAR] %asi, %g2
  
          KSTACK_CHECK
  
          tl1_split
          clr     %o1
          set     trap, %o2
          mov     %g2, %o3
          ba      %xcc, tl1_trap
           mov    T_DATA_MISS | T_KERNEL, %o0
  END(tl1_dmmu_miss_trap)
  
  ENTRY(tl1_dmmu_miss_direct)
          /*
           * Mask off the high bits of the virtual address to get the physical
           * address, and or in the TTE bits.  The virtual address bits that
           * correspond to the TTE valid and page size bits are left set, so
           * they don't have to be included in the TTE bits below.  We know they
           * are set because the virtual address is in the upper va hole.
           * NB: if we are taking advantage of the ASI_ATOMIC_QUAD_LDD_PHYS
           * and we get a miss on the directly accessed kernel TSB we must not
           * set TD_CV in order to access it uniformly bypassing the D$.
           */
          setx    TLB_DIRECT_ADDRESS_MASK, %g7, %g4
          and     %g5, %g4, %g4
          setx    TLB_DIRECT_TO_TTE_MASK, %g7, %g6
          and     %g5, %g6, %g5
          .globl  tl1_dmmu_miss_direct_patch_tsb_phys_1
  tl1_dmmu_miss_direct_patch_tsb_phys_1:
          sethi   %uhi(TSB_KERNEL_PHYS), %g3
          or      %g3, %ulo(TSB_KERNEL_PHYS), %g3
          sllx    %g3, 32, %g3
          sethi   %hi(TSB_KERNEL_PHYS), %g3
          or      %g7, %g3, %g7
          cmp     %g4, %g7
          bl,pt   %xcc, 1f
           or     %g5, TD_CP | TD_W, %g5
          .globl  tl1_dmmu_miss_direct_patch_tsb_phys_end_1
  tl1_dmmu_miss_direct_patch_tsb_phys_end_1:
          sethi   %uhi(TSB_KERNEL_PHYS_END), %g3
          or      %g3, %ulo(TSB_KERNEL_PHYS_END), %g3
          sllx    %g3, 32, %g3
          sethi   %hi(TSB_KERNEL_PHYS_END), %g7
          or      %g7, %g3, %g7
          cmp     %g4, %g7
          bg,a,pt %xcc, 1f
           nop
          ba,pt   %xcc, 2f
           nop
  1:      or      %g5, TD_CV, %g5
  
          /*
           * Load the TTE data into the TLB and retry the instruction.
           */
  2:      stxa    %g5, [%g0] ASI_DTLB_DATA_IN_REG
          retry
  END(tl1_dmmu_miss_direct)
  
          .macro  tl1_dmmu_prot
          ba,a    %xcc, tl1_dmmu_prot_1
           nop
          .align  128
          .endm
  
  ENTRY(tl1_dmmu_prot_1)
          /*
           * Load the context and the virtual page number from the tag access
           * register.
           */
          wr      %g0, ASI_DMMU, %asi
          ldxa    [%g0 + AA_DMMU_TAR] %asi, %g5
  
          /*
           * Extract the context from the contents of the tag access register.
           * If it's non-zero this is a fault on a user address.  Note that the
           * faulting address is passed in %g1.
           */
          sllx    %g5, 64 - TAR_VPN_SHIFT, %g6
          brnz,a,pn %g6, tl1_dmmu_prot_user
           mov    %g5, %g1
  
          /*
           * Compute the address of the TTE.  The TSB mask and address of the
           * TSB are patched at startup.
           */
          .globl  tl1_dmmu_prot_patch_tsb_1
  tl1_dmmu_prot_patch_tsb_1:
          sethi   %uhi(TSB_KERNEL), %g6
          or      %g6, %ulo(TSB_KERNEL), %g6
          sllx    %g6, 32, %g6
          sethi   %hi(TSB_KERNEL), %g7
          or      %g7, %g6, %g7
          .globl  tl1_dmmu_prot_patch_tsb_mask_1
  tl1_dmmu_prot_patch_tsb_mask_1:
          sethi   %hi(TSB_KERNEL_MASK), %g6
          or      %g6, %lo(TSB_KERNEL_MASK), %g6
  
          srlx    %g5, TAR_VPN_SHIFT, %g5
          and     %g5, %g6, %g6
          sllx    %g6, TTE_SHIFT, %g6
          add     %g6, %g7, %g6
  
          /*
           * Load the TTE.
           */
          .globl  tl1_dmmu_prot_patch_quad_ldd_1
  tl1_dmmu_prot_patch_quad_ldd_1:
          ldda    [%g6] TSB_QUAD_LDD, %g6 /*, %g7 */
  
          /*
           * Check that it's valid and writeable and that the virtual page
           * numbers match.
           */
          brgez,pn %g7, tl1_dmmu_prot_trap
           andcc  %g7, TD_SW, %g0
          bz,pn   %xcc, tl1_dmmu_prot_trap
           srlx   %g6, TV_SIZE_BITS, %g6
          cmp     %g5, %g6
          bne,pn  %xcc, tl1_dmmu_prot_trap
           EMPTY
  
          /*
           * Delete the old TLB entry and clear the SFSR.
           */
           sllx   %g5, TAR_VPN_SHIFT, %g6
          or      %g6, TLB_DEMAP_NUCLEUS, %g6
          stxa    %g0, [%g6] ASI_DMMU_DEMAP
          stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
          membar  #Sync
  
          /*
           * Recompute the TTE address, which we clobbered loading the TTE.
           * The TSB mask and address of the TSB are patched at startup.
           */
          .globl  tl1_dmmu_prot_patch_tsb_2
  tl1_dmmu_prot_patch_tsb_2:
          sethi   %uhi(TSB_KERNEL), %g6
          or      %g6, %ulo(TSB_KERNEL), %g6
          sllx    %g6, 32, %g6
          sethi   %hi(TSB_KERNEL), %g7
          or      %g7, %g6, %g7
          .globl  tl1_dmmu_prot_patch_tsb_mask_2
  tl1_dmmu_prot_patch_tsb_mask_2:
          sethi   %hi(TSB_KERNEL_MASK), %g6
          or      %g6, %lo(TSB_KERNEL_MASK), %g6
          and     %g5, %g6, %g5
          sllx    %g5, TTE_SHIFT, %g5
          add     %g5, %g7, %g5
  
          /*
           * Set the hardware write bit.
           */
          .globl  tl1_dmmu_prot_patch_asi_1
  tl1_dmmu_prot_patch_asi_1:
          wr      %g0, TSB_ASI, %asi
          TTE_SET_W(%g5, %g6, %g7, a, %asi)
  
          /*
           * May have become invalid during casxa, in which case start over.
           */
          brgez,pn %g6, 1f
           or     %g6, TD_W, %g6
  
          /*
           * Load the TTE data into the TLB and retry the instruction.
           */
          stxa    %g6, [%g0] ASI_DTLB_DATA_IN_REG
  1:      retry
  END(tl1_dmmu_prot_1)
  
  ENTRY(tl1_dmmu_prot_trap)
          /*
           * Switch to alternate globals.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
  
          /*
           * Load the SFAR, SFSR and TAR.  Clear the SFSR.
           */
          ldxa    [%g0 + AA_DMMU_TAR] %asi, %g2
          ldxa    [%g0 + AA_DMMU_SFAR] %asi, %g3
          ldxa    [%g0 + AA_DMMU_SFSR] %asi, %g4
          stxa    %g0, [%g0 + AA_DMMU_SFSR] %asi
          membar  #Sync
  
          tl1_split
          clr     %o1
          set     trap, %o2
          mov     %g2, %o3
          mov     %g3, %o4
          mov     %g4, %o5
          ba      %xcc, tl1_trap
           mov    T_DATA_PROTECTION | T_KERNEL, %o0
  END(tl1_dmmu_prot_trap)
  
          .macro  tl1_spill_0_n
          SPILL(stx, %sp + SPOFF, 8, EMPTY)
          saved
          retry
          .align  32
          RSF_FATAL(T_SPILL)
          RSF_FATAL(T_SPILL)
          .endm
  
          .macro  tl1_spill_2_n
          wr      %g0, ASI_AIUP, %asi
          SPILL(stxa, %sp + SPOFF, 8, %asi)
          saved
          retry
          .align  32
          RSF_SPILL_TOPCB
          RSF_SPILL_TOPCB
          .endm
  
          .macro  tl1_spill_3_n
          wr      %g0, ASI_AIUP, %asi
          SPILL(stwa, %sp, 4, %asi)
          saved
          retry
          .align  32
          RSF_SPILL_TOPCB
          RSF_SPILL_TOPCB
          .endm
  
          .macro  tl1_spill_7_n
          btst    1, %sp
          bnz,a,pn %xcc, tl1_spill_0_n
           nop
          srl     %sp, 0, %sp
          SPILL(stw, %sp, 4, EMPTY)
          saved
          retry
          .align  32
          RSF_FATAL(T_SPILL)
          RSF_FATAL(T_SPILL)
          .endm
  
          .macro  tl1_spill_0_o
          wr      %g0, ASI_AIUP, %asi
          SPILL(stxa, %sp + SPOFF, 8, %asi)
          saved
          retry
          .align  32
          RSF_SPILL_TOPCB
          RSF_SPILL_TOPCB
          .endm
  
          .macro  tl1_spill_1_o
          wr      %g0, ASI_AIUP, %asi
          SPILL(stwa, %sp, 4, %asi)
          saved
          retry
          .align  32
          RSF_SPILL_TOPCB
          RSF_SPILL_TOPCB
          .endm
  
          .macro  tl1_spill_2_o
          RSF_SPILL_TOPCB
          .align  128
          .endm
  
          .macro  tl1_fill_0_n
          FILL(ldx, %sp + SPOFF, 8, EMPTY)
          restored
          retry
          .align  32
          RSF_FATAL(T_FILL)
          RSF_FATAL(T_FILL)
          .endm
  
          .macro  tl1_fill_2_n
          wr      %g0, ASI_AIUP, %asi
          FILL(ldxa, %sp + SPOFF, 8, %asi)
          restored
          retry
          .align 32
          RSF_FILL_MAGIC
          RSF_FILL_MAGIC
          .endm
  
          .macro  tl1_fill_3_n
          wr      %g0, ASI_AIUP, %asi
          FILL(lduwa, %sp, 4, %asi)
          restored
          retry
          .align 32
          RSF_FILL_MAGIC
          RSF_FILL_MAGIC
          .endm
  
          .macro  tl1_fill_7_n
          btst    1, %sp
          bnz,a,pt %xcc, tl1_fill_0_n
           nop
          srl     %sp, 0, %sp
          FILL(lduw, %sp, 4, EMPTY)
          restored
          retry
          .align  32
          RSF_FATAL(T_FILL)
          RSF_FATAL(T_FILL)
          .endm
  
  /*
   * This is used to spill windows that are still occupied with user
   * data on kernel entry to the pcb.
   */
  ENTRY(tl1_spill_topcb)
          wrpr    %g0, PSTATE_ALT, %pstate
  
          /* Free some globals for our use. */
          dec     24, ASP_REG
          stx     %g1, [ASP_REG + 0]
          stx     %g2, [ASP_REG + 8]
          stx     %g3, [ASP_REG + 16]
  
          ldx     [PCB_REG + PCB_NSAVED], %g1
  
          sllx    %g1, PTR_SHIFT, %g2
          add     %g2, PCB_REG, %g2
          stx     %sp, [%g2 + PCB_RWSP]
  
          sllx    %g1, RW_SHIFT, %g2
          add     %g2, PCB_REG, %g2
          SPILL(stx, %g2 + PCB_RW, 8, EMPTY)
  
          inc     %g1
          stx     %g1, [PCB_REG + PCB_NSAVED]
  
  #if KTR_COMPILE & KTR_TRAP
          CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d"
             , %g1, %g2, %g3, 7, 8, 9)
          rdpr    %tpc, %g2
          stx     %g2, [%g1 + KTR_PARM1]
          rdpr    %tnpc, %g2
          stx     %g2, [%g1 + KTR_PARM2]
          stx     %sp, [%g1 + KTR_PARM3]
          ldx     [PCB_REG + PCB_NSAVED], %g2
          stx     %g2, [%g1 + KTR_PARM4]
  9:
  #endif
  
          saved
  
          ldx     [ASP_REG + 16], %g3
          ldx     [ASP_REG + 8], %g2
          ldx     [ASP_REG + 0], %g1
          inc     24, ASP_REG
          retry
  END(tl1_spill_topcb)
  
          .macro  tl1_spill_bad   count
          .rept   \count
          sir
          .align  128
          .endr
          .endm
  
          .macro  tl1_fill_bad    count
          .rept   \count
          sir
          .align  128
          .endr
          .endm
  
          .macro  tl1_soft        count
          .rept   \count
          tl1_gen T_SOFT | T_KERNEL
          .endr
          .endm
  
          .sect   .trap
          .globl  tl_trap_begin
  tl_trap_begin:
          nop
  
          .align  0x8000
          .globl  tl0_base
  
  tl0_base:
          tl0_reserved    8                               ! 0x0-0x7
  tl0_insn_excptn:
          tl0_insn_excptn                                 ! 0x8
          tl0_reserved    1                               ! 0x9
  tl0_insn_error:
          tl0_gen         T_INSTRUCTION_ERROR             ! 0xa
          tl0_reserved    5                               ! 0xb-0xf
  tl0_insn_illegal:
          tl0_gen         T_ILLEGAL_INSTRUCTION           ! 0x10
  tl0_priv_opcode:
          tl0_gen         T_PRIVILEGED_OPCODE             ! 0x11
          tl0_reserved    14                              ! 0x12-0x1f
  tl0_fp_disabled:
          tl0_gen         T_FP_DISABLED                   ! 0x20
  tl0_fp_ieee:
          tl0_gen         T_FP_EXCEPTION_IEEE_754         ! 0x21
  tl0_fp_other:
          tl0_gen         T_FP_EXCEPTION_OTHER            ! 0x22
  tl0_tag_ovflw:
          tl0_gen         T_TAG_OVERFLOW                  ! 0x23
  tl0_clean_window:
          clean_window                                    ! 0x24
  tl0_divide:
          tl0_gen         T_DIVISION_BY_ZERO              ! 0x28
          tl0_reserved    7                               ! 0x29-0x2f
  tl0_data_excptn:
          tl0_data_excptn                                 ! 0x30
          tl0_reserved    1                               ! 0x31
  tl0_data_error:
          tl0_gen         T_DATA_ERROR                    ! 0x32
          tl0_reserved    1                               ! 0x33
  tl0_align:
          tl0_align                                       ! 0x34
  tl0_align_lddf:
          tl0_gen         T_RESERVED                      ! 0x35
  tl0_align_stdf:
          tl0_gen         T_RESERVED                      ! 0x36
  tl0_priv_action:
          tl0_gen         T_PRIVILEGED_ACTION             ! 0x37
          tl0_reserved    9                               ! 0x38-0x40
  tl0_intr_level:
          tl0_intr_level                                  ! 0x41-0x4f
          tl0_reserved    16                              ! 0x50-0x5f
  tl0_intr_vector:
          intr_vector                                     ! 0x60
  tl0_watch_phys:
          tl0_gen         T_PA_WATCHPOINT                 ! 0x61
  tl0_watch_virt:
          tl0_gen         T_VA_WATCHPOINT                 ! 0x62
  tl0_ecc:
          tl0_gen         T_CORRECTED_ECC_ERROR           ! 0x63
  tl0_immu_miss:
          tl0_immu_miss                                   ! 0x64
  tl0_dmmu_miss:
          tl0_dmmu_miss                                   ! 0x68
  tl0_dmmu_prot:
          tl0_dmmu_prot                                   ! 0x6c
          tl0_reserved    16                              ! 0x70-0x7f
  tl0_spill_0_n:
          tl0_spill_0_n                                   ! 0x80
  tl0_spill_1_n:
          tl0_spill_1_n                                   ! 0x84
          tl0_spill_bad   14                              ! 0x88-0xbf
  tl0_fill_0_n:
          tl0_fill_0_n                                    ! 0xc0
  tl0_fill_1_n:
          tl0_fill_1_n                                    ! 0xc4
          tl0_fill_bad    14                              ! 0xc8-0xff
  tl0_soft:
          tl0_gen         T_SYSCALL                       ! 0x100
          tl0_gen         T_BREAKPOINT                    ! 0x101
          tl0_gen         T_DIVISION_BY_ZERO              ! 0x102
          tl0_reserved    1                               ! 0x103
          tl0_gen         T_CLEAN_WINDOW                  ! 0x104
          tl0_gen         T_RANGE_CHECK                   ! 0x105
          tl0_gen         T_FIX_ALIGNMENT                 ! 0x106
          tl0_gen         T_INTEGER_OVERFLOW              ! 0x107
          tl0_gen         T_SYSCALL                       ! 0x108
          tl0_gen         T_SYSCALL                       ! 0x109
          tl0_fp_restore                                  ! 0x10a
          tl0_reserved    5                               ! 0x10b-0x10f
          tl0_gen         T_TRAP_INSTRUCTION_16           ! 0x110
          tl0_gen         T_TRAP_INSTRUCTION_17           ! 0x111
          tl0_gen         T_TRAP_INSTRUCTION_18           ! 0x112
          tl0_gen         T_TRAP_INSTRUCTION_19           ! 0x113
          tl0_gen         T_TRAP_INSTRUCTION_20           ! 0x114
          tl0_gen         T_TRAP_INSTRUCTION_21           ! 0x115
          tl0_gen         T_TRAP_INSTRUCTION_22           ! 0x116
          tl0_gen         T_TRAP_INSTRUCTION_23           ! 0x117
          tl0_gen         T_TRAP_INSTRUCTION_24           ! 0x118
          tl0_gen         T_TRAP_INSTRUCTION_25           ! 0x119
          tl0_gen         T_TRAP_INSTRUCTION_26           ! 0x11a
          tl0_gen         T_TRAP_INSTRUCTION_27           ! 0x11b
          tl0_gen         T_TRAP_INSTRUCTION_28           ! 0x11c
          tl0_gen         T_TRAP_INSTRUCTION_29           ! 0x11d
          tl0_gen         T_TRAP_INSTRUCTION_30           ! 0x11e
          tl0_gen         T_TRAP_INSTRUCTION_31           ! 0x11f
          tl0_reserved    32                              ! 0x120-0x13f
          tl0_gen         T_SYSCALL                       ! 0x140
          tl0_syscall                                     ! 0x141
          tl0_gen         T_SYSCALL                       ! 0x142
          tl0_gen         T_SYSCALL                       ! 0x143
          tl0_reserved    188                             ! 0x144-0x1ff
  
  tl1_base:
          tl1_reserved    8                               ! 0x200-0x207
  tl1_insn_excptn:
          tl1_insn_excptn                                 ! 0x208
          tl1_reserved    1                               ! 0x209
  tl1_insn_error:
          tl1_gen         T_INSTRUCTION_ERROR             ! 0x20a
          tl1_reserved    5                               ! 0x20b-0x20f
  tl1_insn_illegal:
          tl1_gen         T_ILLEGAL_INSTRUCTION           ! 0x210
  tl1_priv_opcode:
          tl1_gen         T_PRIVILEGED_OPCODE             ! 0x211
          tl1_reserved    14                              ! 0x212-0x21f
  tl1_fp_disabled:
          tl1_fp_disabled                                 ! 0x220
  tl1_fp_ieee:
          tl1_gen         T_FP_EXCEPTION_IEEE_754         ! 0x221
  tl1_fp_other:
          tl1_gen         T_FP_EXCEPTION_OTHER            ! 0x222
  tl1_tag_ovflw:
          tl1_gen         T_TAG_OVERFLOW                  ! 0x223
  tl1_clean_window:
          clean_window                                    ! 0x224
  tl1_divide:
          tl1_gen         T_DIVISION_BY_ZERO              ! 0x228
          tl1_reserved    7                               ! 0x229-0x22f
  tl1_data_excptn:
          tl1_data_excptn                                 ! 0x230
          tl1_reserved    1                               ! 0x231
  tl1_data_error:
          tl1_gen         T_DATA_ERROR                    ! 0x232
          tl1_reserved    1                               ! 0x233
  tl1_align:
          tl1_align                                       ! 0x234
  tl1_align_lddf:
          tl1_gen         T_RESERVED                      ! 0x235
  tl1_align_stdf:
          tl1_gen         T_RESERVED                      ! 0x236
  tl1_priv_action:
          tl1_gen         T_PRIVILEGED_ACTION             ! 0x237
          tl1_reserved    9                               ! 0x238-0x240
  tl1_intr_level:
          tl1_intr_level                                  ! 0x241-0x24f
          tl1_reserved    16                              ! 0x250-0x25f
  tl1_intr_vector:
          intr_vector                                     ! 0x260
  tl1_watch_phys:
          tl1_gen         T_PA_WATCHPOINT                 ! 0x261
  tl1_watch_virt:
          tl1_gen         T_VA_WATCHPOINT                 ! 0x262
  tl1_ecc:
          tl1_gen         T_CORRECTED_ECC_ERROR           ! 0x263
  tl1_immu_miss:
          tl1_immu_miss                                   ! 0x264
  tl1_dmmu_miss:
          tl1_dmmu_miss                                   ! 0x268
  tl1_dmmu_prot:
          tl1_dmmu_prot                                   ! 0x26c
          tl1_reserved    16                              ! 0x270-0x27f
  tl1_spill_0_n:
          tl1_spill_0_n                                   ! 0x280
          tl1_spill_bad   1                               ! 0x284
  tl1_spill_2_n:
          tl1_spill_2_n                                   ! 0x288
  tl1_spill_3_n:
          tl1_spill_3_n                                   ! 0x28c
          tl1_spill_bad   3                               ! 0x290-0x29b
  tl1_spill_7_n:
          tl1_spill_7_n                                   ! 0x29c
  tl1_spill_0_o:
          tl1_spill_0_o                                   ! 0x2a0
  tl1_spill_1_o:
          tl1_spill_1_o                                   ! 0x2a4
  tl1_spill_2_o:
          tl1_spill_2_o                                   ! 0x2a8
          tl1_spill_bad   5                               ! 0x2ac-0x2bf
  tl1_fill_0_n:
          tl1_fill_0_n                                    ! 0x2c0
          tl1_fill_bad    1                               ! 0x2c4
  tl1_fill_2_n:
          tl1_fill_2_n                                    ! 0x2c8
  tl1_fill_3_n:
          tl1_fill_3_n                                    ! 0x2cc
          tl1_fill_bad    3                               ! 0x2d0-0x2db
  tl1_fill_7_n:
          tl1_fill_7_n                                    ! 0x2dc
          tl1_fill_bad    8                               ! 0x2e0-0x2ff
          tl1_reserved    1                               ! 0x300
  tl1_breakpoint:
          tl1_gen         T_BREAKPOINT                    ! 0x301
          tl1_gen         T_RSTRWP_PHYS                   ! 0x302
          tl1_gen         T_RSTRWP_VIRT                   ! 0x303
          tl1_reserved    252                             ! 0x304-0x3ff
  
          .globl  tl_trap_end
  tl_trap_end:
          nop
  
  /*
   * User trap entry point
   *
   * void tl0_utrap(u_long type, u_long o1, u_long o2, u_long tar, u_long sfar,
   *     u_long sfsr)
   *
   * This handles redirecting a trap back to usermode as a user trap.  The user
   * program must have first registered a trap handler with the kernel using
   * sysarch(SPARC_UTRAP_INSTALL).  The trap handler is passed enough state
   * for it to return to the trapping code directly, it will not return through
   * the kernel.  The trap type is passed in %o0, all out registers must be
   * passed through to tl0_trap or to usermode untouched.  Note that the
   * parameters passed in out registers may be used by the user trap handler.
   * Do not change the registers they are passed in or you will break the ABI.
   *
   * If the trap type allows user traps, setup state to execute the user trap
   * handler and bounce back to usermode, otherwise branch to tl0_trap.
   */
  ENTRY(tl0_utrap)
          /*
           * Check if the trap type allows user traps.
           */
          cmp     %o0, UT_MAX
          bge,a,pt %xcc, tl0_trap
           nop
  
          /*
           * Load the user trap handler from the utrap table.
           */
          ldx     [PCPU(CURTHREAD)], %l0
          ldx     [%l0 + TD_PROC], %l0
          ldx     [%l0 + P_MD + MD_UTRAP], %l0
          brz,pt  %l0, tl0_trap
           sllx   %o0, PTR_SHIFT, %l1
          ldx     [%l0 + %l1], %l0
          brz,a,pt %l0, tl0_trap
           nop
  
          /*
           * If the save we did on entry to the kernel had to spill a window
           * to the pcb, pretend we took a spill trap instead.  Any windows
           * that are in the pcb must be copied out or the fill handler will
           * not be able to find them, since the user trap handler returns
           * directly to the trapping code.  Note that we only support precise
           * user traps, which implies that the condition that caused the trap
           * in the first place is still valid, so it will occur again when we
           * re-execute the trapping instruction.
           */
          ldx     [PCB_REG + PCB_NSAVED], %l1
          brnz,a,pn %l1, tl0_trap
           mov    T_SPILL, %o0
  
          /*
           * Pass %fsr in %l4, %tstate in %l5, %tpc in %l6 and %tnpc in %l7.
           * The ABI specifies only %l6 and %l7, but we need to pass %fsr or
           * it may be clobbered by an interrupt before the user trap code
           * can read it, and we must pass %tstate in order to restore %ccr
           * and %asi.  The %fsr must be stored to memory, so we use the
           * temporary stack for that.
           */
          rd      %fprs, %l1
          or      %l1, FPRS_FEF, %l2
          wr      %l2, 0, %fprs
          dec     8, ASP_REG
          stx     %fsr, [ASP_REG]
          ldx     [ASP_REG], %l4
          inc     8, ASP_REG
          wr      %l1, 0, %fprs
  
          rdpr    %tstate, %l5
          rdpr    %tpc, %l6
          rdpr    %tnpc, %l7
  
          /*
           * Setup %tnpc to return to.
           */
          wrpr    %l0, 0, %tnpc
  
          /*
           * Setup %wstate for return, clear WSTATE_TRANSITION.
           */
          rdpr    %wstate, %l1
          and     %l1, WSTATE_NORMAL_MASK, %l1
          wrpr    %l1, 0, %wstate
  
          /*
           * Setup %tstate for return, change the saved cwp to point to the
           * current window instead of the window at the time of the trap.
           */
          andn    %l5, TSTATE_CWP_MASK, %l1
          rdpr    %cwp, %l2
          wrpr    %l1, %l2, %tstate
  
          /*
           * Setup %sp.  Userland processes will crash if this is not setup.
           */
          sub     %fp, CCFSZ, %sp
  
          /*
           * Execute the user trap handler.
           */
          done
  END(tl0_utrap)
  
  /*
   * (Real) User trap entry point
   *
   * void tl0_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfsr,
   *     u_int sfsr)
   *
   * The following setup has been performed:
   *      - the windows have been split and the active user window has been saved
   *        (maybe just to the pcb)
   *      - we are on alternate globals and interrupts are disabled
   *
   * We switch to the kernel stack, build a trapframe, switch to normal
   * globals, enable interrupts and call trap.
   *
   * NOTE: We must be very careful setting up the per-cpu pointer.  We know that
   * it has been pre-set in alternate globals, so we read it from there and setup
   * the normal %g7 *before* enabling interrupts.  This avoids any possibility
   * of cpu migration and using the wrong pcpup.
   */
  ENTRY(tl0_trap)
          /*
           * Force kernel store order.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
  
          rdpr    %tstate, %l0
          rdpr    %tpc, %l1
          rdpr    %tnpc, %l2
          rd      %y, %l3
          rd      %fprs, %l4
          rdpr    %wstate, %l5
  
  #if KTR_COMPILE & KTR_TRAP
          CATR(KTR_TRAP,
              "tl0_trap: td=%p type=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx"
              , %g1, %g2, %g3, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g2
          stx     %g2, [%g1 + KTR_PARM1]
          stx     %o0, [%g1 + KTR_PARM2]
          rdpr    %pil, %g2
          stx     %g2, [%g1 + KTR_PARM3]
          stx     %l1, [%g1 + KTR_PARM4]
          stx     %l2, [%g1 + KTR_PARM5]
          stx     %i6, [%g1 + KTR_PARM6]
  9:
  #endif
  
  1:      and     %l5, WSTATE_NORMAL_MASK, %l5
          sllx    %l5, WSTATE_OTHER_SHIFT, %l5
          wrpr    %l5, WSTATE_KERNEL, %wstate
          rdpr    %canrestore, %l6
          wrpr    %l6, 0, %otherwin
          wrpr    %g0, 0, %canrestore
  
          sub     PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp
  
          stx     %o0, [%sp + SPOFF + CCFSZ + TF_TYPE]
          stx     %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL]
          stx     %o3, [%sp + SPOFF + CCFSZ + TF_TAR]
          stx     %o4, [%sp + SPOFF + CCFSZ + TF_SFAR]
          stx     %o5, [%sp + SPOFF + CCFSZ + TF_SFSR]
  
          stx     %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE]
          stx     %l1, [%sp + SPOFF + CCFSZ + TF_TPC]
          stx     %l2, [%sp + SPOFF + CCFSZ + TF_TNPC]
          stx     %l3, [%sp + SPOFF + CCFSZ + TF_Y]
          stx     %l4, [%sp + SPOFF + CCFSZ + TF_FPRS]
          stx     %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE]
  
          wr      %g0, FPRS_FEF, %fprs
          stx     %fsr, [%sp + SPOFF + CCFSZ + TF_FSR]
          rd      %gsr, %l6
          stx     %l6, [%sp + SPOFF + CCFSZ + TF_GSR]
          wr      %g0, 0, %fprs
  
          mov     PCB_REG, %l0
          mov     PCPU_REG, %l1
          wrpr    %g0, PSTATE_NORMAL, %pstate
  
          stx     %g6, [%sp + SPOFF + CCFSZ + TF_G6]
          stx     %g7, [%sp + SPOFF + CCFSZ + TF_G7]
  
          mov     %l0, PCB_REG
          mov     %l1, PCPU_REG
          wrpr    %g0, PSTATE_KERNEL, %pstate
  
          stx     %i0, [%sp + SPOFF + CCFSZ + TF_O0]
          stx     %i1, [%sp + SPOFF + CCFSZ + TF_O1]
          stx     %i2, [%sp + SPOFF + CCFSZ + TF_O2]
          stx     %i3, [%sp + SPOFF + CCFSZ + TF_O3]
          stx     %i4, [%sp + SPOFF + CCFSZ + TF_O4]
          stx     %i5, [%sp + SPOFF + CCFSZ + TF_O5]
          stx     %i6, [%sp + SPOFF + CCFSZ + TF_O6]
          stx     %i7, [%sp + SPOFF + CCFSZ + TF_O7]
  
          stx     %g1, [%sp + SPOFF + CCFSZ + TF_G1]
          stx     %g2, [%sp + SPOFF + CCFSZ + TF_G2]
          stx     %g3, [%sp + SPOFF + CCFSZ + TF_G3]
          stx     %g4, [%sp + SPOFF + CCFSZ + TF_G4]
          stx     %g5, [%sp + SPOFF + CCFSZ + TF_G5]
  
          set     tl0_ret - 8, %o7
          jmpl    %o2, %g0
           add    %sp, CCFSZ + SPOFF, %o0
  END(tl0_trap)
  
  /*
   * void tl0_intr(u_int level, u_int mask)
   */
  ENTRY(tl0_intr)
          /*
           * Force kernel store order.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
  
          rdpr    %tstate, %l0
          rdpr    %tpc, %l1
          rdpr    %tnpc, %l2
          rd      %y, %l3
          rd      %fprs, %l4
          rdpr    %wstate, %l5
  
  #if KTR_COMPILE & KTR_INTR
          CATR(KTR_INTR,
              "tl0_intr: td=%p level=%#x pil=%#lx pc=%#lx npc=%#lx sp=%#lx"
              , %g1, %g2, %g3, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g2
          stx     %g2, [%g1 + KTR_PARM1]
          stx     %o0, [%g1 + KTR_PARM2]
          rdpr    %pil, %g2
          stx     %g2, [%g1 + KTR_PARM3]
          stx     %l1, [%g1 + KTR_PARM4]
          stx     %l2, [%g1 + KTR_PARM5]
          stx     %i6, [%g1 + KTR_PARM6]
  9:
  #endif
  
          wrpr    %o0, 0, %pil
          wr      %o1, 0, %clear_softint
  
          and     %l5, WSTATE_NORMAL_MASK, %l5
          sllx    %l5, WSTATE_OTHER_SHIFT, %l5
          wrpr    %l5, WSTATE_KERNEL, %wstate
          rdpr    %canrestore, %l6
          wrpr    %l6, 0, %otherwin
          wrpr    %g0, 0, %canrestore
  
          sub     PCB_REG, SPOFF + CCFSZ + TF_SIZEOF, %sp
  
          stx     %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE]
          stx     %l1, [%sp + SPOFF + CCFSZ + TF_TPC]
          stx     %l2, [%sp + SPOFF + CCFSZ + TF_TNPC]
          stx     %l3, [%sp + SPOFF + CCFSZ + TF_Y]
          stx     %l4, [%sp + SPOFF + CCFSZ + TF_FPRS]
          stx     %l5, [%sp + SPOFF + CCFSZ + TF_WSTATE]
  
          wr      %g0, FPRS_FEF, %fprs
          stx     %fsr, [%sp + SPOFF + CCFSZ + TF_FSR]
          rd      %gsr, %l6
          stx     %l6, [%sp + SPOFF + CCFSZ + TF_GSR]
          wr      %g0, 0, %fprs
  
          mov     %o0, %l3
          mov     T_INTERRUPT, %o1
  
          stx     %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL]
          stx     %o1, [%sp + SPOFF + CCFSZ + TF_TYPE]
  
          mov     PCB_REG, %l0
          mov     PCPU_REG, %l1
          wrpr    %g0, PSTATE_NORMAL, %pstate
  
          stx     %g1, [%sp + SPOFF + CCFSZ + TF_G1]
          stx     %g2, [%sp + SPOFF + CCFSZ + TF_G2]
          stx     %g3, [%sp + SPOFF + CCFSZ + TF_G3]
          stx     %g4, [%sp + SPOFF + CCFSZ + TF_G4]
          stx     %g5, [%sp + SPOFF + CCFSZ + TF_G5]
          stx     %g6, [%sp + SPOFF + CCFSZ + TF_G6]
          stx     %g7, [%sp + SPOFF + CCFSZ + TF_G7]
  
          mov     %l0, PCB_REG
          mov     %l1, PCPU_REG
          wrpr    %g0, PSTATE_KERNEL, %pstate
  
          stx     %i0, [%sp + SPOFF + CCFSZ + TF_O0]
          stx     %i1, [%sp + SPOFF + CCFSZ + TF_O1]
          stx     %i2, [%sp + SPOFF + CCFSZ + TF_O2]
          stx     %i3, [%sp + SPOFF + CCFSZ + TF_O3]
          stx     %i4, [%sp + SPOFF + CCFSZ + TF_O4]
          stx     %i5, [%sp + SPOFF + CCFSZ + TF_O5]
          stx     %i6, [%sp + SPOFF + CCFSZ + TF_O6]
          stx     %i7, [%sp + SPOFF + CCFSZ + TF_O7]
  
          SET(intr_handlers, %l1, %l0)
          sllx    %l3, IH_SHIFT, %l1
          ldx     [%l0 + %l1], %l1
          KASSERT(%l1, "tl0_intr: ih null")
          call    %l1
           add    %sp, CCFSZ + SPOFF, %o0
  
          /* %l3 contains PIL */
          SET(intrcnt, %l1, %l2)
          prefetcha [%l2] ASI_N, 1
          SET(pil_countp, %l1, %l0)
          sllx    %l3, 1, %l1
          lduh    [%l0 + %l1], %l0
          sllx    %l0, 3, %l0
          add     %l0, %l2, %l0
          ldx     [%l0], %l1
          inc     %l1
          stx     %l1, [%l0]
  
          lduw    [PCPU(CNT) + V_INTR], %l0
          inc     %l0
          stw     %l0, [PCPU(CNT) + V_INTR]
  
          ba,a    %xcc, tl0_ret
           nop
  END(tl0_intr)
  
  /*
   * Initiate return to usermode.
   *
   * Called with a trapframe on the stack.  The window that was setup in
   * tl0_trap may have been used by "fast" trap handlers that pretend to be
   * leaf functions, so all ins and locals may have been clobbered since
   * then.
   *
   * This code is rather long and complicated.
   */
  ENTRY(tl0_ret)
          /*
           * Check for pending asts atomically with returning.  We must raise
           * the PIL before checking, and if no asts are found the PIL must
           * remain raised until the retry is executed, or we risk missing asts
           * caused by interrupts occurring after the test.  If the PIL is
           * lowered, as it is when we call ast, the check must be re-executed.
           */
          wrpr    %g0, PIL_TICK, %pil
          ldx     [PCPU(CURTHREAD)], %l0
          lduw    [%l0 + TD_FLAGS], %l1
          set     TDF_ASTPENDING | TDF_NEEDRESCHED, %l2
          and     %l1, %l2, %l1
          brz,a,pt %l1, 1f
           nop
  
          /*
           * We have an AST.  Re-enable interrupts and handle it, then restart
           * the return sequence.
           */
          wrpr    %g0, 0, %pil
          call    ast
           add    %sp, CCFSZ + SPOFF, %o0
          ba,a    %xcc, tl0_ret
           nop
  
          /*
           * Check for windows that were spilled to the pcb and need to be
           * copied out.  This must be the last thing that is done before the
           * return to usermode.  If there are still user windows in the cpu
           * and we call a nested function after this, which causes them to be
           * spilled to the pcb, they will not be copied out and the stack will
           * be inconsistent.
           */
  1:      ldx     [PCB_REG + PCB_NSAVED], %l1
          brz,a,pt %l1, 2f
           nop
          wrpr    %g0, 0, %pil
          mov     T_SPILL, %o0
          stx     %o0, [%sp + SPOFF + CCFSZ + TF_TYPE]
          call    trap
           add    %sp, SPOFF + CCFSZ, %o0
          ba,a    %xcc, tl0_ret
           nop
  
          /*
           * Restore the out and most global registers from the trapframe.
           * The ins will become the outs when we restore below.
           */
  2:      ldx     [%sp + SPOFF + CCFSZ + TF_O0], %i0
          ldx     [%sp + SPOFF + CCFSZ + TF_O1], %i1
          ldx     [%sp + SPOFF + CCFSZ + TF_O2], %i2
          ldx     [%sp + SPOFF + CCFSZ + TF_O3], %i3
          ldx     [%sp + SPOFF + CCFSZ + TF_O4], %i4
          ldx     [%sp + SPOFF + CCFSZ + TF_O5], %i5
          ldx     [%sp + SPOFF + CCFSZ + TF_O6], %i6
          ldx     [%sp + SPOFF + CCFSZ + TF_O7], %i7
  
          ldx     [%sp + SPOFF + CCFSZ + TF_G1], %g1
          ldx     [%sp + SPOFF + CCFSZ + TF_G2], %g2
          ldx     [%sp + SPOFF + CCFSZ + TF_G3], %g3
          ldx     [%sp + SPOFF + CCFSZ + TF_G4], %g4
          ldx     [%sp + SPOFF + CCFSZ + TF_G5], %g5
  
          /*
           * Load everything we need to restore below before disabling
           * interrupts.
           */
          ldx     [%sp + SPOFF + CCFSZ + TF_FPRS], %l0
          ldx     [%sp + SPOFF + CCFSZ + TF_GSR], %l1
          ldx     [%sp + SPOFF + CCFSZ + TF_TNPC], %l2
          ldx     [%sp + SPOFF + CCFSZ + TF_TPC], %l3
          ldx     [%sp + SPOFF + CCFSZ + TF_TSTATE], %l4
          ldx     [%sp + SPOFF + CCFSZ + TF_Y], %l5
          ldx     [%sp + SPOFF + CCFSZ + TF_WSTATE], %l6
  
          /*
           * Disable interrupts to restore the special globals.  They are not
           * saved and restored for all kernel traps, so an interrupt at the
           * wrong time would clobber them.
           */
          wrpr    %g0, PSTATE_NORMAL, %pstate
  
          ldx     [%sp + SPOFF + CCFSZ + TF_G6], %g6
          ldx     [%sp + SPOFF + CCFSZ + TF_G7], %g7
  
          /*
           * Switch to alternate globals.  This frees up some registers we
           * can use after the restore changes our window.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
  
          /*
           * Drop %pil to zero.  It must have been zero at the time of the
           * trap, since we were in usermode, but it was raised above in
           * order to check for asts atomically.  We have interrupts disabled
           * so any interrupts will not be serviced until we complete the
           * return to usermode.
           */
          wrpr    %g0, 0, %pil
  
          /*
           * Save %fprs in an alternate global so it can be restored after the
           * restore instruction below.  If we restore it before the restore,
           * and the restore traps we may run for a while with floating point
           * enabled in the kernel, which we want to avoid.
           */
          mov     %l0, %g1
  
          /*
           * Restore %fsr and %gsr.  These need floating point enabled in %fprs,
           * so we set it temporarily and then clear it.
           */
          wr      %g0, FPRS_FEF, %fprs
          ldx     [%sp + SPOFF + CCFSZ + TF_FSR], %fsr
          wr      %l1, 0, %gsr
          wr      %g0, 0, %fprs
  
          /*
           * Restore program counters.  This could be done after the restore
           * but we're out of alternate globals to store them in...
           */
          wrpr    %l2, 0, %tnpc
          wrpr    %l3, 0, %tpc
  
          /*
           * Save %tstate in an alternate global and clear the %cwp field.  %cwp
           * will be affected by the restore below and we need to make sure it
           * points to the current window at that time, not the window that was
           * active at the time of the trap.
           */
          andn    %l4, TSTATE_CWP_MASK, %g2
  
          /*
           * Restore %y.  Could also be below if we had more alternate globals.
           */
          wr      %l5, 0, %y
  
          /*
           * Setup %wstate for return.  We need to restore the user window state
           * which we saved in wstate.other when we trapped.  We also need to
           * set the transition bit so the restore will be handled specially
           * if it traps, use the xor feature of wrpr to do that.
           */
          srlx    %l6, WSTATE_OTHER_SHIFT, %g3
          wrpr    %g3, WSTATE_TRANSITION, %wstate
  
          /*
           * Setup window management registers for return.  If not all user
           * windows were spilled in the kernel %otherwin will be non-zero,
           * so we need to transfer it to %canrestore to correctly restore
           * those windows.  Otherwise everything gets set to zero and the
           * restore below will fill a window directly from the user stack.
           */
          rdpr    %otherwin, %o0
          wrpr    %o0, 0, %canrestore
          wrpr    %g0, 0, %otherwin
          wrpr    %o0, 0, %cleanwin
  
          /*
           * Now do the restore.  If this instruction causes a fill trap which
           * fails to fill a window from the user stack, we will resume at
           * tl0_ret_fill_end and call back into the kernel.
           */
          restore
  tl0_ret_fill:
  
          /*
           * We made it.  We're back in the window that was active at the time
           * of the trap, and ready to return to usermode.
           */
  
          /*
           * Restore %frps.  This was saved in an alternate global above.
           */
          wr      %g1, 0, %fprs
  
          /*
           * Fixup %tstate so the saved %cwp points to the current window and
           * restore it.
           */
          rdpr    %cwp, %g4
          wrpr    %g2, %g4, %tstate
  
          /*
           * Restore the user window state.  The transition bit was set above
           * for special handling of the restore, this clears it.
           */
          wrpr    %g3, 0, %wstate
  
  #if KTR_COMPILE & KTR_TRAP
          CATR(KTR_TRAP, "tl0_ret: td=%#lx pil=%#lx pc=%#lx npc=%#lx sp=%#lx"
              , %g2, %g3, %g4, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g3
          stx     %g3, [%g2 + KTR_PARM1]
          rdpr    %pil, %g3
          stx     %g3, [%g2 + KTR_PARM2]
          rdpr    %tpc, %g3
          stx     %g3, [%g2 + KTR_PARM3]
          rdpr    %tnpc, %g3
          stx     %g3, [%g2 + KTR_PARM4]
          stx     %sp, [%g2 + KTR_PARM5]
  9:
  #endif
  
          /*
           * Return to usermode.
           */
          retry
  tl0_ret_fill_end:
  
  #if KTR_COMPILE & KTR_TRAP
          CATR(KTR_TRAP, "tl0_ret: fill magic ps=%#lx ws=%#lx sp=%#lx"
              , %l0, %l1, %l2, 7, 8, 9)
          rdpr    %pstate, %l1
          stx     %l1, [%l0 + KTR_PARM1]
          stx     %l6, [%l0 + KTR_PARM2]
          stx     %sp, [%l0 + KTR_PARM3]
  9:
  #endif
  
          /*
           * The restore above caused a fill trap and the fill handler was
           * unable to fill a window from the user stack.  The special fill
           * handler recognized this and punted, sending us here.  We need
           * to carefully undo any state that was restored before the restore
           * was executed and call trap again.  Trap will copyin a window
           * from the user stack which will fault in the page we need so the
           * restore above will succeed when we try again.  If this fails
           * the process has trashed its stack, so we kill it.
           */
  
          /*
           * Restore the kernel window state.  This was saved in %l6 above, and
           * since the restore failed we're back in the same window.
           */
          wrpr    %l6, 0, %wstate
  
          /*
           * Restore the normal globals which have predefined values in the
           * kernel.  We clobbered them above restoring the user's globals
           * so this is very important.
           * XXX PSTATE_ALT must already be set.
           */
          wrpr    %g0, PSTATE_ALT, %pstate
          mov     PCB_REG, %o0
          mov     PCPU_REG, %o1
          wrpr    %g0, PSTATE_NORMAL, %pstate
          mov     %o0, PCB_REG
          mov     %o1, PCPU_REG
          wrpr    %g0, PSTATE_KERNEL, %pstate
  
          /*
           * Simulate a fill trap and then start the whole return sequence over
           * again.  This is special because it only copies in 1 window, not 2
           * as we would for a normal failed fill.  This may be the first time
           * the process has been run, so there may not be 2 windows worth of
           * stack to copyin.
           */
          mov     T_FILL_RET, %o0
          stx     %o0, [%sp + SPOFF + CCFSZ + TF_TYPE]
          call    trap
           add    %sp, SPOFF + CCFSZ, %o0
          ba,a    %xcc, tl0_ret
           nop
  END(tl0_ret)
  
  /*
   * Kernel trap entry point
   *
   * void tl1_trap(u_int type, u_long o1, u_long o2, u_long tar, u_long sfar,
   *     u_int sfsr)
   *
   * This is easy because the stack is already setup and the windows don't need
   * to be split.  We build a trapframe and call trap(), the same as above, but
   * the outs don't need to be saved.
   */
  ENTRY(tl1_trap)
          rdpr    %tstate, %l0
          rdpr    %tpc, %l1
          rdpr    %tnpc, %l2
          rdpr    %pil, %l3
          rd      %y, %l4
          rdpr    %wstate, %l5
  
  #if KTR_COMPILE & KTR_TRAP
          CATR(KTR_TRAP, "tl1_trap: td=%p type=%#lx pil=%#lx pc=%#lx sp=%#lx"
              , %g1, %g2, %g3, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g2
          stx     %g2, [%g1 + KTR_PARM1]
          stx     %o0, [%g1 + KTR_PARM2]
          stx     %l3, [%g1 + KTR_PARM3]
          stx     %l1, [%g1 + KTR_PARM4]
          stx     %i6, [%g1 + KTR_PARM5]
  9:
  #endif
  
          wrpr    %g0, 1, %tl
  
          and     %l5, WSTATE_OTHER_MASK, %l5
          wrpr    %l5, WSTATE_KERNEL, %wstate
  
          stx     %o0, [%sp + SPOFF + CCFSZ + TF_TYPE]
          stx     %o1, [%sp + SPOFF + CCFSZ + TF_LEVEL]
          stx     %o3, [%sp + SPOFF + CCFSZ + TF_TAR]
          stx     %o4, [%sp + SPOFF + CCFSZ + TF_SFAR]
          stx     %o5, [%sp + SPOFF + CCFSZ + TF_SFSR]
  
          stx     %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE]
          stx     %l1, [%sp + SPOFF + CCFSZ + TF_TPC]
          stx     %l2, [%sp + SPOFF + CCFSZ + TF_TNPC]
          stx     %l3, [%sp + SPOFF + CCFSZ + TF_PIL]
          stx     %l4, [%sp + SPOFF + CCFSZ + TF_Y]
  
          mov     PCB_REG, %l0
          mov     PCPU_REG, %l1
          wrpr    %g0, PSTATE_NORMAL, %pstate
  
          stx     %g6, [%sp + SPOFF + CCFSZ + TF_G6]
          stx     %g7, [%sp + SPOFF + CCFSZ + TF_G7]
  
          mov     %l0, PCB_REG
          mov     %l1, PCPU_REG
          wrpr    %g0, PSTATE_KERNEL, %pstate
  
          stx     %i0, [%sp + SPOFF + CCFSZ + TF_O0]
          stx     %i1, [%sp + SPOFF + CCFSZ + TF_O1]
          stx     %i2, [%sp + SPOFF + CCFSZ + TF_O2]
          stx     %i3, [%sp + SPOFF + CCFSZ + TF_O3]
          stx     %i4, [%sp + SPOFF + CCFSZ + TF_O4]
          stx     %i5, [%sp + SPOFF + CCFSZ + TF_O5]
          stx     %i6, [%sp + SPOFF + CCFSZ + TF_O6]
          stx     %i7, [%sp + SPOFF + CCFSZ + TF_O7]
  
          stx     %g1, [%sp + SPOFF + CCFSZ + TF_G1]
          stx     %g2, [%sp + SPOFF + CCFSZ + TF_G2]
          stx     %g3, [%sp + SPOFF + CCFSZ + TF_G3]
          stx     %g4, [%sp + SPOFF + CCFSZ + TF_G4]
          stx     %g5, [%sp + SPOFF + CCFSZ + TF_G5]
  
          set     tl1_ret - 8, %o7
          jmpl    %o2, %g0
           add    %sp, CCFSZ + SPOFF, %o0
  END(tl1_trap)
  
  ENTRY(tl1_ret)
          ldx     [%sp + SPOFF + CCFSZ + TF_O0], %i0
          ldx     [%sp + SPOFF + CCFSZ + TF_O1], %i1
          ldx     [%sp + SPOFF + CCFSZ + TF_O2], %i2
          ldx     [%sp + SPOFF + CCFSZ + TF_O3], %i3
          ldx     [%sp + SPOFF + CCFSZ + TF_O4], %i4
          ldx     [%sp + SPOFF + CCFSZ + TF_O5], %i5
          ldx     [%sp + SPOFF + CCFSZ + TF_O6], %i6
          ldx     [%sp + SPOFF + CCFSZ + TF_O7], %i7
  
          ldx     [%sp + SPOFF + CCFSZ + TF_G1], %g1
          ldx     [%sp + SPOFF + CCFSZ + TF_G2], %g2
          ldx     [%sp + SPOFF + CCFSZ + TF_G3], %g3
          ldx     [%sp + SPOFF + CCFSZ + TF_G4], %g4
          ldx     [%sp + SPOFF + CCFSZ + TF_G5], %g5
  
          ldx     [%sp + SPOFF + CCFSZ + TF_TSTATE], %l0
          ldx     [%sp + SPOFF + CCFSZ + TF_TPC], %l1
          ldx     [%sp + SPOFF + CCFSZ + TF_TNPC], %l2
          ldx     [%sp + SPOFF + CCFSZ + TF_PIL], %l3
          ldx     [%sp + SPOFF + CCFSZ + TF_Y], %l4
  
          set     VM_MIN_PROM_ADDRESS, %l5
          cmp     %l1, %l5
          bl,a,pt %xcc, 1f
           nop
          set     VM_MAX_PROM_ADDRESS, %l5
          cmp     %l1, %l5
          bg,a,pt %xcc, 1f
           nop
  
          wrpr    %g0, PSTATE_NORMAL, %pstate
  
          ldx     [%sp + SPOFF + CCFSZ + TF_G6], %g6
          ldx     [%sp + SPOFF + CCFSZ + TF_G7], %g7
  
  1:      wrpr    %g0, PSTATE_ALT, %pstate
  
          andn    %l0, TSTATE_CWP_MASK, %g1
          mov     %l1, %g2
          mov     %l2, %g3
  
          wrpr    %l3, 0, %pil
          wr      %l4, 0, %y
  
          restore
  
          wrpr    %g0, 2, %tl
  
          rdpr    %cwp, %g4
          wrpr    %g1, %g4, %tstate
          wrpr    %g2, 0, %tpc
          wrpr    %g3, 0, %tnpc
  
  #if KTR_COMPILE & KTR_TRAP
          CATR(KTR_TRAP, "tl1_ret: td=%#lx pil=%#lx ts=%#lx pc=%#lx sp=%#lx"
              , %g2, %g3, %g4, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g3
          stx     %g3, [%g2 + KTR_PARM1]
          rdpr    %pil, %g3
          stx     %g3, [%g2 + KTR_PARM2]
          rdpr    %tstate, %g3
          stx     %g3, [%g2 + KTR_PARM3]
          rdpr    %tpc, %g3
          stx     %g3, [%g2 + KTR_PARM4]
          stx     %sp, [%g2 + KTR_PARM5]
  9:
  #endif
  
          retry
  END(tl1_ret)
  
  /*
   * void tl1_intr(u_int level, u_int mask)
   */
  ENTRY(tl1_intr)
          rdpr    %tstate, %l0
          rdpr    %tpc, %l1
          rdpr    %tnpc, %l2
          rdpr    %pil, %l3
          rd      %y, %l4
          rdpr    %wstate, %l5
  
  #if KTR_COMPILE & KTR_INTR
          CATR(KTR_INTR,
              "tl1_intr: td=%p level=%#x pil=%#lx pc=%#lx sp=%#lx"
              , %g1, %g2, %g3, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g2
          stx     %g2, [%g1 + KTR_PARM1]
          stx     %o0, [%g1 + KTR_PARM2]
          stx     %l3, [%g1 + KTR_PARM3]
          stx     %l1, [%g1 + KTR_PARM4]
          stx     %i6, [%g1 + KTR_PARM5]
  9:
  #endif
  
          wrpr    %o0, 0, %pil
          wr      %o1, 0, %clear_softint
  
          wrpr    %g0, 1, %tl
  
          and     %l5, WSTATE_OTHER_MASK, %l5
          wrpr    %l5, WSTATE_KERNEL, %wstate
  
          stx     %l0, [%sp + SPOFF + CCFSZ + TF_TSTATE]
          stx     %l1, [%sp + SPOFF + CCFSZ + TF_TPC]
          stx     %l2, [%sp + SPOFF + CCFSZ + TF_TNPC]
          stx     %l3, [%sp + SPOFF + CCFSZ + TF_PIL]
          stx     %l4, [%sp + SPOFF + CCFSZ + TF_Y]
  
          mov     %o0, %l7
          mov     T_INTERRUPT | T_KERNEL, %o1
  
          stx     %o0, [%sp + SPOFF + CCFSZ + TF_LEVEL]
          stx     %o1, [%sp + SPOFF + CCFSZ + TF_TYPE]
  
          stx     %i6, [%sp + SPOFF + CCFSZ + TF_O6]
          stx     %i7, [%sp + SPOFF + CCFSZ + TF_O7]
  
          mov     PCB_REG, %l4
          mov     PCPU_REG, %l5
          wrpr    %g0, PSTATE_NORMAL, %pstate
  
          stx     %g1, [%sp + SPOFF + CCFSZ + TF_G1]
          stx     %g2, [%sp + SPOFF + CCFSZ + TF_G2]
          stx     %g3, [%sp + SPOFF + CCFSZ + TF_G3]
          stx     %g4, [%sp + SPOFF + CCFSZ + TF_G4]
          stx     %g5, [%sp + SPOFF + CCFSZ + TF_G5]
  
          mov     %l4, PCB_REG
          mov     %l5, PCPU_REG
          wrpr    %g0, PSTATE_KERNEL, %pstate
  
          SET(intr_handlers, %l5, %l4)
          sllx    %l7, IH_SHIFT, %l5
          ldx     [%l4 + %l5], %l5
          KASSERT(%l5, "tl1_intr: ih null")
          call    %l5
           add    %sp, CCFSZ + SPOFF, %o0
  
          /* %l7 contains PIL */
          SET(intrcnt, %l5, %l4)
          prefetcha [%l4] ASI_N, 1
          SET(pil_countp, %l5, %l6)
          sllx    %l7, 1, %l5
          lduh    [%l5 + %l6], %l5
          sllx    %l5, 3, %l5
          add     %l5, %l4, %l4
          ldx     [%l4], %l5
          inc     %l5
          stx     %l5, [%l4]
  
          lduw    [PCPU(CNT) + V_INTR], %l4
          inc     %l4
          stw     %l4, [PCPU(CNT) + V_INTR]
  
          ldx     [%sp + SPOFF + CCFSZ + TF_Y], %l4
  
          ldx     [%sp + SPOFF + CCFSZ + TF_G1], %g1
          ldx     [%sp + SPOFF + CCFSZ + TF_G2], %g2
          ldx     [%sp + SPOFF + CCFSZ + TF_G3], %g3
          ldx     [%sp + SPOFF + CCFSZ + TF_G4], %g4
          ldx     [%sp + SPOFF + CCFSZ + TF_G5], %g5
  
          wrpr    %g0, PSTATE_ALT, %pstate
  
          andn    %l0, TSTATE_CWP_MASK, %g1
          mov     %l1, %g2
          mov     %l2, %g3
          wrpr    %l3, 0, %pil
          wr      %l4, 0, %y
  
          restore
  
          wrpr    %g0, 2, %tl
  
          rdpr    %cwp, %g4
          wrpr    %g1, %g4, %tstate
          wrpr    %g2, 0, %tpc
          wrpr    %g3, 0, %tnpc
  
  #if KTR_COMPILE & KTR_INTR
          CATR(KTR_INTR, "tl1_intr: td=%#x pil=%#lx ts=%#lx pc=%#lx sp=%#lx"
              , %g2, %g3, %g4, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g3
          stx     %g3, [%g2 + KTR_PARM1]
          rdpr    %pil, %g3
          stx     %g3, [%g2 + KTR_PARM2]
          rdpr    %tstate, %g3
          stx     %g3, [%g2 + KTR_PARM3]
          rdpr    %tpc, %g3
          stx     %g3, [%g2 + KTR_PARM4]
          stx     %sp, [%g2 + KTR_PARM5]
  9:
  #endif
  
          retry
  END(tl1_intr)
  
          .globl  tl_text_end
  tl_text_end:
          nop
  
  /*
   * Freshly forked processes come here when switched to for the first time.
   * The arguments to fork_exit() have been setup in the locals, we must move
   * them to the outs.
   */
  ENTRY(fork_trampoline)
  #if KTR_COMPILE & KTR_PROC
          CATR(KTR_PROC, "fork_trampoline: td=%p (%s) cwp=%#lx"
              , %g1, %g2, %g3, 7, 8, 9)
          ldx     [PCPU(CURTHREAD)], %g2
          stx     %g2, [%g1 + KTR_PARM1]
          ldx     [%g2 + TD_PROC], %g2
          add     %g2, P_COMM, %g2
          stx     %g2, [%g1 + KTR_PARM2]
          rdpr    %cwp, %g2
          stx     %g2, [%g1 + KTR_PARM3]
  9:
  #endif
          mov     %l0, %o0
          mov     %l1, %o1
          call    fork_exit
           mov    %l2, %o2
          ba,a    %xcc, tl0_ret
           nop
  END(fork_trampoline)

Cache object: a6b639875ef14a5a5976c25ec4d22f60