The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/osfmk/ppc/Emulate.s

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
    3  *
    4  * @APPLE_LICENSE_HEADER_START@
    5  * 
    6  * The contents of this file constitute Original Code as defined in and
    7  * are subject to the Apple Public Source License Version 1.1 (the
    8  * "License").  You may not use this file except in compliance with the
    9  * License.  Please obtain a copy of the License at
   10  * http://www.apple.com/publicsource and read it before using this file.
   11  * 
   12  * This Original Code and all software distributed under the License are
   13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
   14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
   15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
   16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
   17  * License for the specific language governing rights and limitations
   18  * under the License.
   19  * 
   20  * @APPLE_LICENSE_HEADER_END@
   21  */
   22 /*                                                                                                                                                                                      
   23         Emulate.s 
   24 
   25         Emulate instructions and traps.
   26 
   27         Lovingly crafted by Bill Angell using traditional methods and only natural or recycled materials.
   28         No animal products are used other than rendered otter bile and deep fried pork lard.
   29 
   30 */
   31 
   32 #include <ppc/asm.h>
   33 #include <ppc/proc_reg.h>
   34 #include <ppc/exception.h>
   35 #include <ppc/cpu_capabilities.h>
   36 #include <mach/machine/vm_param.h>
   37 #include <assym.s>
   38 
   39 #define traceInst 30
   40 #define dssAllDone 29
   41 
   42 ;                       General stuff what happens here:
   43 ;                               1)      All general context saved, interrupts off, translation off
   44 ;                               2)      Vector and floating point disabled, but there may be live context.
   45 ;                                       This code is responsible for saving and restoring what is used. This
   46 ;                                       includes exception states, java mode, etc.
   47 ;                               3)      No attempt is made to resolve page faults.  PTE misses are handled
   48 ;                                       automatically, but actual faults (ala copyin/copyout) are not. If 
   49 ;                                       a fault does occur, the exception that caused entry to the emulation
   50 ;                                       routine is remapped to either an instruction or data miss (depending
   51 ;                                       upon the stage detected) and redrived through the exception handler.
   52 ;                                       The only time that an instruction fault can happen is when a different
   53 ;                                       processor removes a mapping between our original fault and when we
   54 ;                                       fetch the assisted instruction. For an assisted instruction, data
   55 ;                                       faults should not occur (except in the MP case).  For a purely
   56 ;                                       emulated instruction, faults can occur.
   57 ;
   58 ;
   59 
   60 
   61                         .align  5
   62                         .globl  EXT(Emulate)
   63 
   64 LEXT(Emulate)
   65 
   66                         bf--    pf64Bitb,emn64                                  ; Skip if not 64-bit
   67                         b               EXT(Emulate64)                                  ; Jump to the 64-bit code...
   68                         
   69 emn64:          mfsprg  r31,0                                                   ; Get the per_proc
   70                         lwz             r12,savesrr1+4(r13)                             ; Get the exception info
   71                         rlwinm. r0,r12,0,SRR1_PRG_ILL_INS_BIT,SRR1_PRG_ILL_INS_BIT      ; Emulation candidate?
   72                         lwz             r30,dgFlags(0)                                  ; Get the flags
   73                         beq+    eExit                                                   ; Nope, do not try to emulate...
   74 
   75                         rlwinm. r0,r30,0,enaDiagEMb,enaDiagEMb  ; Do we want to try to emulate something?
   76                         mfsprg  r28,2                                                   ; Get the processor features
   77                         beq+    eExit                                                   ; No emulation allowed...
   78 
   79                         rlwinm. r28,r28,0,pfAltivecb,pfAltivecb ; Do we have Altivec on this machine?
   80                         beq             eNoVect                                                 ; Nope, no Altivec...
   81                         
   82                         dssall                                                                  ; We need to kill streams because we are going to flip to problem state
   83                         sync
   84 
   85 eNoVect:        bl              eIFetch                                                 ; Get the instruction image
   86                         bne-    eRedriveAsISI                                   ; Go redrive this as an ISI...  
   87 
   88                         rlwinm. r0,r10,0,0,5                                    ; See if we have the "special" op code here
   89                         rlwinm  r20,r10,16,22,31                                ; Set rS/rD and rA
   90                         bne+    eExit                                                   ; Not special op, ignore...
   91 
   92                         rlwinm  r0,r10,31,22,31                                 ; Extract the sub op code
   93                         crclr   cr1_eq                                                  ; Clear
   94                         rlwimi  r20,r10,14,15,16                                ; Move bits 29 and 30 of instruction to 15 and 16 of DSISR
   95                         cmplwi  r0,790                                                  ; lhbrx?
   96                         rlwimi  r20,r10,8,17,17                                 ; Move bit 25 to bit 17
   97                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
   98                         cmplwi  r0,534                                                  ; lwbrx?
   99                         rlwimi  r20,r10,3,18,21                                 ; Move bit 21-24 to bit 18-21
  100                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  101                         cmplwi  r0,918                                                  ; sthbrx?
  102                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  103                         cmplwi  r0,662                                                  ; stwbrx?
  104                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  105                         cmplwi  r0,1014                                                 ; dcbz?
  106                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  107                         cmplwi  r0,533                                                  ; lswx?
  108                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  109                         cmplwi  r0,661                                                  ; stswx?
  110                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  111                         bne             cr1_eq,eNotIndex                                ; Go check non-index forms...
  112 
  113                         rlwinm. r21,r10,19,24,28                                ; Extract index to rA to build EA
  114                         rlwinm  r22,r10,24,24,28                                ; Extract index to rB
  115                         addi    r24,r13,saver0+4                                ; Point to the start of registers
  116                         li              r19,0                                                   ; Assume 0 base
  117                         beq             eZeroBase                                               ; Yes...
  118                         lwzx    r19,r24,r21                                             ; Get the base register value
  119                         
  120 eZeroBase:      lwzx    r22,r24,r22                                             ; Get the index value
  121                         add             r22,r22,r19                                             ; Get DAR
  122                         b               eFinishUp                                               ; Done, go finish up...
  123                                                 
  124 eNotIndex:      cmplwi  r0,725                                                  ; stswi?
  125                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  126                         cmplwi  r0,597                                                  ; lswi?
  127                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  128                         bne             cr1,eExit                                               ; Not one we handle...
  129         
  130                         rlwinm. r21,r10,19,24,28                                ; Extract index to rA to build EA
  131                         addi    r24,r13,saver0+4                                ; Point to the start of registers
  132                         li              r22,0                                                   ; Assume 0 base
  133                         beq             eFinishUp                                               ; Yes, it is...
  134                         lwzx    r22,r24,r21                                             ; Get the base register value
  135                         
  136 eFinishUp:      stw             r20,savedsisr(r13)                              ; Set the DSISR
  137                         li              r11,T_ALIGNMENT                                 ; Get the exception code
  138                         stw             r22,savedar+4(r13)                              ; Save the DAR
  139                         stw             r11,saveexception(r13)                  ; Set the exception code
  140                         b               EXT(AlignAssist)                                ; Go emulate the handler...
  141 
  142 
  143 eExit:          b               EXT(EmulExit)                                   ; Just return for now...
  144 
  145 
  146 ;
  147 ;                       Fetch the failing instruction.
  148 ;                       Image returned in R10 if CR0_EQ is false, otherwise, an ISI should be generated/
  149 ;                       The cr bit kernAccess is set if this was a kernel access.
  150 ;                       R1 has the DSISR if access failed.
  151 ;
  152 
  153                         .align  5
  154 
  155 eIFetch:        lwz             r23,savesrr1+4(r13)                             ; Get old MSR
  156                         mflr    r28                                                             ; Save return
  157 
  158                         rlwinm. r22,r23,0,MSR_PR_BIT,MSR_PR_BIT ; Within kernel?
  159 
  160                         mfmsr   r30                                                             ; Save the MSR for now
  161                         lwz             r23,savesrr0+4(r13)                             ; Get instruction address
  162                         
  163                         ori             r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI))         ; Set RI and DR onto access MSR
  164 
  165                         crset   cr0_eq                                                  ; Set this to see if we failed
  166                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  167                         isync
  168                         
  169                         lwz             r10,0(r23)                                              ; Fetch the instruction
  170                         
  171                         mtmsr   r30                                                             ; Trans and RI off
  172                         isync
  173                         
  174                         mtlr    r28                                                             ; Restore the LR
  175                         blr                                                                             ; Return with instruction image in R10
  176 
  177 
  178 ;
  179 ;                       Redrive as an ISI
  180 ;
  181 
  182 eRedriveAsISI:
  183                         lwz             r6,savesrr1+4(r13)                              ; Get the srr1 value
  184                         lwz             r4,SAVflags(r13)                                ; Pick up the flags
  185                         li              r11,T_INSTRUCTION_ACCESS                ; Set failing instruction fetch code
  186                         rlwimi  r6,r1,0,0,4                                             ; Move the DSISR bits to the SRR1
  187                         oris    r4,r4,hi16(SAVredrive)                  ; Set the redrive bit
  188                         stw             r11,saveexception(r13)                  ; Set the replacement code
  189                         stw             r4,SAVflags(r13)                                ; Set redrive request
  190                         stw             r6,savesrr1+4(r13)                              ; Set the srr1 value
  191                         b               EXT(EmulExit)                                   ; Bail out to handle ISI...
  192 
  193 
  194 ;
  195 ;                       This code emulates instructions that have failed because of operand 
  196 ;                       alignment.  We decode the DSISR to figure out what we need to do.
  197 ;
  198 ;                       DSISR:
  199 ;                               0001FC00 - Instruction designation
  200 #define iFloat 12
  201 #define iOptype1 15
  202 #define iOptype2 16
  203 #define iOptype3 18
  204 #define iOptype4 19
  205 #define iUpdate 17
  206 #define iStore 20
  207 #define iDouble 21
  208 #define iNotify 22
  209 ;                               000003E0 - Target/Source register
  210 ;                               0000001F - Register to update if update form
  211 ;
  212 
  213                         .align  5
  214                         .globl  EXT(AlignAssist)
  215 
  216 LEXT(AlignAssist)
  217                         bf--    pf64Bitb,aan64                                  ; Skip if not 64-bit
  218                         b               EXT(AlignAssist64)                              ; Jump to the 64-bit code...
  219                         
  220 aan64:          lwz             r20,savedsisr(r13)                              ; Get the DSISR
  221                         li              r0,0                                                    ; Assume we emulate
  222                         mfsprg  r31,0                                                   ; Get the per_proc
  223                         mtcrf   0x10,r20                                                ; Put instruction ID in CR for later
  224                         lwz             r21,spcFlags(r31)                               ; Grab the special flags
  225                         stw             r0,savemisc3(r13)                               ; Assume that we emulate ok
  226                         mtcrf   0x08,r20                                                ; Put instruction ID in CR for later
  227                         rlwinm. r0,r21,0,runningVMbit,runningVMbit      ; Are we running a VM?
  228                         mtcrf   0x04,r20                                                ; Put instruction ID in CR for later
  229                         lwz             r22,savesrr1+4(r13)                             ; Get the SRR1
  230                         bne-    aaPassAlong                                             ; We are in a VM, no emulation for alignment exceptions...
  231                         lwz             r19,dgFlags(0)                                  ; Get the diagnostics flags
  232                         crxor   iFloat,iOptype1,iOptype2                ; Set this to 0 if both bits are either 0 or 1
  233                         mr              r26,r20                                                 ; Save the DSISR
  234                         rlwinm. r0,r22,0,MSR_SE_BIT,MSR_SE_BIT  ; Were we single stepping?
  235                         lwz             r23,savedar+4(r13)                              ; Pick up the address that we want to access
  236                         crnot   traceInst,cr0_eq                                ; Remember if trace is on
  237                         
  238                         rlwinm. r0,r19,0,enaNotifyEMb,enaNotifyEMb      ; Should we notify that an alignment exception happened?
  239                         mfmsr   r30                                                             ; Save the MSR for now
  240                         crnot   iNotify,cr0_eq                                  ; Remember to tell someone we did this                          
  241                         li              r29,emfp0                                               ; Point to work area
  242                         crxor   iFloat,iFloat,iOptype3                  ; Set true if we have a floating point instruction
  243                         dcbz    r29,r31                                                 ; Clear and allocate a cache line for us to work in
  244                         rlwinm  r24,r20,3,24,28                                 ; Get displacement to register to update if update form
  245                         rlwimi  r20,r20,24,28,28                                ; Move load/store indication to the bottom of index
  246                         ori             r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI))         ; Set RI onto access MSR
  247                         rlwimi  r20,r20,26,27,27                                ; Move single/double indication to just above the bottom
  248                         lis             r29,hi16(EXT(aaFPopTable))              ; High part of FP branch table
  249                         bf-             iFloat,aaNotFloat                               ; This is not a floating point instruction...
  250                         ori             r29,r29,lo16(EXT(aaFPopTable))  ; Low part of FP branch table
  251                         
  252                         rlwimi  r29,r20,0,22,28                                 ; Index into table based upon register||iDouble||iStore
  253                         mtctr   r29                                                             ; Get set to call the function  
  254                         bt              iStore,aaFPstore                                ; This is an FP store...
  255                 
  256 ;
  257 ;                       Here we handle floating point loads
  258 ;                       
  259 
  260 aaFPload:       crset   cr0_eq                                                  ; Set this to see if we failed
  261                         mtmsr   r22                                                             ; Flip DR, RI
  262                         isync
  263                         
  264                         lwz             r10,0(r23)                                              ; Get the first word
  265                         bf-             cr0_eq,aaLdNotDbl                               ; Jump out if we DSIed...
  266                         bf              iDouble,aaLdNotDbl                              ; this is not a double...
  267                         lwz             r11,4(r23)                                              ; Get the second half
  268                         
  269 aaLdNotDbl:     mr              r4,r0                                                   ; Save the DAR if we failed the access
  270                         
  271                         mtmsr   r30                                                             ; Turn off translation again
  272                         isync
  273                         
  274                         bf-             cr0_eq,aaRedriveAsDSI                   ; Go redrive this as a DSI...   
  275                         
  276                         stw             r10,emfp0(r31)                                  ; Save the first half
  277                         stw             r11,emfp0+4(r31)                                ; Save the second half, just in case we need it
  278                         
  279                         bctrl                                                                   ; Go set the target FP register
  280 
  281                         b               aaComExit                                               ; All done, go exit...                  
  282                 
  283 ;
  284 ;                       Here we handle floating point stores
  285 ;                       
  286 
  287                         .align  5
  288 
  289 aaFPstore:      bctrl                                                                   ; Go save the source FP register
  290                         
  291                         lwz             r10,emfp0(r31)                                  ; Get first word
  292                         crandc  iDouble,iDouble,iOptype4                ; Change to 4-byte access if stfiwx
  293                         lwz             r11,emfp0+4(r31)                                ; and the second
  294                         bf+             iOptype4,aaNotstfiwx                    ; This is not a stfiwx...
  295                         mr              r10,r11                                                 ; The stfiwx wants to store the second half
  296 
  297 aaNotstfiwx:
  298                         crset   cr0_eq                                                  ; Set this to see if we failed
  299                         mtmsr   r22                                                             ; Flip DR, RI
  300                         isync
  301                         
  302                         stw             r10,0(r23)                                              ; Save the first word
  303                         bf-             cr0_eq,aaStNotDbl                               ; Jump out if we DSIed...
  304                         bf              iDouble,aaStNotDbl                              ; this is not a double...
  305                         stw             r11,4(r23)                                              ; Save the second half
  306                         
  307 aaStNotDbl:     mr              r4,r0                                                   ; Save the DAR if we failed the access
  308                         mtmsr   r30                                                             ; Turn off
  309                         isync
  310                         
  311                         bf-             cr0_eq,aaRedriveAsDSI                   ; Go redrive this as a DSI...
  312 
  313 ;
  314 ;                       Common exit routines
  315 ;
  316 
  317 aaComExit:      lwz             r10,savesrr0+4(r13)                             ; Get the failing instruction address
  318                         add             r24,r24,r13                                             ; Offset to update register
  319                         li              r11,T_IN_VAIN                                   ; Assume we are all done
  320                         addi    r10,r10,4                                               ; Step to the next instruction
  321                         bf              iUpdate,aaComExNU                               ; Skip if not an update form...
  322                         stw             r23,saver0+4(r24)                               ; Update the target
  323                         
  324 aaComExNU:      lwz             r9,SAVflags(r13)                                ; Get the flags
  325                         stw             r10,savesrr0+4(r13)                             ; Set new PC
  326                         bt-             traceInst,aaComExitrd                   ; We are tracing, go emulate trace...
  327                         bf+             iNotify,aaComExGo                               ; Nothing special here, go...
  328         
  329                         li              r11,T_ALIGNMENT                                 ; Set the we just did an alignment exception....
  330                         
  331 aaComExGo:      b               EXT(EmulExit)                                   ; We are done, no tracing on...
  332 
  333 
  334 ;
  335 ;                       This is not a floating point operation
  336 ;
  337 ;                       The table of these emulation routines is indexed by taking the low order 4 bits of
  338 ;                       the instruction code in the DSISR and subtracting 7.  If this comes up negative,
  339 ;                       the instruction is not to be emulated.  Then we add bit 0 of the code * 4.  This
  340 ;                       gives us a fairly compact and almost unique index.  Both lwm and stmw map to 0 so
  341 ;                       that one needs to be further reduced, and we end up with holes at a few indexes.
  342 ;
  343 
  344                         .align  5
  345 
  346 aaNotFloat:
  347                         lis             r19,hi16(aaEmTable)                             ; Point to high part of table address
  348                         rlwinm  r3,r26,24,26,29                                 ; Isolate last 4 bits of op type * 4
  349                         rlwimi  r19,r26,20,27,27                                ; Get bit 0 of instruction code * 4 into bottom of table base
  350                         addic.  r3,r3,-28                                               ; Subtract 7*4 to adjust index
  351                         ori             r19,r19,lo16(aaEmTable)                 ; Low part of table address
  352                         blt-    aaPassAlong                                             ; We do not handle any of these (lwarx, stwcx., eciwx, ecowx)...
  353                         add             r19,r19,r3                                              ; Point to emulation routine
  354                         rlwinm  r18,r26,30,24,28                                ; Get the target/source register displacement
  355 
  356                         mtctr   r19                                                             ; Set the routine address
  357                         
  358                         bctr                                                                    ; Go emulate the instruction...
  359 
  360 ;
  361 ;                       This is the table of non-floating point emulation routines.
  362 ;                       It is indexed by the code immediately above.
  363                 
  364                         .align  5                                                       
  365 
  366 aaEmTable:
  367                         b               aaLmwStmw                                               ; This for lmw/stmw
  368                         b               aaLswx                                                  ; This for lwwx
  369                         b               aaLswi                                                  ; This for lswi
  370                         b               aaStswx                                                 ; This for stswx
  371                         b               aaStswi                                                 ; This for stswi
  372                         b               aaLwbrx                                                 ; This for lwbrx
  373                         b               aaPassAlong                                             ; This an invalid index (6)
  374                         b               aaStwbrx                                                ; This for stwbrx
  375                         b               aaPassAlong                                             ; This an invalid index (8)
  376                         b               aaLhbrx                                                 ; This for lhbrx
  377                         b               aaPassAlong                                             ; This an invalid index (A)
  378                         b               aaSthbrx                                                ; This for sthbrx
  379                         b               aaDcbz                                                  ; This for dcbz
  380                         b               aaPassAlong                                             ; This an invalid index (D)
  381                         b               aaPassAlong                                             ; This an invalid index (E)
  382                         b               aaPassAlong                                             ; This an invalid index (F)
  383 
  384 
  385 ;
  386 ;                       Here we handle the set up for the lmw and stmw.  After that, we split off to the
  387 ;                       individual routines.
  388 ;
  389 ;                       Note also that after some set up, all of the string instructions come through here as well.
  390 ;
  391                         .align  5
  392                                                 
  393 aaLmwStmw:
  394                         rlwinm  r17,r18,31,1,29                                 ; Convert doublword based index to words
  395                         li              r28,0                                                   ; Set no extra bytes to move (used for string instructions)
  396                         subfic  r17,r17,32*4                                    ; Calculate the length of the transfer
  397 
  398 aaLSComm:       addi    r19,r13,saver0+4                                ; Offset to registers in savearea
  399                         mr              r16,r23                                                 ; Make a hunk pointer
  400                         
  401                         bt              iUpdate,aaStmw                                  ; This is the stmw...
  402                         
  403 ;
  404 ;                       Load multiple word
  405 ;
  406 
  407 aaLmwNxt:       cmplwi  cr1,r17,8*4                                             ; Is there enough to move 8?
  408                         blt-    cr1,aaLmwNxtH                                   ; Not enough for a full hunk...
  409                         subi    r17,r17,8*4                                             ; Back off for another hunk
  410                         
  411                         crset   cr0_eq                                                  ; Set this to see if we failed
  412                         mtmsr   r22                                                             ; Flip DR, RI
  413                         isync
  414                 
  415                         lwz             r2,0(r16)                                               ; Load word 0
  416                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  417                         lwz             r15,4(r16)                                              ; Load word 1
  418                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  419                         lwz             r14,8(r16)                                              ; Load word 2
  420                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  421                         lwz             r5,12(r16)                                              ; Load word 3
  422                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  423                         lwz             r6,16(r16)                                              ; Load word 4
  424                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  425                         lwz             r7,20(r16)                                              ; Load word 5
  426                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  427                         lwz             r8,24(r16)                                              ; Load word 6
  428                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  429                         lwz             r9,28(r16)                                              ; Load word 7
  430                 
  431 aaLmwB1:        mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  432                         mtmsr   r30                                                             ; Turn off DR, RI
  433                         isync
  434 
  435                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  436 
  437                         addi    r16,r16,8*4                                             ; Point up to next input aread
  438                 
  439                         stwx    r2,r19,r18                                              ; Store register
  440                         addi    r18,r18,8                                               ; Next register
  441                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  442                         stwx    r15,r19,r18                                             ; Store register
  443                         addi    r18,r18,8                                               ; Next register
  444                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  445                         stwx    r14,r19,r18                                             ; Store register
  446                         addi    r18,r18,8                                               ; Next register
  447                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  448                         stwx    r5,r19,r18                                              ; Store register
  449                         addi    r18,r18,8                                               ; Next register
  450                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  451                         stwx    r6,r19,r18                                              ; Store register
  452                         addi    r18,r18,8                                               ; Next register
  453                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  454                         stwx    r7,r19,r18                                              ; Store register
  455                         addi    r18,r18,8                                               ; Next register
  456                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  457                         stwx    r8,r19,r18                                              ; Store register
  458                         addi    r18,r18,8                                               ; Next register
  459                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  460                         stwx    r9,r19,r18                                              ; Store register
  461                         addi    r18,r18,8                                               ; Next register
  462                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  463 
  464                         b               aaLmwNxt                                                ; Do the next hunk...
  465 
  466                         .align  5
  467                         
  468 aaLmwNxtH:      cmplwi  cr1,r17,4*4                                             ; Do we have 4 left?
  469                         blt             cr1,aaLmwL4                                             ; Nope...
  470 
  471                         subi    r17,r17,4*4                                             ; Set count properly
  472                         
  473                         crset   cr0_eq                                                  ; Set this to see if we failed
  474                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  475                         isync
  476                 
  477                         lwz             r2,0(r16)                                               ; Load word 0
  478                         bf-             cr0_eq,aaLmwB2                                  ; Error, bail...
  479                         lwz             r15,4(r16)                                              ; Load word 1
  480                         bf-             cr0_eq,aaLmwB2                                  ; Error, bail...
  481                         lwz             r14,8(r16)                                              ; Load word 2
  482                         bf-             cr0_eq,aaLmwB2                                  ; Error, bail...
  483                         lwz             r5,12(r16)                                              ; Load word 3
  484                 
  485 aaLmwB2:        mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  486                         mtmsr   r30                                                             ; Turn off DR, RI
  487                         isync
  488 
  489                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  490                 
  491                         addi    r16,r16,4*4                                             ; Point up to next input aread
  492                         
  493                         stwx    r2,r19,r18                                              ; Store register
  494                         addi    r18,r18,8                                               ; Next register
  495                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  496                         stwx    r15,r19,r18                                             ; Store register
  497                         addi    r18,r18,8                                               ; Next register
  498                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  499                         stwx    r14,r19,r18                                             ; Store register
  500                         addi    r18,r18,8                                               ; Next register
  501                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  502                         stwx    r5,r19,r18                                              ; Store register
  503                         addi    r18,r18,8                                               ; Next register
  504                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  505 
  506 aaLmwL4:        or.             r5,r17,r28                                              ; Do we have anything left?
  507                         cmplwi  cr1,r17,(2*4)                                   ; Do we have one, two, or three full words left?
  508                         cmplwi  cr2,r17,0                                               ; Do we have no full words left?
  509                         beq             aaComExit                                               ; Nothing left...
  510 
  511                         crset   cr0_eq                                                  ; Set this to see if we failed
  512                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  513                         isync
  514 
  515                         beq-    cr2,aaLmwBy                                             ; No full words, get bytes...
  516                         
  517                         lwz             r2,0(r16)                                               ; Pick up first word
  518                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  519                         addi    r16,r16,4                                               ; Next input location
  520                         blt             cr1,aaLmwBy                                             ; We only had one, we are done...
  521 
  522                         lwz             r15,0(r16)                                              ; Pick up second word
  523                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  524                         addi    r16,r16,4                                               ; Next input location
  525                         beq             cr1,aaLmwBy                                             ; We had two, we are done...
  526 
  527                         lwz             r14,0(r16)                                              ; Load word 3
  528                         addi    r16,r16,4                                               ; Next input location
  529 
  530 aaLmwBy:        cmplwi  cr2,r28,0                                               ; Any trailing bytes to do?
  531                         li              r8,0                                                    ; Clear second trailing byte
  532                         cmplwi  cr1,r28,2                                               ; Check for 1, 2, or 3
  533                         li              r9,0                                                    ; Clear third trailing byte
  534                         beq+    cr2,aaLmwDn                                             ; No trailing bytes...
  535                         
  536                         lbz             r5,0(r16)                                               ; Pick up first trailing byte
  537                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  538                         blt             cr1,aaLmwDn                                             ; We only had one, we are done...
  539 
  540                         lbz             r8,1(r16)                                               ; Pick up second trailing byte
  541                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  542                         beq             cr1,aaLmwDn                                             ; We had two, we are done...
  543 
  544                         lbz             r9,2(r16)                                               ; Get last trailing byte
  545                         
  546 
  547 aaLmwDn:        rlwinm  r5,r5,24,0,7                                    ; Move first byte to top
  548                         cmplwi  cr2,r17,0                                               ; Any full words to do?
  549                         mr              r4,r0                                                   ; Remember DAR, just in case we failed the access
  550                         rlwimi  r9,r8,8,16,23                                   ; Move second byte above third byte
  551                         cmplwi  cr1,r17,(2*4)                                   ; Do we have one, two, or three full words left?
  552                         mr              r3,r30                                                  ; Set the normal MSR
  553                         rlwimi  r5,r9,8,8,23                                    ; Move bytes 1 and 2 after 0
  554 
  555                         mtmsr   r30                                                             ; Turn off DR, RI
  556                         isync
  557 
  558                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  559 
  560                         beq-    cr2,aaLmwCb                                             ; No full words, copy bytes...
  561 
  562                         stwx    r2,r19,r18                                              ; Store register
  563                         addi    r18,r18,8                                               ; Next register
  564                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  565                         blt             cr1,aaLmwCb                                             ; We only had one, we are done...
  566                         
  567                         stwx    r15,r19,r18                                             ; Store register
  568                         addi    r18,r18,8                                               ; Next register
  569                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  570                         beq             cr1,aaLmwCb                                             ; We had two, we are done...
  571 
  572                         stwx    r14,r19,r18                                             ; Store register
  573                         addi    r18,r18,8                                               ; Next register
  574                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  575 
  576 aaLmwCb:        mr.             r28,r28                                                 ; Any trailing bytes to do?
  577                         beq+    aaComExit                                               ; Nope, leave...
  578 
  579                         stwx    r5,r19,r18                                              ; Store register
  580                                                 
  581                         b               aaComExit                                               ; We are done....
  582 
  583 ;
  584 ;                       Store multiple word
  585 ;
  586 
  587                         .align  5
  588 
  589 aaStmw:
  590                         crclr   iUpdate                                                 ; Make sure we do not think this is an update form
  591 
  592 aaStmwNxt:      cmplwi  cr1,r17,8*4                                             ; Is there enough to move 8?
  593                         blt-    cr1,aaStmwNxtH                                  ; Not enough for a full hunk...
  594                         subi    r17,r17,8*4                                             ; Back off for another hunk
  595                 
  596                         lwzx    r2,r19,r18                                              ; Store register
  597                         addi    r18,r18,8                                               ; Next register
  598                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  599                         lwzx    r15,r19,r18                                             ; Store register
  600                         addi    r18,r18,8                                               ; Next register
  601                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  602                         lwzx    r14,r19,r18                                             ; Store register
  603                         addi    r18,r18,8                                               ; Next register
  604                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  605                         lwzx    r5,r19,r18                                              ; Store register
  606                         addi    r18,r18,8                                               ; Next register
  607                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  608                         lwzx    r6,r19,r18                                              ; Store register
  609                         addi    r18,r18,8                                               ; Next register
  610                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  611                         lwzx    r7,r19,r18                                              ; Store register
  612                         addi    r18,r18,8                                               ; Next register
  613                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  614                         lwzx    r8,r19,r18                                              ; Store register
  615                         addi    r18,r18,8                                               ; Next register
  616                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  617                         lwzx    r9,r19,r18                                              ; Store register
  618                         addi    r18,r18,8                                               ; Next register
  619                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  620                         
  621                         crset   cr0_eq                                                  ; Set this to see if we failed
  622                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  623                         isync
  624 
  625                         stw             r2,0(r16)                                               ; Store word 0
  626                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  627                         stw             r15,4(r16)                                              ; Store word 1
  628                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  629                         stw             r14,8(r16)                                              ; Store word 2
  630                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  631                         stw             r5,12(r16)                                              ; Store word 3
  632                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  633                         stw             r6,16(r16)                                              ; Store word 4
  634                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  635                         stw             r7,20(r16)                                              ; Store word 5
  636                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  637                         stw             r8,24(r16)                                              ; Store word 6
  638                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  639                         stw             r9,28(r16)                                              ; Store word 7
  640                 
  641                         addi    r16,r16,8*4                                             ; Point up to next output aread
  642                 
  643                 
  644 aaStmwB1:       mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  645                         mtmsr   r30                                                             ; Normal MSR
  646                         isync
  647 
  648                         bt-             cr0_eq,aaStmwNxt                                ; We have more to do and no failed access...
  649                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...
  650 
  651                         .align  5
  652                         
  653 aaStmwNxtH:     cmplwi  cr1,r17,(4*4)                                   ; Do we have at least 4 left?
  654                         blt             cr1,aaStmwL4                                    ; Nope...
  655                         subi    r17,r17,4*4                                             ; Set count properly
  656 
  657                         lwzx    r2,r19,r18                                              ; Store register
  658                         addi    r18,r18,8                                               ; Next register
  659                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  660                         lwzx    r15,r19,r18                                             ; Store register
  661                         addi    r18,r18,8                                               ; Next register
  662                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  663                         lwzx    r14,r19,r18                                             ; Store register
  664                         addi    r18,r18,8                                               ; Next register
  665                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  666                         lwzx    r5,r19,r18                                              ; Store register
  667                         addi    r18,r18,8                                               ; Next register
  668                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  669                         
  670                         crset   cr0_eq                                                  ; Set this to see if we failed
  671                         mtmsr   r22                                                             ; Flip DR, RI
  672                         isync
  673                 
  674                         stw             r2,0(r16)                                               ; Store word 0
  675                         bf-             cr0_eq,aaStmwB2                                 ; Error, bail...
  676                         stw             r15,4(r16)                                              ; Store word 1
  677                         bf-             cr0_eq,aaStmwB2                                 ; Error, bail...
  678                         stw             r14,8(r16)                                              ; Store word 2
  679                         bf-             cr0_eq,aaStmwB2                                 ; Error, bail...
  680                         stw             r5,12(r16)                                              ; Store word 3
  681 
  682                         addi    r16,r16,4*4                                             ; Point up to next input aread
  683                 
  684 aaStmwB2:       mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  685                         mtmsr   r30                                                             ; Normal MSR
  686                         isync
  687 
  688                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  689 
  690 aaStmwL4:       or.             r5,r17,r28                                              ; Do we have anything left to do?
  691                         cmplwi  cr1,r17,(2*4)                                   ; Do we have one, two, or three left?
  692                         cmplwi  cr2,r17,0                                               ; Do we have no full words left?
  693                         beq             aaComExit                                               ; Nothing left...
  694 
  695                         beq-    cr2,aaStmwBy1                                   ; No full words, check out bytes
  696 
  697                         lwzx    r2,r19,r18                                              ; Store register
  698                         addi    r18,r18,8                                               ; Next register
  699                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  700                         blt             cr1,aaStmwBy1                                   ; We only had one, go save it...
  701                         
  702                         lwzx    r15,r19,r18                                             ; Store register
  703                         addi    r18,r18,8                                               ; Next register
  704                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  705                         beq             cr1,aaStmwBy1                                   ; We had two, go save it...
  706                         
  707                         lwzx    r14,r19,r18                                             ; Store register
  708                         addi    r18,r18,8                                               ; Next register
  709                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  710                         
  711 aaStmwBy1:      mr.             r28,r28                                                 ; Do we have any trailing bytes?
  712                         beq+    aaStmwSt                                                ; Nope...
  713                         
  714                         lwzx    r5,r19,r18                                              ; Yes, pick up one extra register
  715                         
  716 aaStmwSt:       crset   cr0_eq                                                  ; Set this to see if we failed
  717                         mtmsr   r22                                                             ; Flip DR, RI
  718                         isync
  719 
  720                         beq-    cr2,aaStmwBy2                                   ; No words, check trailing bytes...                                     
  721 
  722                         stw             r2,0(r16)                                               ; Save first word
  723                         bf-             cr0_eq,aaStmwDn                                 ; Store failed, escape...
  724                         addi    r16,r16,4                                               ; Bump sink
  725                         blt             cr1,aaStmwBy2                                   ; We only had one, we are done...
  726 
  727                         stw             r15,0(r16)                                              ; Save second word
  728                         bf-             cr0_eq,aaStmwDn                                 ; Store failed, escape...
  729                         addi    r16,r16,4                                               ; Bump sink
  730                         beq             cr1,aaStmwBy2                                   ; We had two, we are done...
  731 
  732                         stw             r14,0(r16)                                              ; Save third word
  733                         bf-             cr0_eq,aaStmwDn                                 ; Store failed, escape...
  734                         addi    r16,r16,4                                               ; Bump sink
  735                                 
  736 aaStmwBy2:      rlwinm  r2,r5,8,24,31                                   ; Get byte 0
  737                         cmplwi  cr2,r28,0                                               ; Any trailing bytes to do?
  738                         rlwinm  r14,r5,24,24,31                                 ; Get byte 3
  739                         li              r8,0                                                    ; Clear second trailing byte
  740                         cmplwi  cr1,r28,2                                               ; Check for 1, 2, or 3
  741                         li              r9,0                                                    ; Clear third trailing byte
  742                         beq+    cr2,aaStmwDn                                    ; No trailing bytes...
  743                         rlwinm  r15,r5,16,24,31                                 ; Get byte 1
  744 
  745                         stb             r2,0(r16)                                               ; Save first byte
  746                         bf-             cr0_eq,aaStmwDn                                 ; Read failed, escape...
  747                         blt             cr1,aaStmwDn                                    ; We only had one, we are done...
  748 
  749                         stb             r15,1(r16)                                              ; Save second byte
  750                         bf-             cr0_eq,aaStmwDn                                 ; Read failed, escape...
  751                         beq             cr1,aaStmwDn                                    ; We had two, we are done...
  752 
  753                         stb             r14,2(r16)                                              ; Save third byte
  754 
  755 aaStmwDn:       mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  756                         mtmsr   r30                                                             ; Normal MSR
  757                         isync
  758 
  759                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  760 
  761                         b               aaComExit                                               ; We are done....
  762 
  763         
  764 ;
  765 ;                       Load String Indexed
  766 ;
  767 
  768                         .align  5
  769                         
  770 aaLswx:         lwz             r17,savexer+4(r13)                              ; Pick up the XER
  771                         crclr   iUpdate                                                 ; Make sure we think this the load form
  772                         rlwinm. r25,r17,0,25,31                                 ; Get the number of bytes to load
  773                         rlwinm  r28,r17,0,30,31                                 ; Get the number of bytes past an even word
  774                         beq-    aaComExit                                               ; Do nothing if 0 length...
  775                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  776                         b               aaLSComm                                                ; Join up with common load/store code...
  777 
  778         
  779 ;
  780 ;                       Load String Immediate
  781 ;
  782 
  783                         .align  5
  784 
  785 aaLswi:         mr              r9,r23                                                  ; Save the DAR
  786                         bl              eIFetch                                                 ; Get the instruction image
  787                         bne-    eRedriveAsISI                                   ; Go redrive this as an ISI...  
  788                         rlwinm  r25,r10,21,27,31                                ; Get the number of bytes to load
  789                         crclr   iUpdate                                                 ; Make sure we think this the load form
  790                         subi    r25,r25,1                                               ; Back off by 1
  791                         rlwinm  r25,r25,0,27,31                                 ; Clear back down
  792                         addi    r25,r25,1                                               ; Add back the 1 to convert 0 to 32
  793                         rlwinm  r28,r25,0,30,31                                 ; Get the number of bytes past an even word
  794                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  795                         mr              r23,r9                                                  ; Move back the DAR
  796                         b               aaLSComm                                                ; Join up with common load/store code...
  797         
  798 ;
  799 ;                       Store String Indexed
  800 ;
  801 
  802                         .align  5
  803 
  804 aaStswx:        lwz             r17,savexer+4(r13)                              ; Pick up the XER
  805                         crclr   iUpdate                                                 ; Make sure this is clear in case we have 0 length
  806                         rlwinm. r25,r17,0,25,31                                 ; Get the number of bytes to load
  807                         rlwinm  r28,r17,0,30,31                                 ; Get the number of bytes past an even word
  808                         beq-    aaComExit                                               ; Do nothing if 0 length...
  809                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  810                         crset   iUpdate                                                 ; Make sure we think this the store form
  811                         b               aaLSComm                                                ; Join up with common load/store code...
  812 
  813         
  814 ;
  815 ;                       Store String Immediate
  816 ;
  817 
  818                         .align  5
  819 
  820 aaStswi:        mr              r9,r23                                                  ; Save the DAR
  821                         bl              eIFetch                                                 ; Get the instruction image
  822                         bne-    eRedriveAsISI                                   ; Go redrive this as an ISI...  
  823                         rlwinm  r25,r10,21,27,31                                ; Get the number of bytes to load
  824                         crclr   iUpdate                                                 ; Make sure we think this the load form
  825                         subi    r25,r25,1                                               ; Back off by 1
  826                         rlwinm  r25,r25,0,27,31                                 ; Clear back down
  827                         addi    r25,r25,1                                               ; Add back the 1 to convert 0 to 32
  828                         rlwinm  r28,r25,21,30,31                                ; Get the number of bytes past an even word
  829                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  830                         mr              r23,r9                                                  ; Move back the DAR
  831                         b               aaLSComm                                                ; Join up with common load/store code...
  832         
  833 
  834 ;
  835 ;                       Load byte-reversed word
  836 ;
  837 
  838                         .align  5
  839 
  840 aaLwbrx:
  841                         add             r18,r18,r13                                             ; Index to source register
  842 
  843                         crset   cr0_eq                                                  ; Set this to see if we failed
  844                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  845                         isync
  846                 
  847                         lwz             r11,0(r23)                                              ; Load the word
  848                 
  849                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  850                         mtmsr   r30                                                             ; Restore normal MSR
  851                         isync
  852                         
  853                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  854                         
  855                         rlwinm  r10,r11,8,0,31                                  ; Get byte 0 to 3 and byte 2 to 1
  856                         rlwimi  r10,r11,24,16,23                                ; Move byte 1 to byte 2
  857                         rlwimi  r10,r11,24,0,7                                  ; Move byte 3 to byte 0
  858                 
  859                         stw             r10,saver0+4(r18)                               ; Set the register
  860 
  861                         b               aaComExit                                               ; All done, go exit...
  862 
  863 
  864 
  865 ;
  866 ;                       Store byte-reversed word
  867 ;
  868 
  869                         .align  5
  870 
  871 aaStwbrx:
  872                         add             r18,r18,r13                                             ; Index to source register
  873                         lwz             r11,saver0+4(r18)                               ; Get the register to store
  874 
  875                         rlwinm  r10,r11,8,0,31                                  ; Get byte 0 to 3 and byte 2 to 1
  876                         rlwimi  r10,r11,24,16,23                                ; Move byte 1 to byte 2
  877                         rlwimi  r10,r11,24,0,7                                  ; Move byte 3 to byte 0
  878                         
  879                         crset   cr0_eq                                                  ; Set this to see if we failed
  880                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  881                         isync
  882                 
  883                         stw             r10,0(r23)                                              ; Store the reversed halfword
  884                 
  885                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  886                         mtmsr   r30                                                             ; Restore normal MSR
  887                         isync
  888                         
  889                         bt+             cr0_eq,aaComExit                                ; All done, go exit...
  890                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...        
  891 
  892 
  893 
  894 ;
  895 ;                       Load byte-reversed halfword
  896 ;
  897 
  898                         .align  5
  899 
  900 aaLhbrx:
  901                         add             r18,r18,r13                                             ; Index to source register
  902 
  903                         crset   cr0_eq                                                  ; Set this to see if we failed
  904                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  905                         isync
  906                 
  907                         lhz             r11,0(r23)                                              ; Load the halfword
  908                 
  909                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  910                         mtmsr   r30                                                             ; Restore normal MSR
  911                         isync
  912 
  913                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  914                         
  915                         rlwinm  r10,r11,8,16,23                                 ; Rotate bottom byte up one and clear everything else
  916                         rlwimi  r10,r11,24,24,31                                ; Put old second from bottom into bottom
  917                 
  918                         stw             r10,saver0+4(r18)                               ; Set the register
  919 
  920                         b               aaComExit                                               ; All done, go exit...
  921 
  922 
  923 ;
  924 ;                       Store byte-reversed halfword
  925 ;
  926 
  927                         .align  5
  928 
  929 aaSthbrx:
  930                         add             r18,r18,r13                                             ; Index to source register
  931                         lwz             r10,saver0+4(r18)                               ; Get the register to store
  932                         rlwinm  r10,r10,8,0,31                                  ; Rotate bottom byte up one
  933                         rlwimi  r10,r10,16,24,31                                ; Put old second from bottom into bottom
  934                         
  935                         crset   cr0_eq                                                  ; Set this to see if we failed
  936                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  937                         isync
  938                 
  939                         sth             r10,0(r23)                                              ; Store the reversed halfword
  940                 
  941                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  942                         mtmsr   r30                                                             ; Restore normal MSR
  943                         isync
  944 
  945                         bt+             cr0_eq,aaComExit                                ; All done, go exit...
  946                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...        
  947 
  948 ;
  949 ;                       Data cache block zero
  950 ;
  951 
  952                         .align  5
  953 
  954 aaDcbz:                 
  955             lwz     r0,savesrr0+4(r13)              ; get instruction address
  956             li          r4,_COMM_PAGE_BASE_ADDRESS
  957                         rlwinm  r23,r23,0,0,26                                  ; Round EA back to a 32-byte boundary
  958             sub     r4,r0,r4                        ; compute instruction offset from base of commpage
  959             cmplwi  r4,_COMM_PAGE_AREA_USED         ; did fault occur in commpage?
  960             bge+    aaDcbz1                         ; skip if not in commpage
  961             lwz         r4,savecr(r13)                  ; if we take a dcbz in the commpage...
  962             rlwinm      r4,r4,0,0,27                    ; ...clear users cr7 as a flag for commpage code
  963             stw         r4,savecr(r13)
  964 aaDcbz1:
  965                         crset   cr0_eq                                                  ; Set this to see if we failed
  966                         li              r0,0                                                    ; Clear this out
  967                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  968                         isync
  969                         
  970                         stw             r0,0(r23)                                               ; Clear word    
  971                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  972                         stw             r0,4(r23)                                               ; Clear word            
  973                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  974                         stw             r0,8(r23)                                               ; Clear word            
  975                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  976                         stw             r0,12(r23)                                              ; Clear word            
  977                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  978                         stw             r0,16(r23)                                              ; Clear word            
  979                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  980                         stw             r0,20(r23)                                              ; Clear word            
  981                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  982                         stw             r0,24(r23)                                              ; Clear word            
  983                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  984                         stw             r0,28(r23)                                              ; Clear word            
  985                         
  986 aaDcbzXit:      mr              r4,r0                                                   ; Save the DAR if we failed the access
  987                         mtmsr   r30                                                             ; Restore normal MSR
  988                         isync
  989 
  990                         crclr   iUpdate                                                 ; Make sure we do not think this is an update form
  991                         
  992                         bt+             cr0_eq,aaComExit                                ; All done, go exit...
  993                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...        
  994 
  995 
  996 ;
  997 ;                       Unhandled alignment exception, pass it along
  998 ;
  999 
 1000 aaPassAlong:
 1001                         li              r0,1                                                    ; Indicate that we failed to emulate
 1002                         stw             r0,savemisc3(r13)                               ; Assume that we emulate ok
 1003                         b               EXT(EmulExit)                                   
 1004 
 1005 
 1006 
 1007 
 1008 ;
 1009 ;                       We go here to emulate a trace exception after we have handled alignment error
 1010 ;
 1011 
 1012                         .align  5
 1013                         
 1014 aaComExitrd:
 1015                         oris    r9,r9,hi16(SAVredrive)                  ; Set the redrive bit
 1016                         li              r11,T_TRACE                                             ; Set trace interrupt
 1017                         rlwinm  r12,r12,0,16,31                                 ; Clear top half of SRR1
 1018                         stw             r9,SAVflags(r13)                                ; Set the flags
 1019                         stw             r11,saveexception(r13)                  ; Set the exception code
 1020                         b               EXT(EmulExit)                                   ; Exit and do trace interrupt...
 1021                         
 1022 
 1023 
 1024 ;
 1025 ;                       Redrive as a DSI
 1026 
 1027 aaRedriveAsDSI:
 1028                         mr              r20,r1                                                  ; Save the DSISR
 1029                         mr              r21,r4
 1030                         lwz             r4,SAVflags(r13)                                ; Pick up the flags
 1031                         li              r11,T_DATA_ACCESS                               ; Set failing data access code
 1032                         oris    r4,r4,hi16(SAVredrive)                  ; Set the redrive bit
 1033                         stw             r20,savedsisr(r13)                              ; Set the DSISR of failed access
 1034                         stw             r21,savedar+4(r13)                              ; Set the address of the failed access
 1035                         stw             r11,saveexception(r13)                  ; Set the replacement code
 1036                         stw             r4,SAVflags(r13)                                ; Set redrive request
 1037                         b               EXT(EmulExit)                                   ; Bail out to handle ISI...
 1038 
 1039 
 1040 
 1041 ;
 1042 ;                       Table of functions to load or store floating point registers
 1043 ;                       This table is indexed reg||size||dir.  That means that each
 1044 ;                       like load/store pair (e.g., lfd f31/stfd f31) are within the same
 1045 ;                       quadword, which is the current ifetch size.  We expect most of the
 1046 ;                       unaligned accesses to be part of copies, therefore, with this
 1047 ;                       organization, we will save the ifetch of the store after the load.
 1048 ;
 1049 
 1050                         .align  10                                                              ; Make sure we are on a 1k boundary
 1051                         .globl  EXT(aaFPopTable)
 1052                         
 1053 LEXT(aaFPopTable)
 1054                         lfs             f0,emfp0(r31)                                   ; Load single variant
 1055                         blr
 1056 
 1057                         stfs    f0,emfp0(r31)                                   ; Store single variant
 1058                         blr
 1059                         
 1060                         lfd             f0,emfp0(r31)                                   ; Load double variant
 1061                         blr
 1062                         
 1063                         stfd    f0,emfp0(r31)                                   ; Store double variant
 1064                         blr
 1065 
 1066                         lfs             f1,emfp0(r31)                                   ; Load single variant
 1067                         blr
 1068 
 1069                         stfs    f1,emfp0(r31)                                   ; Store single variant
 1070                         blr
 1071                         
 1072                         lfd             f1,emfp0(r31)                                   ; Load double variant
 1073                         blr
 1074                         
 1075                         stfd    f1,emfp0(r31)                                   ; Store double variant
 1076                         blr
 1077 
 1078                         lfs             f2,emfp0(r31)                                   ; Load single variant
 1079                         blr
 1080 
 1081                         stfs    f2,emfp0(r31)                                   ; Store single variant
 1082                         blr
 1083                         
 1084                         lfd             f2,emfp0(r31)                                   ; Load double variant
 1085                         blr
 1086                         
 1087                         stfd    f2,emfp0(r31)                                   ; Store double variant
 1088                         blr
 1089 
 1090                         lfs             f3,emfp0(r31)                                   ; Load single variant
 1091                         blr
 1092 
 1093                         stfs    f3,emfp0(r31)                                   ; Store single variant
 1094                         blr
 1095                         
 1096                         lfd             f3,emfp0(r31)                                   ; Load double variant
 1097                         blr
 1098                         
 1099                         stfd    f3,emfp0(r31)                                   ; Store double variant
 1100                         blr
 1101 
 1102                         lfs             f4,emfp0(r31)                                   ; Load single variant
 1103                         blr
 1104 
 1105                         stfs    f4,emfp0(r31)                                   ; Store single variant
 1106                         blr
 1107                         
 1108                         lfd             f4,emfp0(r31)                                   ; Load double variant
 1109                         blr
 1110                         
 1111                         stfd    f4,emfp0(r31)                                   ; Store double variant
 1112                         blr
 1113 
 1114                         lfs             f5,emfp0(r31)                                   ; Load single variant
 1115                         blr
 1116 
 1117                         stfs    f5,emfp0(r31)                                   ; Store single variant
 1118                         blr
 1119                         
 1120                         lfd             f5,emfp0(r31)                                   ; Load double variant
 1121                         blr
 1122                         
 1123                         stfd    f5,emfp0(r31)                                   ; Store double variant
 1124                         blr
 1125 
 1126                         lfs             f6,emfp0(r31)                                   ; Load single variant
 1127                         blr
 1128 
 1129                         stfs    f6,emfp0(r31)                                   ; Store single variant
 1130                         blr
 1131                         
 1132                         lfd             f6,emfp0(r31)                                   ; Load double variant
 1133                         blr
 1134                         
 1135                         stfd    f6,emfp0(r31)                                   ; Store double variant
 1136                         blr
 1137 
 1138                         lfs             f7,emfp0(r31)                                   ; Load single variant
 1139                         blr
 1140 
 1141                         stfs    f7,emfp0(r31)                                   ; Store single variant
 1142                         blr
 1143                         
 1144                         lfd             f7,emfp0(r31)                                   ; Load double variant
 1145                         blr
 1146                         
 1147                         stfd    f7,emfp0(r31)                                   ; Store double variant
 1148                         blr
 1149 
 1150                         lfs             f8,emfp0(r31)                                   ; Load single variant
 1151                         blr
 1152 
 1153                         stfs    f8,emfp0(r31)                                   ; Store single variant
 1154                         blr
 1155                         
 1156                         lfd             f8,emfp0(r31)                                   ; Load double variant
 1157                         blr
 1158                         
 1159                         stfd    f8,emfp0(r31)                                   ; Store double variant
 1160                         blr
 1161 
 1162                         lfs             f9,emfp0(r31)                                   ; Load single variant
 1163                         blr
 1164 
 1165                         stfs    f9,emfp0(r31)                                   ; Store single variant
 1166                         blr
 1167                         
 1168                         lfd             f9,emfp0(r31)                                   ; Load double variant
 1169                         blr
 1170                         
 1171                         stfd    f9,emfp0(r31)                                   ; Store double variant
 1172                         blr
 1173 
 1174                         lfs             f10,emfp0(r31)                                  ; Load single variant
 1175                         blr
 1176 
 1177                         stfs    f10,emfp0(r31)                                  ; Store single variant
 1178                         blr
 1179                         
 1180                         lfd             f10,emfp0(r31)                                  ; Load double variant
 1181                         blr
 1182                         
 1183                         stfd    f10,emfp0(r31)                                  ; Store double variant
 1184                         blr
 1185 
 1186                         lfs             f11,emfp0(r31)                                  ; Load single variant
 1187                         blr
 1188 
 1189                         stfs    f11,emfp0(r31)                                  ; Store single variant
 1190                         blr
 1191                         
 1192                         lfd             f11,emfp0(r31)                                  ; Load double variant
 1193                         blr
 1194                         
 1195                         stfd    f11,emfp0(r31)                                  ; Store double variant
 1196                         blr
 1197 
 1198                         lfs             f12,emfp0(r31)                                  ; Load single variant
 1199                         blr
 1200 
 1201                         stfs    f12,emfp0(r31)                                  ; Store single variant
 1202                         blr
 1203                         
 1204                         lfd             f12,emfp0(r31)                                  ; Load double variant
 1205                         blr
 1206                         
 1207                         stfd    f12,emfp0(r31)                                  ; Store double variant
 1208                         blr
 1209 
 1210                         lfs             f13,emfp0(r31)                                  ; Load single variant
 1211                         blr
 1212 
 1213                         stfs    f13,emfp0(r31)                                  ; Store single variant
 1214                         blr
 1215                         
 1216                         lfd             f13,emfp0(r31)                                  ; Load double variant
 1217                         blr
 1218                         
 1219                         stfd    f13,emfp0(r31)                                  ; Store double variant
 1220                         blr
 1221 
 1222                         lfs             f14,emfp0(r31)                                  ; Load single variant
 1223                         blr
 1224 
 1225                         stfs    f14,emfp0(r31)                                  ; Store single variant
 1226                         blr
 1227                         
 1228                         lfd             f14,emfp0(r31)                                  ; Load double variant
 1229                         blr
 1230                         
 1231                         stfd    f14,emfp0(r31)                                  ; Store double variant
 1232                         blr
 1233 
 1234                         lfs             f15,emfp0(r31)                                  ; Load single variant
 1235                         blr
 1236 
 1237                         stfs    f15,emfp0(r31)                                  ; Store single variant
 1238                         blr
 1239                         
 1240                         lfd             f15,emfp0(r31)                                  ; Load double variant
 1241                         blr
 1242                         
 1243                         stfd    f15,emfp0(r31)                                  ; Store double variant
 1244                         blr
 1245 
 1246                         lfs             f16,emfp0(r31)                                  ; Load single variant
 1247                         blr
 1248 
 1249                         stfs    f16,emfp0(r31)                                  ; Store single variant
 1250                         blr
 1251                         
 1252                         lfd             f16,emfp0(r31)                                  ; Load double variant
 1253                         blr
 1254                         
 1255                         stfd    f16,emfp0(r31)                                  ; Store double variant
 1256                         blr
 1257 
 1258                         lfs             f17,emfp0(r31)                                  ; Load single variant
 1259                         blr
 1260 
 1261                         stfs    f17,emfp0(r31)                                  ; Store single variant
 1262                         blr
 1263                         
 1264                         lfd             f17,emfp0(r31)                                  ; Load double variant
 1265                         blr
 1266                         
 1267                         stfd    f17,emfp0(r31)                                  ; Store double variant
 1268                         blr
 1269 
 1270                         lfs             f18,emfp0(r31)                                  ; Load single variant
 1271                         blr
 1272 
 1273                         stfs    f18,emfp0(r31)                                  ; Store single variant
 1274                         blr
 1275                         
 1276                         lfd             f18,emfp0(r31)                                  ; Load double variant
 1277                         blr
 1278                         
 1279                         stfd    f18,emfp0(r31)                                  ; Store double variant
 1280                         blr
 1281 
 1282                         lfs             f19,emfp0(r31)                                  ; Load single variant
 1283                         blr
 1284 
 1285                         stfs    f19,emfp0(r31)                                  ; Store single variant
 1286                         blr
 1287                         
 1288                         lfd             f19,emfp0(r31)                                  ; Load double variant
 1289                         blr
 1290                         
 1291                         stfd    f19,emfp0(r31)                                  ; Store double variant
 1292                         blr
 1293 
 1294                         lfs             f20,emfp0(r31)                                  ; Load single variant
 1295                         blr
 1296 
 1297                         stfs    f20,emfp0(r31)                                  ; Store single variant
 1298                         blr
 1299                         
 1300                         lfd             f20,emfp0(r31)                                  ; Load double variant
 1301                         blr
 1302                         
 1303                         stfd    f20,emfp0(r31)                                  ; Store double variant
 1304                         blr
 1305 
 1306                         lfs             f21,emfp0(r31)                                  ; Load single variant
 1307                         blr
 1308 
 1309                         stfs    f21,emfp0(r31)                                  ; Store single variant
 1310                         blr
 1311                         
 1312                         lfd             f21,emfp0(r31)                                  ; Load double variant
 1313                         blr
 1314                         
 1315                         stfd    f21,emfp0(r31)                                  ; Store double variant
 1316                         blr
 1317 
 1318                         lfs             f22,emfp0(r31)                                  ; Load single variant
 1319                         blr
 1320 
 1321                         stfs    f22,emfp0(r31)                                  ; Store single variant
 1322                         blr
 1323                         
 1324                         lfd             f22,emfp0(r31)                                  ; Load double variant
 1325                         blr
 1326                         
 1327                         stfd    f22,emfp0(r31)                                  ; Store double variant
 1328                         blr
 1329 
 1330                         lfs             f23,emfp0(r31)                                  ; Load single variant
 1331                         blr
 1332 
 1333                         stfs    f23,emfp0(r31)                                  ; Store single variant
 1334                         blr
 1335                         
 1336                         lfd             f23,emfp0(r31)                                  ; Load double variant
 1337                         blr
 1338                         
 1339                         stfd    f23,emfp0(r31)                                  ; Store double variant
 1340                         blr
 1341 
 1342                         lfs             f24,emfp0(r31)                                  ; Load single variant
 1343                         blr
 1344 
 1345                         stfs    f24,emfp0(r31)                                  ; Store single variant
 1346                         blr
 1347                         
 1348                         lfd             f24,emfp0(r31)                                  ; Load double variant
 1349                         blr
 1350                         
 1351                         stfd    f24,emfp0(r31)                                  ; Store double variant
 1352                         blr
 1353 
 1354                         lfs             f25,emfp0(r31)                                  ; Load single variant
 1355                         blr
 1356 
 1357                         stfs    f25,emfp0(r31)                                  ; Store single variant
 1358                         blr
 1359                         
 1360                         lfd             f25,emfp0(r31)                                  ; Load double variant
 1361                         blr
 1362                         
 1363                         stfd    f25,emfp0(r31)                                  ; Store double variant
 1364                         blr
 1365 
 1366                         lfs             f26,emfp0(r31)                                  ; Load single variant
 1367                         blr
 1368 
 1369                         stfs    f26,emfp0(r31)                                  ; Store single variant
 1370                         blr
 1371                         
 1372                         lfd             f26,emfp0(r31)                                  ; Load double variant
 1373                         blr
 1374                         
 1375                         stfd    f26,emfp0(r31)                                  ; Store double variant
 1376                         blr
 1377 
 1378                         lfs             f27,emfp0(r31)                                  ; Load single variant
 1379                         blr
 1380 
 1381                         stfs    f27,emfp0(r31)                                  ; Store single variant
 1382                         blr
 1383                         
 1384                         lfd             f27,emfp0(r31)                                  ; Load double variant
 1385                         blr
 1386                         
 1387                         stfd    f27,emfp0(r31)                                  ; Store double variant
 1388                         blr
 1389 
 1390                         lfs             f28,emfp0(r31)                                  ; Load single variant
 1391                         blr
 1392 
 1393                         stfs    f28,emfp0(r31)                                  ; Store single variant
 1394                         blr
 1395                         
 1396                         lfd             f28,emfp0(r31)                                  ; Load double variant
 1397                         blr
 1398                         
 1399                         stfd    f28,emfp0(r31)                                  ; Store double variant
 1400                         blr
 1401 
 1402                         lfs             f29,emfp0(r31)                                  ; Load single variant
 1403                         blr
 1404 
 1405                         stfs    f29,emfp0(r31)                                  ; Store single variant
 1406                         blr
 1407                         
 1408                         lfd             f29,emfp0(r31)                                  ; Load double variant
 1409                         blr
 1410                         
 1411                         stfd    f29,emfp0(r31)                                  ; Store double variant
 1412                         blr
 1413 
 1414                         lfs             f30,emfp0(r31)                                  ; Load single variant
 1415                         blr
 1416 
 1417                         stfs    f30,emfp0(r31)                                  ; Store single variant
 1418                         blr
 1419                         
 1420                         lfd             f30,emfp0(r31)                                  ; Load double variant
 1421                         blr
 1422                         
 1423                         stfd    f30,emfp0(r31)                                  ; Store double variant
 1424                         blr
 1425 
 1426                         lfs             f31,emfp0(r31)                                  ; Load single variant
 1427                         blr
 1428 
 1429                         stfs    f31,emfp0(r31)                                  ; Store single variant
 1430                         blr
 1431                         
 1432                         lfd             f31,emfp0(r31)                                  ; Load double variant
 1433                         blr
 1434                         
 1435                         stfd    f31,emfp0(r31)                                  ; Store double variant
 1436                         blr
 1437 

Cache object: c68f1506a85b2d42896a21b25559a3de


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.