The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/osfmk/ppc/Emulate.s

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
    3  *
    4  * @APPLE_LICENSE_HEADER_START@
    5  * 
    6  * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
    7  * 
    8  * This file contains Original Code and/or Modifications of Original Code
    9  * as defined in and that are subject to the Apple Public Source License
   10  * Version 2.0 (the 'License'). You may not use this file except in
   11  * compliance with the License. Please obtain a copy of the License at
   12  * http://www.opensource.apple.com/apsl/ and read it before using this
   13  * file.
   14  * 
   15  * The Original Code and all software distributed under the License are
   16  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
   17  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
   18  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
   19  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
   20  * Please see the License for the specific language governing rights and
   21  * limitations under the License.
   22  * 
   23  * @APPLE_LICENSE_HEADER_END@
   24  */
   25 /*                                                                                                                                                                                      
   26         Emulate.s 
   27 
   28         Emulate instructions and traps.
   29 
   30         Lovingly crafted by Bill Angell using traditional methods and only natural or recycled materials.
   31         No animal products are used other than rendered otter bile and deep fried pork lard.
   32 
   33 */
   34 
   35 #include <cpus.h>
   36 #include <ppc/asm.h>
   37 #include <ppc/proc_reg.h>
   38 #include <ppc/exception.h>
   39 #include <mach/machine/vm_param.h>
   40 #include <assym.s>
   41 
   42 #define traceInst 30
   43 #define dssAllDone 29
   44 
   45 ;                       General stuff what happens here:
   46 ;                               1)      All general context saved, interrupts off, translation off
   47 ;                               2)      Vector and floating point disabled, but there may be live context.
   48 ;                                       This code is responsible for saving and restoring what is used. This
   49 ;                                       includes exception states, java mode, etc.
   50 ;                               3)      No attempt is made to resolve page faults.  PTE misses are handled
   51 ;                                       automatically, but actual faults (ala copyin/copyout) are not. If 
   52 ;                                       a fault does occur, the exception that caused entry to the emulation
   53 ;                                       routine is remapped to either an instruction or data miss (depending
   54 ;                                       upon the stage detected) and redrived through the exception handler.
   55 ;                                       The only time that an instruction fault can happen is when a different
   56 ;                                       processor removes a mapping between our original fault and when we
   57 ;                                       fetch the assisted instruction. For an assisted instruction, data
   58 ;                                       faults should not occur (except in the MP case).  For a purely
   59 ;                                       emulated instruction, faults can occur.
   60 ;
   61 ;
   62 
   63 
   64                         .align  5
   65                         .globl  EXT(Emulate)
   66 
   67 LEXT(Emulate)
   68 
   69                         bf--    pf64Bitb,emn64                                  ; Skip if not 64-bit
   70                         b               EXT(Emulate64)                                  ; Jump to the 64-bit code...
   71                         
   72 emn64:          mfsprg  r31,0                                                   ; Get the per_proc
   73                         lwz             r12,savesrr1+4(r13)                             ; Get the exception info
   74                         rlwinm. r0,r12,0,SRR1_PRG_ILL_INS_BIT,SRR1_PRG_ILL_INS_BIT      ; Emulation candidate?
   75                         lwz             r30,dgFlags(0)                                  ; Get the flags
   76                         beq+    eExit                                                   ; Nope, do not try to emulate...
   77 
   78                         rlwinm. r0,r30,0,enaDiagEMb,enaDiagEMb  ; Do we want to try to emulate something?
   79                         mfsprg  r28,2                                                   ; Get the processor features
   80                         beq+    eExit                                                   ; No emulation allowed...
   81 
   82                         rlwinm. r28,r28,0,pfAltivecb,pfAltivecb ; Do we have Altivec on this machine?
   83                         beq             eNoVect                                                 ; Nope, no Altivec...
   84                         
   85                         dssall                                                                  ; We need to kill streams because we are going to flip to problem state
   86                         sync
   87 
   88 eNoVect:        bl              eIFetch                                                 ; Get the instruction image
   89                         bne-    eRedriveAsISI                                   ; Go redrive this as an ISI...  
   90 
   91                         rlwinm. r0,r10,0,0,5                                    ; See if we have the "special" op code here
   92                         rlwinm  r20,r10,16,22,31                                ; Set rS/rD and rA
   93                         bne+    eExit                                                   ; Not special op, ignore...
   94 
   95                         rlwinm  r0,r10,31,22,31                                 ; Extract the sub op code
   96                         crclr   cr1_eq                                                  ; Clear
   97                         rlwimi  r20,r10,14,15,16                                ; Move bits 29 and 30 of instruction to 15 and 16 of DSISR
   98                         cmplwi  r0,790                                                  ; lhbrx?
   99                         rlwimi  r20,r10,8,17,17                                 ; Move bit 25 to bit 17
  100                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  101                         cmplwi  r0,534                                                  ; lwbrx?
  102                         rlwimi  r20,r10,3,18,21                                 ; Move bit 21-24 to bit 18-21
  103                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  104                         cmplwi  r0,918                                                  ; sthbrx?
  105                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  106                         cmplwi  r0,662                                                  ; stwbrx?
  107                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  108                         cmplwi  r0,1014                                                 ; dcbz?
  109                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  110                         cmplwi  r0,533                                                  ; lswx?
  111                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  112                         cmplwi  r0,661                                                  ; stswx?
  113                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  114                         bne             cr1_eq,eNotIndex                                ; Go check non-index forms...
  115 
  116                         rlwinm. r21,r10,19,24,28                                ; Extract index to rA to build EA
  117                         rlwinm  r22,r10,24,24,28                                ; Extract index to rB
  118                         addi    r24,r13,saver0+4                                ; Point to the start of registers
  119                         li              r19,0                                                   ; Assume 0 base
  120                         beq             eZeroBase                                               ; Yes...
  121                         lwzx    r19,r24,r21                                             ; Get the base register value
  122                         
  123 eZeroBase:      lwzx    r22,r24,r22                                             ; Get the index value
  124                         add             r22,r22,r19                                             ; Get DAR
  125                         b               eFinishUp                                               ; Done, go finish up...
  126                                                 
  127 eNotIndex:      cmplwi  r0,725                                                  ; stswi?
  128                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  129                         cmplwi  r0,597                                                  ; lswi?
  130                         cror    cr1_eq,cr1_eq,cr0_eq                    ; Remember
  131                         bne             cr1,eExit                                               ; Not one we handle...
  132         
  133                         rlwinm. r21,r10,19,24,28                                ; Extract index to rA to build EA
  134                         addi    r24,r13,saver0+4                                ; Point to the start of registers
  135                         li              r22,0                                                   ; Assume 0 base
  136                         beq             eFinishUp                                               ; Yes, it is...
  137                         lwzx    r22,r24,r21                                             ; Get the base register value
  138                         
  139 eFinishUp:      stw             r20,savedsisr(r13)                              ; Set the DSISR
  140                         li              r11,T_ALIGNMENT                                 ; Get the exception code
  141                         stw             r22,savedar+4(r13)                              ; Save the DAR
  142                         stw             r11,saveexception(r13)                  ; Set the exception code
  143                         b               EXT(AlignAssist)                                ; Go emulate the handler...
  144 
  145 
  146 eExit:          b               EXT(EmulExit)                                   ; Just return for now...
  147 
  148 
  149 ;
  150 ;                       Fetch the failing instruction.
  151 ;                       Image returned in R10 if CR0_EQ is false, otherwise, an ISI should be generated/
  152 ;                       The cr bit kernAccess is set if this was a kernel access.
  153 ;                       R1 has the DSISR if access failed.
  154 ;
  155 
  156                         .align  5
  157 
  158 eIFetch:        lwz             r23,savesrr1+4(r13)                             ; Get old MSR
  159                         mflr    r28                                                             ; Save return
  160 
  161                         rlwinm. r22,r23,0,MSR_PR_BIT,MSR_PR_BIT ; Within kernel?
  162 
  163                         mfmsr   r30                                                             ; Save the MSR for now
  164                         lwz             r23,savesrr0+4(r13)                             ; Get instruction address
  165                         
  166                         ori             r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI))         ; Set RI and DR onto access MSR
  167 
  168                         crset   cr0_eq                                                  ; Set this to see if we failed
  169                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  170                         isync
  171                         
  172                         lwz             r10,0(r23)                                              ; Fetch the instruction
  173                         
  174                         mtmsr   r30                                                             ; Trans and RI off
  175                         isync
  176                         
  177                         mtlr    r28                                                             ; Restore the LR
  178                         blr                                                                             ; Return with instruction image in R10
  179 
  180 
  181 ;
  182 ;                       Redrive as an ISI
  183 ;
  184 
  185 eRedriveAsISI:
  186                         lwz             r6,savesrr1+4(r13)                              ; Get the srr1 value
  187                         lwz             r4,SAVflags(r13)                                ; Pick up the flags
  188                         li              r11,T_INSTRUCTION_ACCESS                ; Set failing instruction fetch code
  189                         rlwimi  r6,r1,0,0,4                                             ; Move the DSISR bits to the SRR1
  190                         oris    r4,r4,hi16(SAVredrive)                  ; Set the redrive bit
  191                         stw             r11,saveexception(r13)                  ; Set the replacement code
  192                         stw             r4,SAVflags(r13)                                ; Set redrive request
  193                         stw             r6,savesrr1+4(r13)                              ; Set the srr1 value
  194                         b               EXT(EmulExit)                                   ; Bail out to handle ISI...
  195 
  196 
  197 ;
  198 ;                       This code emulates instructions that have failed because of operand 
  199 ;                       alignment.  We decode the DSISR to figure out what we need to do.
  200 ;
  201 ;                       DSISR:
  202 ;                               0001FC00 - Instruction designation
  203 #define iFloat 12
  204 #define iOptype1 15
  205 #define iOptype2 16
  206 #define iOptype3 18
  207 #define iOptype4 19
  208 #define iUpdate 17
  209 #define iStore 20
  210 #define iDouble 21
  211 #define iNotify 22
  212 ;                               000003E0 - Target/Source register
  213 ;                               0000001F - Register to update if update form
  214 ;
  215 
  216                         .align  5
  217                         .globl  EXT(AlignAssist)
  218 
  219 LEXT(AlignAssist)
  220                         bf--    pf64Bitb,aan64                                  ; Skip if not 64-bit
  221                         b               EXT(AlignAssist64)                              ; Jump to the 64-bit code...
  222                         
  223 aan64:          lwz             r20,savedsisr(r13)                              ; Get the DSISR
  224                         mfsprg  r31,0                                                   ; Get the per_proc
  225                         mtcrf   0x10,r20                                                ; Put instruction ID in CR for later
  226                         lwz             r21,spcFlags(r31)                               ; Grab the special flags
  227                         mtcrf   0x08,r20                                                ; Put instruction ID in CR for later
  228                         rlwinm. r0,r21,0,runningVMbit,runningVMbit      ; Are we running a VM?
  229                         mtcrf   0x04,r20                                                ; Put instruction ID in CR for later
  230                         lwz             r22,savesrr1+4(r13)                             ; Get the SRR1
  231                         bne-    aaPassAlong                                             ; We are in a VM, no emulation for alignment exceptions...
  232                         lwz             r19,dgFlags(0)                                  ; Get the diagnostics flags
  233                         crxor   iFloat,iOptype1,iOptype2                ; Set this to 0 if both bits are either 0 or 1
  234                         mr              r26,r20                                                 ; Save the DSISR
  235                         rlwinm. r0,r22,0,MSR_SE_BIT,MSR_SE_BIT  ; Were we single stepping?
  236                         lwz             r23,savedar+4(r13)                              ; Pick up the address that we want to access
  237                         crnot   traceInst,cr0_eq                                ; Remember if trace is on
  238                         
  239                         rlwinm. r0,r19,0,enaNotifyEMb,enaNotifyEMb      ; Should we notify that an alignment exception happened?
  240                         mfmsr   r30                                                             ; Save the MSR for now
  241                         crnot   iNotify,cr0_eq                                  ; Remember to tell someone we did this                          
  242                         li              r29,emfp0                                               ; Point to work area
  243                         crxor   iFloat,iFloat,iOptype3                  ; Set true if we have a floating point instruction
  244                         dcbz    r29,r31                                                 ; Clear and allocate a cache line for us to work in
  245                         rlwinm  r24,r20,3,24,28                                 ; Get displacement to register to update if update form
  246                         rlwimi  r20,r20,24,28,28                                ; Move load/store indication to the bottom of index
  247                         ori             r22,r30,lo16(MASK(MSR_DR)|MASK(MSR_RI))         ; Set RI onto access MSR
  248                         rlwimi  r20,r20,26,27,27                                ; Move single/double indication to just above the bottom
  249                         lis             r29,hi16(EXT(aaFPopTable))              ; High part of FP branch table
  250                         bf-             iFloat,aaNotFloat                               ; This is not a floating point instruction...
  251                         ori             r29,r29,lo16(EXT(aaFPopTable))  ; Low part of FP branch table
  252                         
  253                         rlwimi  r29,r20,0,22,28                                 ; Index into table based upon register||iDouble||iStore
  254                         mtctr   r29                                                             ; Get set to call the function  
  255                         bt              iStore,aaFPstore                                ; This is an FP store...
  256                 
  257 ;
  258 ;                       Here we handle floating point loads
  259 ;                       
  260 
  261 aaFPload:       crset   cr0_eq                                                  ; Set this to see if we failed
  262                         mtmsr   r22                                                             ; Flip DR, RI
  263                         isync
  264                         
  265                         lwz             r10,0(r23)                                              ; Get the first word
  266                         bf-             cr0_eq,aaLdNotDbl                               ; Jump out if we DSIed...
  267                         bf              iDouble,aaLdNotDbl                              ; this is not a double...
  268                         lwz             r11,4(r23)                                              ; Get the second half
  269                         
  270 aaLdNotDbl:     mr              r4,r0                                                   ; Save the DAR if we failed the access
  271                         
  272                         mtmsr   r30                                                             ; Turn off translation again
  273                         isync
  274                         
  275                         bf-             cr0_eq,aaRedriveAsDSI                   ; Go redrive this as a DSI...   
  276                         
  277                         stw             r10,emfp0(r31)                                  ; Save the first half
  278                         stw             r11,emfp0+4(r31)                                ; Save the second half, just in case we need it
  279                         
  280                         bctrl                                                                   ; Go set the target FP register
  281 
  282                         b               aaComExit                                               ; All done, go exit...                  
  283                 
  284 ;
  285 ;                       Here we handle floating point stores
  286 ;                       
  287 
  288                         .align  5
  289 
  290 aaFPstore:      bctrl                                                                   ; Go save the source FP register
  291                         
  292                         lwz             r10,emfp0(r31)                                  ; Get first word
  293                         crandc  iDouble,iDouble,iOptype4                ; Change to 4-byte access if stfiwx
  294                         lwz             r11,emfp0+4(r31)                                ; and the second
  295                         bf+             iOptype4,aaNotstfiwx                    ; This is not a stfiwx...
  296                         mr              r10,r11                                                 ; The stfiwx wants to store the second half
  297 
  298 aaNotstfiwx:
  299                         crset   cr0_eq                                                  ; Set this to see if we failed
  300                         mtmsr   r22                                                             ; Flip DR, RI
  301                         isync
  302                         
  303                         stw             r10,0(r23)                                              ; Save the first word
  304                         bf-             cr0_eq,aaStNotDbl                               ; Jump out if we DSIed...
  305                         bf              iDouble,aaStNotDbl                              ; this is not a double...
  306                         stw             r11,4(r23)                                              ; Save the second half
  307                         
  308 aaStNotDbl:     mr              r4,r0                                                   ; Save the DAR if we failed the access
  309                         mtmsr   r30                                                             ; Turn off
  310                         isync
  311                         
  312                         bf-             cr0_eq,aaRedriveAsDSI                   ; Go redrive this as a DSI...
  313 
  314 ;
  315 ;                       Common exit routines
  316 ;
  317 
  318 aaComExit:      lwz             r10,savesrr0+4(r13)                             ; Get the failing instruction address
  319                         add             r24,r24,r13                                             ; Offset to update register
  320                         li              r11,T_IN_VAIN                                   ; Assume we are all done
  321                         addi    r10,r10,4                                               ; Step to the next instruction
  322                         bf              iUpdate,aaComExNU                               ; Skip if not an update form...
  323                         stw             r23,saver0+4(r24)                               ; Update the target
  324                         
  325 aaComExNU:      lwz             r9,SAVflags(r13)                                ; Get the flags
  326                         stw             r10,savesrr0+4(r13)                             ; Set new PC
  327                         bt-             traceInst,aaComExitrd                   ; We are tracing, go emulate trace...
  328                         bf+             iNotify,aaComExGo                               ; Nothing special here, go...
  329         
  330                         li              r11,T_ALIGNMENT                                 ; Set the we just did an alignment exception....
  331                         
  332 aaComExGo:      b               EXT(EmulExit)                                   ; We are done, no tracing on...
  333 
  334 
  335 ;
  336 ;                       This is not a floating point operation
  337 ;
  338 ;                       The emulation routines for these are positioned every 64 bytes (16 instructions)
  339 ;                       in a 1024-byte aligned table.  It is indexed by taking the low order 4 bits of
  340 ;                       the instruction code in the DSISR and subtracting 7.  If this comes up negative,
  341 ;                       the instruction is not to be emulated.  Then we add bit 0 of the code * 4.  This
  342 ;                       gives us a fairly compact and almost unique index.  Both lwm and stmw map to 0 so
  343 ;                       that one needs to be further reduced, and we end up with holes at index 6, 8, and 10.
  344 ;                       
  345 ;                       If the emulation routine takes more than 16 instructions, it must branch elsewhere
  346 ;                       to finish up.
  347 ;
  348 
  349                         .align  5
  350 
  351 aaNotFloat:
  352                         lis             r19,hi16(aaEmTable)                             ; Point to high part of table address
  353                         rlwinm  r3,r26,24,26,29                                 ; Isolate last 4 bits of op type * 4
  354                         rlwimi  r19,r26,20,27,27                                ; Get bit 0 of instruction code * 4 into bottom of table base
  355                         addic.  r3,r3,-28                                               ; Subtract 7*4 to adjust index
  356                         ori             r19,r19,lo16(aaEmTable)                 ; Low part of table address
  357                         blt-    aaPassAlong                                             ; We do not handle any of these (lwarx, stwcx., eciwx, ecowx)...
  358                         add             r19,r19,r3                                              ; Point to emulation routine
  359                         rlwinm  r18,r26,30,24,28                                ; Get the target/source register displacement
  360 
  361                         mtctr   r19                                                             ; Set the routine address
  362                         
  363                         bctr                                                                    ; Go emulate the instruction...
  364 
  365 ;
  366 ;                       This is the table of non-floating point emulation routines.
  367 ;                       It is indexed by low 4 bits of DSISR op type - 7 + bit 0 of
  368 ;                       op type * 4
  369 ;
  370                 
  371                         .align  5                                                       
  372 
  373 aaEmTable:
  374                         b               aaLmwStmw                                               ; This for lmw/stmw
  375                         b               aaLswx                                                  ; This for lwwx
  376                         b               aaLswi                                                  ; This for lswi
  377                         b               aaStswx                                                 ; This for stswx
  378                         b               aaStswi                                                 ; This for stswi
  379                         b               aaLwbrx                                                 ; This for lwbrx
  380                         b               aaPassAlong                                             ; This an invalid index (6)
  381                         b               aaStwbrx                                                ; This for stwbrx
  382                         b               aaPassAlong                                             ; This an invalid index (8)
  383                         b               aaLhbrx                                                 ; This for lhbrx
  384                         b               aaPassAlong                                             ; This an invalid index (A)
  385                         b               aaSthbrx                                                ; This for sthbrx
  386                         b               aaDcbz                                                  ; This for dcbz
  387                         b               aaPassAlong                                             ; This an invalid index (D)
  388                         b               aaPassAlong                                             ; This an invalid index (E)
  389                         b               aaPassAlong                                             ; This an invalid index (F)
  390 
  391 
  392 ;
  393 ;                       Here we handle the set up for the lmw and stmw.  After that, we split off to the
  394 ;                       individual routines.
  395 ;
  396 ;                       Note also that after some set up, all of the string instructions come through here as well.
  397 ;
  398                         .align  5
  399                                                 
  400 aaLmwStmw:
  401                         rlwinm  r17,r18,31,1,29                                 ; Convert doublword based index to words
  402                         li              r28,0                                                   ; Set no extra bytes to move (used for string instructions)
  403                         subfic  r17,r17,32*4                                    ; Calculate the length of the transfer
  404 
  405 aaLSComm:       addi    r19,r13,saver0+4                                ; Offset to registers in savearea
  406                         mr              r16,r23                                                 ; Make a hunk pointer
  407                         
  408                         bt              iUpdate,aaStmw                                  ; This is the stmw...
  409                         
  410 ;
  411 ;                       Load multiple word
  412 ;
  413 
  414 aaLmwNxt:       cmplwi  cr1,r17,8*4                                             ; Is there enough to move 8?
  415                         blt-    cr1,aaLmwNxtH                                   ; Not enough for a full hunk...
  416                         subi    r17,r17,8*4                                             ; Back off for another hunk
  417                         
  418                         crset   cr0_eq                                                  ; Set this to see if we failed
  419                         mtmsr   r22                                                             ; Flip DR, RI
  420                         isync
  421                 
  422                         lwz             r2,0(r16)                                               ; Load word 0
  423                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  424                         lwz             r15,4(r16)                                              ; Load word 1
  425                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  426                         lwz             r14,8(r16)                                              ; Load word 2
  427                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  428                         lwz             r5,12(r16)                                              ; Load word 3
  429                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  430                         lwz             r6,16(r16)                                              ; Load word 4
  431                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  432                         lwz             r7,20(r16)                                              ; Load word 5
  433                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  434                         lwz             r8,24(r16)                                              ; Load word 6
  435                         bf-             cr0_eq,aaLmwB1                                  ; Error, bail...
  436                         lwz             r9,28(r16)                                              ; Load word 7
  437                 
  438 aaLmwB1:        mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  439                         mtmsr   r30                                                             ; Turn off DR, RI
  440                         isync
  441 
  442                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  443 
  444                         addi    r16,r16,8*4                                             ; Point up to next input aread
  445                 
  446                         stwx    r2,r19,r18                                              ; Store register
  447                         addi    r18,r18,8                                               ; Next register
  448                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  449                         stwx    r15,r19,r18                                             ; Store register
  450                         addi    r18,r18,8                                               ; Next register
  451                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  452                         stwx    r14,r19,r18                                             ; Store register
  453                         addi    r18,r18,8                                               ; Next register
  454                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  455                         stwx    r5,r19,r18                                              ; Store register
  456                         addi    r18,r18,8                                               ; Next register
  457                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  458                         stwx    r6,r19,r18                                              ; Store register
  459                         addi    r18,r18,8                                               ; Next register
  460                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  461                         stwx    r7,r19,r18                                              ; Store register
  462                         addi    r18,r18,8                                               ; Next register
  463                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  464                         stwx    r8,r19,r18                                              ; Store register
  465                         addi    r18,r18,8                                               ; Next register
  466                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  467                         stwx    r9,r19,r18                                              ; Store register
  468                         addi    r18,r18,8                                               ; Next register
  469                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  470 
  471                         b               aaLmwNxt                                                ; Do the next hunk...
  472 
  473                         .align  5
  474                         
  475 aaLmwNxtH:      cmplwi  cr1,r17,4*4                                             ; Do we have 4 left?
  476                         blt             cr1,aaLmwL4                                             ; Nope...
  477 
  478                         subi    r17,r17,4*4                                             ; Set count properly
  479                         
  480                         crset   cr0_eq                                                  ; Set this to see if we failed
  481                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  482                         isync
  483                 
  484                         lwz             r2,0(r16)                                               ; Load word 0
  485                         bf-             cr0_eq,aaLmwB2                                  ; Error, bail...
  486                         lwz             r15,4(r16)                                              ; Load word 1
  487                         bf-             cr0_eq,aaLmwB2                                  ; Error, bail...
  488                         lwz             r14,8(r16)                                              ; Load word 2
  489                         bf-             cr0_eq,aaLmwB2                                  ; Error, bail...
  490                         lwz             r5,12(r16)                                              ; Load word 3
  491                 
  492 aaLmwB2:        mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  493                         mtmsr   r30                                                             ; Turn off DR, RI
  494                         isync
  495 
  496                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  497                 
  498                         addi    r16,r16,4*4                                             ; Point up to next input aread
  499                         
  500                         stwx    r2,r19,r18                                              ; Store register
  501                         addi    r18,r18,8                                               ; Next register
  502                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  503                         stwx    r15,r19,r18                                             ; Store register
  504                         addi    r18,r18,8                                               ; Next register
  505                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  506                         stwx    r14,r19,r18                                             ; Store register
  507                         addi    r18,r18,8                                               ; Next register
  508                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  509                         stwx    r5,r19,r18                                              ; Store register
  510                         addi    r18,r18,8                                               ; Next register
  511                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  512 
  513 aaLmwL4:        or.             r5,r17,r28                                              ; Do we have anything left?
  514                         cmplwi  cr1,r17,(2*4)                                   ; Do we have one, two, or three full words left?
  515                         cmplwi  cr2,r17,0                                               ; Do we have no full words left?
  516                         beq             aaComExit                                               ; Nothing left...
  517 
  518                         crset   cr0_eq                                                  ; Set this to see if we failed
  519                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  520                         isync
  521 
  522                         beq-    cr2,aaLmwBy                                             ; No full words, get bytes...
  523                         
  524                         lwz             r2,0(r16)                                               ; Pick up first word
  525                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  526                         addi    r16,r16,4                                               ; Next input location
  527                         blt             cr1,aaLmwBy                                             ; We only had one, we are done...
  528 
  529                         lwz             r15,0(r16)                                              ; Pick up second word
  530                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  531                         addi    r16,r16,4                                               ; Next input location
  532                         beq             cr1,aaLmwBy                                             ; We had two, we are done...
  533 
  534                         lwz             r14,0(r16)                                              ; Load word 3
  535                         addi    r16,r16,4                                               ; Next input location
  536 
  537 aaLmwBy:        cmplwi  cr2,r28,0                                               ; Any trailing bytes to do?
  538                         li              r8,0                                                    ; Clear second trailing byte
  539                         cmplwi  cr1,r28,2                                               ; Check for 1, 2, or 3
  540                         li              r9,0                                                    ; Clear third trailing byte
  541                         beq+    cr2,aaLmwDn                                             ; No trailing bytes...
  542                         
  543                         lbz             r5,0(r16)                                               ; Pick up first trailing byte
  544                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  545                         blt             cr1,aaLmwDn                                             ; We only had one, we are done...
  546 
  547                         lbz             r8,1(r16)                                               ; Pick up second trailing byte
  548                         bf-             cr0_eq,aaLmwDn                                  ; Read failed, escape...
  549                         beq             cr1,aaLmwDn                                             ; We had two, we are done...
  550 
  551                         lbz             r9,2(r16)                                               ; Get last trailing byte
  552                         
  553 
  554 aaLmwDn:        rlwinm  r5,r5,24,0,7                                    ; Move first byte to top
  555                         cmplwi  cr2,r17,0                                               ; Any full words to do?
  556                         mr              r4,r0                                                   ; Remember DAR, just in case we failed the access
  557                         rlwimi  r9,r8,8,16,23                                   ; Move second byte above third byte
  558                         cmplwi  cr1,r17,(2*4)                                   ; Do we have one, two, or three full words left?
  559                         mr              r3,r30                                                  ; Set the normal MSR
  560                         rlwimi  r5,r9,8,8,23                                    ; Move bytes 1 and 2 after 0
  561 
  562                         mtmsr   r30                                                             ; Turn off DR, RI
  563                         isync
  564 
  565                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  566 
  567                         beq-    cr2,aaLmwCb                                             ; No full words, copy bytes...
  568 
  569                         stwx    r2,r19,r18                                              ; Store register
  570                         addi    r18,r18,8                                               ; Next register
  571                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  572                         blt             cr1,aaLmwCb                                             ; We only had one, we are done...
  573                         
  574                         stwx    r15,r19,r18                                             ; Store register
  575                         addi    r18,r18,8                                               ; Next register
  576                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  577                         beq             cr1,aaLmwCb                                             ; We had two, we are done...
  578 
  579                         stwx    r14,r19,r18                                             ; Store register
  580                         addi    r18,r18,8                                               ; Next register
  581                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  582 
  583 aaLmwCb:        mr.             r28,r28                                                 ; Any trailing bytes to do?
  584                         beq+    aaComExit                                               ; Nope, leave...
  585 
  586                         stwx    r5,r19,r18                                              ; Store register
  587                                                 
  588                         b               aaComExit                                               ; We are done....
  589 
  590 ;
  591 ;                       Store multiple word
  592 ;
  593 
  594                         .align  5
  595 
  596 aaStmw:
  597                         crclr   iUpdate                                                 ; Make sure we do not think this is an update form
  598 
  599 aaStmwNxt:      cmplwi  cr1,r17,8*4                                             ; Is there enough to move 8?
  600                         blt-    cr1,aaStmwNxtH                                  ; Not enough for a full hunk...
  601                         subi    r17,r17,8*4                                             ; Back off for another hunk
  602                 
  603                         lwzx    r2,r19,r18                                              ; Store register
  604                         addi    r18,r18,8                                               ; Next register
  605                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  606                         lwzx    r15,r19,r18                                             ; Store register
  607                         addi    r18,r18,8                                               ; Next register
  608                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  609                         lwzx    r14,r19,r18                                             ; Store register
  610                         addi    r18,r18,8                                               ; Next register
  611                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  612                         lwzx    r5,r19,r18                                              ; Store register
  613                         addi    r18,r18,8                                               ; Next register
  614                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  615                         lwzx    r6,r19,r18                                              ; Store register
  616                         addi    r18,r18,8                                               ; Next register
  617                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  618                         lwzx    r7,r19,r18                                              ; Store register
  619                         addi    r18,r18,8                                               ; Next register
  620                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  621                         lwzx    r8,r19,r18                                              ; Store register
  622                         addi    r18,r18,8                                               ; Next register
  623                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  624                         lwzx    r9,r19,r18                                              ; Store register
  625                         addi    r18,r18,8                                               ; Next register
  626                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  627                         
  628                         crset   cr0_eq                                                  ; Set this to see if we failed
  629                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  630                         isync
  631 
  632                         stw             r2,0(r16)                                               ; Store word 0
  633                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  634                         stw             r15,4(r16)                                              ; Store word 1
  635                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  636                         stw             r14,8(r16)                                              ; Store word 2
  637                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  638                         stw             r5,12(r16)                                              ; Store word 3
  639                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  640                         stw             r6,16(r16)                                              ; Store word 4
  641                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  642                         stw             r7,20(r16)                                              ; Store word 5
  643                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  644                         stw             r8,24(r16)                                              ; Store word 6
  645                         bf-             cr0_eq,aaStmwB1                                 ; Error, bail...
  646                         stw             r9,28(r16)                                              ; Store word 7
  647                 
  648                         addi    r16,r16,8*4                                             ; Point up to next output aread
  649                 
  650                 
  651 aaStmwB1:       mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  652                         mtmsr   r30                                                             ; Normal MSR
  653                         isync
  654 
  655                         bt-             cr0_eq,aaStmwNxt                                ; We have more to do and no failed access...
  656                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...
  657 
  658                         .align  5
  659                         
  660 aaStmwNxtH:     cmplwi  cr1,r17,(4*4)                                   ; Do we have at least 4 left?
  661                         blt             cr1,aaStmwL4                                    ; Nope...
  662                         subi    r17,r17,4*4                                             ; Set count properly
  663 
  664                         lwzx    r2,r19,r18                                              ; Store register
  665                         addi    r18,r18,8                                               ; Next register
  666                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  667                         lwzx    r15,r19,r18                                             ; Store register
  668                         addi    r18,r18,8                                               ; Next register
  669                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  670                         lwzx    r14,r19,r18                                             ; Store register
  671                         addi    r18,r18,8                                               ; Next register
  672                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  673                         lwzx    r5,r19,r18                                              ; Store register
  674                         addi    r18,r18,8                                               ; Next register
  675                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  676                         
  677                         crset   cr0_eq                                                  ; Set this to see if we failed
  678                         mtmsr   r22                                                             ; Flip DR, RI
  679                         isync
  680                 
  681                         stw             r2,0(r16)                                               ; Store word 0
  682                         bf-             cr0_eq,aaStmwB2                                 ; Error, bail...
  683                         stw             r15,4(r16)                                              ; Store word 1
  684                         bf-             cr0_eq,aaStmwB2                                 ; Error, bail...
  685                         stw             r14,8(r16)                                              ; Store word 2
  686                         bf-             cr0_eq,aaStmwB2                                 ; Error, bail...
  687                         stw             r5,12(r16)                                              ; Store word 3
  688 
  689                         addi    r16,r16,4*4                                             ; Point up to next input aread
  690                 
  691 aaStmwB2:       mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  692                         mtmsr   r30                                                             ; Normal MSR
  693                         isync
  694 
  695                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  696 
  697 aaStmwL4:       or.             r5,r17,r28                                              ; Do we have anything left to do?
  698                         cmplwi  cr1,r17,(2*4)                                   ; Do we have one, two, or three left?
  699                         cmplwi  cr2,r17,0                                               ; Do we have no full words left?
  700                         beq             aaComExit                                               ; Nothing left...
  701 
  702                         beq-    cr2,aaStmwBy1                                   ; No full words, check out bytes
  703 
  704                         lwzx    r2,r19,r18                                              ; Store register
  705                         addi    r18,r18,8                                               ; Next register
  706                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  707                         blt             cr1,aaStmwBy1                                   ; We only had one, go save it...
  708                         
  709                         lwzx    r15,r19,r18                                             ; Store register
  710                         addi    r18,r18,8                                               ; Next register
  711                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  712                         beq             cr1,aaStmwBy1                                   ; We had two, go save it...
  713                         
  714                         lwzx    r14,r19,r18                                             ; Store register
  715                         addi    r18,r18,8                                               ; Next register
  716                         rlwinm  r18,r18,0,24,28                                 ; Wrap back to 0 if needed
  717                         
  718 aaStmwBy1:      mr.             r28,r28                                                 ; Do we have any trailing bytes?
  719                         beq+    aaStmwSt                                                ; Nope...
  720                         
  721                         lwzx    r5,r19,r18                                              ; Yes, pick up one extra register
  722                         
  723 aaStmwSt:       crset   cr0_eq                                                  ; Set this to see if we failed
  724                         mtmsr   r22                                                             ; Flip DR, RI
  725                         isync
  726 
  727                         beq-    cr2,aaStmwBy2                                   ; No words, check trailing bytes...                                     
  728 
  729                         stw             r2,0(r16)                                               ; Save first word
  730                         bf-             cr0_eq,aaStmwDn                                 ; Store failed, escape...
  731                         addi    r16,r16,4                                               ; Bump sink
  732                         blt             cr1,aaStmwBy2                                   ; We only had one, we are done...
  733 
  734                         stw             r15,0(r16)                                              ; Save second word
  735                         bf-             cr0_eq,aaStmwDn                                 ; Store failed, escape...
  736                         addi    r16,r16,4                                               ; Bump sink
  737                         beq             cr1,aaStmwBy2                                   ; We had two, we are done...
  738 
  739                         stw             r14,0(r16)                                              ; Save third word
  740                         bf-             cr0_eq,aaStmwDn                                 ; Store failed, escape...
  741                         addi    r16,r16,4                                               ; Bump sink
  742                                 
  743 aaStmwBy2:      rlwinm  r2,r5,8,24,31                                   ; Get byte 0
  744                         cmplwi  cr2,r28,0                                               ; Any trailing bytes to do?
  745                         rlwinm  r14,r5,24,24,31                                 ; Get byte 3
  746                         li              r8,0                                                    ; Clear second trailing byte
  747                         cmplwi  cr1,r28,2                                               ; Check for 1, 2, or 3
  748                         li              r9,0                                                    ; Clear third trailing byte
  749                         beq+    cr2,aaStmwDn                                    ; No trailing bytes...
  750                         rlwinm  r15,r5,16,24,31                                 ; Get byte 1
  751 
  752                         stb             r2,0(r16)                                               ; Save first byte
  753                         bf-             cr0_eq,aaStmwDn                                 ; Read failed, escape...
  754                         blt             cr1,aaStmwDn                                    ; We only had one, we are done...
  755 
  756                         stb             r15,1(r16)                                              ; Save second byte
  757                         bf-             cr0_eq,aaStmwDn                                 ; Read failed, escape...
  758                         beq             cr1,aaStmwDn                                    ; We had two, we are done...
  759 
  760                         stb             r14,2(r16)                                              ; Save third byte
  761 
  762 aaStmwDn:       mr              r4,r0                                                   ; Remember DAR, jus in case we failed the access
  763                         mtmsr   r30                                                             ; Normal MSR
  764                         isync
  765 
  766                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  767 
  768                         b               aaComExit                                               ; We are done....
  769 
  770         
  771 ;
  772 ;                       Load String Indexed
  773 ;
  774 
  775                         .align  5
  776                         
  777 aaLswx:         lwz             r17,savexer+4(r13)                              ; Pick up the XER
  778                         crclr   iUpdate                                                 ; Make sure we think this the load form
  779                         rlwinm. r25,r17,0,25,31                                 ; Get the number of bytes to load
  780                         rlwinm  r28,r17,0,30,31                                 ; Get the number of bytes past an even word
  781                         beq-    aaComExit                                               ; Do nothing if 0 length...
  782                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  783                         b               aaLSComm                                                ; Join up with common load/store code...
  784 
  785         
  786 ;
  787 ;                       Load String Immediate
  788 ;
  789 
  790                         .align  5
  791 
  792 aaLswi:         mr              r9,r23                                                  ; Save the DAR
  793                         bl              eIFetch                                                 ; Get the instruction image
  794                         bne-    eRedriveAsISI                                   ; Go redrive this as an ISI...  
  795                         rlwinm  r25,r10,21,27,31                                ; Get the number of bytes to load
  796                         crclr   iUpdate                                                 ; Make sure we think this the load form
  797                         subi    r25,r25,1                                               ; Back off by 1
  798                         rlwinm  r25,r25,0,27,31                                 ; Clear back down
  799                         addi    r25,r25,1                                               ; Add back the 1 to convert 0 to 32
  800                         rlwinm  r28,r25,0,30,31                                 ; Get the number of bytes past an even word
  801                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  802                         mr              r23,r9                                                  ; Move back the DAR
  803                         b               aaLSComm                                                ; Join up with common load/store code...
  804         
  805 ;
  806 ;                       Store String Indexed
  807 ;
  808 
  809                         .align  5
  810 
  811 aaStswx:        lwz             r17,savexer+4(r13)                              ; Pick up the XER
  812                         crclr   iUpdate                                                 ; Make sure this is clear in case we have 0 length
  813                         rlwinm. r25,r17,0,25,31                                 ; Get the number of bytes to load
  814                         rlwinm  r28,r17,0,30,31                                 ; Get the number of bytes past an even word
  815                         beq-    aaComExit                                               ; Do nothing if 0 length...
  816                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  817                         crset   iUpdate                                                 ; Make sure we think this the store form
  818                         b               aaLSComm                                                ; Join up with common load/store code...
  819 
  820         
  821 ;
  822 ;                       Store String Immediate
  823 ;
  824 
  825                         .align  5
  826 
  827 aaStswi:        mr              r9,r23                                                  ; Save the DAR
  828                         bl              eIFetch                                                 ; Get the instruction image
  829                         bne-    eRedriveAsISI                                   ; Go redrive this as an ISI...  
  830                         rlwinm  r25,r10,21,27,31                                ; Get the number of bytes to load
  831                         crclr   iUpdate                                                 ; Make sure we think this the load form
  832                         subi    r25,r25,1                                               ; Back off by 1
  833                         rlwinm  r25,r25,0,27,31                                 ; Clear back down
  834                         addi    r25,r25,1                                               ; Add back the 1 to convert 0 to 32
  835                         rlwinm  r28,r25,21,30,31                                ; Get the number of bytes past an even word
  836                         xor             r17,r25,r28                                             ; Round down to an even word boundary
  837                         mr              r23,r9                                                  ; Move back the DAR
  838                         b               aaLSComm                                                ; Join up with common load/store code...
  839         
  840 
  841 ;
  842 ;                       Load byte-reversed word
  843 ;
  844 
  845                         .align  5
  846 
  847 aaLwbrx:
  848                         add             r18,r18,r13                                             ; Index to source register
  849 
  850                         crset   cr0_eq                                                  ; Set this to see if we failed
  851                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  852                         isync
  853                 
  854                         lwz             r11,0(r23)                                              ; Load the word
  855                 
  856                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  857                         mtmsr   r30                                                             ; Restore normal MSR
  858                         isync
  859                         
  860                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  861                         
  862                         rlwinm  r10,r11,8,0,31                                  ; Get byte 0 to 3 and byte 2 to 1
  863                         rlwimi  r10,r11,24,16,23                                ; Move byte 1 to byte 2
  864                         rlwimi  r10,r11,24,0,7                                  ; Move byte 3 to byte 0
  865                 
  866                         stw             r10,saver0+4(r18)                               ; Set the register
  867 
  868                         b               aaComExit                                               ; All done, go exit...
  869 
  870 
  871 
  872 ;
  873 ;                       Store byte-reversed word
  874 ;
  875 
  876                         .align  5
  877 
  878 aaStwbrx:
  879                         add             r18,r18,r13                                             ; Index to source register
  880                         lwz             r11,saver0+4(r18)                               ; Get the register to store
  881 
  882                         rlwinm  r10,r11,8,0,31                                  ; Get byte 0 to 3 and byte 2 to 1
  883                         rlwimi  r10,r11,24,16,23                                ; Move byte 1 to byte 2
  884                         rlwimi  r10,r11,24,0,7                                  ; Move byte 3 to byte 0
  885                         
  886                         crset   cr0_eq                                                  ; Set this to see if we failed
  887                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  888                         isync
  889                 
  890                         stw             r10,0(r23)                                              ; Store the reversed halfword
  891                 
  892                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  893                         mtmsr   r30                                                             ; Restore normal MSR
  894                         isync
  895                         
  896                         bt+             cr0_eq,aaComExit                                ; All done, go exit...
  897                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...        
  898 
  899 
  900 
  901 ;
  902 ;                       Load byte-reversed halfword
  903 ;
  904 
  905                         .align  5
  906 
  907 aaLhbrx:
  908                         add             r18,r18,r13                                             ; Index to source register
  909 
  910                         crset   cr0_eq                                                  ; Set this to see if we failed
  911                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  912                         isync
  913                 
  914                         lhz             r11,0(r23)                                              ; Load the halfword
  915                 
  916                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  917                         mtmsr   r30                                                             ; Restore normal MSR
  918                         isync
  919 
  920                         bf-             cr0_eq,aaRedriveAsDSI                   ; We failed, go redrive this as a DSI...
  921                         
  922                         rlwinm  r10,r11,8,16,23                                 ; Rotate bottom byte up one and clear everything else
  923                         rlwimi  r10,r11,24,24,31                                ; Put old second from bottom into bottom
  924                 
  925                         stw             r10,saver0+4(r18)                               ; Set the register
  926 
  927                         b               aaComExit                                               ; All done, go exit...
  928 
  929 
  930 ;
  931 ;                       Store byte-reversed halfword
  932 ;
  933 
  934                         .align  5
  935 
  936 aaSthbrx:
  937                         add             r18,r18,r13                                             ; Index to source register
  938                         lwz             r10,saver0+4(r18)                               ; Get the register to store
  939                         rlwinm  r10,r10,8,0,31                                  ; Rotate bottom byte up one
  940                         rlwimi  r10,r10,16,24,31                                ; Put old second from bottom into bottom
  941                         
  942                         crset   cr0_eq                                                  ; Set this to see if we failed
  943                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  944                         isync
  945                 
  946                         sth             r10,0(r23)                                              ; Store the reversed halfword
  947                 
  948                         mr              r4,r0                                                   ; Save the DAR if we failed the access
  949                         mtmsr   r30                                                             ; Restore normal MSR
  950                         isync
  951 
  952                         bt+             cr0_eq,aaComExit                                ; All done, go exit...
  953                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...        
  954 
  955 ;
  956 ;                       Data cache block zero
  957 ;
  958 
  959                         .align  5
  960 
  961 aaDcbz:
  962                         rlwinm  r23,r23,0,0,26                                  ; Round back to a 32-byte boundary
  963                         
  964                         crset   cr0_eq                                                  ; Set this to see if we failed
  965                         li              r0,0                                                    ; Clear this out
  966                         mtmsr   r22                                                             ; Flip DR, RI, and maybe PR on
  967                         isync
  968                         
  969                         stw             r0,0(r23)                                               ; Clear word    
  970                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  971                         stw             r0,4(r23)                                               ; Clear word            
  972                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  973                         stw             r0,8(r23)                                               ; Clear word            
  974                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  975                         stw             r0,12(r23)                                              ; Clear word            
  976                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  977                         stw             r0,16(r23)                                              ; Clear word            
  978                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  979                         stw             r0,20(r23)                                              ; Clear word            
  980                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  981                         stw             r0,24(r23)                                              ; Clear word            
  982                         bne-    aaDcbzXit                                               ; Got DSI, we are stopping...   
  983                         stw             r0,28(r23)                                              ; Clear word            
  984                         
  985 aaDcbzXit:      mr              r4,r0                                                   ; Save the DAR if we failed the access
  986                         mtmsr   r30                                                             ; Restore normal MSR
  987                         isync
  988 
  989                         crclr   iUpdate                                                 ; Make sure we do not think this is an update form
  990                         
  991                         bt+             cr0_eq,aaComExit                                ; All done, go exit...
  992                         b               aaRedriveAsDSI                                  ; We failed, go redrive this as a DSI...        
  993 
  994 
  995 ;
  996 ;                       Unhandled alignment exception, pass it along
  997 ;
  998 
  999 aaPassAlong:
 1000                         b               EXT(EmulExit)                                   
 1001 
 1002 
 1003 
 1004 
 1005 ;
 1006 ;                       We go here to emulate a trace exception after we have handled alignment error
 1007 ;
 1008 
 1009                         .align  5
 1010                         
 1011 aaComExitrd:
 1012                         oris    r9,r9,hi16(SAVredrive)                  ; Set the redrive bit
 1013                         li              r11,T_TRACE                                             ; Set trace interrupt
 1014                         rlwinm  r12,r12,0,16,31                                 ; Clear top half of SRR1
 1015                         stw             r9,SAVflags(r13)                                ; Set the flags
 1016                         stw             r11,saveexception(r13)                  ; Set the exception code
 1017                         b               EXT(EmulExit)                                   ; Exit and do trace interrupt...
 1018                         
 1019 
 1020 
 1021 ;
 1022 ;                       Redrive as a DSI
 1023 
 1024 aaRedriveAsDSI:
 1025                         mr              r20,r1                                                  ; Save the DSISR
 1026                         mr              r21,r4
 1027                         lwz             r4,SAVflags(r13)                                ; Pick up the flags
 1028                         li              r11,T_DATA_ACCESS                               ; Set failing data access code
 1029                         oris    r4,r4,hi16(SAVredrive)                  ; Set the redrive bit
 1030                         stw             r20,savedsisr(r13)                              ; Set the DSISR of failed access
 1031                         stw             r21,savedar+4(r13)                              ; Set the address of the failed access
 1032                         stw             r11,saveexception(r13)                  ; Set the replacement code
 1033                         stw             r4,SAVflags(r13)                                ; Set redrive request
 1034                         b               EXT(EmulExit)                                   ; Bail out to handle ISI...
 1035 
 1036 
 1037 
 1038 ;
 1039 ;                       Table of functions to load or store floating point registers
 1040 ;                       This table is indexed reg||size||dir.  That means that each
 1041 ;                       like load/store pair (e.g., lfd f31/stfd f31) are within the same
 1042 ;                       quadword, which is the current ifetch size.  We expect most of the
 1043 ;                       unaligned accesses to be part of copies, therefore, with this
 1044 ;                       organization, we will save the ifetch of the store after the load.
 1045 ;
 1046 
 1047                         .align  10                                                              ; Make sure we are on a 1k boundary
 1048                         .globl  EXT(aaFPopTable)
 1049                         
 1050 LEXT(aaFPopTable)
 1051                         lfs             f0,emfp0(r31)                                   ; Load single variant
 1052                         blr
 1053 
 1054                         stfs    f0,emfp0(r31)                                   ; Store single variant
 1055                         blr
 1056                         
 1057                         lfd             f0,emfp0(r31)                                   ; Load double variant
 1058                         blr
 1059                         
 1060                         stfd    f0,emfp0(r31)                                   ; Store double variant
 1061                         blr
 1062 
 1063                         lfs             f1,emfp0(r31)                                   ; Load single variant
 1064                         blr
 1065 
 1066                         stfs    f1,emfp0(r31)                                   ; Store single variant
 1067                         blr
 1068                         
 1069                         lfd             f1,emfp0(r31)                                   ; Load double variant
 1070                         blr
 1071                         
 1072                         stfd    f1,emfp0(r31)                                   ; Store double variant
 1073                         blr
 1074 
 1075                         lfs             f2,emfp0(r31)                                   ; Load single variant
 1076                         blr
 1077 
 1078                         stfs    f2,emfp0(r31)                                   ; Store single variant
 1079                         blr
 1080                         
 1081                         lfd             f2,emfp0(r31)                                   ; Load double variant
 1082                         blr
 1083                         
 1084                         stfd    f2,emfp0(r31)                                   ; Store double variant
 1085                         blr
 1086 
 1087                         lfs             f3,emfp0(r31)                                   ; Load single variant
 1088                         blr
 1089 
 1090                         stfs    f3,emfp0(r31)                                   ; Store single variant
 1091                         blr
 1092                         
 1093                         lfd             f3,emfp0(r31)                                   ; Load double variant
 1094                         blr
 1095                         
 1096                         stfd    f3,emfp0(r31)                                   ; Store double variant
 1097                         blr
 1098 
 1099                         lfs             f4,emfp0(r31)                                   ; Load single variant
 1100                         blr
 1101 
 1102                         stfs    f4,emfp0(r31)                                   ; Store single variant
 1103                         blr
 1104                         
 1105                         lfd             f4,emfp0(r31)                                   ; Load double variant
 1106                         blr
 1107                         
 1108                         stfd    f4,emfp0(r31)                                   ; Store double variant
 1109                         blr
 1110 
 1111                         lfs             f5,emfp0(r31)                                   ; Load single variant
 1112                         blr
 1113 
 1114                         stfs    f5,emfp0(r31)                                   ; Store single variant
 1115                         blr
 1116                         
 1117                         lfd             f5,emfp0(r31)                                   ; Load double variant
 1118                         blr
 1119                         
 1120                         stfd    f5,emfp0(r31)                                   ; Store double variant
 1121                         blr
 1122 
 1123                         lfs             f6,emfp0(r31)                                   ; Load single variant
 1124                         blr
 1125 
 1126                         stfs    f6,emfp0(r31)                                   ; Store single variant
 1127                         blr
 1128                         
 1129                         lfd             f6,emfp0(r31)                                   ; Load double variant
 1130                         blr
 1131                         
 1132                         stfd    f6,emfp0(r31)                                   ; Store double variant
 1133                         blr
 1134 
 1135                         lfs             f7,emfp0(r31)                                   ; Load single variant
 1136                         blr
 1137 
 1138                         stfs    f7,emfp0(r31)                                   ; Store single variant
 1139                         blr
 1140                         
 1141                         lfd             f7,emfp0(r31)                                   ; Load double variant
 1142                         blr
 1143                         
 1144                         stfd    f7,emfp0(r31)                                   ; Store double variant
 1145                         blr
 1146 
 1147                         lfs             f8,emfp0(r31)                                   ; Load single variant
 1148                         blr
 1149 
 1150                         stfs    f8,emfp0(r31)                                   ; Store single variant
 1151                         blr
 1152                         
 1153                         lfd             f8,emfp0(r31)                                   ; Load double variant
 1154                         blr
 1155                         
 1156                         stfd    f8,emfp0(r31)                                   ; Store double variant
 1157                         blr
 1158 
 1159                         lfs             f9,emfp0(r31)                                   ; Load single variant
 1160                         blr
 1161 
 1162                         stfs    f9,emfp0(r31)                                   ; Store single variant
 1163                         blr
 1164                         
 1165                         lfd             f9,emfp0(r31)                                   ; Load double variant
 1166                         blr
 1167                         
 1168                         stfd    f9,emfp0(r31)                                   ; Store double variant
 1169                         blr
 1170 
 1171                         lfs             f10,emfp0(r31)                                  ; Load single variant
 1172                         blr
 1173 
 1174                         stfs    f10,emfp0(r31)                                  ; Store single variant
 1175                         blr
 1176                         
 1177                         lfd             f10,emfp0(r31)                                  ; Load double variant
 1178                         blr
 1179                         
 1180                         stfd    f10,emfp0(r31)                                  ; Store double variant
 1181                         blr
 1182 
 1183                         lfs             f11,emfp0(r31)                                  ; Load single variant
 1184                         blr
 1185 
 1186                         stfs    f11,emfp0(r31)                                  ; Store single variant
 1187                         blr
 1188                         
 1189                         lfd             f11,emfp0(r31)                                  ; Load double variant
 1190                         blr
 1191                         
 1192                         stfd    f11,emfp0(r31)                                  ; Store double variant
 1193                         blr
 1194 
 1195                         lfs             f12,emfp0(r31)                                  ; Load single variant
 1196                         blr
 1197 
 1198                         stfs    f12,emfp0(r31)                                  ; Store single variant
 1199                         blr
 1200                         
 1201                         lfd             f12,emfp0(r31)                                  ; Load double variant
 1202                         blr
 1203                         
 1204                         stfd    f12,emfp0(r31)                                  ; Store double variant
 1205                         blr
 1206 
 1207                         lfs             f13,emfp0(r31)                                  ; Load single variant
 1208                         blr
 1209 
 1210                         stfs    f13,emfp0(r31)                                  ; Store single variant
 1211                         blr
 1212                         
 1213                         lfd             f13,emfp0(r31)                                  ; Load double variant
 1214                         blr
 1215                         
 1216                         stfd    f13,emfp0(r31)                                  ; Store double variant
 1217                         blr
 1218 
 1219                         lfs             f14,emfp0(r31)                                  ; Load single variant
 1220                         blr
 1221 
 1222                         stfs    f14,emfp0(r31)                                  ; Store single variant
 1223                         blr
 1224                         
 1225                         lfd             f14,emfp0(r31)                                  ; Load double variant
 1226                         blr
 1227                         
 1228                         stfd    f14,emfp0(r31)                                  ; Store double variant
 1229                         blr
 1230 
 1231                         lfs             f15,emfp0(r31)                                  ; Load single variant
 1232                         blr
 1233 
 1234                         stfs    f15,emfp0(r31)                                  ; Store single variant
 1235                         blr
 1236                         
 1237                         lfd             f15,emfp0(r31)                                  ; Load double variant
 1238                         blr
 1239                         
 1240                         stfd    f15,emfp0(r31)                                  ; Store double variant
 1241                         blr
 1242 
 1243                         lfs             f16,emfp0(r31)                                  ; Load single variant
 1244                         blr
 1245 
 1246                         stfs    f16,emfp0(r31)                                  ; Store single variant
 1247                         blr
 1248                         
 1249                         lfd             f16,emfp0(r31)                                  ; Load double variant
 1250                         blr
 1251                         
 1252                         stfd    f16,emfp0(r31)                                  ; Store double variant
 1253                         blr
 1254 
 1255                         lfs             f17,emfp0(r31)                                  ; Load single variant
 1256                         blr
 1257 
 1258                         stfs    f17,emfp0(r31)                                  ; Store single variant
 1259                         blr
 1260                         
 1261                         lfd             f17,emfp0(r31)                                  ; Load double variant
 1262                         blr
 1263                         
 1264                         stfd    f17,emfp0(r31)                                  ; Store double variant
 1265                         blr
 1266 
 1267                         lfs             f18,emfp0(r31)                                  ; Load single variant
 1268                         blr
 1269 
 1270                         stfs    f18,emfp0(r31)                                  ; Store single variant
 1271                         blr
 1272                         
 1273                         lfd             f18,emfp0(r31)                                  ; Load double variant
 1274                         blr
 1275                         
 1276                         stfd    f18,emfp0(r31)                                  ; Store double variant
 1277                         blr
 1278 
 1279                         lfs             f19,emfp0(r31)                                  ; Load single variant
 1280                         blr
 1281 
 1282                         stfs    f19,emfp0(r31)                                  ; Store single variant
 1283                         blr
 1284                         
 1285                         lfd             f19,emfp0(r31)                                  ; Load double variant
 1286                         blr
 1287                         
 1288                         stfd    f19,emfp0(r31)                                  ; Store double variant
 1289                         blr
 1290 
 1291                         lfs             f20,emfp0(r31)                                  ; Load single variant
 1292                         blr
 1293 
 1294                         stfs    f20,emfp0(r31)                                  ; Store single variant
 1295                         blr
 1296                         
 1297                         lfd             f20,emfp0(r31)                                  ; Load double variant
 1298                         blr
 1299                         
 1300                         stfd    f20,emfp0(r31)                                  ; Store double variant
 1301                         blr
 1302 
 1303                         lfs             f21,emfp0(r31)                                  ; Load single variant
 1304                         blr
 1305 
 1306                         stfs    f21,emfp0(r31)                                  ; Store single variant
 1307                         blr
 1308                         
 1309                         lfd             f21,emfp0(r31)                                  ; Load double variant
 1310                         blr
 1311                         
 1312                         stfd    f21,emfp0(r31)                                  ; Store double variant
 1313                         blr
 1314 
 1315                         lfs             f22,emfp0(r31)                                  ; Load single variant
 1316                         blr
 1317 
 1318                         stfs    f22,emfp0(r31)                                  ; Store single variant
 1319                         blr
 1320                         
 1321                         lfd             f22,emfp0(r31)                                  ; Load double variant
 1322                         blr
 1323                         
 1324                         stfd    f22,emfp0(r31)                                  ; Store double variant
 1325                         blr
 1326 
 1327                         lfs             f23,emfp0(r31)                                  ; Load single variant
 1328                         blr
 1329 
 1330                         stfs    f23,emfp0(r31)                                  ; Store single variant
 1331                         blr
 1332                         
 1333                         lfd             f23,emfp0(r31)                                  ; Load double variant
 1334                         blr
 1335                         
 1336                         stfd    f23,emfp0(r31)                                  ; Store double variant
 1337                         blr
 1338 
 1339                         lfs             f24,emfp0(r31)                                  ; Load single variant
 1340                         blr
 1341 
 1342                         stfs    f24,emfp0(r31)                                  ; Store single variant
 1343                         blr
 1344                         
 1345                         lfd             f24,emfp0(r31)                                  ; Load double variant
 1346                         blr
 1347                         
 1348                         stfd    f24,emfp0(r31)                                  ; Store double variant
 1349                         blr
 1350 
 1351                         lfs             f25,emfp0(r31)                                  ; Load single variant
 1352                         blr
 1353 
 1354                         stfs    f25,emfp0(r31)                                  ; Store single variant
 1355                         blr
 1356                         
 1357                         lfd             f25,emfp0(r31)                                  ; Load double variant
 1358                         blr
 1359                         
 1360                         stfd    f25,emfp0(r31)                                  ; Store double variant
 1361                         blr
 1362 
 1363                         lfs             f26,emfp0(r31)                                  ; Load single variant
 1364                         blr
 1365 
 1366                         stfs    f26,emfp0(r31)                                  ; Store single variant
 1367                         blr
 1368                         
 1369                         lfd             f26,emfp0(r31)                                  ; Load double variant
 1370                         blr
 1371                         
 1372                         stfd    f26,emfp0(r31)                                  ; Store double variant
 1373                         blr
 1374 
 1375                         lfs             f27,emfp0(r31)                                  ; Load single variant
 1376                         blr
 1377 
 1378                         stfs    f27,emfp0(r31)                                  ; Store single variant
 1379                         blr
 1380                         
 1381                         lfd             f27,emfp0(r31)                                  ; Load double variant
 1382                         blr
 1383                         
 1384                         stfd    f27,emfp0(r31)                                  ; Store double variant
 1385                         blr
 1386 
 1387                         lfs             f28,emfp0(r31)                                  ; Load single variant
 1388                         blr
 1389 
 1390                         stfs    f28,emfp0(r31)                                  ; Store single variant
 1391                         blr
 1392                         
 1393                         lfd             f28,emfp0(r31)                                  ; Load double variant
 1394                         blr
 1395                         
 1396                         stfd    f28,emfp0(r31)                                  ; Store double variant
 1397                         blr
 1398 
 1399                         lfs             f29,emfp0(r31)                                  ; Load single variant
 1400                         blr
 1401 
 1402                         stfs    f29,emfp0(r31)                                  ; Store single variant
 1403                         blr
 1404                         
 1405                         lfd             f29,emfp0(r31)                                  ; Load double variant
 1406                         blr
 1407                         
 1408                         stfd    f29,emfp0(r31)                                  ; Store double variant
 1409                         blr
 1410 
 1411                         lfs             f30,emfp0(r31)                                  ; Load single variant
 1412                         blr
 1413 
 1414                         stfs    f30,emfp0(r31)                                  ; Store single variant
 1415                         blr
 1416                         
 1417                         lfd             f30,emfp0(r31)                                  ; Load double variant
 1418                         blr
 1419                         
 1420                         stfd    f30,emfp0(r31)                                  ; Store double variant
 1421                         blr
 1422 
 1423                         lfs             f31,emfp0(r31)                                  ; Load single variant
 1424                         blr
 1425 
 1426                         stfs    f31,emfp0(r31)                                  ; Store single variant
 1427                         blr
 1428                         
 1429                         lfd             f31,emfp0(r31)                                  ; Load double variant
 1430                         blr
 1431                         
 1432                         stfd    f31,emfp0(r31)                                  ; Store double variant
 1433                         blr
 1434 

Cache object: 1ceec235dde0876e46b80f4565811039


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.