The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * CDDL HEADER START
    3  *
    4  * The contents of this file are subject to the terms of the
    5  * Common Development and Distribution License (the "License").
    6  * You may not use this file except in compliance with the License.
    7  *
    8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
    9  * or https://opensource.org/licenses/CDDL-1.0.
   10  * See the License for the specific language governing permissions
   11  * and limitations under the License.
   12  *
   13  * When distributing Covered Code, include this CDDL HEADER in each
   14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   15  * If applicable, add the following below this CDDL HEADER, with the
   16  * fields enclosed by brackets "[]" replaced with your own identifying
   17  * information: Portions Copyright [yyyy] [name of copyright owner]
   18  *
   19  * CDDL HEADER END
   20  */
   21 
   22 /*
   23  * Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3
   24  * Copyright (c) 2019-2022 Samuel Neves and Matthew Krupcale
   25  * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
   26  *
   27  * This is converted assembly: SSE2 -> ARMv8-A
   28  * Used tools: SIMDe https://github.com/simd-everywhere/simde
   29  */
   30 
   31 #if defined(__aarch64__)
   32         .text
   33         .section        .rodata.cst16,"aM",@progbits,16
   34         .p2align        4
   35 .LCPI0_0:
   36         .word   1779033703
   37         .word   3144134277
   38         .word   1013904242
   39         .word   2773480762
   40 .LCPI0_1:
   41         .xword  0
   42         .xword  -4294967296
   43 .LCPI0_2:
   44         .xword  -1
   45         .xword  4294967295
   46         .text
   47         .globl  zfs_blake3_compress_in_place_sse2
   48         .p2align        2
   49         .type   zfs_blake3_compress_in_place_sse2,@function
   50 zfs_blake3_compress_in_place_sse2:
   51         .cfi_startproc
   52         ldp     q3, q2, [x0]
   53         ldp     q5, q6, [x1]
   54         add     x10, x1, #32
   55         lsr     x11, x3, #32
   56         fmov    s4, w3
   57         ld2     { v17.4s, v18.4s }, [x10]
   58         adrp    x10, .LCPI0_2
   59         and     w8, w2, #0xff
   60         mov     v4.s[1], w11
   61         ldr     q1, [x10, :lo12:.LCPI0_2]
   62         and     w9, w4, #0xff
   63         adrp    x12, .LCPI0_0
   64         mov     v4.s[2], w8
   65         uzp1    v19.4s, v5.4s, v6.4s
   66         add     v3.4s, v2.4s, v3.4s
   67         ldr     q7, [x12, :lo12:.LCPI0_0]
   68         mov     v4.s[3], w9
   69         add     v3.4s, v3.4s, v19.4s
   70         uzp2    v5.4s, v5.4s, v6.4s
   71         ext     v21.16b, v18.16b, v18.16b, #12
   72         uzp1    v6.4s, v19.4s, v19.4s
   73         ext     v22.16b, v19.16b, v19.16b, #12
   74         eor     v4.16b, v3.16b, v4.16b
   75         ext     v20.16b, v17.16b, v17.16b, #12
   76         ext     v6.16b, v6.16b, v19.16b, #8
   77         ext     v19.16b, v19.16b, v22.16b, #12
   78         zip1    v22.2d, v21.2d, v5.2d
   79         rev32   v24.8h, v4.8h
   80         mov     v4.16b, v1.16b
   81         zip2    v23.4s, v5.4s, v21.4s
   82         uzp2    v6.4s, v6.4s, v5.4s
   83         bsl     v4.16b, v22.16b, v20.16b
   84         add     v3.4s, v3.4s, v5.4s
   85         zip1    v5.4s, v23.4s, v20.4s
   86         zip1    v22.4s, v20.4s, v23.4s
   87         add     v23.4s, v24.4s, v7.4s
   88         ext     v7.16b, v6.16b, v6.16b, #4
   89         ext     v25.16b, v4.16b, v4.16b, #12
   90         ext     v5.16b, v22.16b, v5.16b, #8
   91         eor     v2.16b, v23.16b, v2.16b
   92         uzp1    v4.4s, v4.4s, v25.4s
   93         uzp1    v22.4s, v7.4s, v7.4s
   94         ext     v25.16b, v7.16b, v7.16b, #12
   95         ext     v22.16b, v22.16b, v7.16b, #8
   96         ext     v7.16b, v7.16b, v25.16b, #12
   97         ushr    v25.4s, v2.4s, #12
   98         shl     v2.4s, v2.4s, #20
   99         orr     v2.16b, v2.16b, v25.16b
  100         add     v3.4s, v3.4s, v2.4s
  101         eor     v24.16b, v3.16b, v24.16b
  102         add     v3.4s, v3.4s, v17.4s
  103         ushr    v17.4s, v24.4s, #8
  104         shl     v18.4s, v24.4s, #24
  105         orr     v17.16b, v18.16b, v17.16b
  106         add     v18.4s, v17.4s, v23.4s
  107         eor     v2.16b, v18.16b, v2.16b
  108         ushr    v23.4s, v2.4s, #7
  109         shl     v2.4s, v2.4s, #25
  110         ext     v3.16b, v3.16b, v3.16b, #12
  111         orr     v2.16b, v2.16b, v23.16b
  112         ext     v17.16b, v17.16b, v17.16b, #8
  113         add     v3.4s, v2.4s, v3.4s
  114         adrp    x11, .LCPI0_1
  115         eor     v17.16b, v3.16b, v17.16b
  116         ldr     q16, [x11, :lo12:.LCPI0_1]
  117         ext     v18.16b, v18.16b, v18.16b, #4
  118         rev32   v24.8h, v17.8h
  119         movi    v0.2d, #0xffffffff00000000
  120         add     v23.4s, v3.4s, v21.4s
  121         mov     v21.s[1], v20.s[2]
  122         add     v20.4s, v18.4s, v24.4s
  123         bit     v19.16b, v21.16b, v0.16b
  124         eor     v3.16b, v20.16b, v2.16b
  125         uzp2    v2.4s, v22.4s, v19.4s
  126         zip1    v17.2d, v5.2d, v19.2d
  127         zip2    v18.4s, v19.4s, v5.4s
  128         ushr    v21.4s, v3.4s, #12
  129         shl     v3.4s, v3.4s, #20
  130         ext     v22.16b, v2.16b, v2.16b, #4
  131         bsl     v16.16b, v4.16b, v17.16b
  132         zip1    v17.4s, v18.4s, v4.4s
  133         zip1    v18.4s, v4.4s, v18.4s
  134         orr     v21.16b, v3.16b, v21.16b
  135         ext     v25.16b, v16.16b, v16.16b, #12
  136         ext     v3.16b, v18.16b, v17.16b, #8
  137         uzp1    v18.4s, v22.4s, v22.4s
  138         ext     v26.16b, v22.16b, v22.16b, #12
  139         add     v23.4s, v23.4s, v21.4s
  140         uzp1    v17.4s, v16.4s, v25.4s
  141         ext     v16.16b, v18.16b, v22.16b, #8
  142         ext     v18.16b, v22.16b, v26.16b, #12
  143         eor     v22.16b, v23.16b, v24.16b
  144         add     v6.4s, v23.4s, v6.4s
  145         ushr    v23.4s, v22.4s, #8
  146         shl     v22.4s, v22.4s, #24
  147         orr     v22.16b, v22.16b, v23.16b
  148         add     v20.4s, v22.4s, v20.4s
  149         eor     v21.16b, v20.16b, v21.16b
  150         ushr    v23.4s, v21.4s, #7
  151         shl     v21.4s, v21.4s, #25
  152         ext     v6.16b, v6.16b, v6.16b, #4
  153         orr     v21.16b, v21.16b, v23.16b
  154         ext     v22.16b, v22.16b, v22.16b, #8
  155         add     v6.4s, v21.4s, v6.4s
  156         eor     v22.16b, v6.16b, v22.16b
  157         ext     v20.16b, v20.16b, v20.16b, #12
  158         add     v6.4s, v6.4s, v19.4s
  159         rev32   v19.8h, v22.8h
  160         add     v20.4s, v20.4s, v19.4s
  161         eor     v21.16b, v20.16b, v21.16b
  162         ushr    v22.4s, v21.4s, #12
  163         shl     v21.4s, v21.4s, #20
  164         orr     v21.16b, v21.16b, v22.16b
  165         add     v6.4s, v6.4s, v21.4s
  166         eor     v19.16b, v6.16b, v19.16b
  167         ushr    v22.4s, v19.4s, #8
  168         shl     v19.4s, v19.4s, #24
  169         orr     v19.16b, v19.16b, v22.16b
  170         add     v20.4s, v19.4s, v20.4s
  171         eor     v21.16b, v20.16b, v21.16b
  172         ext     v6.16b, v6.16b, v6.16b, #12
  173         ushr    v22.4s, v21.4s, #7
  174         shl     v21.4s, v21.4s, #25
  175         add     v6.4s, v6.4s, v4.4s
  176         orr     v21.16b, v21.16b, v22.16b
  177         ext     v19.16b, v19.16b, v19.16b, #8
  178         add     v6.4s, v6.4s, v21.4s
  179         eor     v19.16b, v6.16b, v19.16b
  180         ext     v20.16b, v20.16b, v20.16b, #4
  181         rev32   v19.8h, v19.8h
  182         add     v20.4s, v20.4s, v19.4s
  183         add     v6.4s, v6.4s, v5.4s
  184         mov     v5.s[1], v4.s[2]
  185         eor     v4.16b, v20.16b, v21.16b
  186         ushr    v21.4s, v4.4s, #12
  187         shl     v4.4s, v4.4s, #20
  188         orr     v21.16b, v4.16b, v21.16b
  189         add     v6.4s, v6.4s, v21.4s
  190         eor     v19.16b, v6.16b, v19.16b
  191         add     v2.4s, v6.4s, v2.4s
  192         ushr    v6.4s, v19.4s, #8
  193         shl     v19.4s, v19.4s, #24
  194         orr     v6.16b, v19.16b, v6.16b
  195         add     v19.4s, v6.4s, v20.4s
  196         eor     v20.16b, v19.16b, v21.16b
  197         ushr    v21.4s, v20.4s, #7
  198         shl     v20.4s, v20.4s, #25
  199         ext     v2.16b, v2.16b, v2.16b, #4
  200         orr     v20.16b, v20.16b, v21.16b
  201         ext     v6.16b, v6.16b, v6.16b, #8
  202         add     v2.4s, v20.4s, v2.4s
  203         eor     v6.16b, v2.16b, v6.16b
  204         ext     v19.16b, v19.16b, v19.16b, #12
  205         rev32   v6.8h, v6.8h
  206         add     v19.4s, v19.4s, v6.4s
  207         mov     v22.16b, v0.16b
  208         eor     v20.16b, v19.16b, v20.16b
  209         bsl     v22.16b, v5.16b, v7.16b
  210         ushr    v21.4s, v20.4s, #12
  211         shl     v20.4s, v20.4s, #20
  212         add     v2.4s, v2.4s, v22.4s
  213         orr     v20.16b, v20.16b, v21.16b
  214         add     v2.4s, v2.4s, v20.4s
  215         eor     v6.16b, v2.16b, v6.16b
  216         ushr    v21.4s, v6.4s, #8
  217         shl     v6.4s, v6.4s, #24
  218         orr     v6.16b, v6.16b, v21.16b
  219         add     v19.4s, v6.4s, v19.4s
  220         eor     v20.16b, v19.16b, v20.16b
  221         ext     v2.16b, v2.16b, v2.16b, #12
  222         ushr    v21.4s, v20.4s, #7
  223         shl     v20.4s, v20.4s, #25
  224         add     v2.4s, v2.4s, v17.4s
  225         orr     v20.16b, v20.16b, v21.16b
  226         ext     v6.16b, v6.16b, v6.16b, #8
  227         add     v2.4s, v2.4s, v20.4s
  228         eor     v6.16b, v2.16b, v6.16b
  229         uzp2    v5.4s, v16.4s, v22.4s
  230         zip1    v7.2d, v3.2d, v22.2d
  231         zip2    v16.4s, v22.4s, v3.4s
  232         ext     v19.16b, v19.16b, v19.16b, #4
  233         rev32   v22.8h, v6.8h
  234         ext     v23.16b, v5.16b, v5.16b, #4
  235         bif     v7.16b, v17.16b, v1.16b
  236         zip1    v24.4s, v16.4s, v17.4s
  237         zip1    v16.4s, v17.4s, v16.4s
  238         add     v21.4s, v2.4s, v3.4s
  239         mov     v3.s[1], v17.s[2]
  240         add     v17.4s, v19.4s, v22.4s
  241         mov     v19.16b, v0.16b
  242         ext     v25.16b, v7.16b, v7.16b, #12
  243         ext     v4.16b, v16.16b, v24.16b, #8
  244         uzp1    v16.4s, v23.4s, v23.4s
  245         bsl     v19.16b, v3.16b, v18.16b
  246         eor     v2.16b, v17.16b, v20.16b
  247         uzp1    v7.4s, v7.4s, v25.4s
  248         ext     v25.16b, v16.16b, v23.16b, #8
  249         zip1    v3.2d, v4.2d, v19.2d
  250         ushr    v20.4s, v2.4s, #12
  251         shl     v2.4s, v2.4s, #20
  252         ext     v24.16b, v23.16b, v23.16b, #12
  253         uzp2    v6.4s, v25.4s, v19.4s
  254         zip2    v18.4s, v19.4s, v4.4s
  255         bif     v3.16b, v7.16b, v1.16b
  256         orr     v20.16b, v2.16b, v20.16b
  257         ext     v16.16b, v23.16b, v24.16b, #12
  258         ext     v23.16b, v6.16b, v6.16b, #4
  259         zip1    v24.4s, v18.4s, v7.4s
  260         zip1    v18.4s, v7.4s, v18.4s
  261         ext     v25.16b, v3.16b, v3.16b, #12
  262         add     v21.4s, v21.4s, v20.4s
  263         ext     v2.16b, v18.16b, v24.16b, #8
  264         uzp1    v18.4s, v23.4s, v23.4s
  265         ext     v24.16b, v23.16b, v23.16b, #12
  266         uzp1    v3.4s, v3.4s, v25.4s
  267         eor     v22.16b, v21.16b, v22.16b
  268         ext     v25.16b, v18.16b, v23.16b, #8
  269         dup     v18.4s, v2.s[3]
  270         ext     v23.16b, v23.16b, v24.16b, #12
  271         add     v5.4s, v21.4s, v5.4s
  272         trn1    v21.4s, v3.4s, v3.4s
  273         ushr    v24.4s, v22.4s, #8
  274         shl     v22.4s, v22.4s, #24
  275         ext     v18.16b, v21.16b, v18.16b, #8
  276         orr     v21.16b, v22.16b, v24.16b
  277         add     v17.4s, v21.4s, v17.4s
  278         eor     v20.16b, v17.16b, v20.16b
  279         ushr    v22.4s, v20.4s, #7
  280         shl     v20.4s, v20.4s, #25
  281         ext     v5.16b, v5.16b, v5.16b, #4
  282         orr     v20.16b, v20.16b, v22.16b
  283         ext     v21.16b, v21.16b, v21.16b, #8
  284         add     v5.4s, v20.4s, v5.4s
  285         eor     v21.16b, v5.16b, v21.16b
  286         ext     v17.16b, v17.16b, v17.16b, #12
  287         add     v5.4s, v5.4s, v19.4s
  288         rev32   v19.8h, v21.8h
  289         add     v17.4s, v17.4s, v19.4s
  290         eor     v20.16b, v17.16b, v20.16b
  291         ushr    v21.4s, v20.4s, #12
  292         shl     v20.4s, v20.4s, #20
  293         orr     v20.16b, v20.16b, v21.16b
  294         add     v5.4s, v5.4s, v20.4s
  295         eor     v19.16b, v5.16b, v19.16b
  296         ushr    v21.4s, v19.4s, #8
  297         shl     v19.4s, v19.4s, #24
  298         orr     v19.16b, v19.16b, v21.16b
  299         add     v17.4s, v19.4s, v17.4s
  300         eor     v20.16b, v17.16b, v20.16b
  301         ext     v5.16b, v5.16b, v5.16b, #12
  302         ushr    v21.4s, v20.4s, #7
  303         shl     v20.4s, v20.4s, #25
  304         add     v5.4s, v5.4s, v7.4s
  305         orr     v20.16b, v20.16b, v21.16b
  306         ext     v19.16b, v19.16b, v19.16b, #8
  307         add     v5.4s, v5.4s, v20.4s
  308         eor     v19.16b, v5.16b, v19.16b
  309         ext     v17.16b, v17.16b, v17.16b, #4
  310         rev32   v22.8h, v19.8h
  311         add     v21.4s, v5.4s, v4.4s
  312         mov     v4.s[1], v7.s[2]
  313         add     v19.4s, v17.4s, v22.4s
  314         bit     v16.16b, v4.16b, v0.16b
  315         eor     v5.16b, v19.16b, v20.16b
  316         uzp2    v4.4s, v25.4s, v16.4s
  317         zip1    v7.2d, v2.2d, v16.2d
  318         zip2    v17.4s, v16.4s, v2.4s
  319         ushr    v20.4s, v5.4s, #12
  320         shl     v5.4s, v5.4s, #20
  321         ext     v24.16b, v4.16b, v4.16b, #4
  322         bif     v7.16b, v3.16b, v1.16b
  323         zip1    v25.4s, v17.4s, v3.4s
  324         zip1    v17.4s, v3.4s, v17.4s
  325         orr     v20.16b, v5.16b, v20.16b
  326         ext     v26.16b, v7.16b, v7.16b, #12
  327         ext     v5.16b, v17.16b, v25.16b, #8
  328         uzp1    v17.4s, v24.4s, v24.4s
  329         ext     v25.16b, v24.16b, v24.16b, #12
  330         bit     v23.16b, v18.16b, v0.16b
  331         add     v21.4s, v21.4s, v20.4s
  332         uzp1    v7.4s, v7.4s, v26.4s
  333         ext     v26.16b, v17.16b, v24.16b, #8
  334         ext     v17.16b, v24.16b, v25.16b, #12
  335         eor     v22.16b, v21.16b, v22.16b
  336         add     v6.4s, v21.4s, v6.4s
  337         zip1    v21.2d, v5.2d, v23.2d
  338         zip2    v24.4s, v23.4s, v5.4s
  339         bif     v21.16b, v7.16b, v1.16b
  340         zip1    v1.4s, v24.4s, v7.4s
  341         zip1    v24.4s, v7.4s, v24.4s
  342         ext     v1.16b, v24.16b, v1.16b, #8
  343         ushr    v24.4s, v22.4s, #8
  344         shl     v22.4s, v22.4s, #24
  345         orr     v22.16b, v22.16b, v24.16b
  346         add     v19.4s, v22.4s, v19.4s
  347         ext     v24.16b, v21.16b, v21.16b, #12
  348         eor     v20.16b, v19.16b, v20.16b
  349         uzp1    v21.4s, v21.4s, v24.4s
  350         ushr    v24.4s, v20.4s, #7
  351         shl     v20.4s, v20.4s, #25
  352         orr     v20.16b, v20.16b, v24.16b
  353         ext     v6.16b, v6.16b, v6.16b, #4
  354         ext     v22.16b, v22.16b, v22.16b, #8
  355         add     v6.4s, v20.4s, v6.4s
  356         eor     v22.16b, v6.16b, v22.16b
  357         ext     v19.16b, v19.16b, v19.16b, #12
  358         add     v6.4s, v6.4s, v16.4s
  359         rev32   v16.8h, v22.8h
  360         add     v19.4s, v19.4s, v16.4s
  361         eor     v20.16b, v19.16b, v20.16b
  362         ushr    v22.4s, v20.4s, #12
  363         shl     v20.4s, v20.4s, #20
  364         orr     v20.16b, v20.16b, v22.16b
  365         add     v6.4s, v6.4s, v20.4s
  366         eor     v16.16b, v6.16b, v16.16b
  367         ext     v6.16b, v6.16b, v6.16b, #12
  368         add     v3.4s, v6.4s, v3.4s
  369         ushr    v6.4s, v16.4s, #8
  370         shl     v16.4s, v16.4s, #24
  371         orr     v6.16b, v16.16b, v6.16b
  372         add     v16.4s, v6.4s, v19.4s
  373         eor     v19.16b, v16.16b, v20.16b
  374         ushr    v20.4s, v19.4s, #7
  375         shl     v19.4s, v19.4s, #25
  376         orr     v19.16b, v19.16b, v20.16b
  377         ext     v6.16b, v6.16b, v6.16b, #8
  378         add     v3.4s, v3.4s, v19.4s
  379         eor     v6.16b, v3.16b, v6.16b
  380         ext     v16.16b, v16.16b, v16.16b, #4
  381         add     v2.4s, v3.4s, v2.4s
  382         rev32   v3.8h, v6.8h
  383         add     v6.4s, v16.4s, v3.4s
  384         eor     v16.16b, v6.16b, v19.16b
  385         ushr    v19.4s, v16.4s, #12
  386         shl     v16.4s, v16.4s, #20
  387         orr     v16.16b, v16.16b, v19.16b
  388         add     v2.4s, v2.4s, v16.4s
  389         eor     v3.16b, v2.16b, v3.16b
  390         add     v2.4s, v2.4s, v4.4s
  391         ushr    v4.4s, v3.4s, #8
  392         shl     v3.4s, v3.4s, #24
  393         orr     v3.16b, v3.16b, v4.16b
  394         add     v4.4s, v3.4s, v6.4s
  395         eor     v6.16b, v4.16b, v16.16b
  396         ushr    v16.4s, v6.4s, #7
  397         shl     v6.4s, v6.4s, #25
  398         ext     v2.16b, v2.16b, v2.16b, #4
  399         orr     v6.16b, v6.16b, v16.16b
  400         ext     v3.16b, v3.16b, v3.16b, #8
  401         add     v2.4s, v6.4s, v2.4s
  402         eor     v3.16b, v2.16b, v3.16b
  403         ext     v4.16b, v4.16b, v4.16b, #12
  404         rev32   v3.8h, v3.8h
  405         add     v4.4s, v4.4s, v3.4s
  406         eor     v6.16b, v4.16b, v6.16b
  407         ushr    v16.4s, v6.4s, #12
  408         shl     v6.4s, v6.4s, #20
  409         add     v2.4s, v2.4s, v23.4s
  410         orr     v6.16b, v6.16b, v16.16b
  411         add     v2.4s, v2.4s, v6.4s
  412         eor     v3.16b, v2.16b, v3.16b
  413         ushr    v16.4s, v3.4s, #8
  414         shl     v3.4s, v3.4s, #24
  415         orr     v3.16b, v3.16b, v16.16b
  416         add     v4.4s, v3.4s, v4.4s
  417         eor     v6.16b, v4.16b, v6.16b
  418         ext     v2.16b, v2.16b, v2.16b, #12
  419         ushr    v16.4s, v6.4s, #7
  420         shl     v6.4s, v6.4s, #25
  421         add     v2.4s, v2.4s, v7.4s
  422         orr     v6.16b, v6.16b, v16.16b
  423         ext     v3.16b, v3.16b, v3.16b, #8
  424         add     v2.4s, v2.4s, v6.4s
  425         eor     v3.16b, v2.16b, v3.16b
  426         ext     v4.16b, v4.16b, v4.16b, #4
  427         rev32   v3.8h, v3.8h
  428         add     v2.4s, v2.4s, v5.4s
  429         mov     v5.s[1], v7.s[2]
  430         add     v4.4s, v4.4s, v3.4s
  431         bsl     v0.16b, v5.16b, v17.16b
  432         eor     v5.16b, v4.16b, v6.16b
  433         ushr    v6.4s, v5.4s, #12
  434         shl     v5.4s, v5.4s, #20
  435         orr     v5.16b, v5.16b, v6.16b
  436         add     v2.4s, v2.4s, v5.4s
  437         eor     v3.16b, v2.16b, v3.16b
  438         ushr    v6.4s, v3.4s, #8
  439         shl     v3.4s, v3.4s, #24
  440         orr     v3.16b, v3.16b, v6.16b
  441         add     v4.4s, v3.4s, v4.4s
  442         uzp2    v18.4s, v26.4s, v18.4s
  443         eor     v5.16b, v4.16b, v5.16b
  444         add     v2.4s, v2.4s, v18.4s
  445         ushr    v6.4s, v5.4s, #7
  446         shl     v5.4s, v5.4s, #25
  447         ext     v2.16b, v2.16b, v2.16b, #4
  448         orr     v5.16b, v5.16b, v6.16b
  449         ext     v3.16b, v3.16b, v3.16b, #8
  450         add     v2.4s, v5.4s, v2.4s
  451         eor     v3.16b, v2.16b, v3.16b
  452         ext     v4.16b, v4.16b, v4.16b, #12
  453         add     v0.4s, v2.4s, v0.4s
  454         rev32   v2.8h, v3.8h
  455         add     v3.4s, v4.4s, v2.4s
  456         eor     v4.16b, v3.16b, v5.16b
  457         ushr    v5.4s, v4.4s, #12
  458         shl     v4.4s, v4.4s, #20
  459         orr     v4.16b, v4.16b, v5.16b
  460         add     v0.4s, v0.4s, v4.4s
  461         eor     v2.16b, v0.16b, v2.16b
  462         ushr    v5.4s, v2.4s, #8
  463         shl     v2.4s, v2.4s, #24
  464         orr     v2.16b, v2.16b, v5.16b
  465         add     v3.4s, v2.4s, v3.4s
  466         eor     v4.16b, v3.16b, v4.16b
  467         ext     v0.16b, v0.16b, v0.16b, #12
  468         ushr    v5.4s, v4.4s, #7
  469         shl     v4.4s, v4.4s, #25
  470         add     v0.4s, v0.4s, v21.4s
  471         orr     v4.16b, v4.16b, v5.16b
  472         ext     v2.16b, v2.16b, v2.16b, #8
  473         add     v0.4s, v0.4s, v4.4s
  474         eor     v2.16b, v0.16b, v2.16b
  475         ext     v3.16b, v3.16b, v3.16b, #4
  476         add     v0.4s, v0.4s, v1.4s
  477         rev32   v1.8h, v2.8h
  478         add     v2.4s, v3.4s, v1.4s
  479         eor     v3.16b, v2.16b, v4.16b
  480         ushr    v4.4s, v3.4s, #12
  481         shl     v3.4s, v3.4s, #20
  482         orr     v3.16b, v3.16b, v4.16b
  483         add     v0.4s, v0.4s, v3.4s
  484         eor     v1.16b, v0.16b, v1.16b
  485         ushr    v4.4s, v1.4s, #8
  486         shl     v1.4s, v1.4s, #24
  487         orr     v1.16b, v1.16b, v4.16b
  488         add     v2.4s, v1.4s, v2.4s
  489         eor     v3.16b, v2.16b, v3.16b
  490         ext     v0.16b, v0.16b, v0.16b, #4
  491         ext     v2.16b, v2.16b, v2.16b, #12
  492         ushr    v4.4s, v3.4s, #7
  493         shl     v3.4s, v3.4s, #25
  494         ext     v1.16b, v1.16b, v1.16b, #8
  495         eor     v0.16b, v2.16b, v0.16b
  496         orr     v2.16b, v3.16b, v4.16b
  497         eor     v1.16b, v2.16b, v1.16b
  498         stp     q0, q1, [x0]
  499         ret
  500 .Lfunc_end0:
  501         .size   zfs_blake3_compress_in_place_sse2, .Lfunc_end0-zfs_blake3_compress_in_place_sse2
  502         .cfi_endproc
  503 
  504         .section        .rodata.cst16,"aM",@progbits,16
  505         .p2align        4
  506 .LCPI1_0:
  507         .word   1779033703
  508         .word   3144134277
  509         .word   1013904242
  510         .word   2773480762
  511 .LCPI1_1:
  512         .xword  0
  513         .xword  -4294967296
  514 .LCPI1_2:
  515         .xword  -1
  516         .xword  4294967295
  517         .text
  518         .globl  zfs_blake3_compress_xof_sse2
  519         .p2align        2
  520         .type   zfs_blake3_compress_xof_sse2,@function
  521 zfs_blake3_compress_xof_sse2:
  522         .cfi_startproc
  523         ldp     q3, q2, [x0]
  524         ldp     q5, q6, [x1]
  525         add     x10, x1, #32
  526         lsr     x11, x3, #32
  527         fmov    s4, w3
  528         ld2     { v17.4s, v18.4s }, [x10]
  529         adrp    x10, .LCPI1_2
  530         and     w8, w2, #0xff
  531         mov     v4.s[1], w11
  532         ldr     q1, [x10, :lo12:.LCPI1_2]
  533         and     w9, w4, #0xff
  534         adrp    x12, .LCPI1_0
  535         mov     v4.s[2], w8
  536         uzp1    v19.4s, v5.4s, v6.4s
  537         add     v3.4s, v2.4s, v3.4s
  538         ldr     q7, [x12, :lo12:.LCPI1_0]
  539         mov     v4.s[3], w9
  540         add     v3.4s, v3.4s, v19.4s
  541         uzp2    v5.4s, v5.4s, v6.4s
  542         ext     v21.16b, v18.16b, v18.16b, #12
  543         uzp1    v6.4s, v19.4s, v19.4s
  544         ext     v22.16b, v19.16b, v19.16b, #12
  545         eor     v4.16b, v3.16b, v4.16b
  546         ext     v20.16b, v17.16b, v17.16b, #12
  547         ext     v6.16b, v6.16b, v19.16b, #8
  548         ext     v19.16b, v19.16b, v22.16b, #12
  549         zip1    v22.2d, v21.2d, v5.2d
  550         rev32   v24.8h, v4.8h
  551         mov     v4.16b, v1.16b
  552         zip2    v23.4s, v5.4s, v21.4s
  553         uzp2    v6.4s, v6.4s, v5.4s
  554         bsl     v4.16b, v22.16b, v20.16b
  555         add     v3.4s, v3.4s, v5.4s
  556         zip1    v5.4s, v23.4s, v20.4s
  557         zip1    v22.4s, v20.4s, v23.4s
  558         add     v23.4s, v24.4s, v7.4s
  559         ext     v7.16b, v6.16b, v6.16b, #4
  560         ext     v25.16b, v4.16b, v4.16b, #12
  561         ext     v5.16b, v22.16b, v5.16b, #8
  562         eor     v2.16b, v23.16b, v2.16b
  563         uzp1    v4.4s, v4.4s, v25.4s
  564         uzp1    v22.4s, v7.4s, v7.4s
  565         ext     v25.16b, v7.16b, v7.16b, #12
  566         ext     v22.16b, v22.16b, v7.16b, #8
  567         ext     v7.16b, v7.16b, v25.16b, #12
  568         ushr    v25.4s, v2.4s, #12
  569         shl     v2.4s, v2.4s, #20
  570         orr     v2.16b, v2.16b, v25.16b
  571         add     v3.4s, v3.4s, v2.4s
  572         eor     v24.16b, v3.16b, v24.16b
  573         add     v3.4s, v3.4s, v17.4s
  574         ushr    v17.4s, v24.4s, #8
  575         shl     v18.4s, v24.4s, #24
  576         orr     v17.16b, v18.16b, v17.16b
  577         add     v18.4s, v17.4s, v23.4s
  578         eor     v2.16b, v18.16b, v2.16b
  579         ushr    v23.4s, v2.4s, #7
  580         shl     v2.4s, v2.4s, #25
  581         ext     v3.16b, v3.16b, v3.16b, #12
  582         orr     v2.16b, v2.16b, v23.16b
  583         ext     v17.16b, v17.16b, v17.16b, #8
  584         add     v3.4s, v2.4s, v3.4s
  585         adrp    x11, .LCPI1_1
  586         eor     v17.16b, v3.16b, v17.16b
  587         ldr     q16, [x11, :lo12:.LCPI1_1]
  588         ext     v18.16b, v18.16b, v18.16b, #4
  589         rev32   v24.8h, v17.8h
  590         movi    v0.2d, #0xffffffff00000000
  591         add     v23.4s, v3.4s, v21.4s
  592         mov     v21.s[1], v20.s[2]
  593         add     v20.4s, v18.4s, v24.4s
  594         bit     v19.16b, v21.16b, v0.16b
  595         eor     v3.16b, v20.16b, v2.16b
  596         uzp2    v2.4s, v22.4s, v19.4s
  597         zip1    v17.2d, v5.2d, v19.2d
  598         zip2    v18.4s, v19.4s, v5.4s
  599         ushr    v21.4s, v3.4s, #12
  600         shl     v3.4s, v3.4s, #20
  601         ext     v22.16b, v2.16b, v2.16b, #4
  602         bsl     v16.16b, v4.16b, v17.16b
  603         zip1    v17.4s, v18.4s, v4.4s
  604         zip1    v18.4s, v4.4s, v18.4s
  605         orr     v21.16b, v3.16b, v21.16b
  606         ext     v25.16b, v16.16b, v16.16b, #12
  607         ext     v3.16b, v18.16b, v17.16b, #8
  608         uzp1    v18.4s, v22.4s, v22.4s
  609         ext     v26.16b, v22.16b, v22.16b, #12
  610         add     v23.4s, v23.4s, v21.4s
  611         uzp1    v17.4s, v16.4s, v25.4s
  612         ext     v16.16b, v18.16b, v22.16b, #8
  613         ext     v18.16b, v22.16b, v26.16b, #12
  614         eor     v22.16b, v23.16b, v24.16b
  615         add     v6.4s, v23.4s, v6.4s
  616         ushr    v23.4s, v22.4s, #8
  617         shl     v22.4s, v22.4s, #24
  618         orr     v22.16b, v22.16b, v23.16b
  619         add     v20.4s, v22.4s, v20.4s
  620         eor     v21.16b, v20.16b, v21.16b
  621         ushr    v23.4s, v21.4s, #7
  622         shl     v21.4s, v21.4s, #25
  623         ext     v6.16b, v6.16b, v6.16b, #4
  624         orr     v21.16b, v21.16b, v23.16b
  625         ext     v22.16b, v22.16b, v22.16b, #8
  626         add     v6.4s, v21.4s, v6.4s
  627         eor     v22.16b, v6.16b, v22.16b
  628         ext     v20.16b, v20.16b, v20.16b, #12
  629         add     v6.4s, v6.4s, v19.4s
  630         rev32   v19.8h, v22.8h
  631         add     v20.4s, v20.4s, v19.4s
  632         eor     v21.16b, v20.16b, v21.16b
  633         ushr    v22.4s, v21.4s, #12
  634         shl     v21.4s, v21.4s, #20
  635         orr     v21.16b, v21.16b, v22.16b
  636         add     v6.4s, v6.4s, v21.4s
  637         eor     v19.16b, v6.16b, v19.16b
  638         ushr    v22.4s, v19.4s, #8
  639         shl     v19.4s, v19.4s, #24
  640         orr     v19.16b, v19.16b, v22.16b
  641         add     v20.4s, v19.4s, v20.4s
  642         eor     v21.16b, v20.16b, v21.16b
  643         ext     v6.16b, v6.16b, v6.16b, #12
  644         ushr    v22.4s, v21.4s, #7
  645         shl     v21.4s, v21.4s, #25
  646         add     v6.4s, v6.4s, v4.4s
  647         orr     v21.16b, v21.16b, v22.16b
  648         ext     v19.16b, v19.16b, v19.16b, #8
  649         add     v6.4s, v6.4s, v21.4s
  650         eor     v19.16b, v6.16b, v19.16b
  651         ext     v20.16b, v20.16b, v20.16b, #4
  652         rev32   v19.8h, v19.8h
  653         add     v20.4s, v20.4s, v19.4s
  654         add     v6.4s, v6.4s, v5.4s
  655         mov     v5.s[1], v4.s[2]
  656         eor     v4.16b, v20.16b, v21.16b
  657         ushr    v21.4s, v4.4s, #12
  658         shl     v4.4s, v4.4s, #20
  659         orr     v21.16b, v4.16b, v21.16b
  660         add     v6.4s, v6.4s, v21.4s
  661         eor     v19.16b, v6.16b, v19.16b
  662         add     v2.4s, v6.4s, v2.4s
  663         ushr    v6.4s, v19.4s, #8
  664         shl     v19.4s, v19.4s, #24
  665         orr     v6.16b, v19.16b, v6.16b
  666         add     v19.4s, v6.4s, v20.4s
  667         eor     v20.16b, v19.16b, v21.16b
  668         ushr    v21.4s, v20.4s, #7
  669         shl     v20.4s, v20.4s, #25
  670         ext     v2.16b, v2.16b, v2.16b, #4
  671         orr     v20.16b, v20.16b, v21.16b
  672         ext     v6.16b, v6.16b, v6.16b, #8
  673         add     v2.4s, v20.4s, v2.4s
  674         eor     v6.16b, v2.16b, v6.16b
  675         ext     v19.16b, v19.16b, v19.16b, #12
  676         rev32   v6.8h, v6.8h
  677         add     v19.4s, v19.4s, v6.4s
  678         mov     v22.16b, v0.16b
  679         eor     v20.16b, v19.16b, v20.16b
  680         bsl     v22.16b, v5.16b, v7.16b
  681         ushr    v21.4s, v20.4s, #12
  682         shl     v20.4s, v20.4s, #20
  683         add     v2.4s, v2.4s, v22.4s
  684         orr     v20.16b, v20.16b, v21.16b
  685         add     v2.4s, v2.4s, v20.4s
  686         eor     v6.16b, v2.16b, v6.16b
  687         ushr    v21.4s, v6.4s, #8
  688         shl     v6.4s, v6.4s, #24
  689         orr     v6.16b, v6.16b, v21.16b
  690         add     v19.4s, v6.4s, v19.4s
  691         eor     v20.16b, v19.16b, v20.16b
  692         ext     v2.16b, v2.16b, v2.16b, #12
  693         ushr    v21.4s, v20.4s, #7
  694         shl     v20.4s, v20.4s, #25
  695         add     v2.4s, v2.4s, v17.4s
  696         orr     v20.16b, v20.16b, v21.16b
  697         ext     v6.16b, v6.16b, v6.16b, #8
  698         add     v2.4s, v2.4s, v20.4s
  699         eor     v6.16b, v2.16b, v6.16b
  700         uzp2    v5.4s, v16.4s, v22.4s
  701         zip1    v7.2d, v3.2d, v22.2d
  702         zip2    v16.4s, v22.4s, v3.4s
  703         ext     v19.16b, v19.16b, v19.16b, #4
  704         rev32   v22.8h, v6.8h
  705         ext     v23.16b, v5.16b, v5.16b, #4
  706         bif     v7.16b, v17.16b, v1.16b
  707         zip1    v24.4s, v16.4s, v17.4s
  708         zip1    v16.4s, v17.4s, v16.4s
  709         add     v21.4s, v2.4s, v3.4s
  710         mov     v3.s[1], v17.s[2]
  711         add     v17.4s, v19.4s, v22.4s
  712         mov     v19.16b, v0.16b
  713         ext     v25.16b, v7.16b, v7.16b, #12
  714         ext     v4.16b, v16.16b, v24.16b, #8
  715         uzp1    v16.4s, v23.4s, v23.4s
  716         bsl     v19.16b, v3.16b, v18.16b
  717         eor     v2.16b, v17.16b, v20.16b
  718         uzp1    v7.4s, v7.4s, v25.4s
  719         ext     v25.16b, v16.16b, v23.16b, #8
  720         zip1    v3.2d, v4.2d, v19.2d
  721         ushr    v20.4s, v2.4s, #12
  722         shl     v2.4s, v2.4s, #20
  723         ext     v24.16b, v23.16b, v23.16b, #12
  724         uzp2    v6.4s, v25.4s, v19.4s
  725         zip2    v18.4s, v19.4s, v4.4s
  726         bif     v3.16b, v7.16b, v1.16b
  727         orr     v20.16b, v2.16b, v20.16b
  728         ext     v16.16b, v23.16b, v24.16b, #12
  729         ext     v23.16b, v6.16b, v6.16b, #4
  730         zip1    v24.4s, v18.4s, v7.4s
  731         zip1    v18.4s, v7.4s, v18.4s
  732         ext     v25.16b, v3.16b, v3.16b, #12
  733         add     v21.4s, v21.4s, v20.4s
  734         ext     v2.16b, v18.16b, v24.16b, #8
  735         uzp1    v18.4s, v23.4s, v23.4s
  736         ext     v24.16b, v23.16b, v23.16b, #12
  737         uzp1    v3.4s, v3.4s, v25.4s
  738         eor     v22.16b, v21.16b, v22.16b
  739         ext     v25.16b, v18.16b, v23.16b, #8
  740         dup     v18.4s, v2.s[3]
  741         ext     v23.16b, v23.16b, v24.16b, #12
  742         add     v5.4s, v21.4s, v5.4s
  743         trn1    v21.4s, v3.4s, v3.4s
  744         ushr    v24.4s, v22.4s, #8
  745         shl     v22.4s, v22.4s, #24
  746         ext     v18.16b, v21.16b, v18.16b, #8
  747         orr     v21.16b, v22.16b, v24.16b
  748         add     v17.4s, v21.4s, v17.4s
  749         eor     v20.16b, v17.16b, v20.16b
  750         ushr    v22.4s, v20.4s, #7
  751         shl     v20.4s, v20.4s, #25
  752         ext     v5.16b, v5.16b, v5.16b, #4
  753         orr     v20.16b, v20.16b, v22.16b
  754         ext     v21.16b, v21.16b, v21.16b, #8
  755         add     v5.4s, v20.4s, v5.4s
  756         eor     v21.16b, v5.16b, v21.16b
  757         ext     v17.16b, v17.16b, v17.16b, #12
  758         add     v5.4s, v5.4s, v19.4s
  759         rev32   v19.8h, v21.8h
  760         add     v17.4s, v17.4s, v19.4s
  761         eor     v20.16b, v17.16b, v20.16b
  762         ushr    v21.4s, v20.4s, #12
  763         shl     v20.4s, v20.4s, #20
  764         orr     v20.16b, v20.16b, v21.16b
  765         add     v5.4s, v5.4s, v20.4s
  766         eor     v19.16b, v5.16b, v19.16b
  767         ushr    v21.4s, v19.4s, #8
  768         shl     v19.4s, v19.4s, #24
  769         orr     v19.16b, v19.16b, v21.16b
  770         add     v17.4s, v19.4s, v17.4s
  771         eor     v20.16b, v17.16b, v20.16b
  772         ext     v5.16b, v5.16b, v5.16b, #12
  773         ushr    v21.4s, v20.4s, #7
  774         shl     v20.4s, v20.4s, #25
  775         add     v5.4s, v5.4s, v7.4s
  776         orr     v20.16b, v20.16b, v21.16b
  777         ext     v19.16b, v19.16b, v19.16b, #8
  778         add     v5.4s, v5.4s, v20.4s
  779         eor     v19.16b, v5.16b, v19.16b
  780         ext     v17.16b, v17.16b, v17.16b, #4
  781         rev32   v22.8h, v19.8h
  782         add     v21.4s, v5.4s, v4.4s
  783         mov     v4.s[1], v7.s[2]
  784         add     v19.4s, v17.4s, v22.4s
  785         bit     v16.16b, v4.16b, v0.16b
  786         eor     v5.16b, v19.16b, v20.16b
  787         uzp2    v4.4s, v25.4s, v16.4s
  788         zip1    v7.2d, v2.2d, v16.2d
  789         zip2    v17.4s, v16.4s, v2.4s
  790         ushr    v20.4s, v5.4s, #12
  791         shl     v5.4s, v5.4s, #20
  792         ext     v24.16b, v4.16b, v4.16b, #4
  793         bif     v7.16b, v3.16b, v1.16b
  794         zip1    v25.4s, v17.4s, v3.4s
  795         zip1    v17.4s, v3.4s, v17.4s
  796         orr     v20.16b, v5.16b, v20.16b
  797         ext     v26.16b, v7.16b, v7.16b, #12
  798         ext     v5.16b, v17.16b, v25.16b, #8
  799         uzp1    v17.4s, v24.4s, v24.4s
  800         ext     v25.16b, v24.16b, v24.16b, #12
  801         bit     v23.16b, v18.16b, v0.16b
  802         add     v21.4s, v21.4s, v20.4s
  803         uzp1    v7.4s, v7.4s, v26.4s
  804         ext     v26.16b, v17.16b, v24.16b, #8
  805         ext     v17.16b, v24.16b, v25.16b, #12
  806         eor     v22.16b, v21.16b, v22.16b
  807         add     v6.4s, v21.4s, v6.4s
  808         zip1    v21.2d, v5.2d, v23.2d
  809         zip2    v24.4s, v23.4s, v5.4s
  810         bif     v21.16b, v7.16b, v1.16b
  811         zip1    v1.4s, v24.4s, v7.4s
  812         zip1    v24.4s, v7.4s, v24.4s
  813         ext     v1.16b, v24.16b, v1.16b, #8
  814         ushr    v24.4s, v22.4s, #8
  815         shl     v22.4s, v22.4s, #24
  816         orr     v22.16b, v22.16b, v24.16b
  817         add     v19.4s, v22.4s, v19.4s
  818         ext     v24.16b, v21.16b, v21.16b, #12
  819         eor     v20.16b, v19.16b, v20.16b
  820         uzp1    v21.4s, v21.4s, v24.4s
  821         ushr    v24.4s, v20.4s, #7
  822         shl     v20.4s, v20.4s, #25
  823         orr     v20.16b, v20.16b, v24.16b
  824         ext     v6.16b, v6.16b, v6.16b, #4
  825         ext     v22.16b, v22.16b, v22.16b, #8
  826         add     v6.4s, v20.4s, v6.4s
  827         eor     v22.16b, v6.16b, v22.16b
  828         ext     v19.16b, v19.16b, v19.16b, #12
  829         add     v6.4s, v6.4s, v16.4s
  830         rev32   v16.8h, v22.8h
  831         add     v19.4s, v19.4s, v16.4s
  832         eor     v20.16b, v19.16b, v20.16b
  833         ushr    v22.4s, v20.4s, #12
  834         shl     v20.4s, v20.4s, #20
  835         orr     v20.16b, v20.16b, v22.16b
  836         add     v6.4s, v6.4s, v20.4s
  837         eor     v16.16b, v6.16b, v16.16b
  838         ext     v6.16b, v6.16b, v6.16b, #12
  839         add     v3.4s, v6.4s, v3.4s
  840         ushr    v6.4s, v16.4s, #8
  841         shl     v16.4s, v16.4s, #24
  842         orr     v6.16b, v16.16b, v6.16b
  843         add     v16.4s, v6.4s, v19.4s
  844         eor     v19.16b, v16.16b, v20.16b
  845         ushr    v20.4s, v19.4s, #7
  846         shl     v19.4s, v19.4s, #25
  847         orr     v19.16b, v19.16b, v20.16b
  848         ext     v6.16b, v6.16b, v6.16b, #8
  849         add     v3.4s, v3.4s, v19.4s
  850         eor     v6.16b, v3.16b, v6.16b
  851         ext     v16.16b, v16.16b, v16.16b, #4
  852         add     v2.4s, v3.4s, v2.4s
  853         rev32   v3.8h, v6.8h
  854         add     v6.4s, v16.4s, v3.4s
  855         eor     v16.16b, v6.16b, v19.16b
  856         ushr    v19.4s, v16.4s, #12
  857         shl     v16.4s, v16.4s, #20
  858         orr     v16.16b, v16.16b, v19.16b
  859         add     v2.4s, v2.4s, v16.4s
  860         eor     v3.16b, v2.16b, v3.16b
  861         add     v2.4s, v2.4s, v4.4s
  862         ushr    v4.4s, v3.4s, #8
  863         shl     v3.4s, v3.4s, #24
  864         orr     v3.16b, v3.16b, v4.16b
  865         add     v4.4s, v3.4s, v6.4s
  866         eor     v6.16b, v4.16b, v16.16b
  867         ushr    v16.4s, v6.4s, #7
  868         shl     v6.4s, v6.4s, #25
  869         ext     v2.16b, v2.16b, v2.16b, #4
  870         orr     v6.16b, v6.16b, v16.16b
  871         ext     v3.16b, v3.16b, v3.16b, #8
  872         add     v2.4s, v6.4s, v2.4s
  873         eor     v3.16b, v2.16b, v3.16b
  874         ext     v4.16b, v4.16b, v4.16b, #12
  875         rev32   v3.8h, v3.8h
  876         add     v4.4s, v4.4s, v3.4s
  877         eor     v6.16b, v4.16b, v6.16b
  878         ushr    v16.4s, v6.4s, #12
  879         shl     v6.4s, v6.4s, #20
  880         add     v2.4s, v2.4s, v23.4s
  881         orr     v6.16b, v6.16b, v16.16b
  882         add     v2.4s, v2.4s, v6.4s
  883         eor     v3.16b, v2.16b, v3.16b
  884         ushr    v16.4s, v3.4s, #8
  885         shl     v3.4s, v3.4s, #24
  886         orr     v3.16b, v3.16b, v16.16b
  887         add     v4.4s, v3.4s, v4.4s
  888         eor     v6.16b, v4.16b, v6.16b
  889         ext     v2.16b, v2.16b, v2.16b, #12
  890         ushr    v16.4s, v6.4s, #7
  891         shl     v6.4s, v6.4s, #25
  892         add     v2.4s, v2.4s, v7.4s
  893         orr     v6.16b, v6.16b, v16.16b
  894         ext     v3.16b, v3.16b, v3.16b, #8
  895         add     v2.4s, v2.4s, v6.4s
  896         eor     v3.16b, v2.16b, v3.16b
  897         ext     v4.16b, v4.16b, v4.16b, #4
  898         rev32   v3.8h, v3.8h
  899         add     v2.4s, v2.4s, v5.4s
  900         mov     v5.s[1], v7.s[2]
  901         add     v4.4s, v4.4s, v3.4s
  902         bsl     v0.16b, v5.16b, v17.16b
  903         eor     v5.16b, v4.16b, v6.16b
  904         ushr    v6.4s, v5.4s, #12
  905         shl     v5.4s, v5.4s, #20
  906         orr     v5.16b, v5.16b, v6.16b
  907         add     v2.4s, v2.4s, v5.4s
  908         eor     v3.16b, v2.16b, v3.16b
  909         ushr    v6.4s, v3.4s, #8
  910         shl     v3.4s, v3.4s, #24
  911         orr     v3.16b, v3.16b, v6.16b
  912         add     v4.4s, v3.4s, v4.4s
  913         uzp2    v18.4s, v26.4s, v18.4s
  914         eor     v5.16b, v4.16b, v5.16b
  915         add     v2.4s, v2.4s, v18.4s
  916         ushr    v6.4s, v5.4s, #7
  917         shl     v5.4s, v5.4s, #25
  918         ext     v2.16b, v2.16b, v2.16b, #4
  919         orr     v5.16b, v5.16b, v6.16b
  920         ext     v3.16b, v3.16b, v3.16b, #8
  921         add     v2.4s, v5.4s, v2.4s
  922         eor     v3.16b, v2.16b, v3.16b
  923         ext     v4.16b, v4.16b, v4.16b, #12
  924         add     v0.4s, v2.4s, v0.4s
  925         rev32   v2.8h, v3.8h
  926         add     v3.4s, v4.4s, v2.4s
  927         eor     v4.16b, v3.16b, v5.16b
  928         ushr    v5.4s, v4.4s, #12
  929         shl     v4.4s, v4.4s, #20
  930         orr     v4.16b, v4.16b, v5.16b
  931         add     v0.4s, v0.4s, v4.4s
  932         eor     v2.16b, v0.16b, v2.16b
  933         ushr    v5.4s, v2.4s, #8
  934         shl     v2.4s, v2.4s, #24
  935         orr     v2.16b, v2.16b, v5.16b
  936         add     v3.4s, v2.4s, v3.4s
  937         eor     v4.16b, v3.16b, v4.16b
  938         ext     v0.16b, v0.16b, v0.16b, #12
  939         ushr    v5.4s, v4.4s, #7
  940         shl     v4.4s, v4.4s, #25
  941         add     v0.4s, v0.4s, v21.4s
  942         orr     v4.16b, v4.16b, v5.16b
  943         ext     v2.16b, v2.16b, v2.16b, #8
  944         add     v0.4s, v0.4s, v4.4s
  945         eor     v2.16b, v0.16b, v2.16b
  946         ext     v3.16b, v3.16b, v3.16b, #4
  947         add     v0.4s, v0.4s, v1.4s
  948         rev32   v1.8h, v2.8h
  949         add     v2.4s, v3.4s, v1.4s
  950         eor     v3.16b, v2.16b, v4.16b
  951         ushr    v4.4s, v3.4s, #12
  952         shl     v3.4s, v3.4s, #20
  953         orr     v3.16b, v3.16b, v4.16b
  954         add     v0.4s, v0.4s, v3.4s
  955         eor     v1.16b, v0.16b, v1.16b
  956         ushr    v4.4s, v1.4s, #8
  957         shl     v1.4s, v1.4s, #24
  958         orr     v1.16b, v1.16b, v4.16b
  959         add     v2.4s, v1.4s, v2.4s
  960         eor     v3.16b, v2.16b, v3.16b
  961         ushr    v4.4s, v3.4s, #7
  962         shl     v3.4s, v3.4s, #25
  963         ext     v0.16b, v0.16b, v0.16b, #4
  964         ext     v1.16b, v1.16b, v1.16b, #8
  965         ext     v2.16b, v2.16b, v2.16b, #12
  966         orr     v3.16b, v3.16b, v4.16b
  967         eor     v0.16b, v2.16b, v0.16b
  968         eor     v3.16b, v3.16b, v1.16b
  969         stp     q0, q3, [x5]
  970         ldr     q0, [x0]
  971         eor     v0.16b, v0.16b, v2.16b
  972         str     q0, [x5, #32]
  973         ldr     q0, [x0, #16]
  974         eor     v0.16b, v0.16b, v1.16b
  975         str     q0, [x5, #48]
  976         ret
  977 .Lfunc_end1:
  978         .size   zfs_blake3_compress_xof_sse2, .Lfunc_end1-zfs_blake3_compress_xof_sse2
  979         .cfi_endproc
  980 
  981         .section        .rodata.cst16,"aM",@progbits,16
  982         .p2align        4
  983 .LCPI2_0:
  984         .word   0
  985         .word   1
  986         .word   2
  987         .word   3
  988         .text
  989         .globl  zfs_blake3_hash_many_sse2
  990         .p2align        2
  991         .type   zfs_blake3_hash_many_sse2,@function
  992 zfs_blake3_hash_many_sse2:
  993         .cfi_startproc
  994         stp     d15, d14, [sp, #-160]!
  995         stp     d13, d12, [sp, #16]
  996         stp     d11, d10, [sp, #32]
  997         stp     d9, d8, [sp, #48]
  998         stp     x29, x30, [sp, #64]
  999         stp     x28, x27, [sp, #80]
 1000         stp     x26, x25, [sp, #96]
 1001         stp     x24, x23, [sp, #112]
 1002         stp     x22, x21, [sp, #128]
 1003         stp     x20, x19, [sp, #144]
 1004         mov     x29, sp
 1005         sub     sp, sp, #384
 1006         .cfi_def_cfa w29, 160
 1007         .cfi_offset w19, -8
 1008         .cfi_offset w20, -16
 1009         .cfi_offset w21, -24
 1010         .cfi_offset w22, -32
 1011         .cfi_offset w23, -40
 1012         .cfi_offset w24, -48
 1013         .cfi_offset w25, -56
 1014         .cfi_offset w26, -64
 1015         .cfi_offset w27, -72
 1016         .cfi_offset w28, -80
 1017         .cfi_offset w30, -88
 1018         .cfi_offset w29, -96
 1019         .cfi_offset b8, -104
 1020         .cfi_offset b9, -112
 1021         .cfi_offset b10, -120
 1022         .cfi_offset b11, -128
 1023         .cfi_offset b12, -136
 1024         .cfi_offset b13, -144
 1025         .cfi_offset b14, -152
 1026         .cfi_offset b15, -160
 1027         ldr     x26, [x29, #168]
 1028         ldrb    w27, [x29, #160]
 1029         mov     w19, w6
 1030         mov     x20, x4
 1031         mov     x22, x2
 1032         mov     x28, x1
 1033         cmp     x1, #4
 1034         mov     x24, x0
 1035         str     x3, [sp, #40]
 1036         b.lo    .LBB2_8
 1037         adrp    x9, .LCPI2_0
 1038         ldr     q0, [x9, :lo12:.LCPI2_0]
 1039         sbfx    w11, w5, #0, #1
 1040         dup     v1.4s, w11
 1041         mov     w9, #58983
 1042         mov     w10, #44677
 1043         and     v0.16b, v1.16b, v0.16b
 1044         mov     w11, #62322
 1045         mov     w12, #62778
 1046         orr     w8, w7, w19
 1047         movk    w9, #27145, lsl #16
 1048         movk    w10, #47975, lsl #16
 1049         movk    w11, #15470, lsl #16
 1050         str     q0, [sp, #16]
 1051         orr     v0.4s, #128, lsl #24
 1052         movk    w12, #42319, lsl #16
 1053         str     q0, [sp]
 1054 .LBB2_2:
 1055         ldr     x0, [sp, #40]
 1056         mov     x13, x0
 1057         ld1r    { v20.4s }, [x13], #4
 1058         add     x14, x0, #8
 1059         add     x15, x0, #12
 1060         add     x16, x0, #16
 1061         add     x17, x0, #20
 1062         add     x18, x0, #24
 1063         add     x0, x0, #28
 1064         ld1r    { v17.4s }, [x14]
 1065         ld1r    { v6.4s }, [x15]
 1066         ld1r    { v8.4s }, [x16]
 1067         ld1r    { v9.4s }, [x17]
 1068         ld1r    { v31.4s }, [x18]
 1069         ld1r    { v26.4s }, [x13]
 1070         ld1r    { v15.4s }, [x0]
 1071         cbz     x22, .LBB2_7
 1072         ldr     q1, [sp, #16]
 1073         dup     v0.4s, w20
 1074         ldp     x13, x14, [x24]
 1075         ldp     x15, x16, [x24, #16]
 1076         add     v1.4s, v0.4s, v1.4s
 1077         movi    v0.4s, #128, lsl #24
 1078         str     q1, [sp, #64]
 1079         eor     v0.16b, v1.16b, v0.16b
 1080         ldr     q1, [sp]
 1081         lsr     x18, x20, #32
 1082         mov     x17, xzr
 1083         cmgt    v0.4s, v1.4s, v0.4s
 1084         dup     v1.4s, w18
 1085         sub     v0.4s, v1.4s, v0.4s
 1086         mov     w18, w8
 1087         str     q0, [sp, #48]
 1088 .LBB2_4:
 1089         mov     w2, #16
 1090         bfi     x2, x17, #6, #58
 1091         ldr     q1, [x13, x2]
 1092         ldr     q3, [x14, x2]
 1093         ldr     q2, [x15, x2]
 1094         ldr     q4, [x16, x2]
 1095         mov     w2, #32
 1096         bfi     x2, x17, #6, #58
 1097         ldr     q5, [x13, x2]
 1098         ldr     q18, [x14, x2]
 1099         ldr     q19, [x15, x2]
 1100         ldr     q23, [x16, x2]
 1101         mov     w2, #48
 1102         lsl     x3, x17, #6
 1103         bfi     x2, x17, #6, #58
 1104         add     x17, x17, #1
 1105         ldr     q0, [x13, x3]
 1106         ldr     q21, [x14, x3]
 1107         ldr     q7, [x15, x3]
 1108         ldr     q16, [x16, x3]
 1109         cmp     x17, x22
 1110         ldr     q13, [x13, x2]
 1111         ldr     q14, [x14, x2]
 1112         ldr     q29, [x15, x2]
 1113         ldr     q10, [x16, x2]
 1114         csel    w2, w27, wzr, eq
 1115         orr     w18, w2, w18
 1116         mov     x0, xzr
 1117         and     w18, w18, #0xff
 1118         add     x3, x3, #256
 1119 .LBB2_5:
 1120         ldr     x2, [x24, x0]
 1121         add     x0, x0, #8
 1122         cmp     x0, #32
 1123         add     x2, x2, x3
 1124         prfm    pldl1keep, [x2]
 1125         b.ne    .LBB2_5
 1126         dup     v22.4s, w18
 1127         str     q22, [sp, #192]
 1128         zip1    v27.4s, v0.4s, v21.4s
 1129         zip2    v21.4s, v0.4s, v21.4s
 1130         zip1    v0.4s, v7.4s, v16.4s
 1131         zip2    v22.4s, v7.4s, v16.4s
 1132         zip1    v7.4s, v1.4s, v3.4s
 1133         zip1    v25.4s, v2.4s, v4.4s
 1134         zip2    v16.4s, v2.4s, v4.4s
 1135         zip1    v11.4s, v19.4s, v23.4s
 1136         zip2    v12.4s, v19.4s, v23.4s
 1137         zip1    v19.4s, v13.4s, v14.4s
 1138         zip2    v23.4s, v13.4s, v14.4s
 1139         zip1    v13.4s, v29.4s, v10.4s
 1140         zip2    v14.4s, v29.4s, v10.4s
 1141         add     v10.4s, v20.4s, v8.4s
 1142         add     v2.4s, v26.4s, v9.4s
 1143         ext     v20.16b, v22.16b, v21.16b, #8
 1144         ext     v26.16b, v25.16b, v7.16b, #8
 1145         zip2    v24.4s, v1.4s, v3.4s
 1146         add     v1.4s, v6.4s, v15.4s
 1147         ext     v6.16b, v0.16b, v27.16b, #8
 1148         ext     v20.16b, v21.16b, v20.16b, #8
 1149         mov     v21.d[1], v22.d[0]
 1150         ext     v22.16b, v7.16b, v26.16b, #8
 1151         mov     v7.d[1], v25.d[0]
 1152         add     v3.4s, v17.4s, v31.4s
 1153         str     q1, [sp, #144]
 1154         ext     v1.16b, v27.16b, v6.16b, #8
 1155         mov     v6.16b, v7.16b
 1156         zip1    v28.4s, v5.4s, v18.4s
 1157         stur    q1, [x29, #-80]
 1158         mov     v1.16b, v27.16b
 1159         mov     v27.16b, v24.16b
 1160         add     v3.4s, v3.4s, v6.4s
 1161         ldr     q6, [sp, #64]
 1162         ext     v29.16b, v16.16b, v24.16b, #8
 1163         mov     v1.d[1], v0.d[0]
 1164         ext     v0.16b, v11.16b, v28.16b, #8
 1165         mov     v27.d[1], v16.d[0]
 1166         ext     v16.16b, v14.16b, v23.16b, #8
 1167         stur    q7, [x29, #-144]
 1168         ext     v7.16b, v24.16b, v29.16b, #8
 1169         ext     v29.16b, v28.16b, v0.16b, #8
 1170         ext     v0.16b, v23.16b, v16.16b, #8
 1171         mov     v23.d[1], v14.d[0]
 1172         stp     q0, q23, [sp, #80]
 1173         add     v0.4s, v10.4s, v1.4s
 1174         eor     v16.16b, v0.16b, v6.16b
 1175         ldr     q6, [sp, #48]
 1176         add     v2.4s, v2.4s, v21.4s
 1177         mov     v28.d[1], v11.d[0]
 1178         zip2    v18.4s, v5.4s, v18.4s
 1179         eor     v10.16b, v2.16b, v6.16b
 1180         movi    v6.4s, #64
 1181         eor     v11.16b, v3.16b, v6.16b
 1182         ldr     q6, [sp, #144]
 1183         dup     v17.4s, w9
 1184         ext     v30.16b, v12.16b, v18.16b, #8
 1185         rev32   v16.8h, v16.8h
 1186         dup     v5.4s, w10
 1187         ext     v25.16b, v18.16b, v30.16b, #8
 1188         mov     v30.16b, v23.16b
 1189         mov     v23.16b, v1.16b
 1190         str     q1, [sp, #160]
 1191         rev32   v10.8h, v10.8h
 1192         add     v1.4s, v16.4s, v17.4s
 1193         add     v17.4s, v6.4s, v27.4s
 1194         ldr     q6, [sp, #192]
 1195         dup     v4.4s, w11
 1196         rev32   v11.8h, v11.8h
 1197         add     v5.4s, v10.4s, v5.4s
 1198         eor     v8.16b, v1.16b, v8.16b
 1199         stur    q21, [x29, #-128]
 1200         mov     v18.d[1], v12.d[0]
 1201         add     v4.4s, v11.4s, v4.4s
 1202         eor     v9.16b, v5.16b, v9.16b
 1203         ushr    v12.4s, v8.4s, #12
 1204         shl     v8.4s, v8.4s, #20
 1205         ldur    q21, [x29, #-80]
 1206         ext     v26.16b, v13.16b, v19.16b, #8
 1207         eor     v31.16b, v4.16b, v31.16b
 1208         orr     v8.16b, v8.16b, v12.16b
 1209         ushr    v12.4s, v9.4s, #12
 1210         shl     v9.4s, v9.4s, #20
 1211         ext     v26.16b, v19.16b, v26.16b, #8
 1212         mov     v19.d[1], v13.d[0]
 1213         orr     v9.16b, v9.16b, v12.16b
 1214         ushr    v12.4s, v31.4s, #12
 1215         shl     v31.4s, v31.4s, #20
 1216         eor     v13.16b, v17.16b, v6.16b
 1217         orr     v31.16b, v31.16b, v12.16b
 1218         dup     v12.4s, w12
 1219         rev32   v13.8h, v13.8h
 1220         add     v12.4s, v13.4s, v12.4s
 1221         add     v0.4s, v0.4s, v21.4s
 1222         eor     v14.16b, v12.16b, v15.16b
 1223         add     v0.4s, v0.4s, v8.4s
 1224         add     v2.4s, v2.4s, v20.4s
 1225         ushr    v15.4s, v14.4s, #12
 1226         shl     v14.4s, v14.4s, #20
 1227         eor     v16.16b, v0.16b, v16.16b
 1228         add     v2.4s, v2.4s, v9.4s
 1229         add     v3.4s, v3.4s, v22.4s
 1230         orr     v14.16b, v14.16b, v15.16b
 1231         ushr    v15.4s, v16.4s, #8
 1232         shl     v16.4s, v16.4s, #24
 1233         eor     v10.16b, v2.16b, v10.16b
 1234         add     v3.4s, v3.4s, v31.4s
 1235         add     v17.4s, v17.4s, v7.4s
 1236         orr     v16.16b, v16.16b, v15.16b
 1237         ushr    v15.4s, v10.4s, #8
 1238         shl     v10.4s, v10.4s, #24
 1239         eor     v11.16b, v3.16b, v11.16b
 1240         add     v17.4s, v17.4s, v14.4s
 1241         orr     v10.16b, v10.16b, v15.16b
 1242         ushr    v15.4s, v11.4s, #8
 1243         shl     v11.4s, v11.4s, #24
 1244         eor     v13.16b, v17.16b, v13.16b
 1245         add     v1.4s, v16.4s, v1.4s
 1246         orr     v11.16b, v11.16b, v15.16b
 1247         ushr    v15.4s, v13.4s, #8
 1248         shl     v13.4s, v13.4s, #24
 1249         eor     v8.16b, v1.16b, v8.16b
 1250         add     v5.4s, v10.4s, v5.4s
 1251         orr     v13.16b, v13.16b, v15.16b
 1252         ushr    v15.4s, v8.4s, #7
 1253         shl     v8.4s, v8.4s, #25
 1254         eor     v9.16b, v5.16b, v9.16b
 1255         add     v4.4s, v11.4s, v4.4s
 1256         orr     v8.16b, v8.16b, v15.16b
 1257         ushr    v15.4s, v9.4s, #7
 1258         shl     v9.4s, v9.4s, #25
 1259         eor     v31.16b, v4.16b, v31.16b
 1260         add     v12.4s, v13.4s, v12.4s
 1261         orr     v9.16b, v9.16b, v15.16b
 1262         ushr    v15.4s, v31.4s, #7
 1263         shl     v31.4s, v31.4s, #25
 1264         eor     v14.16b, v12.16b, v14.16b
 1265         add     v0.4s, v0.4s, v28.4s
 1266         orr     v31.16b, v31.16b, v15.16b
 1267         ushr    v15.4s, v14.4s, #7
 1268         shl     v14.4s, v14.4s, #25
 1269         add     v0.4s, v0.4s, v9.4s
 1270         add     v2.4s, v2.4s, v18.4s
 1271         orr     v14.16b, v14.16b, v15.16b
 1272         eor     v13.16b, v0.16b, v13.16b
 1273         add     v2.4s, v2.4s, v31.4s
 1274         add     v3.4s, v3.4s, v19.4s
 1275         rev32   v13.8h, v13.8h
 1276         eor     v16.16b, v2.16b, v16.16b
 1277         add     v3.4s, v3.4s, v14.4s
 1278         add     v17.4s, v17.4s, v30.4s
 1279         add     v4.4s, v4.4s, v13.4s
 1280         rev32   v16.8h, v16.8h
 1281         eor     v10.16b, v3.16b, v10.16b
 1282         add     v17.4s, v17.4s, v8.4s
 1283         eor     v9.16b, v4.16b, v9.16b
 1284         add     v12.4s, v12.4s, v16.4s
 1285         rev32   v10.8h, v10.8h
 1286         eor     v11.16b, v17.16b, v11.16b
 1287         mov     v24.16b, v7.16b
 1288         stur    q7, [x29, #-112]
 1289         ushr    v15.4s, v9.4s, #12
 1290         shl     v9.4s, v9.4s, #20
 1291         eor     v31.16b, v12.16b, v31.16b
 1292         add     v1.4s, v1.4s, v10.4s
 1293         rev32   v11.8h, v11.8h
 1294         mov     v7.16b, v26.16b
 1295         add     v3.4s, v3.4s, v26.4s
 1296         ldr     q26, [sp, #80]
 1297         orr     v9.16b, v9.16b, v15.16b
 1298         ushr    v15.4s, v31.4s, #12
 1299         shl     v31.4s, v31.4s, #20
 1300         eor     v14.16b, v1.16b, v14.16b
 1301         add     v5.4s, v5.4s, v11.4s
 1302         add     v0.4s, v0.4s, v29.4s
 1303         orr     v31.16b, v31.16b, v15.16b
 1304         ushr    v15.4s, v14.4s, #12
 1305         shl     v14.4s, v14.4s, #20
 1306         eor     v8.16b, v5.16b, v8.16b
 1307         add     v0.4s, v0.4s, v9.4s
 1308         add     v2.4s, v2.4s, v25.4s
 1309         orr     v14.16b, v14.16b, v15.16b
 1310         ushr    v15.4s, v8.4s, #12
 1311         shl     v8.4s, v8.4s, #20
 1312         eor     v13.16b, v0.16b, v13.16b
 1313         add     v2.4s, v2.4s, v31.4s
 1314         orr     v8.16b, v8.16b, v15.16b
 1315         ushr    v15.4s, v13.4s, #8
 1316         shl     v13.4s, v13.4s, #24
 1317         eor     v16.16b, v2.16b, v16.16b
 1318         add     v3.4s, v3.4s, v14.4s
 1319         add     v17.4s, v17.4s, v26.4s
 1320         orr     v13.16b, v13.16b, v15.16b
 1321         ushr    v15.4s, v16.4s, #8
 1322         shl     v16.4s, v16.4s, #24
 1323         eor     v10.16b, v3.16b, v10.16b
 1324         add     v17.4s, v17.4s, v8.4s
 1325         orr     v16.16b, v16.16b, v15.16b
 1326         ushr    v15.4s, v10.4s, #8
 1327         shl     v10.4s, v10.4s, #24
 1328         eor     v11.16b, v17.16b, v11.16b
 1329         add     v4.4s, v13.4s, v4.4s
 1330         orr     v10.16b, v10.16b, v15.16b
 1331         ushr    v15.4s, v11.4s, #8
 1332         shl     v11.4s, v11.4s, #24
 1333         eor     v9.16b, v4.16b, v9.16b
 1334         add     v12.4s, v16.4s, v12.4s
 1335         str     q22, [sp, #128]
 1336         orr     v11.16b, v11.16b, v15.16b
 1337         ushr    v15.4s, v9.4s, #7
 1338         shl     v9.4s, v9.4s, #25
 1339         eor     v31.16b, v12.16b, v31.16b
 1340         add     v1.4s, v10.4s, v1.4s
 1341         ldur    q22, [x29, #-128]
 1342         orr     v9.16b, v9.16b, v15.16b
 1343         ushr    v15.4s, v31.4s, #7
 1344         shl     v31.4s, v31.4s, #25
 1345         eor     v14.16b, v1.16b, v14.16b
 1346         add     v5.4s, v11.4s, v5.4s
 1347         orr     v31.16b, v31.16b, v15.16b
 1348         ushr    v15.4s, v14.4s, #7
 1349         shl     v14.4s, v14.4s, #25
 1350         eor     v8.16b, v5.16b, v8.16b
 1351         mov     v6.16b, v18.16b
 1352         orr     v14.16b, v14.16b, v15.16b
 1353         ushr    v15.4s, v8.4s, #7
 1354         shl     v8.4s, v8.4s, #25
 1355         ldur    q18, [x29, #-144]
 1356         orr     v8.16b, v8.16b, v15.16b
 1357         add     v0.4s, v0.4s, v22.4s
 1358         add     v0.4s, v0.4s, v8.4s
 1359         add     v2.4s, v2.4s, v20.4s
 1360         eor     v16.16b, v0.16b, v16.16b
 1361         add     v2.4s, v2.4s, v9.4s
 1362         add     v3.4s, v3.4s, v24.4s
 1363         rev32   v16.8h, v16.8h
 1364         eor     v10.16b, v2.16b, v10.16b
 1365         add     v3.4s, v3.4s, v31.4s
 1366         add     v17.4s, v17.4s, v18.4s
 1367         add     v1.4s, v1.4s, v16.4s
 1368         rev32   v10.8h, v10.8h
 1369         eor     v11.16b, v3.16b, v11.16b
 1370         add     v17.4s, v17.4s, v14.4s
 1371         eor     v8.16b, v1.16b, v8.16b
 1372         add     v5.4s, v5.4s, v10.4s
 1373         rev32   v11.8h, v11.8h
 1374         eor     v13.16b, v17.16b, v13.16b
 1375         ushr    v15.4s, v8.4s, #12
 1376         shl     v8.4s, v8.4s, #20
 1377         eor     v9.16b, v5.16b, v9.16b
 1378         add     v4.4s, v4.4s, v11.4s
 1379         rev32   v13.8h, v13.8h
 1380         orr     v8.16b, v8.16b, v15.16b
 1381         ushr    v15.4s, v9.4s, #12
 1382         shl     v9.4s, v9.4s, #20
 1383         eor     v31.16b, v4.16b, v31.16b
 1384         add     v12.4s, v12.4s, v13.4s
 1385         add     v0.4s, v0.4s, v27.4s
 1386         orr     v9.16b, v9.16b, v15.16b
 1387         ushr    v15.4s, v31.4s, #12
 1388         shl     v31.4s, v31.4s, #20
 1389         eor     v14.16b, v12.16b, v14.16b
 1390         add     v0.4s, v0.4s, v8.4s
 1391         add     v2.4s, v2.4s, v6.4s
 1392         orr     v31.16b, v31.16b, v15.16b
 1393         ushr    v15.4s, v14.4s, #12
 1394         shl     v14.4s, v14.4s, #20
 1395         eor     v16.16b, v0.16b, v16.16b
 1396         add     v2.4s, v2.4s, v9.4s
 1397         add     v3.4s, v3.4s, v23.4s
 1398         orr     v14.16b, v14.16b, v15.16b
 1399         ushr    v15.4s, v16.4s, #8
 1400         shl     v16.4s, v16.4s, #24
 1401         eor     v10.16b, v2.16b, v10.16b
 1402         add     v3.4s, v3.4s, v31.4s
 1403         add     v17.4s, v17.4s, v7.4s
 1404         orr     v16.16b, v16.16b, v15.16b
 1405         ushr    v15.4s, v10.4s, #8
 1406         shl     v10.4s, v10.4s, #24
 1407         eor     v11.16b, v3.16b, v11.16b
 1408         add     v17.4s, v17.4s, v14.4s
 1409         orr     v10.16b, v10.16b, v15.16b
 1410         ushr    v15.4s, v11.4s, #8
 1411         shl     v11.4s, v11.4s, #24
 1412         eor     v13.16b, v17.16b, v13.16b
 1413         add     v1.4s, v16.4s, v1.4s
 1414         orr     v11.16b, v11.16b, v15.16b
 1415         ushr    v15.4s, v13.4s, #8
 1416         shl     v13.4s, v13.4s, #24
 1417         eor     v8.16b, v1.16b, v8.16b
 1418         add     v5.4s, v10.4s, v5.4s
 1419         orr     v13.16b, v13.16b, v15.16b
 1420         ushr    v15.4s, v8.4s, #7
 1421         shl     v8.4s, v8.4s, #25
 1422         eor     v9.16b, v5.16b, v9.16b
 1423         add     v4.4s, v11.4s, v4.4s
 1424         orr     v8.16b, v8.16b, v15.16b
 1425         ushr    v15.4s, v9.4s, #7
 1426         shl     v9.4s, v9.4s, #25
 1427         eor     v31.16b, v4.16b, v31.16b
 1428         add     v12.4s, v13.4s, v12.4s
 1429         orr     v9.16b, v9.16b, v15.16b
 1430         ushr    v15.4s, v31.4s, #7
 1431         shl     v31.4s, v31.4s, #25
 1432         eor     v14.16b, v12.16b, v14.16b
 1433         add     v0.4s, v0.4s, v21.4s
 1434         orr     v31.16b, v31.16b, v15.16b
 1435         ushr    v15.4s, v14.4s, #7
 1436         shl     v14.4s, v14.4s, #25
 1437         add     v0.4s, v0.4s, v9.4s
 1438         add     v2.4s, v2.4s, v19.4s
 1439         orr     v14.16b, v14.16b, v15.16b
 1440         eor     v13.16b, v0.16b, v13.16b
 1441         add     v2.4s, v2.4s, v31.4s
 1442         add     v3.4s, v3.4s, v29.4s
 1443         str     q28, [sp, #112]
 1444         rev32   v13.8h, v13.8h
 1445         eor     v16.16b, v2.16b, v16.16b
 1446         add     v3.4s, v3.4s, v14.4s
 1447         add     v17.4s, v17.4s, v26.4s
 1448         add     v4.4s, v4.4s, v13.4s
 1449         rev32   v16.8h, v16.8h
 1450         eor     v10.16b, v3.16b, v10.16b
 1451         add     v17.4s, v17.4s, v8.4s
 1452         ldp     q28, q23, [sp, #112]
 1453         eor     v9.16b, v4.16b, v9.16b
 1454         add     v12.4s, v12.4s, v16.4s
 1455         rev32   v10.8h, v10.8h
 1456         eor     v11.16b, v17.16b, v11.16b
 1457         ldr     q21, [sp, #96]
 1458         ushr    v15.4s, v9.4s, #12
 1459         shl     v9.4s, v9.4s, #20
 1460         eor     v31.16b, v12.16b, v31.16b
 1461         add     v1.4s, v1.4s, v10.4s
 1462         rev32   v11.8h, v11.8h
 1463         orr     v9.16b, v9.16b, v15.16b
 1464         ushr    v15.4s, v31.4s, #12
 1465         shl     v31.4s, v31.4s, #20
 1466         eor     v14.16b, v1.16b, v14.16b
 1467         add     v5.4s, v5.4s, v11.4s
 1468         add     v0.4s, v0.4s, v25.4s
 1469         orr     v31.16b, v31.16b, v15.16b
 1470         ushr    v15.4s, v14.4s, #12
 1471         shl     v14.4s, v14.4s, #20
 1472         eor     v8.16b, v5.16b, v8.16b
 1473         add     v0.4s, v0.4s, v9.4s
 1474         add     v2.4s, v2.4s, v23.4s
 1475         orr     v14.16b, v14.16b, v15.16b
 1476         ushr    v15.4s, v8.4s, #12
 1477         shl     v8.4s, v8.4s, #20
 1478         eor     v13.16b, v0.16b, v13.16b
 1479         add     v2.4s, v2.4s, v31.4s
 1480         add     v3.4s, v3.4s, v21.4s
 1481         orr     v8.16b, v8.16b, v15.16b
 1482         ushr    v15.4s, v13.4s, #8
 1483         shl     v13.4s, v13.4s, #24
 1484         eor     v16.16b, v2.16b, v16.16b
 1485         add     v3.4s, v3.4s, v14.4s
 1486         add     v17.4s, v17.4s, v28.4s
 1487         orr     v13.16b, v13.16b, v15.16b
 1488         ushr    v15.4s, v16.4s, #8
 1489         shl     v16.4s, v16.4s, #24
 1490         eor     v10.16b, v3.16b, v10.16b
 1491         add     v17.4s, v17.4s, v8.4s
 1492         orr     v16.16b, v16.16b, v15.16b
 1493         ushr    v15.4s, v10.4s, #8
 1494         shl     v10.4s, v10.4s, #24
 1495         eor     v11.16b, v17.16b, v11.16b
 1496         add     v4.4s, v13.4s, v4.4s
 1497         orr     v10.16b, v10.16b, v15.16b
 1498         ushr    v15.4s, v11.4s, #8
 1499         shl     v11.4s, v11.4s, #24
 1500         eor     v9.16b, v4.16b, v9.16b
 1501         add     v12.4s, v16.4s, v12.4s
 1502         orr     v11.16b, v11.16b, v15.16b
 1503         ushr    v15.4s, v9.4s, #7
 1504         shl     v9.4s, v9.4s, #25
 1505         eor     v31.16b, v12.16b, v31.16b
 1506         add     v1.4s, v10.4s, v1.4s
 1507         orr     v9.16b, v9.16b, v15.16b
 1508         ushr    v15.4s, v31.4s, #7
 1509         shl     v31.4s, v31.4s, #25
 1510         eor     v14.16b, v1.16b, v14.16b
 1511         add     v5.4s, v11.4s, v5.4s
 1512         orr     v31.16b, v31.16b, v15.16b
 1513         ushr    v15.4s, v14.4s, #7
 1514         shl     v14.4s, v14.4s, #25
 1515         eor     v8.16b, v5.16b, v8.16b
 1516         mov     v30.16b, v29.16b
 1517         mov     v29.16b, v25.16b
 1518         orr     v14.16b, v14.16b, v15.16b
 1519         ushr    v15.4s, v8.4s, #7
 1520         shl     v8.4s, v8.4s, #25
 1521         ldur    q25, [x29, #-112]
 1522         orr     v8.16b, v8.16b, v15.16b
 1523         add     v0.4s, v0.4s, v20.4s
 1524         add     v0.4s, v0.4s, v8.4s
 1525         add     v2.4s, v2.4s, v6.4s
 1526         eor     v16.16b, v0.16b, v16.16b
 1527         add     v2.4s, v2.4s, v9.4s
 1528         add     v3.4s, v3.4s, v7.4s
 1529         rev32   v16.8h, v16.8h
 1530         eor     v10.16b, v2.16b, v10.16b
 1531         add     v3.4s, v3.4s, v31.4s
 1532         add     v17.4s, v17.4s, v25.4s
 1533         add     v1.4s, v1.4s, v16.4s
 1534         rev32   v10.8h, v10.8h
 1535         eor     v11.16b, v3.16b, v11.16b
 1536         add     v17.4s, v17.4s, v14.4s
 1537         eor     v8.16b, v1.16b, v8.16b
 1538         add     v5.4s, v5.4s, v10.4s
 1539         rev32   v11.8h, v11.8h
 1540         eor     v13.16b, v17.16b, v13.16b
 1541         ushr    v15.4s, v8.4s, #12
 1542         shl     v8.4s, v8.4s, #20
 1543         eor     v9.16b, v5.16b, v9.16b
 1544         add     v4.4s, v4.4s, v11.4s
 1545         rev32   v13.8h, v13.8h
 1546         orr     v8.16b, v8.16b, v15.16b
 1547         ushr    v15.4s, v9.4s, #12
 1548         shl     v9.4s, v9.4s, #20
 1549         eor     v31.16b, v4.16b, v31.16b
 1550         add     v12.4s, v12.4s, v13.4s
 1551         add     v0.4s, v0.4s, v18.4s
 1552         orr     v9.16b, v9.16b, v15.16b
 1553         ushr    v15.4s, v31.4s, #12
 1554         shl     v31.4s, v31.4s, #20
 1555         eor     v14.16b, v12.16b, v14.16b
 1556         add     v0.4s, v0.4s, v8.4s
 1557         add     v2.4s, v2.4s, v19.4s
 1558         orr     v31.16b, v31.16b, v15.16b
 1559         ushr    v15.4s, v14.4s, #12
 1560         shl     v14.4s, v14.4s, #20
 1561         eor     v16.16b, v0.16b, v16.16b
 1562         add     v2.4s, v2.4s, v9.4s
 1563         add     v3.4s, v3.4s, v22.4s
 1564         orr     v14.16b, v14.16b, v15.16b
 1565         ushr    v15.4s, v16.4s, #8
 1566         shl     v16.4s, v16.4s, #24
 1567         eor     v10.16b, v2.16b, v10.16b
 1568         add     v3.4s, v3.4s, v31.4s
 1569         add     v17.4s, v17.4s, v21.4s
 1570         orr     v16.16b, v16.16b, v15.16b
 1571         ushr    v15.4s, v10.4s, #8
 1572         shl     v10.4s, v10.4s, #24
 1573         eor     v11.16b, v3.16b, v11.16b
 1574         add     v17.4s, v17.4s, v14.4s
 1575         orr     v10.16b, v10.16b, v15.16b
 1576         ushr    v15.4s, v11.4s, #8
 1577         shl     v11.4s, v11.4s, #24
 1578         eor     v13.16b, v17.16b, v13.16b
 1579         add     v1.4s, v16.4s, v1.4s
 1580         orr     v11.16b, v11.16b, v15.16b
 1581         ushr    v15.4s, v13.4s, #8
 1582         shl     v13.4s, v13.4s, #24
 1583         eor     v8.16b, v1.16b, v8.16b
 1584         add     v5.4s, v10.4s, v5.4s
 1585         orr     v13.16b, v13.16b, v15.16b
 1586         ushr    v15.4s, v8.4s, #7
 1587         shl     v8.4s, v8.4s, #25
 1588         eor     v9.16b, v5.16b, v9.16b
 1589         add     v4.4s, v11.4s, v4.4s
 1590         orr     v8.16b, v8.16b, v15.16b
 1591         ushr    v15.4s, v9.4s, #7
 1592         shl     v9.4s, v9.4s, #25
 1593         eor     v31.16b, v4.16b, v31.16b
 1594         add     v12.4s, v13.4s, v12.4s
 1595         orr     v9.16b, v9.16b, v15.16b
 1596         ushr    v15.4s, v31.4s, #7
 1597         shl     v31.4s, v31.4s, #25
 1598         eor     v14.16b, v12.16b, v14.16b
 1599         add     v0.4s, v0.4s, v27.4s
 1600         orr     v31.16b, v31.16b, v15.16b
 1601         ushr    v15.4s, v14.4s, #7
 1602         shl     v14.4s, v14.4s, #25
 1603         add     v0.4s, v0.4s, v9.4s
 1604         add     v2.4s, v2.4s, v30.4s
 1605         orr     v14.16b, v14.16b, v15.16b
 1606         eor     v13.16b, v0.16b, v13.16b
 1607         add     v2.4s, v2.4s, v31.4s
 1608         add     v3.4s, v3.4s, v29.4s
 1609         rev32   v13.8h, v13.8h
 1610         eor     v16.16b, v2.16b, v16.16b
 1611         add     v3.4s, v3.4s, v14.4s
 1612         add     v17.4s, v17.4s, v28.4s
 1613         add     v4.4s, v4.4s, v13.4s
 1614         rev32   v16.8h, v16.8h
 1615         eor     v10.16b, v3.16b, v10.16b
 1616         add     v17.4s, v17.4s, v8.4s
 1617         eor     v9.16b, v4.16b, v9.16b
 1618         add     v12.4s, v12.4s, v16.4s
 1619         rev32   v10.8h, v10.8h
 1620         eor     v11.16b, v17.16b, v11.16b
 1621         ushr    v15.4s, v9.4s, #12
 1622         shl     v9.4s, v9.4s, #20
 1623         eor     v31.16b, v12.16b, v31.16b
 1624         add     v1.4s, v1.4s, v10.4s
 1625         rev32   v11.8h, v11.8h
 1626         ldr     q24, [sp, #160]
 1627         orr     v9.16b, v9.16b, v15.16b
 1628         ushr    v15.4s, v31.4s, #12
 1629         shl     v31.4s, v31.4s, #20
 1630         eor     v14.16b, v1.16b, v14.16b
 1631         add     v5.4s, v5.4s, v11.4s
 1632         stur    q7, [x29, #-64]
 1633         orr     v31.16b, v31.16b, v15.16b
 1634         ushr    v15.4s, v14.4s, #12
 1635         shl     v14.4s, v14.4s, #20
 1636         eor     v8.16b, v5.16b, v8.16b
 1637         mov     v7.16b, v26.16b
 1638         add     v3.4s, v3.4s, v26.4s
 1639         ldur    q26, [x29, #-80]
 1640         orr     v14.16b, v14.16b, v15.16b
 1641         ushr    v15.4s, v8.4s, #12
 1642         shl     v8.4s, v8.4s, #20
 1643         add     v0.4s, v0.4s, v23.4s
 1644         orr     v8.16b, v8.16b, v15.16b
 1645         add     v15.4s, v0.4s, v9.4s
 1646         add     v2.4s, v2.4s, v24.4s
 1647         eor     v0.16b, v15.16b, v13.16b
 1648         add     v2.4s, v2.4s, v31.4s
 1649         ushr    v13.4s, v0.4s, #8
 1650         shl     v0.4s, v0.4s, #24
 1651         eor     v16.16b, v2.16b, v16.16b
 1652         add     v3.4s, v3.4s, v14.4s
 1653         add     v17.4s, v17.4s, v26.4s
 1654         orr     v0.16b, v0.16b, v13.16b
 1655         ushr    v13.4s, v16.4s, #8
 1656         shl     v16.4s, v16.4s, #24
 1657         eor     v10.16b, v3.16b, v10.16b
 1658         add     v17.4s, v17.4s, v8.4s
 1659         orr     v16.16b, v16.16b, v13.16b
 1660         ushr    v13.4s, v10.4s, #8
 1661         shl     v10.4s, v10.4s, #24
 1662         eor     v11.16b, v17.16b, v11.16b
 1663         add     v4.4s, v0.4s, v4.4s
 1664         orr     v10.16b, v10.16b, v13.16b
 1665         ushr    v13.4s, v11.4s, #8
 1666         shl     v11.4s, v11.4s, #24
 1667         eor     v9.16b, v4.16b, v9.16b
 1668         add     v12.4s, v16.4s, v12.4s
 1669         orr     v11.16b, v11.16b, v13.16b
 1670         ushr    v13.4s, v9.4s, #7
 1671         shl     v9.4s, v9.4s, #25
 1672         eor     v31.16b, v12.16b, v31.16b
 1673         orr     v9.16b, v9.16b, v13.16b
 1674         ushr    v13.4s, v31.4s, #7
 1675         shl     v31.4s, v31.4s, #25
 1676         add     v1.4s, v10.4s, v1.4s
 1677         orr     v31.16b, v31.16b, v13.16b
 1678         eor     v13.16b, v1.16b, v14.16b
 1679         add     v5.4s, v11.4s, v5.4s
 1680         ushr    v14.4s, v13.4s, #7
 1681         shl     v13.4s, v13.4s, #25
 1682         eor     v8.16b, v5.16b, v8.16b
 1683         orr     v13.16b, v13.16b, v14.16b
 1684         ushr    v14.4s, v8.4s, #7
 1685         shl     v8.4s, v8.4s, #25
 1686         stur    q6, [x29, #-96]
 1687         orr     v8.16b, v8.16b, v14.16b
 1688         add     v14.4s, v15.4s, v6.4s
 1689         ldur    q6, [x29, #-64]
 1690         mov     v18.16b, v19.16b
 1691         add     v14.4s, v14.4s, v8.4s
 1692         add     v2.4s, v2.4s, v18.4s
 1693         eor     v16.16b, v14.16b, v16.16b
 1694         add     v2.4s, v2.4s, v9.4s
 1695         add     v3.4s, v3.4s, v21.4s
 1696         rev32   v16.8h, v16.8h
 1697         eor     v10.16b, v2.16b, v10.16b
 1698         add     v3.4s, v3.4s, v31.4s
 1699         add     v17.4s, v17.4s, v6.4s
 1700         add     v1.4s, v1.4s, v16.4s
 1701         rev32   v10.8h, v10.8h
 1702         eor     v11.16b, v3.16b, v11.16b
 1703         add     v17.4s, v17.4s, v13.4s
 1704         eor     v8.16b, v1.16b, v8.16b
 1705         add     v5.4s, v5.4s, v10.4s
 1706         rev32   v11.8h, v11.8h
 1707         eor     v0.16b, v17.16b, v0.16b
 1708         ushr    v15.4s, v8.4s, #12
 1709         shl     v8.4s, v8.4s, #20
 1710         eor     v9.16b, v5.16b, v9.16b
 1711         add     v4.4s, v4.4s, v11.4s
 1712         rev32   v0.8h, v0.8h
 1713         str     q27, [sp, #176]
 1714         mov     v27.16b, v30.16b
 1715         orr     v8.16b, v8.16b, v15.16b
 1716         ushr    v15.4s, v9.4s, #12
 1717         shl     v9.4s, v9.4s, #20
 1718         eor     v31.16b, v4.16b, v31.16b
 1719         add     v12.4s, v12.4s, v0.4s
 1720         add     v14.4s, v14.4s, v25.4s
 1721         orr     v9.16b, v9.16b, v15.16b
 1722         ushr    v15.4s, v31.4s, #12
 1723         shl     v31.4s, v31.4s, #20
 1724         eor     v13.16b, v12.16b, v13.16b
 1725         add     v14.4s, v14.4s, v8.4s
 1726         add     v2.4s, v2.4s, v27.4s
 1727         orr     v31.16b, v31.16b, v15.16b
 1728         ushr    v15.4s, v13.4s, #12
 1729         shl     v13.4s, v13.4s, #20
 1730         eor     v16.16b, v14.16b, v16.16b
 1731         add     v2.4s, v2.4s, v9.4s
 1732         add     v3.4s, v3.4s, v20.4s
 1733         orr     v13.16b, v13.16b, v15.16b
 1734         ushr    v15.4s, v16.4s, #8
 1735         shl     v16.4s, v16.4s, #24
 1736         eor     v10.16b, v2.16b, v10.16b
 1737         add     v3.4s, v3.4s, v31.4s
 1738         add     v17.4s, v17.4s, v7.4s
 1739         orr     v16.16b, v16.16b, v15.16b
 1740         ushr    v15.4s, v10.4s, #8
 1741         shl     v10.4s, v10.4s, #24
 1742         eor     v11.16b, v3.16b, v11.16b
 1743         add     v17.4s, v17.4s, v13.4s
 1744         mov     v30.16b, v23.16b
 1745         orr     v10.16b, v10.16b, v15.16b
 1746         ushr    v15.4s, v11.4s, #8
 1747         shl     v11.4s, v11.4s, #24
 1748         eor     v0.16b, v17.16b, v0.16b
 1749         add     v1.4s, v16.4s, v1.4s
 1750         ldur    q23, [x29, #-144]
 1751         orr     v11.16b, v11.16b, v15.16b
 1752         ushr    v15.4s, v0.4s, #8
 1753         shl     v0.4s, v0.4s, #24
 1754         eor     v8.16b, v1.16b, v8.16b
 1755         add     v5.4s, v10.4s, v5.4s
 1756         orr     v0.16b, v0.16b, v15.16b
 1757         ushr    v15.4s, v8.4s, #7
 1758         shl     v8.4s, v8.4s, #25
 1759         eor     v9.16b, v5.16b, v9.16b
 1760         add     v4.4s, v11.4s, v4.4s
 1761         orr     v8.16b, v8.16b, v15.16b
 1762         ushr    v15.4s, v9.4s, #7
 1763         shl     v9.4s, v9.4s, #25
 1764         eor     v31.16b, v4.16b, v31.16b
 1765         add     v12.4s, v0.4s, v12.4s
 1766         orr     v9.16b, v9.16b, v15.16b
 1767         ushr    v15.4s, v31.4s, #7
 1768         shl     v31.4s, v31.4s, #25
 1769         eor     v13.16b, v12.16b, v13.16b
 1770         add     v14.4s, v14.4s, v23.4s
 1771         orr     v31.16b, v31.16b, v15.16b
 1772         ushr    v15.4s, v13.4s, #7
 1773         shl     v13.4s, v13.4s, #25
 1774         add     v14.4s, v14.4s, v9.4s
 1775         add     v2.4s, v2.4s, v29.4s
 1776         orr     v13.16b, v13.16b, v15.16b
 1777         eor     v0.16b, v14.16b, v0.16b
 1778         add     v2.4s, v2.4s, v31.4s
 1779         add     v3.4s, v3.4s, v30.4s
 1780         rev32   v0.8h, v0.8h
 1781         eor     v16.16b, v2.16b, v16.16b
 1782         add     v3.4s, v3.4s, v13.4s
 1783         add     v17.4s, v17.4s, v26.4s
 1784         add     v4.4s, v4.4s, v0.4s
 1785         rev32   v16.8h, v16.8h
 1786         eor     v10.16b, v3.16b, v10.16b
 1787         add     v17.4s, v17.4s, v8.4s
 1788         ldur    q22, [x29, #-128]
 1789         eor     v9.16b, v4.16b, v9.16b
 1790         add     v12.4s, v12.4s, v16.4s
 1791         rev32   v10.8h, v10.8h
 1792         eor     v11.16b, v17.16b, v11.16b
 1793         ushr    v15.4s, v9.4s, #12
 1794         shl     v9.4s, v9.4s, #20
 1795         eor     v31.16b, v12.16b, v31.16b
 1796         add     v1.4s, v1.4s, v10.4s
 1797         rev32   v11.8h, v11.8h
 1798         ldr     q26, [sp, #176]
 1799         orr     v9.16b, v9.16b, v15.16b
 1800         ushr    v15.4s, v31.4s, #12
 1801         shl     v31.4s, v31.4s, #20
 1802         eor     v13.16b, v1.16b, v13.16b
 1803         add     v5.4s, v5.4s, v11.4s
 1804         add     v14.4s, v14.4s, v24.4s
 1805         orr     v31.16b, v31.16b, v15.16b
 1806         ushr    v15.4s, v13.4s, #12
 1807         shl     v13.4s, v13.4s, #20
 1808         eor     v8.16b, v5.16b, v8.16b
 1809         add     v14.4s, v14.4s, v9.4s
 1810         add     v2.4s, v2.4s, v22.4s
 1811         orr     v13.16b, v13.16b, v15.16b
 1812         ushr    v15.4s, v8.4s, #12
 1813         shl     v8.4s, v8.4s, #20
 1814         eor     v0.16b, v14.16b, v0.16b
 1815         add     v2.4s, v2.4s, v31.4s
 1816         add     v3.4s, v3.4s, v28.4s
 1817         orr     v8.16b, v8.16b, v15.16b
 1818         ushr    v15.4s, v0.4s, #8
 1819         shl     v0.4s, v0.4s, #24
 1820         eor     v16.16b, v2.16b, v16.16b
 1821         add     v3.4s, v3.4s, v13.4s
 1822         add     v17.4s, v17.4s, v26.4s
 1823         orr     v0.16b, v0.16b, v15.16b
 1824         ushr    v15.4s, v16.4s, #8
 1825         shl     v16.4s, v16.4s, #24
 1826         eor     v10.16b, v3.16b, v10.16b
 1827         add     v17.4s, v17.4s, v8.4s
 1828         orr     v16.16b, v16.16b, v15.16b
 1829         ushr    v15.4s, v10.4s, #8
 1830         shl     v10.4s, v10.4s, #24
 1831         eor     v11.16b, v17.16b, v11.16b
 1832         add     v4.4s, v0.4s, v4.4s
 1833         orr     v10.16b, v10.16b, v15.16b
 1834         ushr    v15.4s, v11.4s, #8
 1835         shl     v11.4s, v11.4s, #24
 1836         eor     v9.16b, v4.16b, v9.16b
 1837         add     v12.4s, v16.4s, v12.4s
 1838         orr     v11.16b, v11.16b, v15.16b
 1839         ushr    v15.4s, v9.4s, #7
 1840         shl     v9.4s, v9.4s, #25
 1841         eor     v31.16b, v12.16b, v31.16b
 1842         add     v1.4s, v10.4s, v1.4s
 1843         orr     v9.16b, v9.16b, v15.16b
 1844         ushr    v15.4s, v31.4s, #7
 1845         shl     v31.4s, v31.4s, #25
 1846         eor     v13.16b, v1.16b, v13.16b
 1847         add     v5.4s, v11.4s, v5.4s
 1848         orr     v31.16b, v31.16b, v15.16b
 1849         ushr    v15.4s, v13.4s, #7
 1850         shl     v13.4s, v13.4s, #25
 1851         eor     v8.16b, v5.16b, v8.16b
 1852         orr     v13.16b, v13.16b, v15.16b
 1853         ushr    v15.4s, v8.4s, #7
 1854         shl     v8.4s, v8.4s, #25
 1855         orr     v8.16b, v8.16b, v15.16b
 1856         add     v14.4s, v14.4s, v18.4s
 1857         add     v14.4s, v14.4s, v8.4s
 1858         add     v2.4s, v2.4s, v27.4s
 1859         eor     v16.16b, v14.16b, v16.16b
 1860         add     v2.4s, v2.4s, v9.4s
 1861         add     v3.4s, v3.4s, v7.4s
 1862         rev32   v16.8h, v16.8h
 1863         eor     v10.16b, v2.16b, v10.16b
 1864         add     v3.4s, v3.4s, v31.4s
 1865         add     v17.4s, v17.4s, v21.4s
 1866         add     v1.4s, v1.4s, v16.4s
 1867         rev32   v10.8h, v10.8h
 1868         eor     v11.16b, v3.16b, v11.16b
 1869         add     v17.4s, v17.4s, v13.4s
 1870         eor     v8.16b, v1.16b, v8.16b
 1871         add     v5.4s, v5.4s, v10.4s
 1872         rev32   v11.8h, v11.8h
 1873         eor     v0.16b, v17.16b, v0.16b
 1874         add     v14.4s, v14.4s, v6.4s
 1875         ldur    q6, [x29, #-96]
 1876         ushr    v15.4s, v8.4s, #12
 1877         shl     v8.4s, v8.4s, #20
 1878         eor     v9.16b, v5.16b, v9.16b
 1879         add     v4.4s, v4.4s, v11.4s
 1880         rev32   v0.8h, v0.8h
 1881         stur    q20, [x29, #-160]
 1882         mov     v20.16b, v29.16b
 1883         orr     v8.16b, v8.16b, v15.16b
 1884         ushr    v15.4s, v9.4s, #12
 1885         shl     v9.4s, v9.4s, #20
 1886         eor     v31.16b, v4.16b, v31.16b
 1887         add     v12.4s, v12.4s, v0.4s
 1888         mov     v19.16b, v29.16b
 1889         orr     v9.16b, v9.16b, v15.16b
 1890         ushr    v15.4s, v31.4s, #12
 1891         shl     v31.4s, v31.4s, #20
 1892         eor     v13.16b, v12.16b, v13.16b
 1893         add     v14.4s, v14.4s, v8.4s
 1894         add     v2.4s, v2.4s, v20.4s
 1895         mov     v19.16b, v28.16b
 1896         orr     v31.16b, v31.16b, v15.16b
 1897         ushr    v15.4s, v13.4s, #12
 1898         shl     v13.4s, v13.4s, #20
 1899         eor     v16.16b, v14.16b, v16.16b
 1900         add     v2.4s, v2.4s, v9.4s
 1901         add     v3.4s, v3.4s, v6.4s
 1902         orr     v13.16b, v13.16b, v15.16b
 1903         ushr    v15.4s, v16.4s, #8
 1904         shl     v16.4s, v16.4s, #24
 1905         eor     v10.16b, v2.16b, v10.16b
 1906         add     v3.4s, v3.4s, v31.4s
 1907         add     v17.4s, v17.4s, v19.4s
 1908         orr     v16.16b, v16.16b, v15.16b
 1909         ushr    v15.4s, v10.4s, #8
 1910         shl     v10.4s, v10.4s, #24
 1911         eor     v11.16b, v3.16b, v11.16b
 1912         add     v17.4s, v17.4s, v13.4s
 1913         orr     v10.16b, v10.16b, v15.16b
 1914         ushr    v15.4s, v11.4s, #8
 1915         shl     v11.4s, v11.4s, #24
 1916         eor     v0.16b, v17.16b, v0.16b
 1917         add     v1.4s, v16.4s, v1.4s
 1918         orr     v11.16b, v11.16b, v15.16b
 1919         ushr    v15.4s, v0.4s, #8
 1920         shl     v0.4s, v0.4s, #24
 1921         eor     v8.16b, v1.16b, v8.16b
 1922         add     v5.4s, v10.4s, v5.4s
 1923         orr     v0.16b, v0.16b, v15.16b
 1924         ushr    v15.4s, v8.4s, #7
 1925         shl     v8.4s, v8.4s, #25
 1926         eor     v9.16b, v5.16b, v9.16b
 1927         add     v4.4s, v11.4s, v4.4s
 1928         orr     v8.16b, v8.16b, v15.16b
 1929         ushr    v15.4s, v9.4s, #7
 1930         shl     v9.4s, v9.4s, #25
 1931         eor     v31.16b, v4.16b, v31.16b
 1932         add     v12.4s, v0.4s, v12.4s
 1933         orr     v9.16b, v9.16b, v15.16b
 1934         ushr    v15.4s, v31.4s, #7
 1935         shl     v31.4s, v31.4s, #25
 1936         eor     v13.16b, v12.16b, v13.16b
 1937         add     v14.4s, v14.4s, v25.4s
 1938         orr     v31.16b, v31.16b, v15.16b
 1939         ushr    v15.4s, v13.4s, #7
 1940         shl     v13.4s, v13.4s, #25
 1941         add     v14.4s, v14.4s, v9.4s
 1942         add     v2.4s, v2.4s, v30.4s
 1943         orr     v13.16b, v13.16b, v15.16b
 1944         eor     v0.16b, v14.16b, v0.16b
 1945         add     v2.4s, v2.4s, v31.4s
 1946         add     v3.4s, v3.4s, v24.4s
 1947         rev32   v0.8h, v0.8h
 1948         eor     v16.16b, v2.16b, v16.16b
 1949         add     v3.4s, v3.4s, v13.4s
 1950         add     v17.4s, v17.4s, v26.4s
 1951         mov     v29.16b, v27.16b
 1952         add     v4.4s, v4.4s, v0.4s
 1953         rev32   v16.8h, v16.8h
 1954         eor     v10.16b, v3.16b, v10.16b
 1955         add     v17.4s, v17.4s, v8.4s
 1956         ldur    q27, [x29, #-160]
 1957         eor     v9.16b, v4.16b, v9.16b
 1958         add     v12.4s, v12.4s, v16.4s
 1959         rev32   v10.8h, v10.8h
 1960         eor     v11.16b, v17.16b, v11.16b
 1961         ldur    q6, [x29, #-80]
 1962         ushr    v15.4s, v9.4s, #12
 1963         shl     v9.4s, v9.4s, #20
 1964         eor     v31.16b, v12.16b, v31.16b
 1965         add     v1.4s, v1.4s, v10.4s
 1966         rev32   v11.8h, v11.8h
 1967         orr     v9.16b, v9.16b, v15.16b
 1968         ushr    v15.4s, v31.4s, #12
 1969         shl     v31.4s, v31.4s, #20
 1970         eor     v13.16b, v1.16b, v13.16b
 1971         add     v5.4s, v5.4s, v11.4s
 1972         add     v14.4s, v14.4s, v22.4s
 1973         orr     v31.16b, v31.16b, v15.16b
 1974         ushr    v15.4s, v13.4s, #12
 1975         shl     v13.4s, v13.4s, #20
 1976         eor     v8.16b, v5.16b, v8.16b
 1977         add     v14.4s, v14.4s, v9.4s
 1978         add     v2.4s, v2.4s, v27.4s
 1979         orr     v13.16b, v13.16b, v15.16b
 1980         ushr    v15.4s, v8.4s, #12
 1981         shl     v8.4s, v8.4s, #20
 1982         eor     v0.16b, v14.16b, v0.16b
 1983         add     v2.4s, v2.4s, v31.4s
 1984         add     v3.4s, v3.4s, v6.4s
 1985         orr     v8.16b, v8.16b, v15.16b
 1986         ushr    v15.4s, v0.4s, #8
 1987         shl     v0.4s, v0.4s, #24
 1988         eor     v16.16b, v2.16b, v16.16b
 1989         add     v3.4s, v3.4s, v13.4s
 1990         add     v17.4s, v17.4s, v23.4s
 1991         orr     v0.16b, v0.16b, v15.16b
 1992         ushr    v15.4s, v16.4s, #8
 1993         shl     v16.4s, v16.4s, #24
 1994         eor     v10.16b, v3.16b, v10.16b
 1995         add     v17.4s, v17.4s, v8.4s
 1996         orr     v16.16b, v16.16b, v15.16b
 1997         ushr    v15.4s, v10.4s, #8
 1998         shl     v10.4s, v10.4s, #24
 1999         eor     v11.16b, v17.16b, v11.16b
 2000         add     v4.4s, v0.4s, v4.4s
 2001         orr     v10.16b, v10.16b, v15.16b
 2002         ushr    v15.4s, v11.4s, #8
 2003         shl     v11.4s, v11.4s, #24
 2004         eor     v9.16b, v4.16b, v9.16b
 2005         add     v12.4s, v16.4s, v12.4s
 2006         orr     v11.16b, v11.16b, v15.16b
 2007         ushr    v15.4s, v9.4s, #7
 2008         shl     v9.4s, v9.4s, #25
 2009         eor     v31.16b, v12.16b, v31.16b
 2010         add     v1.4s, v10.4s, v1.4s
 2011         orr     v9.16b, v9.16b, v15.16b
 2012         ushr    v15.4s, v31.4s, #7
 2013         shl     v31.4s, v31.4s, #25
 2014         eor     v13.16b, v1.16b, v13.16b
 2015         add     v5.4s, v11.4s, v5.4s
 2016         orr     v31.16b, v31.16b, v15.16b
 2017         ushr    v15.4s, v13.4s, #7
 2018         shl     v13.4s, v13.4s, #25
 2019         eor     v8.16b, v5.16b, v8.16b
 2020         orr     v13.16b, v13.16b, v15.16b
 2021         ushr    v15.4s, v8.4s, #7
 2022         shl     v8.4s, v8.4s, #25
 2023         orr     v8.16b, v8.16b, v15.16b
 2024         add     v14.4s, v14.4s, v29.4s
 2025         add     v14.4s, v14.4s, v8.4s
 2026         add     v2.4s, v2.4s, v20.4s
 2027         mov     v28.16b, v7.16b
 2028         eor     v16.16b, v14.16b, v16.16b
 2029         add     v2.4s, v2.4s, v9.4s
 2030         add     v3.4s, v3.4s, v19.4s
 2031         rev32   v16.8h, v16.8h
 2032         eor     v10.16b, v2.16b, v10.16b
 2033         add     v3.4s, v3.4s, v31.4s
 2034         add     v17.4s, v17.4s, v28.4s
 2035         add     v1.4s, v1.4s, v16.4s
 2036         rev32   v10.8h, v10.8h
 2037         eor     v11.16b, v3.16b, v11.16b
 2038         add     v17.4s, v17.4s, v13.4s
 2039         eor     v8.16b, v1.16b, v8.16b
 2040         add     v5.4s, v5.4s, v10.4s
 2041         rev32   v11.8h, v11.8h
 2042         eor     v0.16b, v17.16b, v0.16b
 2043         ushr    v15.4s, v8.4s, #12
 2044         shl     v8.4s, v8.4s, #20
 2045         eor     v9.16b, v5.16b, v9.16b
 2046         add     v4.4s, v4.4s, v11.4s
 2047         rev32   v0.8h, v0.8h
 2048         orr     v8.16b, v8.16b, v15.16b
 2049         ushr    v15.4s, v9.4s, #12
 2050         shl     v9.4s, v9.4s, #20
 2051         eor     v31.16b, v4.16b, v31.16b
 2052         add     v12.4s, v12.4s, v0.4s
 2053         add     v14.4s, v14.4s, v21.4s
 2054         orr     v9.16b, v9.16b, v15.16b
 2055         ushr    v15.4s, v31.4s, #12
 2056         shl     v31.4s, v31.4s, #20
 2057         eor     v13.16b, v12.16b, v13.16b
 2058         add     v14.4s, v14.4s, v8.4s
 2059         add     v2.4s, v2.4s, v30.4s
 2060         orr     v31.16b, v31.16b, v15.16b
 2061         ushr    v15.4s, v13.4s, #12
 2062         shl     v13.4s, v13.4s, #20
 2063         eor     v16.16b, v14.16b, v16.16b
 2064         add     v2.4s, v2.4s, v9.4s
 2065         orr     v13.16b, v13.16b, v15.16b
 2066         ushr    v15.4s, v16.4s, #8
 2067         shl     v16.4s, v16.4s, #24
 2068         eor     v10.16b, v2.16b, v10.16b
 2069         orr     v16.16b, v16.16b, v15.16b
 2070         ushr    v15.4s, v10.4s, #8
 2071         shl     v10.4s, v10.4s, #24
 2072         add     v3.4s, v3.4s, v18.4s
 2073         orr     v10.16b, v10.16b, v15.16b
 2074         add     v15.4s, v3.4s, v31.4s
 2075         eor     v3.16b, v15.16b, v11.16b
 2076         ushr    v11.4s, v3.4s, #8
 2077         shl     v3.4s, v3.4s, #24
 2078         orr     v11.16b, v3.16b, v11.16b
 2079         add     v3.4s, v17.4s, v6.4s
 2080         add     v17.4s, v3.4s, v13.4s
 2081         eor     v0.16b, v17.16b, v0.16b
 2082         ushr    v3.4s, v0.4s, #8
 2083         shl     v0.4s, v0.4s, #24
 2084         add     v1.4s, v16.4s, v1.4s
 2085         orr     v0.16b, v0.16b, v3.16b
 2086         eor     v3.16b, v1.16b, v8.16b
 2087         ushr    v8.4s, v3.4s, #7
 2088         shl     v3.4s, v3.4s, #25
 2089         add     v5.4s, v10.4s, v5.4s
 2090         orr     v8.16b, v3.16b, v8.16b
 2091         eor     v3.16b, v5.16b, v9.16b
 2092         add     v4.4s, v11.4s, v4.4s
 2093         ushr    v9.4s, v3.4s, #7
 2094         shl     v3.4s, v3.4s, #25
 2095         eor     v31.16b, v4.16b, v31.16b
 2096         mov     v7.16b, v23.16b
 2097         mov     v23.16b, v28.16b
 2098         mov     v28.16b, v6.16b
 2099         orr     v3.16b, v3.16b, v9.16b
 2100         ushr    v9.4s, v31.4s, #7
 2101         shl     v31.4s, v31.4s, #25
 2102         ldur    q6, [x29, #-64]
 2103         orr     v31.16b, v31.16b, v9.16b
 2104         add     v9.4s, v0.4s, v12.4s
 2105         eor     v12.16b, v9.16b, v13.16b
 2106         ushr    v13.4s, v12.4s, #7
 2107         shl     v12.4s, v12.4s, #25
 2108         orr     v12.16b, v12.16b, v13.16b
 2109         add     v13.4s, v14.4s, v6.4s
 2110         add     v13.4s, v13.4s, v3.4s
 2111         eor     v0.16b, v13.16b, v0.16b
 2112         add     v2.4s, v2.4s, v24.4s
 2113         rev32   v14.8h, v0.8h
 2114         add     v0.4s, v2.4s, v31.4s
 2115         add     v6.4s, v4.4s, v14.4s
 2116         eor     v2.16b, v0.16b, v16.16b
 2117         eor     v3.16b, v6.16b, v3.16b
 2118         rev32   v16.8h, v2.8h
 2119         ushr    v4.4s, v3.4s, #12
 2120         shl     v3.4s, v3.4s, #20
 2121         add     v2.4s, v9.4s, v16.4s
 2122         orr     v4.16b, v3.16b, v4.16b
 2123         eor     v3.16b, v2.16b, v31.16b
 2124         ushr    v31.4s, v3.4s, #12
 2125         shl     v3.4s, v3.4s, #20
 2126         orr     v3.16b, v3.16b, v31.16b
 2127         add     v31.4s, v15.4s, v22.4s
 2128         add     v31.4s, v31.4s, v12.4s
 2129         add     v17.4s, v17.4s, v7.4s
 2130         eor     v9.16b, v31.16b, v10.16b
 2131         add     v17.4s, v17.4s, v8.4s
 2132         rev32   v9.8h, v9.8h
 2133         eor     v11.16b, v17.16b, v11.16b
 2134         add     v1.4s, v1.4s, v9.4s
 2135         rev32   v11.8h, v11.8h
 2136         eor     v10.16b, v1.16b, v12.16b
 2137         add     v5.4s, v5.4s, v11.4s
 2138         ushr    v12.4s, v10.4s, #12
 2139         shl     v10.4s, v10.4s, #20
 2140         eor     v8.16b, v5.16b, v8.16b
 2141         orr     v10.16b, v10.16b, v12.16b
 2142         ushr    v12.4s, v8.4s, #12
 2143         shl     v8.4s, v8.4s, #20
 2144         orr     v8.16b, v8.16b, v12.16b
 2145         add     v12.4s, v13.4s, v27.4s
 2146         add     v12.4s, v12.4s, v4.4s
 2147         eor     v13.16b, v12.16b, v14.16b
 2148         ldur    q14, [x29, #-96]
 2149         mov     v25.16b, v29.16b
 2150         add     v29.4s, v12.4s, v20.4s
 2151         add     v20.4s, v31.4s, v26.4s
 2152         add     v0.4s, v0.4s, v14.4s
 2153         add     v0.4s, v0.4s, v3.4s
 2154         eor     v16.16b, v0.16b, v16.16b
 2155         add     v0.4s, v0.4s, v30.4s
 2156         ldur    q30, [x29, #-112]
 2157         add     v20.4s, v20.4s, v10.4s
 2158         eor     v31.16b, v20.16b, v9.16b
 2159         add     v20.4s, v20.4s, v28.4s
 2160         add     v17.4s, v17.4s, v30.4s
 2161         add     v17.4s, v17.4s, v8.4s
 2162         eor     v9.16b, v17.16b, v11.16b
 2163         ushr    v28.4s, v13.4s, #8
 2164         shl     v11.4s, v13.4s, #24
 2165         orr     v28.16b, v11.16b, v28.16b
 2166         ushr    v11.4s, v16.4s, #8
 2167         shl     v16.4s, v16.4s, #24
 2168         orr     v16.16b, v16.16b, v11.16b
 2169         ushr    v11.4s, v31.4s, #8
 2170         shl     v31.4s, v31.4s, #24
 2171         add     v6.4s, v28.4s, v6.4s
 2172         orr     v31.16b, v31.16b, v11.16b
 2173         ushr    v11.4s, v9.4s, #8
 2174         shl     v9.4s, v9.4s, #24
 2175         add     v2.4s, v16.4s, v2.4s
 2176         eor     v4.16b, v6.16b, v4.16b
 2177         orr     v9.16b, v9.16b, v11.16b
 2178         add     v1.4s, v31.4s, v1.4s
 2179         eor     v3.16b, v2.16b, v3.16b
 2180         ushr    v11.4s, v4.4s, #7
 2181         shl     v4.4s, v4.4s, #25
 2182         add     v5.4s, v9.4s, v5.4s
 2183         eor     v10.16b, v1.16b, v10.16b
 2184         orr     v4.16b, v4.16b, v11.16b
 2185         ushr    v11.4s, v3.4s, #7
 2186         shl     v3.4s, v3.4s, #25
 2187         eor     v8.16b, v5.16b, v8.16b
 2188         orr     v3.16b, v3.16b, v11.16b
 2189         ushr    v11.4s, v10.4s, #7
 2190         shl     v10.4s, v10.4s, #25
 2191         orr     v10.16b, v10.16b, v11.16b
 2192         ushr    v11.4s, v8.4s, #7
 2193         shl     v8.4s, v8.4s, #25
 2194         orr     v8.16b, v8.16b, v11.16b
 2195         add     v29.4s, v29.4s, v8.4s
 2196         eor     v16.16b, v29.16b, v16.16b
 2197         add     v0.4s, v0.4s, v4.4s
 2198         mov     v12.16b, v26.16b
 2199         add     v17.4s, v17.4s, v19.4s
 2200         add     v26.4s, v29.4s, v23.4s
 2201         eor     v29.16b, v0.16b, v31.16b
 2202         add     v20.4s, v20.4s, v3.4s
 2203         rev32   v16.8h, v16.8h
 2204         stur    q18, [x29, #-176]
 2205         mov     v18.16b, v27.16b
 2206         add     v0.4s, v0.4s, v24.4s
 2207         eor     v27.16b, v20.16b, v9.16b
 2208         add     v17.4s, v17.4s, v10.4s
 2209         rev32   v24.8h, v29.8h
 2210         add     v1.4s, v1.4s, v16.4s
 2211         add     v20.4s, v20.4s, v25.4s
 2212         eor     v25.16b, v17.16b, v28.16b
 2213         rev32   v27.8h, v27.8h
 2214         add     v5.4s, v5.4s, v24.4s
 2215         eor     v28.16b, v1.16b, v8.16b
 2216         rev32   v25.8h, v25.8h
 2217         add     v6.4s, v6.4s, v27.4s
 2218         eor     v4.16b, v5.16b, v4.16b
 2219         ushr    v31.4s, v28.4s, #12
 2220         shl     v28.4s, v28.4s, #20
 2221         add     v2.4s, v2.4s, v25.4s
 2222         eor     v3.16b, v6.16b, v3.16b
 2223         orr     v28.16b, v28.16b, v31.16b
 2224         ushr    v31.4s, v4.4s, #12
 2225         shl     v4.4s, v4.4s, #20
 2226         eor     v29.16b, v2.16b, v10.16b
 2227         orr     v4.16b, v4.16b, v31.16b
 2228         ushr    v31.4s, v3.4s, #12
 2229         shl     v3.4s, v3.4s, #20
 2230         add     v26.4s, v26.4s, v28.4s
 2231         orr     v3.16b, v3.16b, v31.16b
 2232         ushr    v31.4s, v29.4s, #12
 2233         shl     v29.4s, v29.4s, #20
 2234         eor     v16.16b, v26.16b, v16.16b
 2235         add     v0.4s, v0.4s, v4.4s
 2236         add     v17.4s, v17.4s, v12.4s
 2237         orr     v29.16b, v29.16b, v31.16b
 2238         eor     v24.16b, v0.16b, v24.16b
 2239         add     v0.4s, v0.4s, v22.4s
 2240         add     v20.4s, v20.4s, v3.4s
 2241         ushr    v22.4s, v16.4s, #8
 2242         shl     v16.4s, v16.4s, #24
 2243         add     v23.4s, v26.4s, v21.4s
 2244         eor     v21.16b, v20.16b, v27.16b
 2245         add     v17.4s, v17.4s, v29.4s
 2246         orr     v16.16b, v16.16b, v22.16b
 2247         ushr    v22.4s, v24.4s, #8
 2248         shl     v24.4s, v24.4s, #24
 2249         eor     v25.16b, v17.16b, v25.16b
 2250         orr     v22.16b, v24.16b, v22.16b
 2251         ushr    v24.4s, v21.4s, #8
 2252         shl     v21.4s, v21.4s, #24
 2253         orr     v21.16b, v21.16b, v24.16b
 2254         ushr    v24.4s, v25.4s, #8
 2255         shl     v25.4s, v25.4s, #24
 2256         add     v1.4s, v16.4s, v1.4s
 2257         orr     v24.16b, v25.16b, v24.16b
 2258         add     v5.4s, v22.4s, v5.4s
 2259         eor     v25.16b, v1.16b, v28.16b
 2260         add     v6.4s, v21.4s, v6.4s
 2261         eor     v4.16b, v5.16b, v4.16b
 2262         ushr    v27.4s, v25.4s, #7
 2263         shl     v25.4s, v25.4s, #25
 2264         add     v2.4s, v24.4s, v2.4s
 2265         eor     v3.16b, v6.16b, v3.16b
 2266         orr     v25.16b, v25.16b, v27.16b
 2267         ushr    v27.4s, v4.4s, #7
 2268         shl     v4.4s, v4.4s, #25
 2269         ldur    q19, [x29, #-176]
 2270         eor     v26.16b, v2.16b, v29.16b
 2271         orr     v4.16b, v4.16b, v27.16b
 2272         ushr    v27.4s, v3.4s, #7
 2273         shl     v3.4s, v3.4s, #25
 2274         orr     v3.16b, v3.16b, v27.16b
 2275         ushr    v27.4s, v26.4s, #7
 2276         shl     v26.4s, v26.4s, #25
 2277         add     v20.4s, v20.4s, v18.4s
 2278         add     v17.4s, v17.4s, v30.4s
 2279         orr     v26.16b, v26.16b, v27.16b
 2280         add     v0.4s, v0.4s, v3.4s
 2281         eor     v16.16b, v0.16b, v16.16b
 2282         add     v0.4s, v0.4s, v19.4s
 2283         add     v19.4s, v20.4s, v26.4s
 2284         add     v17.4s, v17.4s, v25.4s
 2285         eor     v20.16b, v19.16b, v22.16b
 2286         add     v7.4s, v19.4s, v7.4s
 2287         eor     v19.16b, v17.16b, v21.16b
 2288         ldur    q21, [x29, #-64]
 2289         add     v23.4s, v23.4s, v4.4s
 2290         eor     v24.16b, v23.16b, v24.16b
 2291         rev32   v16.8h, v16.8h
 2292         add     v17.4s, v17.4s, v21.4s
 2293         rev32   v21.8h, v24.8h
 2294         add     v6.4s, v6.4s, v21.4s
 2295         rev32   v20.8h, v20.8h
 2296         add     v2.4s, v2.4s, v16.4s
 2297         eor     v4.16b, v6.16b, v4.16b
 2298         rev32   v19.8h, v19.8h
 2299         add     v1.4s, v1.4s, v20.4s
 2300         eor     v3.16b, v2.16b, v3.16b
 2301         ushr    v24.4s, v4.4s, #12
 2302         shl     v4.4s, v4.4s, #20
 2303         add     v5.4s, v5.4s, v19.4s
 2304         eor     v22.16b, v1.16b, v26.16b
 2305         orr     v4.16b, v4.16b, v24.16b
 2306         ushr    v24.4s, v3.4s, #12
 2307         shl     v3.4s, v3.4s, #20
 2308         add     v18.4s, v23.4s, v14.4s
 2309         eor     v23.16b, v5.16b, v25.16b
 2310         orr     v3.16b, v3.16b, v24.16b
 2311         ushr    v24.4s, v22.4s, #12
 2312         shl     v22.4s, v22.4s, #20
 2313         orr     v22.16b, v22.16b, v24.16b
 2314         ushr    v24.4s, v23.4s, #12
 2315         shl     v23.4s, v23.4s, #20
 2316         orr     v23.16b, v23.16b, v24.16b
 2317         add     v18.4s, v18.4s, v4.4s
 2318         add     v0.4s, v0.4s, v3.4s
 2319         add     v24.4s, v17.4s, v23.4s
 2320         eor     v17.16b, v18.16b, v21.16b
 2321         add     v7.4s, v7.4s, v22.4s
 2322         eor     v16.16b, v0.16b, v16.16b
 2323         ushr    v21.4s, v17.4s, #8
 2324         shl     v17.4s, v17.4s, #24
 2325         eor     v20.16b, v7.16b, v20.16b
 2326         orr     v21.16b, v17.16b, v21.16b
 2327         ushr    v17.4s, v16.4s, #8
 2328         shl     v16.4s, v16.4s, #24
 2329         eor     v19.16b, v24.16b, v19.16b
 2330         orr     v16.16b, v16.16b, v17.16b
 2331         ushr    v17.4s, v20.4s, #8
 2332         shl     v20.4s, v20.4s, #24
 2333         orr     v25.16b, v20.16b, v17.16b
 2334         ushr    v17.4s, v19.4s, #8
 2335         shl     v19.4s, v19.4s, #24
 2336         orr     v19.16b, v19.16b, v17.16b
 2337         add     v1.4s, v25.4s, v1.4s
 2338         eor     v22.16b, v1.16b, v22.16b
 2339         eor     v20.16b, v1.16b, v18.16b
 2340         add     v1.4s, v19.4s, v5.4s
 2341         eor     v26.16b, v1.16b, v0.16b
 2342         add     v0.4s, v21.4s, v6.4s
 2343         eor     v5.16b, v1.16b, v23.16b
 2344         eor     v1.16b, v0.16b, v4.16b
 2345         eor     v17.16b, v0.16b, v7.16b
 2346         add     v0.4s, v16.4s, v2.4s
 2347         eor     v2.16b, v0.16b, v3.16b
 2348         eor     v6.16b, v0.16b, v24.16b
 2349         ushr    v0.4s, v1.4s, #7
 2350         shl     v1.4s, v1.4s, #25
 2351         orr     v0.16b, v1.16b, v0.16b
 2352         ushr    v1.4s, v2.4s, #7
 2353         shl     v2.4s, v2.4s, #25
 2354         orr     v1.16b, v2.16b, v1.16b
 2355         ushr    v2.4s, v22.4s, #7
 2356         shl     v3.4s, v22.4s, #25
 2357         orr     v2.16b, v3.16b, v2.16b
 2358         ushr    v3.4s, v5.4s, #7
 2359         shl     v4.4s, v5.4s, #25
 2360         orr     v3.16b, v4.16b, v3.16b
 2361         eor     v8.16b, v16.16b, v3.16b
 2362         eor     v9.16b, v25.16b, v0.16b
 2363         eor     v31.16b, v1.16b, v19.16b
 2364         cmp     x17, x22
 2365         eor     v15.16b, v2.16b, v21.16b
 2366         mov     w18, w19
 2367         b.ne    .LBB2_4
 2368 .LBB2_7:
 2369         zip1    v0.4s, v20.4s, v26.4s
 2370         zip2    v1.4s, v20.4s, v26.4s
 2371         zip1    v2.4s, v17.4s, v6.4s
 2372         zip2    v3.4s, v17.4s, v6.4s
 2373         zip1    v4.4s, v8.4s, v9.4s
 2374         zip2    v5.4s, v8.4s, v9.4s
 2375         zip1    v6.4s, v31.4s, v15.4s
 2376         zip2    v7.4s, v31.4s, v15.4s
 2377         add     x13, x20, #4
 2378         tst     w5, #0x1
 2379         sub     x28, x28, #4
 2380         zip1    v16.2d, v0.2d, v2.2d
 2381         zip2    v0.2d, v0.2d, v2.2d
 2382         zip1    v2.2d, v1.2d, v3.2d
 2383         zip2    v1.2d, v1.2d, v3.2d
 2384         zip1    v3.2d, v4.2d, v6.2d
 2385         zip2    v4.2d, v4.2d, v6.2d
 2386         zip1    v6.2d, v5.2d, v7.2d
 2387         zip2    v5.2d, v5.2d, v7.2d
 2388         add     x24, x24, #32
 2389         csel    x20, x13, x20, ne
 2390         cmp     x28, #3
 2391         stp     q16, q3, [x26]
 2392         stp     q0, q4, [x26, #32]
 2393         stp     q2, q6, [x26, #64]
 2394         stp     q1, q5, [x26, #96]
 2395         add     x26, x26, #128
 2396         b.hi    .LBB2_2
 2397 .LBB2_8:
 2398         cbz     x28, .LBB2_16
 2399         orr     w8, w7, w19
 2400         and     x21, x5, #0x1
 2401         stur    w8, [x29, #-64]
 2402 .LBB2_10:
 2403         ldr     x8, [sp, #40]
 2404         ldr     x25, [x24]
 2405         ldur    w4, [x29, #-64]
 2406         ldp     q1, q0, [x8]
 2407         mov     x8, x22
 2408         stp     q1, q0, [x29, #-48]
 2409 .LBB2_11:
 2410         subs    x23, x8, #1
 2411         b.eq    .LBB2_13
 2412         cbnz    x8, .LBB2_14
 2413         b       .LBB2_15
 2414 .LBB2_13:
 2415         orr     w4, w4, w27
 2416 .LBB2_14:
 2417         sub     x0, x29, #48
 2418         mov     w2, #64
 2419         mov     x1, x25
 2420         mov     x3, x20
 2421         bl      zfs_blake3_compress_in_place_sse2
 2422         add     x25, x25, #64
 2423         mov     x8, x23
 2424         mov     w4, w19
 2425         b       .LBB2_11
 2426 .LBB2_15:
 2427         ldp     q0, q1, [x29, #-48]
 2428         add     x20, x20, x21
 2429         add     x24, x24, #8
 2430         subs    x28, x28, #1
 2431         stp     q0, q1, [x26], #32
 2432         b.ne    .LBB2_10
 2433 .LBB2_16:
 2434         add     sp, sp, #384
 2435         ldp     x20, x19, [sp, #144]
 2436         ldp     x22, x21, [sp, #128]
 2437         ldp     x24, x23, [sp, #112]
 2438         ldp     x26, x25, [sp, #96]
 2439         ldp     x28, x27, [sp, #80]
 2440         ldp     x29, x30, [sp, #64]
 2441         ldp     d9, d8, [sp, #48]
 2442         ldp     d11, d10, [sp, #32]
 2443         ldp     d13, d12, [sp, #16]
 2444         ldp     d15, d14, [sp], #160
 2445         ret
 2446 .Lfunc_end2:
 2447         .size   zfs_blake3_hash_many_sse2, .Lfunc_end2-zfs_blake3_hash_many_sse2
 2448         .cfi_endproc
 2449         .section        ".note.GNU-stack","",@progbits
 2450 #endif

Cache object: 54f87ea69a1e53077cd4e9f9911099dc


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.