sha512c_arm64.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10

    1 /*-
    2  * Copyright (c) 2021 The FreeBSD Foundation
    3  *
    4  * This software was developed by Andrew Turner under sponsorship from
    5  * the FreeBSD Foundation.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  */
   28 
   29 #include <sys/cdefs.h>
   30 __FBSDID("$FreeBSD$");
   31 
   32 #include <sys/types.h>
   33 
   34 #include <arm_neon.h>
   35 
   36 #include "sha512.h"
   37 #include "sha512c_impl.h"
   38 
   39 void __hidden
   40 SHA512_Transform_arm64_impl(uint64_t * state,
   41     const unsigned char block[SHA512_BLOCK_LENGTH], const uint64_t K[80])
   42 {
   43         uint64x2_t W[8];
   44         uint64x2_t S[4];
   45         uint64x2_t S_start[4];
   46         uint64x2_t K_tmp, S_tmp;
   47         int i;
   48 
   49 #define A64_LOAD_W(x)                                                   \
   50     W[x] = vld1q_u64((const uint64_t *)(&block[(x) * 16]));             \
   51     W[x] = vreinterpretq_u64_u8(vrev64q_u8(vreinterpretq_u8_u64(W[x])))
   52 
   53         /* 1. Prepare the first part of the message schedule W. */
   54         A64_LOAD_W(0);
   55         A64_LOAD_W(1);
   56         A64_LOAD_W(2);
   57         A64_LOAD_W(3);
   58         A64_LOAD_W(4);
   59         A64_LOAD_W(5);
   60         A64_LOAD_W(6);
   61         A64_LOAD_W(7);
   62 
   63         /* 2. Initialize working variables. */
   64         S[0] = vld1q_u64(&state[0]);
   65         S[1] = vld1q_u64(&state[2]);
   66         S[2] = vld1q_u64(&state[4]);
   67         S[3] = vld1q_u64(&state[6]);
   68 
   69         S_start[0] = S[0];
   70         S_start[1] = S[1];
   71         S_start[2] = S[2];
   72         S_start[3] = S[3];
   73 
   74         /* 3. Mix. */
   75         for (i = 0; i < 80; i += 16) {
   76                 /*
   77                  * The schedule array has 4 vectors:
   78                  *  ab = S[( 8 - i) % 4]
   79                  *  cd = S[( 9 - i) % 4]
   80                  *  ef = S[(10 - i) % 4]
   81                  *  gh = S[(11 - i) % 4]
   82                  *
   83                  * The following maacro:
   84                  *  - Loads the round constants
   85                  *  - Add them to schedule words
   86                  *  - Rotates the total to switch the order of the two halves
   87                  *    so they are in the correct order for gh
   88                  *  - Fix the alignment
   89                  *   - Extract fg from ef and gh
   90                  *   - Extract de from cd and ef
   91                  * - Pass these into the first part of the sha512 calculation
   92                  *   to calculate the Sigma 1 and Ch steps
   93                  * - Calculate the Sigma 0 and Maj steps and store to gh
   94                  * - Add the first part to the cd vector
   95                  */
   96 #define A64_RNDr(S, W, i, ii)                                           \
   97     K_tmp = vld1q_u64(K + (i * 2) + ii);                                \
   98     K_tmp = vaddq_u64(W[i], K_tmp);                                     \
   99     K_tmp = vextq_u64(K_tmp, K_tmp, 1);                                 \
  100     K_tmp = vaddq_u64(K_tmp, S[(11 - i) % 4]);                          \
  101     S_tmp = vsha512hq_u64(K_tmp,                                        \
  102       vextq_u64(S[(10 - i) % 4], S[(11 - i) % 4], 1),                   \
  103       vextq_u64(S[(9 - i) % 4], S[(10 - i) % 4], 1));                   \
  104     S[(11 - i) % 4] = vsha512h2q_u64(S_tmp, S[(9 - i) % 4], S[(8 - i) % 4]); \
  105     S[(9 - i) % 4] = vaddq_u64(S[(9 - i) % 4], S_tmp)
  106 
  107                 A64_RNDr(S, W, 0, i);
  108                 A64_RNDr(S, W, 1, i);
  109                 A64_RNDr(S, W, 2, i);
  110                 A64_RNDr(S, W, 3, i);
  111                 A64_RNDr(S, W, 4, i);
  112                 A64_RNDr(S, W, 5, i);
  113                 A64_RNDr(S, W, 6, i);
  114                 A64_RNDr(S, W, 7, i);
  115 
  116                 if (i == 64)
  117                         break;
  118 
  119                 /*
  120                  * Perform the Message schedule computation:
  121                  * - vsha512su0q_u64 performs the sigma 0 half and add it to
  122                  *   the old value
  123                  * - vextq_u64 fixes the alignment of the vectors
  124                  * - vsha512su1q_u64 performs the sigma 1 half and adds it
  125                  *   and both the above all together
  126                  */
  127 #define A64_MSCH(x)                                                     \
  128     W[x] = vsha512su1q_u64(                                             \
  129       vsha512su0q_u64(W[x], W[(x + 1) % 8]),                            \
  130       W[(x + 7) % 8],                                                   \
  131       vextq_u64(W[(x + 4) % 8], W[(x + 5) % 8], 1))
  132 
  133                 A64_MSCH(0);
  134                 A64_MSCH(1);
  135                 A64_MSCH(2);
  136                 A64_MSCH(3);
  137                 A64_MSCH(4);
  138                 A64_MSCH(5);
  139                 A64_MSCH(6);
  140                 A64_MSCH(7);
  141         }
  142 
  143         /* 4. Mix local working variables into global state */
  144         S[0] = vaddq_u64(S[0], S_start[0]);
  145         S[1] = vaddq_u64(S[1], S_start[1]);
  146         S[2] = vaddq_u64(S[2], S_start[2]);
  147         S[3] = vaddq_u64(S[3], S_start[3]);
  148 
  149         vst1q_u64(&state[0], S[0]);
  150         vst1q_u64(&state[2], S[1]);
  151         vst1q_u64(&state[4], S[2]);
  152         vst1q_u64(&state[6], S[3]);
  153 }

Cache object: 95b3b9cf5a3cef3caea340f8466c57be

[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.

FreeBSD/Linux Kernel Cross Reference sys/crypto/sha2/sha512c_arm64.c

FreeBSD/Linux Kernel Cross Reference
sys/crypto/sha2/sha512c_arm64.c