FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/icp/asm-x86_64/aes/aes_aesni.S

     /*
      * ====================================================================
      * Written by Intel Corporation for the OpenSSL project to add support
      * for Intel AES-NI instructions. Rights for redistribution and usage
      * in source and binary forms are granted according to the OpenSSL
      * license.
      *
      *   Author: Huang Ying <ying.huang at intel dot com>
      *           Vinodh Gopal <vinodh.gopal at intel dot com>
     *           Kahraman Akdemir
     *
     * Intel AES-NI is a new set of Single Instruction Multiple Data (SIMD)
     * instructions that are going to be introduced in the next generation
     * of Intel processor, as of 2009. These instructions enable fast and
     * secure data encryption and decryption, using the Advanced Encryption
     * Standard (AES), defined by FIPS Publication number 197. The
     * architecture introduces six instructions that offer full hardware
     * support for AES. Four of them support high performance data
     * encryption and decryption, and the other two instructions support
     * the AES key expansion procedure.
     * ====================================================================
     */
    
    /*
     * ====================================================================
     * Copyright (c) 1998-2008 The OpenSSL Project.  All rights reserved.
     *
     * Redistribution and use in source and binary forms, with or without
     * modification, are permitted provided that the following conditions
     * are met:
     *
     * 1. Redistributions of source code must retain the above copyright
     *    notice, this list of conditions and the following disclaimer.
     *
     * 2. Redistributions in binary form must reproduce the above copyright
     *    notice, this list of conditions and the following disclaimer in
     *    the documentation and/or other materials provided with the
     *    distribution.
     *
     * 3. All advertising materials mentioning features or use of this
     *    software must display the following acknowledgment:
     *    "This product includes software developed by the OpenSSL Project
     *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
     *
     * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
     *    endorse or promote products derived from this software without
     *    prior written permission. For written permission, please contact
     *    openssl-core@openssl.org.
     *
     * 5. Products derived from this software may not be called "OpenSSL"
     *    nor may "OpenSSL" appear in their names without prior written
     *    permission of the OpenSSL Project.
     *
     * 6. Redistributions of any form whatsoever must retain the following
     *    acknowledgment:
     *    "This product includes software developed by the OpenSSL Project
     *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
     *
     * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
     * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
     * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
     * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
     * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
     * OF THE POSSIBILITY OF SUCH DAMAGE.
     * ====================================================================
     */
    
    /*
     * ====================================================================
     * OpenSolaris OS modifications
     *
     * This source originates as files aes-intel.S and eng_aesni_asm.pl, in
     * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
     * Huang Ying of Intel to the openssl-dev mailing list under the subject
     * of "Add support to Intel AES-NI instruction set for x86_64 platform".
     *
     * This OpenSolaris version has these major changes from the original source:
     *
     * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
     * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function
     * definitions for lint.
     *
     * 2. Formatted code, added comments, and added #includes and #defines.
     *
     * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
     * calling kpreempt_disable() and kpreempt_enable().
     * If the TS bit is not set, Save and restore %xmm registers at the beginning
     * and end of function calls (%xmm* registers are not saved and restored by
     * during kernel thread preemption).
     *
     * 4. Renamed functions, reordered parameters, and changed return value
     * to match OpenSolaris:
     *
    * OpenSSL interface:
    *      int intel_AES_set_encrypt_key(const unsigned char *userKey,
    *              const int bits, AES_KEY *key);
    *      int intel_AES_set_decrypt_key(const unsigned char *userKey,
    *              const int bits, AES_KEY *key);
    *      Return values for above are non-zero on error, 0 on success.
    *
    *      void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
    *              const AES_KEY *key);
    *      void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
    *              const AES_KEY *key);
    *      typedef struct aes_key_st {
    *              unsigned int    rd_key[4 *(AES_MAXNR + 1)];
    *              int             rounds;
    *              unsigned int    pad[3];
    *      } AES_KEY;
    * Note: AES_LONG is undefined (that is, Intel uses 32-bit key schedules
    * (ks32) instead of 64-bit (ks64).
    * Number of rounds (aka round count) is at offset 240 of AES_KEY.
    *
    * OpenSolaris OS interface (#ifdefs removed for readability):
    *      int rijndael_key_setup_dec_intel(uint32_t rk[],
    *              const uint32_t cipherKey[], uint64_t keyBits);
    *      int rijndael_key_setup_enc_intel(uint32_t rk[],
    *              const uint32_t cipherKey[], uint64_t keyBits);
    *      Return values for above are 0 on error, number of rounds on success.
    *
    *      void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
    *              const uint32_t pt[4], uint32_t ct[4]);
    *      void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
    *              const uint32_t pt[4], uint32_t ct[4]);
    *      typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4];
    *               uint32_t ks32[(MAX_AES_NR + 1) * 4]; } aes_ks_t;
    *
    *      typedef union {
    *              uint32_t        ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
    *      } aes_ks_t;
    *      typedef struct aes_key {
    *              aes_ks_t        encr_ks, decr_ks;
    *              long double     align128;
    *              int             flags, nr, type;
    *      } aes_key_t;
    *
    * Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text,
    * ct is crypto text, and MAX_AES_NR is 14.
    * For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64.
    *
    * Note2: aes_ks_t must be aligned on a 0 mod 128 byte boundary.
    *
    * ====================================================================
    */
   
   
   #if defined(lint) || defined(__lint)
   
   #include <sys/types.h>
   
   void
   aes_encrypt_intel(const uint32_t rk[], int Nr, const uint32_t pt[4],
       uint32_t ct[4]) {
           (void) rk, (void) Nr, (void) pt, (void) ct;
   }
   void
   aes_decrypt_intel(const uint32_t rk[], int Nr, const uint32_t ct[4],
       uint32_t pt[4]) {
           (void) rk, (void) Nr, (void) ct, (void) pt;
   }
   int
   rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
       uint64_t keyBits) {
           (void) rk, (void) cipherKey, (void) keyBits;
           return (0);
   }
   int
   rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
      uint64_t keyBits) {
           (void) rk, (void) cipherKey, (void) keyBits;
           return (0);
   }
   
   
   #elif defined(HAVE_AES) /* guard by instruction set */
   
   #define _ASM
   #include <sys/asm_linkage.h>
   
   /*
    * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(),
    * _key_expansion_256a(), _key_expansion_256b()
    *
    * Helper functions called by rijndael_key_setup_inc_intel().
    * Also used indirectly by rijndael_key_setup_dec_intel().
    *
    * Input:
    * %xmm0        User-provided cipher key
    * %xmm1        Round constant
    * Output:
    * (%rcx)       AES key
    */
   
   ENTRY_NP2(_key_expansion_128, _key_expansion_256a)
   _key_expansion_128_local:
   _key_expansion_256a_local:
           pshufd  $0b11111111, %xmm1, %xmm1
           shufps  $0b00010000, %xmm0, %xmm4
           pxor    %xmm4, %xmm0
           shufps  $0b10001100, %xmm0, %xmm4
           pxor    %xmm4, %xmm0
           pxor    %xmm1, %xmm0
           movups  %xmm0, (%rcx)
           add     $0x10, %rcx
           RET
           nop
   SET_SIZE(_key_expansion_128)
   SET_SIZE(_key_expansion_256a)
   
   
   ENTRY_NP(_key_expansion_192a)
   _key_expansion_192a_local:
           pshufd  $0b01010101, %xmm1, %xmm1
           shufps  $0b00010000, %xmm0, %xmm4
           pxor    %xmm4, %xmm0
           shufps  $0b10001100, %xmm0, %xmm4
           pxor    %xmm4, %xmm0
           pxor    %xmm1, %xmm0
   
           movups  %xmm2, %xmm5
           movups  %xmm2, %xmm6
           pslldq  $4, %xmm5
           pshufd  $0b11111111, %xmm0, %xmm3
           pxor    %xmm3, %xmm2
           pxor    %xmm5, %xmm2
   
           movups  %xmm0, %xmm1
           shufps  $0b01000100, %xmm0, %xmm6
           movups  %xmm6, (%rcx)
           shufps  $0b01001110, %xmm2, %xmm1
           movups  %xmm1, 0x10(%rcx)
           add     $0x20, %rcx
           RET
   SET_SIZE(_key_expansion_192a)
   
   
   ENTRY_NP(_key_expansion_192b)
   _key_expansion_192b_local:
           pshufd  $0b01010101, %xmm1, %xmm1
           shufps  $0b00010000, %xmm0, %xmm4
           pxor    %xmm4, %xmm0
           shufps  $0b10001100, %xmm0, %xmm4
           pxor    %xmm4, %xmm0
           pxor    %xmm1, %xmm0
   
           movups  %xmm2, %xmm5
           pslldq  $4, %xmm5
           pshufd  $0b11111111, %xmm0, %xmm3
           pxor    %xmm3, %xmm2
           pxor    %xmm5, %xmm2
   
           movups  %xmm0, (%rcx)
           add     $0x10, %rcx
           RET
   SET_SIZE(_key_expansion_192b)
   
   
   ENTRY_NP(_key_expansion_256b)
   _key_expansion_256b_local:
           pshufd  $0b10101010, %xmm1, %xmm1
           shufps  $0b00010000, %xmm2, %xmm4
           pxor    %xmm4, %xmm2
           shufps  $0b10001100, %xmm2, %xmm4
           pxor    %xmm4, %xmm2
           pxor    %xmm1, %xmm2
           movups  %xmm2, (%rcx)
           add     $0x10, %rcx
           RET
   SET_SIZE(_key_expansion_256b)
   
   
   /*
    * rijndael_key_setup_enc_intel()
    * Expand the cipher key into the encryption key schedule.
    *
    * For kernel code, caller is responsible for ensuring kpreempt_disable()
    * has been called.  This is because %xmm registers are not saved/restored.
    * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
    * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
    * on the stack.
    *
    * OpenSolaris interface:
    * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[],
    *      uint64_t keyBits);
    * Return value is 0 on error, number of rounds on success.
    *
    * Original Intel OpenSSL interface:
    * int intel_AES_set_encrypt_key(const unsigned char *userKey,
    *      const int bits, AES_KEY *key);
    * Return value is non-zero on error, 0 on success.
    */
   
   #ifdef  OPENSSL_INTERFACE
   #define rijndael_key_setup_enc_intel    intel_AES_set_encrypt_key
   #define rijndael_key_setup_dec_intel    intel_AES_set_decrypt_key
   
   #define USERCIPHERKEY           rdi     /* P1, 64 bits */
   #define KEYSIZE32               esi     /* P2, 32 bits */
   #define KEYSIZE64               rsi     /* P2, 64 bits */
   #define AESKEY                  rdx     /* P3, 64 bits */
   
   #else   /* OpenSolaris Interface */
   #define AESKEY                  rdi     /* P1, 64 bits */
   #define USERCIPHERKEY           rsi     /* P2, 64 bits */
   #define KEYSIZE32               edx     /* P3, 32 bits */
   #define KEYSIZE64               rdx     /* P3, 64 bits */
   #endif  /* OPENSSL_INTERFACE */
   
   #define ROUNDS32                KEYSIZE32       /* temp */
   #define ROUNDS64                KEYSIZE64       /* temp */
   #define ENDAESKEY               USERCIPHERKEY   /* temp */
   
   ENTRY_NP(rijndael_key_setup_enc_intel)
   rijndael_key_setup_enc_intel_local:
           FRAME_BEGIN
           // NULL pointer sanity check
           test    %USERCIPHERKEY, %USERCIPHERKEY
           jz      .Lenc_key_invalid_param
           test    %AESKEY, %AESKEY
           jz      .Lenc_key_invalid_param
   
           movups  (%USERCIPHERKEY), %xmm0 // user key (first 16 bytes)
           movups  %xmm0, (%AESKEY)
           lea     0x10(%AESKEY), %rcx     // key addr
           pxor    %xmm4, %xmm4            // xmm4 is assumed 0 in _key_expansion_x
   
           cmp     $256, %KEYSIZE32
           jnz     .Lenc_key192
   
           // AES 256: 14 rounds in encryption key schedule
   #ifdef OPENSSL_INTERFACE
           mov     $14, %ROUNDS32
           movl    %ROUNDS32, 240(%AESKEY)         // key.rounds = 14
   #endif  /* OPENSSL_INTERFACE */
   
           movups  0x10(%USERCIPHERKEY), %xmm2     // other user key (2nd 16 bytes)
           movups  %xmm2, (%rcx)
           add     $0x10, %rcx
   
           aeskeygenassist $0x1, %xmm2, %xmm1      // expand the key
           call    _key_expansion_256a_local
           aeskeygenassist $0x1, %xmm0, %xmm1
           call    _key_expansion_256b_local
           aeskeygenassist $0x2, %xmm2, %xmm1      // expand the key
           call    _key_expansion_256a_local
           aeskeygenassist $0x2, %xmm0, %xmm1
           call    _key_expansion_256b_local
           aeskeygenassist $0x4, %xmm2, %xmm1      // expand the key
           call    _key_expansion_256a_local
           aeskeygenassist $0x4, %xmm0, %xmm1
           call    _key_expansion_256b_local
           aeskeygenassist $0x8, %xmm2, %xmm1      // expand the key
           call    _key_expansion_256a_local
           aeskeygenassist $0x8, %xmm0, %xmm1
           call    _key_expansion_256b_local
           aeskeygenassist $0x10, %xmm2, %xmm1     // expand the key
           call    _key_expansion_256a_local
           aeskeygenassist $0x10, %xmm0, %xmm1
           call    _key_expansion_256b_local
           aeskeygenassist $0x20, %xmm2, %xmm1     // expand the key
           call    _key_expansion_256a_local
           aeskeygenassist $0x20, %xmm0, %xmm1
           call    _key_expansion_256b_local
           aeskeygenassist $0x40, %xmm2, %xmm1     // expand the key
           call    _key_expansion_256a_local
   
   #ifdef  OPENSSL_INTERFACE
           xor     %rax, %rax                      // return 0 (OK)
   #else   /* Open Solaris Interface */
           mov     $14, %rax                       // return # rounds = 14
   #endif
           FRAME_END
           RET
   
   .balign 4
   .Lenc_key192:
           cmp     $192, %KEYSIZE32
           jnz     .Lenc_key128
   
           // AES 192: 12 rounds in encryption key schedule
   #ifdef OPENSSL_INTERFACE
           mov     $12, %ROUNDS32
           movl    %ROUNDS32, 240(%AESKEY) // key.rounds = 12
   #endif  /* OPENSSL_INTERFACE */
   
           movq    0x10(%USERCIPHERKEY), %xmm2     // other user key
           aeskeygenassist $0x1, %xmm2, %xmm1      // expand the key
           call    _key_expansion_192a_local
           aeskeygenassist $0x2, %xmm2, %xmm1      // expand the key
           call    _key_expansion_192b_local
           aeskeygenassist $0x4, %xmm2, %xmm1      // expand the key
           call    _key_expansion_192a_local
           aeskeygenassist $0x8, %xmm2, %xmm1      // expand the key
           call    _key_expansion_192b_local
           aeskeygenassist $0x10, %xmm2, %xmm1     // expand the key
           call    _key_expansion_192a_local
           aeskeygenassist $0x20, %xmm2, %xmm1     // expand the key
           call    _key_expansion_192b_local
           aeskeygenassist $0x40, %xmm2, %xmm1     // expand the key
           call    _key_expansion_192a_local
           aeskeygenassist $0x80, %xmm2, %xmm1     // expand the key
           call    _key_expansion_192b_local
   
   #ifdef  OPENSSL_INTERFACE
           xor     %rax, %rax                      // return 0 (OK)
   #else   /* OpenSolaris Interface */
           mov     $12, %rax                       // return # rounds = 12
   #endif
           FRAME_END
           RET
   
   .balign 4
   .Lenc_key128:
           cmp $128, %KEYSIZE32
           jnz .Lenc_key_invalid_key_bits
   
           // AES 128: 10 rounds in encryption key schedule
   #ifdef OPENSSL_INTERFACE
           mov     $10, %ROUNDS32
           movl    %ROUNDS32, 240(%AESKEY)         // key.rounds = 10
   #endif  /* OPENSSL_INTERFACE */
   
           aeskeygenassist $0x1, %xmm0, %xmm1      // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x2, %xmm0, %xmm1      // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x4, %xmm0, %xmm1      // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x8, %xmm0, %xmm1      // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x10, %xmm0, %xmm1     // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x20, %xmm0, %xmm1     // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x40, %xmm0, %xmm1     // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x80, %xmm0, %xmm1     // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x1b, %xmm0, %xmm1     // expand the key
           call    _key_expansion_128_local
           aeskeygenassist $0x36, %xmm0, %xmm1     // expand the key
           call    _key_expansion_128_local
   
   #ifdef  OPENSSL_INTERFACE
           xor     %rax, %rax                      // return 0 (OK)
   #else   /* OpenSolaris Interface */
           mov     $10, %rax                       // return # rounds = 10
   #endif
           FRAME_END
           RET
   
   .Lenc_key_invalid_param:
   #ifdef  OPENSSL_INTERFACE
           mov     $-1, %rax       // user key or AES key pointer is NULL
           FRAME_END
           RET
   #else
           /* FALLTHROUGH */
   #endif  /* OPENSSL_INTERFACE */
   
   .Lenc_key_invalid_key_bits:
   #ifdef  OPENSSL_INTERFACE
           mov     $-2, %rax       // keysize is invalid
   #else   /* Open Solaris Interface */
           xor     %rax, %rax      // a key pointer is NULL or invalid keysize
   #endif  /* OPENSSL_INTERFACE */
           FRAME_END
           RET
           SET_SIZE(rijndael_key_setup_enc_intel)
   
   
   /*
    * rijndael_key_setup_dec_intel()
    * Expand the cipher key into the decryption key schedule.
    *
    * For kernel code, caller is responsible for ensuring kpreempt_disable()
    * has been called.  This is because %xmm registers are not saved/restored.
    * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
    * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
    * on the stack.
    *
    * OpenSolaris interface:
    * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
    *      uint64_t keyBits);
    * Return value is 0 on error, number of rounds on success.
    * P1->P2, P2->P3, P3->P1
    *
    * Original Intel OpenSSL interface:
    * int intel_AES_set_decrypt_key(const unsigned char *userKey,
    *      const int bits, AES_KEY *key);
    * Return value is non-zero on error, 0 on success.
    */
   
   ENTRY_NP(rijndael_key_setup_dec_intel)
   FRAME_BEGIN
           // Generate round keys used for encryption
           call    rijndael_key_setup_enc_intel_local
           test    %rax, %rax
   #ifdef  OPENSSL_INTERFACE
           jnz     .Ldec_key_exit  // Failed if returned non-0
   #else   /* OpenSolaris Interface */
           jz      .Ldec_key_exit  // Failed if returned 0
   #endif  /* OPENSSL_INTERFACE */
   
           /*
            * Convert round keys used for encryption
            * to a form usable for decryption
            */
   #ifndef OPENSSL_INTERFACE               /* OpenSolaris Interface */
           mov     %rax, %ROUNDS64         // set # rounds (10, 12, or 14)
                                           // (already set for OpenSSL)
   #endif
   
           lea     0x10(%AESKEY), %rcx     // key addr
           shl     $4, %ROUNDS32
           add     %AESKEY, %ROUNDS64
           mov     %ROUNDS64, %ENDAESKEY
   
   .balign 4
   .Ldec_key_reorder_loop:
           movups  (%AESKEY), %xmm0
           movups  (%ROUNDS64), %xmm1
           movups  %xmm0, (%ROUNDS64)
           movups  %xmm1, (%AESKEY)
           lea     0x10(%AESKEY), %AESKEY
           lea     -0x10(%ROUNDS64), %ROUNDS64
           cmp     %AESKEY, %ROUNDS64
           ja      .Ldec_key_reorder_loop
   
   .balign 4
   .Ldec_key_inv_loop:
           movups  (%rcx), %xmm0
           // Convert an encryption round key to a form usable for decryption
           // with the "AES Inverse Mix Columns" instruction
           aesimc  %xmm0, %xmm1
           movups  %xmm1, (%rcx)
           lea     0x10(%rcx), %rcx
           cmp     %ENDAESKEY, %rcx
           jnz     .Ldec_key_inv_loop
   
   .Ldec_key_exit:
           // OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
           // OpenSSL: rax = 0 for OK, or non-zero for error
           FRAME_END
           RET
           SET_SIZE(rijndael_key_setup_dec_intel)
   
   
   /*
    * aes_encrypt_intel()
    * Encrypt a single block (in and out can overlap).
    *
    * For kernel code, caller is responsible for ensuring kpreempt_disable()
    * has been called.  This is because %xmm registers are not saved/restored.
    * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
    * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
    * on the stack.
    *
    * Temporary register usage:
    * %xmm0        State
    * %xmm1        Key
    *
    * Original OpenSolaris Interface:
    * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
    *      const uint32_t pt[4], uint32_t ct[4])
    *
    * Original Intel OpenSSL Interface:
    * void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
    *      const AES_KEY *key)
    */
   
   #ifdef  OPENSSL_INTERFACE
   #define aes_encrypt_intel       intel_AES_encrypt
   #define aes_decrypt_intel       intel_AES_decrypt
   
   #define INP             rdi     /* P1, 64 bits */
   #define OUTP            rsi     /* P2, 64 bits */
   #define KEYP            rdx     /* P3, 64 bits */
   
   /* No NROUNDS parameter--offset 240 from KEYP saved in %ecx:  */
   #define NROUNDS32       ecx     /* temporary, 32 bits */
   #define NROUNDS         cl      /* temporary,  8 bits */
   
   #else   /* OpenSolaris Interface */
   #define KEYP            rdi     /* P1, 64 bits */
   #define NROUNDS         esi     /* P2, 32 bits */
   #define INP             rdx     /* P3, 64 bits */
   #define OUTP            rcx     /* P4, 64 bits */
   #endif  /* OPENSSL_INTERFACE */
   
   #define STATE           xmm0    /* temporary, 128 bits */
   #define KEY             xmm1    /* temporary, 128 bits */
   
   
   ENTRY_NP(aes_encrypt_intel)
   
           movups  (%INP), %STATE                  // input
           movups  (%KEYP), %KEY                   // key
   #ifdef  OPENSSL_INTERFACE
           mov     240(%KEYP), %NROUNDS32          // round count
   #else   /* OpenSolaris Interface */
           /* Round count is already present as P2 in %rsi/%esi */
   #endif  /* OPENSSL_INTERFACE */
   
           pxor    %KEY, %STATE                    // round 0
           lea     0x30(%KEYP), %KEYP
           cmp     $12, %NROUNDS
           jb      .Lenc128
           lea     0x20(%KEYP), %KEYP
           je      .Lenc192
   
           // AES 256
           lea     0x20(%KEYP), %KEYP
           movups  -0x60(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  -0x50(%KEYP), %KEY
           aesenc  %KEY, %STATE
   
   .balign 4
   .Lenc192:
           // AES 192 and 256
           movups  -0x40(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  -0x30(%KEYP), %KEY
           aesenc  %KEY, %STATE
   
   .balign 4
   .Lenc128:
           // AES 128, 192, and 256
           movups  -0x20(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  -0x10(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  (%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  0x10(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  0x20(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  0x30(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  0x40(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  0x50(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  0x60(%KEYP), %KEY
           aesenc  %KEY, %STATE
           movups  0x70(%KEYP), %KEY
           aesenclast       %KEY, %STATE           // last round
           movups  %STATE, (%OUTP)                 // output
   
           RET
           SET_SIZE(aes_encrypt_intel)
   
   
   /*
    * aes_decrypt_intel()
    * Decrypt a single block (in and out can overlap).
    *
    * For kernel code, caller is responsible for ensuring kpreempt_disable()
    * has been called.  This is because %xmm registers are not saved/restored.
    * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
    * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
    * on the stack.
    *
    * Temporary register usage:
    * %xmm0        State
    * %xmm1        Key
    *
    * Original OpenSolaris Interface:
    * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
    *      const uint32_t pt[4], uint32_t ct[4])/
    *
    * Original Intel OpenSSL Interface:
    * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
    *      const AES_KEY *key);
    */
   ENTRY_NP(aes_decrypt_intel)
   
           movups  (%INP), %STATE                  // input
           movups  (%KEYP), %KEY                   // key
   #ifdef  OPENSSL_INTERFACE
           mov     240(%KEYP), %NROUNDS32          // round count
   #else   /* OpenSolaris Interface */
           /* Round count is already present as P2 in %rsi/%esi */
   #endif  /* OPENSSL_INTERFACE */
   
           pxor    %KEY, %STATE                    // round 0
           lea     0x30(%KEYP), %KEYP
           cmp     $12, %NROUNDS
           jb      .Ldec128
           lea     0x20(%KEYP), %KEYP
           je      .Ldec192
   
           // AES 256
           lea     0x20(%KEYP), %KEYP
           movups  -0x60(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  -0x50(%KEYP), %KEY
           aesdec  %KEY, %STATE
   
   .balign 4
   .Ldec192:
           // AES 192 and 256
           movups  -0x40(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  -0x30(%KEYP), %KEY
           aesdec  %KEY, %STATE
   
   .balign 4
   .Ldec128:
           // AES 128, 192, and 256
           movups  -0x20(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  -0x10(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  (%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  0x10(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  0x20(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  0x30(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  0x40(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  0x50(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  0x60(%KEYP), %KEY
           aesdec  %KEY, %STATE
           movups  0x70(%KEYP), %KEY
           aesdeclast      %KEY, %STATE            // last round
           movups  %STATE, (%OUTP)                 // output
   
           RET
           SET_SIZE(aes_decrypt_intel)
   
   #endif  /* lint || __lint */
   
   #ifdef __ELF__
   .section .note.GNU-stack,"",%progbits
   #endif

Cache object: 69d1125648b6d32b2addaefec76b3899