The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/mips/cavium/cryptocteon/cavium_crypto.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*
    2  * vim:sw=4 ts=8
    3  */
    4 /*
    5  * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com>
    6  *
    7  * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
    8  * reserved.
    9  * 
   10  * Redistribution and use in source and binary forms, with or without
   11  * modification, are permitted provided that the following conditions are met:
   12  * 1. Redistributions of source code must retain the above copyright notice,
   13  * this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright notice,
   15  * this list of conditions and the following disclaimer in the documentation
   16  * and/or other materials provided with the distribution.
   17  * 3. All advertising materials mentioning features or use of this software
   18  * must display the following acknowledgement:
   19  * This product includes software developed by Cavium Networks
   20  * 4. Cavium Networks' name may not be used to endorse or promote products
   21  * derived from this software without specific prior written permission.
   22  * 
   23  * This Software, including technical data, may be subject to U.S. export
   24  * control laws, including the U.S. Export Administration Act and its
   25  * associated regulations, and may be subject to export or import regulations
   26  * in other countries. You warrant that You will comply strictly in all
   27  * respects with all such regulations and acknowledge that you have the
   28  * responsibility to obtain licenses to export, re-export or import the
   29  * Software.
   30  * 
   31  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND
   32  * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES,
   33  * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE
   34  * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
   35  * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
   36  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
   37  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
   38  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
   39  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
   40  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
   41 */
   42 /****************************************************************************/
   43 
   44 #include <sys/cdefs.h>
   45 __FBSDID("$FreeBSD: releng/10.3/sys/mips/cavium/cryptocteon/cavium_crypto.c 229677 2012-01-06 01:23:26Z gonzo $");
   46 
   47 #include <sys/param.h>
   48 #include <sys/systm.h>
   49 #include <sys/kernel.h>
   50 #include <sys/module.h>
   51 #include <sys/malloc.h>
   52 #include <sys/uio.h>
   53 
   54 #include <opencrypto/cryptodev.h>
   55 
   56 #include <contrib/octeon-sdk/cvmx.h>
   57 
   58 #include <mips/cavium/cryptocteon/cryptocteonvar.h>
   59 
   60 /****************************************************************************/
   61 
   62 #define IOV_INIT(iov, ptr, idx, len)                                    \
   63         do {                                                            \
   64             (idx) = 0;                                                  \
   65             (ptr) = (iov)[(idx)].iov_base;                              \
   66             (len) = (iov)[(idx)].iov_len;                               \
   67         } while (0)
   68 
   69 /*
   70  * XXX
   71  * It would be better if this were an IOV_READ/IOV_WRITE macro instead so
   72  * that we could detect overflow before it happens rather than right after,
   73  * which is especially bad since there is usually no IOV_CONSUME after the
   74  * final read or write.
   75  */
   76 #define IOV_CONSUME(iov, ptr, idx, len)                                 \
   77         do {                                                            \
   78             if ((len) > sizeof *(ptr)) {                                \
   79                 (len) -= sizeof *(ptr);                                 \
   80                 (ptr)++;                                                \
   81             } else {                                                    \
   82                 if ((len) != sizeof *(ptr))                             \
   83                         panic("%s: went past end of iovec.", __func__); \
   84                 (idx)++;                                                \
   85                 (ptr) = (iov)[(idx)].iov_base;                          \
   86                 (len) = (iov)[(idx)].iov_len;                           \
   87             }                                                           \
   88         } while (0)
   89 
   90 #define ESP_HEADER_LENGTH     8
   91 #define DES_CBC_IV_LENGTH     8
   92 #define AES_CBC_IV_LENGTH     16
   93 #define ESP_HMAC_LEN          12
   94 
   95 #define ESP_HEADER_LENGTH 8
   96 #define DES_CBC_IV_LENGTH 8
   97 
   98 /****************************************************************************/
   99 
  100 #define CVM_LOAD_SHA_UNIT(dat, next)  { \
  101    if (next == 0) {                     \
  102       next = 1;                         \
  103       CVMX_MT_HSH_DAT (dat, 0);         \
  104    } else if (next == 1) {              \
  105       next = 2;                         \
  106       CVMX_MT_HSH_DAT (dat, 1);         \
  107    } else if (next == 2) {              \
  108       next = 3;                    \
  109       CVMX_MT_HSH_DAT (dat, 2);         \
  110    } else if (next == 3) {              \
  111       next = 4;                         \
  112       CVMX_MT_HSH_DAT (dat, 3);         \
  113    } else if (next == 4) {              \
  114       next = 5;                           \
  115       CVMX_MT_HSH_DAT (dat, 4);         \
  116    } else if (next == 5) {              \
  117       next = 6;                         \
  118       CVMX_MT_HSH_DAT (dat, 5);         \
  119    } else if (next == 6) {              \
  120       next = 7;                         \
  121       CVMX_MT_HSH_DAT (dat, 6);         \
  122    } else {                             \
  123      CVMX_MT_HSH_STARTSHA (dat);        \
  124      next = 0;                          \
  125    }                                    \
  126 }
  127 
  128 #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next)  { \
  129    if (next == 0) {                      \
  130       CVMX_MT_HSH_DAT (dat1, 0);         \
  131       CVMX_MT_HSH_DAT (dat2, 1);         \
  132       next = 2;                          \
  133    } else if (next == 1) {               \
  134       CVMX_MT_HSH_DAT (dat1, 1);         \
  135       CVMX_MT_HSH_DAT (dat2, 2);         \
  136       next = 3;                          \
  137    } else if (next == 2) {               \
  138       CVMX_MT_HSH_DAT (dat1, 2);         \
  139       CVMX_MT_HSH_DAT (dat2, 3);         \
  140       next = 4;                          \
  141    } else if (next == 3) {               \
  142       CVMX_MT_HSH_DAT (dat1, 3);         \
  143       CVMX_MT_HSH_DAT (dat2, 4);         \
  144       next = 5;                          \
  145    } else if (next == 4) {               \
  146       CVMX_MT_HSH_DAT (dat1, 4);         \
  147       CVMX_MT_HSH_DAT (dat2, 5);         \
  148       next = 6;                          \
  149    } else if (next == 5) {               \
  150       CVMX_MT_HSH_DAT (dat1, 5);         \
  151       CVMX_MT_HSH_DAT (dat2, 6);         \
  152       next = 7;                          \
  153    } else if (next == 6) {               \
  154       CVMX_MT_HSH_DAT (dat1, 6);         \
  155       CVMX_MT_HSH_STARTSHA (dat2);       \
  156       next = 0;                          \
  157    } else {                              \
  158      CVMX_MT_HSH_STARTSHA (dat1);        \
  159      CVMX_MT_HSH_DAT (dat2, 0);          \
  160      next = 1;                           \
  161    }                                     \
  162 }
  163 
  164 /****************************************************************************/
  165 
  166 #define CVM_LOAD_MD5_UNIT(dat, next)  { \
  167    if (next == 0) {                     \
  168       next = 1;                         \
  169       CVMX_MT_HSH_DAT (dat, 0);         \
  170    } else if (next == 1) {              \
  171       next = 2;                         \
  172       CVMX_MT_HSH_DAT (dat, 1);         \
  173    } else if (next == 2) {              \
  174       next = 3;                    \
  175       CVMX_MT_HSH_DAT (dat, 2);         \
  176    } else if (next == 3) {              \
  177       next = 4;                         \
  178       CVMX_MT_HSH_DAT (dat, 3);         \
  179    } else if (next == 4) {              \
  180       next = 5;                           \
  181       CVMX_MT_HSH_DAT (dat, 4);         \
  182    } else if (next == 5) {              \
  183       next = 6;                         \
  184       CVMX_MT_HSH_DAT (dat, 5);         \
  185    } else if (next == 6) {              \
  186       next = 7;                         \
  187       CVMX_MT_HSH_DAT (dat, 6);         \
  188    } else {                             \
  189      CVMX_MT_HSH_STARTMD5 (dat);        \
  190      next = 0;                          \
  191    }                                    \
  192 }
  193 
  194 #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next)  { \
  195    if (next == 0) {                      \
  196       CVMX_MT_HSH_DAT (dat1, 0);         \
  197       CVMX_MT_HSH_DAT (dat2, 1);         \
  198       next = 2;                          \
  199    } else if (next == 1) {               \
  200       CVMX_MT_HSH_DAT (dat1, 1);         \
  201       CVMX_MT_HSH_DAT (dat2, 2);         \
  202       next = 3;                          \
  203    } else if (next == 2) {               \
  204       CVMX_MT_HSH_DAT (dat1, 2);         \
  205       CVMX_MT_HSH_DAT (dat2, 3);         \
  206       next = 4;                          \
  207    } else if (next == 3) {               \
  208       CVMX_MT_HSH_DAT (dat1, 3);         \
  209       CVMX_MT_HSH_DAT (dat2, 4);         \
  210       next = 5;                          \
  211    } else if (next == 4) {               \
  212       CVMX_MT_HSH_DAT (dat1, 4);         \
  213       CVMX_MT_HSH_DAT (dat2, 5);         \
  214       next = 6;                          \
  215    } else if (next == 5) {               \
  216       CVMX_MT_HSH_DAT (dat1, 5);         \
  217       CVMX_MT_HSH_DAT (dat2, 6);         \
  218       next = 7;                          \
  219    } else if (next == 6) {               \
  220       CVMX_MT_HSH_DAT (dat1, 6);         \
  221       CVMX_MT_HSH_STARTMD5 (dat2);       \
  222       next = 0;                          \
  223    } else {                              \
  224      CVMX_MT_HSH_STARTMD5 (dat1);        \
  225      CVMX_MT_HSH_DAT (dat2, 0);          \
  226      next = 1;                           \
  227    }                                     \
  228 }
  229 
  230 /****************************************************************************/
  231 
  232 void
  233 octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer)
  234 {
  235     uint8_t hash_key[64];
  236     uint64_t *key1;
  237     register uint64_t xor1 = 0x3636363636363636ULL;
  238     register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL;
  239 
  240     dprintf("%s()\n", __func__);
  241 
  242     memset(hash_key, 0, sizeof(hash_key));
  243     memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
  244     key1 = (uint64_t *) hash_key;
  245     if (auth) {
  246        CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
  247        CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
  248        CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
  249     } else {
  250        CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
  251        CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
  252     }
  253 
  254     CVMX_MT_HSH_DAT((*key1 ^ xor1), 0);
  255     key1++;
  256     CVMX_MT_HSH_DAT((*key1 ^ xor1), 1);
  257     key1++;
  258     CVMX_MT_HSH_DAT((*key1 ^ xor1), 2);
  259     key1++;
  260     CVMX_MT_HSH_DAT((*key1 ^ xor1), 3);
  261     key1++;
  262     CVMX_MT_HSH_DAT((*key1 ^ xor1), 4);
  263     key1++;
  264     CVMX_MT_HSH_DAT((*key1 ^ xor1), 5);
  265     key1++;
  266     CVMX_MT_HSH_DAT((*key1 ^ xor1), 6);
  267     key1++;
  268     if (auth)
  269                 CVMX_MT_HSH_STARTSHA((*key1 ^ xor1));
  270     else
  271                 CVMX_MT_HSH_STARTMD5((*key1 ^ xor1));
  272 
  273     CVMX_MF_HSH_IV(inner[0], 0);
  274     CVMX_MF_HSH_IV(inner[1], 1);
  275     if (auth) {
  276                 inner[2] = 0;
  277                 CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2);
  278     }
  279 
  280     memset(hash_key, 0, sizeof(hash_key));
  281     memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
  282     key1 = (uint64_t *) hash_key;
  283     if (auth) {
  284       CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
  285       CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
  286       CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
  287     } else {
  288       CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
  289       CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
  290     }
  291 
  292     CVMX_MT_HSH_DAT((*key1 ^ xor2), 0);
  293     key1++;
  294     CVMX_MT_HSH_DAT((*key1 ^ xor2), 1);
  295     key1++;
  296     CVMX_MT_HSH_DAT((*key1 ^ xor2), 2);
  297     key1++;
  298     CVMX_MT_HSH_DAT((*key1 ^ xor2), 3);
  299     key1++;
  300     CVMX_MT_HSH_DAT((*key1 ^ xor2), 4);
  301     key1++;
  302     CVMX_MT_HSH_DAT((*key1 ^ xor2), 5);
  303     key1++;
  304     CVMX_MT_HSH_DAT((*key1 ^ xor2), 6);
  305     key1++;
  306     if (auth)
  307        CVMX_MT_HSH_STARTSHA((*key1 ^ xor2));
  308     else 
  309        CVMX_MT_HSH_STARTMD5((*key1 ^ xor2));
  310 
  311     CVMX_MF_HSH_IV(outer[0], 0);
  312     CVMX_MF_HSH_IV(outer[1], 1);
  313     if (auth) {
  314       outer[2] = 0;
  315       CVMX_MF_HSH_IV(outer[2], 2);
  316     }
  317     return;
  318 }
  319 
  320 /****************************************************************************/
  321 /* DES functions */
  322 
  323 int
  324 octo_des_cbc_encrypt(
  325     struct octo_sess *od,
  326     struct iovec *iov, size_t iovcnt, size_t iovlen,
  327     int auth_off, int auth_len,
  328     int crypt_off, int crypt_len,
  329     int icv_off, uint8_t *ivp)
  330 {
  331     uint64_t *data;
  332     int data_i, data_l;
  333 
  334     dprintf("%s()\n", __func__);
  335 
  336     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
  337             (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
  338         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  339                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  340                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  341                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  342         return -EINVAL;
  343     }
  344 
  345     IOV_INIT(iov, data, data_i, data_l);
  346 
  347     CVMX_PREFETCH0(ivp);
  348     CVMX_PREFETCH0(od->octo_enckey);
  349 
  350 
  351     /* load 3DES Key */
  352     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
  353     if (od->octo_encklen == 24) {
  354         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
  355         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  356     } else if (od->octo_encklen == 8) {
  357         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
  358         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
  359     } else {
  360         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
  361         return -EINVAL;
  362     }
  363 
  364     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
  365 
  366     while (crypt_off > 0) {
  367         IOV_CONSUME(iov, data, data_i, data_l);
  368         crypt_off -= 8;
  369     }
  370 
  371     while (crypt_len > 0) {
  372         CVMX_MT_3DES_ENC_CBC(*data);
  373         CVMX_MF_3DES_RESULT(*data);
  374         IOV_CONSUME(iov, data, data_i, data_l);
  375         crypt_len -= 8;
  376     }
  377 
  378     return 0;
  379 }
  380 
  381 
  382 int
  383 octo_des_cbc_decrypt(
  384     struct octo_sess *od,
  385     struct iovec *iov, size_t iovcnt, size_t iovlen,
  386     int auth_off, int auth_len,
  387     int crypt_off, int crypt_len,
  388     int icv_off, uint8_t *ivp)
  389 {
  390     uint64_t *data;
  391     int data_i, data_l;
  392 
  393     dprintf("%s()\n", __func__);
  394 
  395     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
  396             (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
  397         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  398                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  399                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  400                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  401         return -EINVAL;
  402     }
  403 
  404     IOV_INIT(iov, data, data_i, data_l);
  405 
  406     CVMX_PREFETCH0(ivp);
  407     CVMX_PREFETCH0(od->octo_enckey);
  408 
  409     /* load 3DES Key */
  410     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
  411     if (od->octo_encklen == 24) {
  412         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
  413         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  414     } else if (od->octo_encklen == 8) {
  415         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
  416         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
  417     } else {
  418         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
  419         return -EINVAL;
  420     }
  421 
  422     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
  423 
  424     while (crypt_off > 0) {
  425         IOV_CONSUME(iov, data, data_i, data_l);
  426         crypt_off -= 8;
  427     }
  428 
  429     while (crypt_len > 0) {
  430         CVMX_MT_3DES_DEC_CBC(*data);
  431         CVMX_MF_3DES_RESULT(*data);
  432         IOV_CONSUME(iov, data, data_i, data_l);
  433         crypt_len -= 8;
  434     }
  435 
  436     return 0;
  437 }
  438 
  439 /****************************************************************************/
  440 /* AES functions */
  441 
  442 int
  443 octo_aes_cbc_encrypt(
  444     struct octo_sess *od,
  445     struct iovec *iov, size_t iovcnt, size_t iovlen,
  446     int auth_off, int auth_len,
  447     int crypt_off, int crypt_len,
  448     int icv_off, uint8_t *ivp)
  449 {
  450     uint64_t *data, *pdata;
  451     int data_i, data_l;
  452 
  453     dprintf("%s()\n", __func__);
  454 
  455     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
  456             (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
  457         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  458                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  459                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  460                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  461         return -EINVAL;
  462     }
  463 
  464     IOV_INIT(iov, data, data_i, data_l);
  465 
  466     CVMX_PREFETCH0(ivp);
  467     CVMX_PREFETCH0(od->octo_enckey);
  468 
  469     /* load AES Key */
  470     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
  471     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
  472 
  473     if (od->octo_encklen == 16) {
  474         CVMX_MT_AES_KEY(0x0, 2);
  475         CVMX_MT_AES_KEY(0x0, 3);
  476     } else if (od->octo_encklen == 24) {
  477         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  478         CVMX_MT_AES_KEY(0x0, 3);
  479     } else if (od->octo_encklen == 32) {
  480         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  481         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
  482     } else {
  483         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
  484         return -EINVAL;
  485     }
  486     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
  487 
  488     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
  489     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
  490 
  491     while (crypt_off > 0) {
  492         IOV_CONSUME(iov, data, data_i, data_l);
  493         crypt_off -= 8;
  494     }
  495 
  496     while (crypt_len > 0) {
  497         pdata = data;
  498         CVMX_MT_AES_ENC_CBC0(*data);
  499         IOV_CONSUME(iov, data, data_i, data_l);
  500         CVMX_MT_AES_ENC_CBC1(*data);
  501         CVMX_MF_AES_RESULT(*pdata, 0);
  502         CVMX_MF_AES_RESULT(*data, 1);
  503         IOV_CONSUME(iov, data, data_i, data_l);
  504         crypt_len -= 16;
  505     }
  506 
  507     return 0;
  508 }
  509 
  510 
  511 int
  512 octo_aes_cbc_decrypt(
  513     struct octo_sess *od,
  514     struct iovec *iov, size_t iovcnt, size_t iovlen,
  515     int auth_off, int auth_len,
  516     int crypt_off, int crypt_len,
  517     int icv_off, uint8_t *ivp)
  518 {
  519     uint64_t *data, *pdata;
  520     int data_i, data_l;
  521 
  522     dprintf("%s()\n", __func__);
  523 
  524     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
  525             (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
  526         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  527                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  528                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  529                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  530         return -EINVAL;
  531     }
  532 
  533     IOV_INIT(iov, data, data_i, data_l);
  534 
  535     CVMX_PREFETCH0(ivp);
  536     CVMX_PREFETCH0(od->octo_enckey);
  537 
  538     /* load AES Key */
  539     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
  540     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
  541 
  542     if (od->octo_encklen == 16) {
  543         CVMX_MT_AES_KEY(0x0, 2);
  544         CVMX_MT_AES_KEY(0x0, 3);
  545     } else if (od->octo_encklen == 24) {
  546         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  547         CVMX_MT_AES_KEY(0x0, 3);
  548     } else if (od->octo_encklen == 32) {
  549         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  550         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
  551     } else {
  552         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
  553         return -EINVAL;
  554     }
  555     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
  556 
  557     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
  558     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
  559 
  560     while (crypt_off > 0) {
  561         IOV_CONSUME(iov, data, data_i, data_l);
  562         crypt_off -= 8;
  563     }
  564 
  565     while (crypt_len > 0) {
  566         pdata = data;
  567         CVMX_MT_AES_DEC_CBC0(*data);
  568         IOV_CONSUME(iov, data, data_i, data_l);
  569         CVMX_MT_AES_DEC_CBC1(*data);
  570         CVMX_MF_AES_RESULT(*pdata, 0);
  571         CVMX_MF_AES_RESULT(*data, 1);
  572         IOV_CONSUME(iov, data, data_i, data_l);
  573         crypt_len -= 16;
  574     }
  575 
  576     return 0;
  577 }
  578 
  579 /****************************************************************************/
  580 /* MD5 */
  581 
  582 int
  583 octo_null_md5_encrypt(
  584     struct octo_sess *od,
  585     struct iovec *iov, size_t iovcnt, size_t iovlen,
  586     int auth_off, int auth_len,
  587     int crypt_off, int crypt_len,
  588     int icv_off, uint8_t *ivp)
  589 {
  590     register int next = 0;
  591     uint64_t *data;
  592     uint64_t tmp1, tmp2;
  593     int data_i, data_l, alen = auth_len;
  594 
  595     dprintf("%s()\n", __func__);
  596 
  597     if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
  598             (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
  599         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  600                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  601                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  602                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  603         return -EINVAL;
  604     }
  605 
  606     IOV_INIT(iov, data, data_i, data_l);
  607 
  608     /* Load MD5 IV */
  609     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
  610     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
  611 
  612     while (auth_off > 0) {
  613         IOV_CONSUME(iov, data, data_i, data_l);
  614         auth_off -= 8;
  615     }
  616 
  617     while (auth_len > 0) {
  618         CVM_LOAD_MD5_UNIT(*data, next);
  619         auth_len -= 8;
  620         IOV_CONSUME(iov, data, data_i, data_l);
  621     }
  622 
  623     /* finish the hash */
  624     CVMX_PREFETCH0(od->octo_hmouter);
  625 #if 0
  626     if (__predict_false(inplen)) {
  627         uint64_t tmp = 0;
  628         uint8_t *p = (uint8_t *) & tmp;
  629         p[inplen] = 0x80;
  630         do {
  631             inplen--;
  632             p[inplen] = ((uint8_t *) data)[inplen];
  633         } while (inplen);
  634         CVM_LOAD_MD5_UNIT(tmp, next);
  635     } else {
  636         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
  637     }
  638 #else
  639     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
  640 #endif
  641 
  642     /* Finish Inner hash */
  643     while (next != 7) {
  644         CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
  645     }
  646     CVMX_ES64(tmp1, ((alen + 64) << 3));
  647     CVM_LOAD_MD5_UNIT(tmp1, next);
  648 
  649     /* Get the inner hash of HMAC */
  650     CVMX_MF_HSH_IV(tmp1, 0);
  651     CVMX_MF_HSH_IV(tmp2, 1);
  652 
  653     /* Initialize hash unit */
  654     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
  655     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
  656 
  657     CVMX_MT_HSH_DAT(tmp1, 0);
  658     CVMX_MT_HSH_DAT(tmp2, 1);
  659     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
  660     CVMX_MT_HSH_DATZ(3);
  661     CVMX_MT_HSH_DATZ(4);
  662     CVMX_MT_HSH_DATZ(5);
  663     CVMX_MT_HSH_DATZ(6);
  664     CVMX_ES64(tmp1, ((64 + 16) << 3));
  665     CVMX_MT_HSH_STARTMD5(tmp1);
  666 
  667     /* save the HMAC */
  668     IOV_INIT(iov, data, data_i, data_l);
  669     while (icv_off > 0) {
  670         IOV_CONSUME(iov, data, data_i, data_l);
  671         icv_off -= 8;
  672     }
  673     CVMX_MF_HSH_IV(*data, 0);
  674     IOV_CONSUME(iov, data, data_i, data_l);
  675     CVMX_MF_HSH_IV(tmp1, 1);
  676     *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
  677 
  678     return 0;
  679 }
  680 
  681 /****************************************************************************/
  682 /* SHA1 */
  683 
  684 int
  685 octo_null_sha1_encrypt(
  686     struct octo_sess *od,
  687     struct iovec *iov, size_t iovcnt, size_t iovlen,
  688     int auth_off, int auth_len,
  689     int crypt_off, int crypt_len,
  690     int icv_off, uint8_t *ivp)
  691 {
  692     register int next = 0;
  693     uint64_t *data;
  694     uint64_t tmp1, tmp2, tmp3;
  695     int data_i, data_l, alen = auth_len;
  696 
  697     dprintf("%s()\n", __func__);
  698 
  699     if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
  700             (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
  701         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  702                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  703                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  704                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  705         return -EINVAL;
  706     }
  707 
  708     IOV_INIT(iov, data, data_i, data_l);
  709 
  710     /* Load SHA1 IV */
  711     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
  712     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
  713     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
  714 
  715     while (auth_off > 0) {
  716         IOV_CONSUME(iov, data, data_i, data_l);
  717         auth_off -= 8;
  718     }
  719 
  720     while (auth_len > 0) {
  721         CVM_LOAD_SHA_UNIT(*data, next);
  722         auth_len -= 8;
  723         IOV_CONSUME(iov, data, data_i, data_l);
  724     }
  725 
  726     /* finish the hash */
  727     CVMX_PREFETCH0(od->octo_hmouter);
  728 #if 0
  729     if (__predict_false(inplen)) {
  730         uint64_t tmp = 0;
  731         uint8_t *p = (uint8_t *) & tmp;
  732         p[inplen] = 0x80;
  733         do {
  734             inplen--;
  735             p[inplen] = ((uint8_t *) data)[inplen];
  736         } while (inplen);
  737         CVM_LOAD_MD5_UNIT(tmp, next);
  738     } else {
  739         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
  740     }
  741 #else
  742     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
  743 #endif
  744 
  745     /* Finish Inner hash */
  746     while (next != 7) {
  747         CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
  748     }
  749         CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
  750 
  751     /* Get the inner hash of HMAC */
  752     CVMX_MF_HSH_IV(tmp1, 0);
  753     CVMX_MF_HSH_IV(tmp2, 1);
  754     tmp3 = 0;
  755     CVMX_MF_HSH_IV(tmp3, 2);
  756 
  757     /* Initialize hash unit */
  758     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
  759     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
  760     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
  761 
  762     CVMX_MT_HSH_DAT(tmp1, 0);
  763     CVMX_MT_HSH_DAT(tmp2, 1);
  764     tmp3 |= 0x0000000080000000;
  765     CVMX_MT_HSH_DAT(tmp3, 2);
  766     CVMX_MT_HSH_DATZ(3);
  767     CVMX_MT_HSH_DATZ(4);
  768     CVMX_MT_HSH_DATZ(5);
  769     CVMX_MT_HSH_DATZ(6);
  770     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
  771 
  772     /* save the HMAC */
  773     IOV_INIT(iov, data, data_i, data_l);
  774     while (icv_off > 0) {
  775         IOV_CONSUME(iov, data, data_i, data_l);
  776         icv_off -= 8;
  777     }
  778     CVMX_MF_HSH_IV(*data, 0);
  779     IOV_CONSUME(iov, data, data_i, data_l);
  780     CVMX_MF_HSH_IV(tmp1, 1);
  781     *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
  782 
  783     return 0;
  784 }
  785 
  786 /****************************************************************************/
  787 /* DES MD5 */
  788 
  789 int
  790 octo_des_cbc_md5_encrypt(
  791     struct octo_sess *od,
  792     struct iovec *iov, size_t iovcnt, size_t iovlen,
  793     int auth_off, int auth_len,
  794     int crypt_off, int crypt_len,
  795     int icv_off, uint8_t *ivp)
  796 {
  797     register int next = 0;
  798     union {
  799         uint32_t data32[2];
  800         uint64_t data64[1];
  801     } mydata;
  802     uint64_t *data = &mydata.data64[0];
  803     uint32_t *data32;
  804     uint64_t tmp1, tmp2;
  805     int data_i, data_l, alen = auth_len;
  806 
  807     dprintf("%s()\n", __func__);
  808 
  809     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
  810             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
  811             (crypt_len  & 0x7) ||
  812             (auth_len  & 0x7) ||
  813             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
  814         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  815                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  816                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  817                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  818         return -EINVAL;
  819     }
  820 
  821     IOV_INIT(iov, data32, data_i, data_l);
  822 
  823     CVMX_PREFETCH0(ivp);
  824     CVMX_PREFETCH0(od->octo_enckey);
  825 
  826     /* load 3DES Key */
  827     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
  828     if (od->octo_encklen == 24) {
  829         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
  830         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  831     } else if (od->octo_encklen == 8) {
  832         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
  833         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
  834     } else {
  835         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
  836         return -EINVAL;
  837     }
  838 
  839     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
  840 
  841     /* Load MD5 IV */
  842     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
  843     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
  844 
  845     while (crypt_off > 0 && auth_off > 0) {
  846         IOV_CONSUME(iov, data32, data_i, data_l);
  847         crypt_off -= 4;
  848         auth_off -= 4;
  849     }
  850 
  851     while (crypt_len > 0 || auth_len > 0) {
  852         uint32_t *first = data32;
  853         mydata.data32[0] = *first;
  854         IOV_CONSUME(iov, data32, data_i, data_l);
  855         mydata.data32[1] = *data32;
  856         if (crypt_off <= 0) {
  857             if (crypt_len > 0) {
  858                 CVMX_MT_3DES_ENC_CBC(*data);
  859                 CVMX_MF_3DES_RESULT(*data);
  860                 crypt_len -= 8;
  861             }
  862         } else
  863             crypt_off -= 8;
  864         if (auth_off <= 0) {
  865             if (auth_len > 0) {
  866                 CVM_LOAD_MD5_UNIT(*data, next);
  867                 auth_len -= 8;
  868             }
  869         } else
  870             auth_off -= 8;
  871         *first = mydata.data32[0];
  872         *data32 = mydata.data32[1];
  873         IOV_CONSUME(iov, data32, data_i, data_l);
  874     }
  875 
  876     /* finish the hash */
  877     CVMX_PREFETCH0(od->octo_hmouter);
  878 #if 0
  879     if (__predict_false(inplen)) {
  880         uint64_t tmp = 0;
  881         uint8_t *p = (uint8_t *) & tmp;
  882         p[inplen] = 0x80;
  883         do {
  884             inplen--;
  885             p[inplen] = ((uint8_t *) data)[inplen];
  886         } while (inplen);
  887         CVM_LOAD_MD5_UNIT(tmp, next);
  888     } else {
  889         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
  890     }
  891 #else
  892     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
  893 #endif
  894 
  895     /* Finish Inner hash */
  896     while (next != 7) {
  897         CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
  898     }
  899     CVMX_ES64(tmp1, ((alen + 64) << 3));
  900     CVM_LOAD_MD5_UNIT(tmp1, next);
  901 
  902     /* Get the inner hash of HMAC */
  903     CVMX_MF_HSH_IV(tmp1, 0);
  904     CVMX_MF_HSH_IV(tmp2, 1);
  905 
  906     /* Initialize hash unit */
  907     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
  908     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
  909 
  910     CVMX_MT_HSH_DAT(tmp1, 0);
  911     CVMX_MT_HSH_DAT(tmp2, 1);
  912     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
  913     CVMX_MT_HSH_DATZ(3);
  914     CVMX_MT_HSH_DATZ(4);
  915     CVMX_MT_HSH_DATZ(5);
  916     CVMX_MT_HSH_DATZ(6);
  917     CVMX_ES64(tmp1, ((64 + 16) << 3));
  918     CVMX_MT_HSH_STARTMD5(tmp1);
  919 
  920     /* save the HMAC */
  921     IOV_INIT(iov, data32, data_i, data_l);
  922     while (icv_off > 0) {
  923         IOV_CONSUME(iov, data32, data_i, data_l);
  924         icv_off -= 4;
  925     }
  926     CVMX_MF_HSH_IV(tmp1, 0);
  927     *data32 = (uint32_t) (tmp1 >> 32);
  928     IOV_CONSUME(iov, data32, data_i, data_l);
  929     *data32 = (uint32_t) tmp1;
  930     IOV_CONSUME(iov, data32, data_i, data_l);
  931     CVMX_MF_HSH_IV(tmp1, 1);
  932     *data32 = (uint32_t) (tmp1 >> 32);
  933 
  934     return 0;
  935 }
  936 
  937 int
  938 octo_des_cbc_md5_decrypt(
  939     struct octo_sess *od,
  940     struct iovec *iov, size_t iovcnt, size_t iovlen,
  941     int auth_off, int auth_len,
  942     int crypt_off, int crypt_len,
  943     int icv_off, uint8_t *ivp)
  944 {
  945     register int next = 0;
  946     union {
  947         uint32_t data32[2];
  948         uint64_t data64[1];
  949     } mydata;
  950     uint64_t *data = &mydata.data64[0];
  951     uint32_t *data32;
  952     uint64_t tmp1, tmp2;
  953     int data_i, data_l, alen = auth_len;
  954 
  955     dprintf("%s()\n", __func__);
  956 
  957     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
  958             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
  959             (crypt_len  & 0x7) ||
  960             (auth_len  & 0x7) ||
  961             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
  962         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
  963                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
  964                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
  965                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
  966         return -EINVAL;
  967     }
  968 
  969     IOV_INIT(iov, data32, data_i, data_l);
  970 
  971     CVMX_PREFETCH0(ivp);
  972     CVMX_PREFETCH0(od->octo_enckey);
  973 
  974     /* load 3DES Key */
  975     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
  976     if (od->octo_encklen == 24) {
  977         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
  978         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
  979     } else if (od->octo_encklen == 8) {
  980         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
  981         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
  982     } else {
  983         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
  984         return -EINVAL;
  985     }
  986 
  987     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
  988 
  989     /* Load MD5 IV */
  990     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
  991     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
  992 
  993     while (crypt_off > 0 && auth_off > 0) {
  994         IOV_CONSUME(iov, data32, data_i, data_l);
  995         crypt_off -= 4;
  996         auth_off -= 4;
  997     }
  998 
  999     while (crypt_len > 0 || auth_len > 0) {
 1000         uint32_t *first = data32;
 1001         mydata.data32[0] = *first;
 1002         IOV_CONSUME(iov, data32, data_i, data_l);
 1003         mydata.data32[1] = *data32;
 1004         if (auth_off <= 0) {
 1005             if (auth_len > 0) {
 1006                 CVM_LOAD_MD5_UNIT(*data, next);
 1007                 auth_len -= 8;
 1008             }
 1009         } else
 1010             auth_off -= 8;
 1011         if (crypt_off <= 0) {
 1012             if (crypt_len > 0) {
 1013                 CVMX_MT_3DES_DEC_CBC(*data);
 1014                 CVMX_MF_3DES_RESULT(*data);
 1015                 crypt_len -= 8;
 1016             }
 1017         } else
 1018             crypt_off -= 8;
 1019         *first = mydata.data32[0];
 1020         *data32 = mydata.data32[1];
 1021         IOV_CONSUME(iov, data32, data_i, data_l);
 1022     }
 1023 
 1024     /* finish the hash */
 1025     CVMX_PREFETCH0(od->octo_hmouter);
 1026 #if 0
 1027     if (__predict_false(inplen)) {
 1028         uint64_t tmp = 0;
 1029         uint8_t *p = (uint8_t *) & tmp;
 1030         p[inplen] = 0x80;
 1031         do {
 1032             inplen--;
 1033             p[inplen] = ((uint8_t *) data)[inplen];
 1034         } while (inplen);
 1035         CVM_LOAD_MD5_UNIT(tmp, next);
 1036     } else {
 1037         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1038     }
 1039 #else
 1040     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1041 #endif
 1042 
 1043     /* Finish Inner hash */
 1044     while (next != 7) {
 1045         CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
 1046     }
 1047     CVMX_ES64(tmp1, ((alen + 64) << 3));
 1048     CVM_LOAD_MD5_UNIT(tmp1, next);
 1049 
 1050     /* Get the inner hash of HMAC */
 1051     CVMX_MF_HSH_IV(tmp1, 0);
 1052     CVMX_MF_HSH_IV(tmp2, 1);
 1053 
 1054     /* Initialize hash unit */
 1055     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
 1056     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 1057 
 1058     CVMX_MT_HSH_DAT(tmp1, 0);
 1059     CVMX_MT_HSH_DAT(tmp2, 1);
 1060     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
 1061     CVMX_MT_HSH_DATZ(3);
 1062     CVMX_MT_HSH_DATZ(4);
 1063     CVMX_MT_HSH_DATZ(5);
 1064     CVMX_MT_HSH_DATZ(6);
 1065     CVMX_ES64(tmp1, ((64 + 16) << 3));
 1066     CVMX_MT_HSH_STARTMD5(tmp1);
 1067 
 1068     /* save the HMAC */
 1069     IOV_INIT(iov, data32, data_i, data_l);
 1070     while (icv_off > 0) {
 1071         IOV_CONSUME(iov, data32, data_i, data_l);
 1072         icv_off -= 4;
 1073     }
 1074     CVMX_MF_HSH_IV(tmp1, 0);
 1075     *data32 = (uint32_t) (tmp1 >> 32);
 1076     IOV_CONSUME(iov, data32, data_i, data_l);
 1077     *data32 = (uint32_t) tmp1;
 1078     IOV_CONSUME(iov, data32, data_i, data_l);
 1079     CVMX_MF_HSH_IV(tmp1, 1);
 1080     *data32 = (uint32_t) (tmp1 >> 32);
 1081 
 1082     return 0;
 1083 }
 1084 
 1085 /****************************************************************************/
 1086 /* DES SHA */
 1087 
 1088 int
 1089 octo_des_cbc_sha1_encrypt(
 1090     struct octo_sess *od,
 1091     struct iovec *iov, size_t iovcnt, size_t iovlen,
 1092     int auth_off, int auth_len,
 1093     int crypt_off, int crypt_len,
 1094     int icv_off, uint8_t *ivp)
 1095 {
 1096     register int next = 0;
 1097     union {
 1098         uint32_t data32[2];
 1099         uint64_t data64[1];
 1100     } mydata;
 1101     uint64_t *data = &mydata.data64[0];
 1102     uint32_t *data32;
 1103     uint64_t tmp1, tmp2, tmp3;
 1104     int data_i, data_l, alen = auth_len;
 1105 
 1106     dprintf("%s()\n", __func__);
 1107 
 1108     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 1109             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 1110             (crypt_len  & 0x7) ||
 1111             (auth_len  & 0x7) ||
 1112             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 1113         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 1114                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
 1115                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
 1116                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
 1117         return -EINVAL;
 1118     }
 1119 
 1120     IOV_INIT(iov, data32, data_i, data_l);
 1121 
 1122     CVMX_PREFETCH0(ivp);
 1123     CVMX_PREFETCH0(od->octo_enckey);
 1124 
 1125     /* load 3DES Key */
 1126     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
 1127     if (od->octo_encklen == 24) {
 1128         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 1129         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1130     } else if (od->octo_encklen == 8) {
 1131         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 1132         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
 1133     } else {
 1134         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 1135         return -EINVAL;
 1136     }
 1137 
 1138     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 1139 
 1140     /* Load SHA1 IV */
 1141     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
 1142     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 1143     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 1144 
 1145     while (crypt_off > 0 && auth_off > 0) {
 1146         IOV_CONSUME(iov, data32, data_i, data_l);
 1147         crypt_off -= 4;
 1148         auth_off -= 4;
 1149     }
 1150 
 1151     while (crypt_len > 0 || auth_len > 0) {
 1152         uint32_t *first = data32;
 1153         mydata.data32[0] = *first;
 1154         IOV_CONSUME(iov, data32, data_i, data_l);
 1155         mydata.data32[1] = *data32;
 1156         if (crypt_off <= 0) {
 1157             if (crypt_len > 0) {
 1158                 CVMX_MT_3DES_ENC_CBC(*data);
 1159                 CVMX_MF_3DES_RESULT(*data);
 1160                 crypt_len -= 8;
 1161             }
 1162         } else
 1163             crypt_off -= 8;
 1164         if (auth_off <= 0) {
 1165             if (auth_len > 0) {
 1166                 CVM_LOAD_SHA_UNIT(*data, next);
 1167                 auth_len -= 8;
 1168             }
 1169         } else
 1170             auth_off -= 8;
 1171         *first = mydata.data32[0];
 1172         *data32 = mydata.data32[1];
 1173         IOV_CONSUME(iov, data32, data_i, data_l);
 1174     }
 1175 
 1176     /* finish the hash */
 1177     CVMX_PREFETCH0(od->octo_hmouter);
 1178 #if 0
 1179     if (__predict_false(inplen)) {
 1180         uint64_t tmp = 0;
 1181         uint8_t *p = (uint8_t *) & tmp;
 1182         p[inplen] = 0x80;
 1183         do {
 1184             inplen--;
 1185             p[inplen] = ((uint8_t *) data)[inplen];
 1186         } while (inplen);
 1187         CVM_LOAD_SHA_UNIT(tmp, next);
 1188     } else {
 1189         CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 1190     }
 1191 #else
 1192     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 1193 #endif
 1194 
 1195     /* Finish Inner hash */
 1196     while (next != 7) {
 1197         CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
 1198     }
 1199         CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 1200 
 1201     /* Get the inner hash of HMAC */
 1202     CVMX_MF_HSH_IV(tmp1, 0);
 1203     CVMX_MF_HSH_IV(tmp2, 1);
 1204     tmp3 = 0;
 1205     CVMX_MF_HSH_IV(tmp3, 2);
 1206 
 1207     /* Initialize hash unit */
 1208     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
 1209     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 1210     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 1211 
 1212     CVMX_MT_HSH_DAT(tmp1, 0);
 1213     CVMX_MT_HSH_DAT(tmp2, 1);
 1214     tmp3 |= 0x0000000080000000;
 1215     CVMX_MT_HSH_DAT(tmp3, 2);
 1216     CVMX_MT_HSH_DATZ(3);
 1217     CVMX_MT_HSH_DATZ(4);
 1218     CVMX_MT_HSH_DATZ(5);
 1219     CVMX_MT_HSH_DATZ(6);
 1220     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 1221 
 1222     /* save the HMAC */
 1223     IOV_INIT(iov, data32, data_i, data_l);
 1224     while (icv_off > 0) {
 1225         IOV_CONSUME(iov, data32, data_i, data_l);
 1226         icv_off -= 4;
 1227     }
 1228     CVMX_MF_HSH_IV(tmp1, 0);
 1229     *data32 = (uint32_t) (tmp1 >> 32);
 1230     IOV_CONSUME(iov, data32, data_i, data_l);
 1231     *data32 = (uint32_t) tmp1;
 1232     IOV_CONSUME(iov, data32, data_i, data_l);
 1233     CVMX_MF_HSH_IV(tmp1, 1);
 1234     *data32 = (uint32_t) (tmp1 >> 32);
 1235 
 1236     return 0;
 1237 }
 1238 
 1239 int
 1240 octo_des_cbc_sha1_decrypt(
 1241     struct octo_sess *od,
 1242     struct iovec *iov, size_t iovcnt, size_t iovlen,
 1243     int auth_off, int auth_len,
 1244     int crypt_off, int crypt_len,
 1245     int icv_off, uint8_t *ivp)
 1246 {
 1247     register int next = 0;
 1248     union {
 1249         uint32_t data32[2];
 1250         uint64_t data64[1];
 1251     } mydata;
 1252     uint64_t *data = &mydata.data64[0];
 1253     uint32_t *data32;
 1254     uint64_t tmp1, tmp2, tmp3;
 1255     int data_i, data_l, alen = auth_len;
 1256 
 1257     dprintf("%s()\n", __func__);
 1258 
 1259     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 1260             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 1261             (crypt_len  & 0x7) ||
 1262             (auth_len  & 0x7) ||
 1263             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 1264         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 1265                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
 1266                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
 1267                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
 1268         return -EINVAL;
 1269     }
 1270 
 1271     IOV_INIT(iov, data32, data_i, data_l);
 1272 
 1273     CVMX_PREFETCH0(ivp);
 1274     CVMX_PREFETCH0(od->octo_enckey);
 1275 
 1276     /* load 3DES Key */
 1277     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
 1278     if (od->octo_encklen == 24) {
 1279         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 1280         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1281     } else if (od->octo_encklen == 8) {
 1282         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 1283         CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
 1284     } else {
 1285         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 1286         return -EINVAL;
 1287     }
 1288 
 1289     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 1290 
 1291     /* Load SHA1 IV */
 1292     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
 1293     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 1294     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 1295 
 1296     while (crypt_off > 0 && auth_off > 0) {
 1297         IOV_CONSUME(iov, data32, data_i, data_l);
 1298         crypt_off -= 4;
 1299         auth_off -= 4;
 1300     }
 1301 
 1302     while (crypt_len > 0 || auth_len > 0) {
 1303         uint32_t *first = data32;
 1304         mydata.data32[0] = *first;
 1305         IOV_CONSUME(iov, data32, data_i, data_l);
 1306         mydata.data32[1] = *data32;
 1307         if (auth_off <= 0) {
 1308             if (auth_len > 0) {
 1309                 CVM_LOAD_SHA_UNIT(*data, next);
 1310                 auth_len -= 8;
 1311             }
 1312         } else
 1313             auth_off -= 8;
 1314         if (crypt_off <= 0) {
 1315             if (crypt_len > 0) {
 1316                 CVMX_MT_3DES_DEC_CBC(*data);
 1317                 CVMX_MF_3DES_RESULT(*data);
 1318                 crypt_len -= 8;
 1319             }
 1320         } else
 1321             crypt_off -= 8;
 1322         *first = mydata.data32[0];
 1323         *data32 = mydata.data32[1];
 1324         IOV_CONSUME(iov, data32, data_i, data_l);
 1325     }
 1326 
 1327     /* finish the hash */
 1328     CVMX_PREFETCH0(od->octo_hmouter);
 1329 #if 0
 1330     if (__predict_false(inplen)) {
 1331         uint64_t tmp = 0;
 1332         uint8_t *p = (uint8_t *) & tmp;
 1333         p[inplen] = 0x80;
 1334         do {
 1335             inplen--;
 1336             p[inplen] = ((uint8_t *) data)[inplen];
 1337         } while (inplen);
 1338         CVM_LOAD_SHA_UNIT(tmp, next);
 1339     } else {
 1340         CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 1341     }
 1342 #else
 1343     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 1344 #endif
 1345 
 1346     /* Finish Inner hash */
 1347     while (next != 7) {
 1348         CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
 1349     }
 1350         CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 1351 
 1352     /* Get the inner hash of HMAC */
 1353     CVMX_MF_HSH_IV(tmp1, 0);
 1354     CVMX_MF_HSH_IV(tmp2, 1);
 1355     tmp3 = 0;
 1356     CVMX_MF_HSH_IV(tmp3, 2);
 1357 
 1358     /* Initialize hash unit */
 1359     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
 1360     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 1361     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 1362 
 1363     CVMX_MT_HSH_DAT(tmp1, 0);
 1364     CVMX_MT_HSH_DAT(tmp2, 1);
 1365     tmp3 |= 0x0000000080000000;
 1366     CVMX_MT_HSH_DAT(tmp3, 2);
 1367     CVMX_MT_HSH_DATZ(3);
 1368     CVMX_MT_HSH_DATZ(4);
 1369     CVMX_MT_HSH_DATZ(5);
 1370     CVMX_MT_HSH_DATZ(6);
 1371     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 1372     /* save the HMAC */
 1373     IOV_INIT(iov, data32, data_i, data_l);
 1374     while (icv_off > 0) {
 1375         IOV_CONSUME(iov, data32, data_i, data_l);
 1376         icv_off -= 4;
 1377     }
 1378     CVMX_MF_HSH_IV(tmp1, 0);
 1379     *data32 = (uint32_t) (tmp1 >> 32);
 1380     IOV_CONSUME(iov, data32, data_i, data_l);
 1381     *data32 = (uint32_t) tmp1;
 1382     IOV_CONSUME(iov, data32, data_i, data_l);
 1383     CVMX_MF_HSH_IV(tmp1, 1);
 1384     *data32 = (uint32_t) (tmp1 >> 32);
 1385 
 1386     return 0;
 1387 }
 1388 
 1389 /****************************************************************************/
 1390 /* AES MD5 */
 1391 
 1392 int
 1393 octo_aes_cbc_md5_encrypt(
 1394     struct octo_sess *od,
 1395     struct iovec *iov, size_t iovcnt, size_t iovlen,
 1396     int auth_off, int auth_len,
 1397     int crypt_off, int crypt_len,
 1398     int icv_off, uint8_t *ivp)
 1399 {
 1400     register int next = 0;
 1401     union {
 1402         uint32_t data32[2];
 1403         uint64_t data64[1];
 1404     } mydata[2];
 1405     uint64_t *pdata = &mydata[0].data64[0];
 1406     uint64_t *data =  &mydata[1].data64[0];
 1407     uint32_t *data32;
 1408     uint64_t tmp1, tmp2;
 1409     int data_i, data_l, alen = auth_len;
 1410 
 1411     dprintf("%s()\n", __func__);
 1412 
 1413     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 1414             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 1415             (crypt_len  & 0x7) ||
 1416             (auth_len  & 0x7) ||
 1417             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 1418         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 1419                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
 1420                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
 1421                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
 1422         return -EINVAL;
 1423     }
 1424 
 1425     IOV_INIT(iov, data32, data_i, data_l);
 1426 
 1427     CVMX_PREFETCH0(ivp);
 1428     CVMX_PREFETCH0(od->octo_enckey);
 1429 
 1430     /* load AES Key */
 1431     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
 1432     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 1433 
 1434     if (od->octo_encklen == 16) {
 1435         CVMX_MT_AES_KEY(0x0, 2);
 1436         CVMX_MT_AES_KEY(0x0, 3);
 1437     } else if (od->octo_encklen == 24) {
 1438         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1439         CVMX_MT_AES_KEY(0x0, 3);
 1440     } else if (od->octo_encklen == 32) {
 1441         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1442         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
 1443     } else {
 1444         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 1445         return -EINVAL;
 1446     }
 1447     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 1448 
 1449     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
 1450     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 1451 
 1452     /* Load MD5 IV */
 1453     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
 1454     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 1455 
 1456     while (crypt_off > 0 && auth_off > 0) {
 1457         IOV_CONSUME(iov, data32, data_i, data_l);
 1458         crypt_off -= 4;
 1459         auth_off -= 4;
 1460     }
 1461 
 1462     while (crypt_len > 0 || auth_len > 0) {
 1463         uint32_t *pdata32[3];
 1464 
 1465         pdata32[0] = data32;
 1466         mydata[0].data32[0] = *data32;
 1467         IOV_CONSUME(iov, data32, data_i, data_l);
 1468 
 1469         pdata32[1] = data32;
 1470         mydata[0].data32[1] = *data32;
 1471         IOV_CONSUME(iov, data32, data_i, data_l);
 1472 
 1473         pdata32[2] = data32;
 1474         mydata[1].data32[0] = *data32;
 1475         IOV_CONSUME(iov, data32, data_i, data_l);
 1476 
 1477         mydata[1].data32[1] = *data32;
 1478 
 1479 
 1480         if (crypt_off <= 0) {
 1481             if (crypt_len > 0) {
 1482                 CVMX_MT_AES_ENC_CBC0(*pdata);
 1483                 CVMX_MT_AES_ENC_CBC1(*data);
 1484                 CVMX_MF_AES_RESULT(*pdata, 0);
 1485                 CVMX_MF_AES_RESULT(*data, 1);
 1486                 crypt_len -= 16;
 1487             }
 1488         } else
 1489             crypt_off -= 16;
 1490 
 1491         if (auth_off <= 0) {
 1492             if (auth_len > 0) {
 1493                 CVM_LOAD_MD5_UNIT(*pdata, next);
 1494                 CVM_LOAD_MD5_UNIT(*data, next);
 1495                 auth_len -= 16;
 1496             }
 1497         } else
 1498             auth_off -= 16;
 1499 
 1500         *pdata32[0] = mydata[0].data32[0];
 1501         *pdata32[1] = mydata[0].data32[1];
 1502         *pdata32[2] = mydata[1].data32[0];
 1503         *data32     = mydata[1].data32[1];
 1504 
 1505         IOV_CONSUME(iov, data32, data_i, data_l);
 1506     }
 1507 
 1508     /* finish the hash */
 1509     CVMX_PREFETCH0(od->octo_hmouter);
 1510 #if 0
 1511     if (__predict_false(inplen)) {
 1512         uint64_t tmp = 0;
 1513         uint8_t *p = (uint8_t *) & tmp;
 1514         p[inplen] = 0x80;
 1515         do {
 1516             inplen--;
 1517             p[inplen] = ((uint8_t *) data)[inplen];
 1518         } while (inplen);
 1519         CVM_LOAD_MD5_UNIT(tmp, next);
 1520     } else {
 1521         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1522     }
 1523 #else
 1524     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1525 #endif
 1526 
 1527     /* Finish Inner hash */
 1528     while (next != 7) {
 1529         CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
 1530     }
 1531     CVMX_ES64(tmp1, ((alen + 64) << 3));
 1532     CVM_LOAD_MD5_UNIT(tmp1, next);
 1533 
 1534     /* Get the inner hash of HMAC */
 1535     CVMX_MF_HSH_IV(tmp1, 0);
 1536     CVMX_MF_HSH_IV(tmp2, 1);
 1537 
 1538     /* Initialize hash unit */
 1539     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
 1540     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 1541 
 1542     CVMX_MT_HSH_DAT(tmp1, 0);
 1543     CVMX_MT_HSH_DAT(tmp2, 1);
 1544     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
 1545     CVMX_MT_HSH_DATZ(3);
 1546     CVMX_MT_HSH_DATZ(4);
 1547     CVMX_MT_HSH_DATZ(5);
 1548     CVMX_MT_HSH_DATZ(6);
 1549     CVMX_ES64(tmp1, ((64 + 16) << 3));
 1550     CVMX_MT_HSH_STARTMD5(tmp1);
 1551 
 1552     /* save the HMAC */
 1553     IOV_INIT(iov, data32, data_i, data_l);
 1554     while (icv_off > 0) {
 1555         IOV_CONSUME(iov, data32, data_i, data_l);
 1556         icv_off -= 4;
 1557     }
 1558     CVMX_MF_HSH_IV(tmp1, 0);
 1559     *data32 = (uint32_t) (tmp1 >> 32);
 1560     IOV_CONSUME(iov, data32, data_i, data_l);
 1561     *data32 = (uint32_t) tmp1;
 1562     IOV_CONSUME(iov, data32, data_i, data_l);
 1563     CVMX_MF_HSH_IV(tmp1, 1);
 1564     *data32 = (uint32_t) (tmp1 >> 32);
 1565 
 1566     return 0;
 1567 }
 1568 
 1569 int
 1570 octo_aes_cbc_md5_decrypt(
 1571     struct octo_sess *od,
 1572     struct iovec *iov, size_t iovcnt, size_t iovlen,
 1573     int auth_off, int auth_len,
 1574     int crypt_off, int crypt_len,
 1575     int icv_off, uint8_t *ivp)
 1576 {
 1577     register int next = 0;
 1578     union {
 1579         uint32_t data32[2];
 1580         uint64_t data64[1];
 1581     } mydata[2];
 1582     uint64_t *pdata = &mydata[0].data64[0];
 1583     uint64_t *data =  &mydata[1].data64[0];
 1584     uint32_t *data32;
 1585     uint64_t tmp1, tmp2;
 1586     int data_i, data_l, alen = auth_len;
 1587 
 1588     dprintf("%s()\n", __func__);
 1589 
 1590     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 1591             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 1592             (crypt_len  & 0x7) ||
 1593             (auth_len  & 0x7) ||
 1594             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 1595         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 1596                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
 1597                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
 1598                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
 1599         return -EINVAL;
 1600     }
 1601 
 1602     IOV_INIT(iov, data32, data_i, data_l);
 1603 
 1604     CVMX_PREFETCH0(ivp);
 1605     CVMX_PREFETCH0(od->octo_enckey);
 1606 
 1607     /* load AES Key */
 1608     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
 1609     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 1610 
 1611     if (od->octo_encklen == 16) {
 1612         CVMX_MT_AES_KEY(0x0, 2);
 1613         CVMX_MT_AES_KEY(0x0, 3);
 1614     } else if (od->octo_encklen == 24) {
 1615         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1616         CVMX_MT_AES_KEY(0x0, 3);
 1617     } else if (od->octo_encklen == 32) {
 1618         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1619         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
 1620     } else {
 1621         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 1622         return -EINVAL;
 1623     }
 1624     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 1625 
 1626     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
 1627     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 1628 
 1629     /* Load MD5 IV */
 1630     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
 1631     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 1632 
 1633     while (crypt_off > 0 && auth_off > 0) {
 1634         IOV_CONSUME(iov, data32, data_i, data_l);
 1635         crypt_off -= 4;
 1636         auth_off -= 4;
 1637     }
 1638 
 1639     while (crypt_len > 0 || auth_len > 0) {
 1640         uint32_t *pdata32[3];
 1641 
 1642         pdata32[0] = data32;
 1643         mydata[0].data32[0] = *data32;
 1644         IOV_CONSUME(iov, data32, data_i, data_l);
 1645         pdata32[1] = data32;
 1646         mydata[0].data32[1] = *data32;
 1647         IOV_CONSUME(iov, data32, data_i, data_l);
 1648         pdata32[2] = data32;
 1649         mydata[1].data32[0] = *data32;
 1650         IOV_CONSUME(iov, data32, data_i, data_l);
 1651         mydata[1].data32[1] = *data32;
 1652 
 1653         if (auth_off <= 0) {
 1654             if (auth_len > 0) {
 1655                 CVM_LOAD_MD5_UNIT(*pdata, next);
 1656                 CVM_LOAD_MD5_UNIT(*data, next);
 1657                 auth_len -= 16;
 1658             }
 1659         } else
 1660             auth_off -= 16;
 1661 
 1662         if (crypt_off <= 0) {
 1663             if (crypt_len > 0) {
 1664                 CVMX_MT_AES_DEC_CBC0(*pdata);
 1665                 CVMX_MT_AES_DEC_CBC1(*data);
 1666                 CVMX_MF_AES_RESULT(*pdata, 0);
 1667                 CVMX_MF_AES_RESULT(*data, 1);
 1668                 crypt_len -= 16;
 1669             }
 1670         } else
 1671             crypt_off -= 16;
 1672 
 1673         *pdata32[0] = mydata[0].data32[0];
 1674         *pdata32[1] = mydata[0].data32[1];
 1675         *pdata32[2] = mydata[1].data32[0];
 1676         *data32     = mydata[1].data32[1];
 1677 
 1678         IOV_CONSUME(iov, data32, data_i, data_l);
 1679     }
 1680 
 1681     /* finish the hash */
 1682     CVMX_PREFETCH0(od->octo_hmouter);
 1683 #if 0
 1684     if (__predict_false(inplen)) {
 1685         uint64_t tmp = 0;
 1686         uint8_t *p = (uint8_t *) & tmp;
 1687         p[inplen] = 0x80;
 1688         do {
 1689             inplen--;
 1690             p[inplen] = ((uint8_t *) data)[inplen];
 1691         } while (inplen);
 1692         CVM_LOAD_MD5_UNIT(tmp, next);
 1693     } else {
 1694         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1695     }
 1696 #else
 1697     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1698 #endif
 1699 
 1700     /* Finish Inner hash */
 1701     while (next != 7) {
 1702         CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
 1703     }
 1704     CVMX_ES64(tmp1, ((alen + 64) << 3));
 1705     CVM_LOAD_MD5_UNIT(tmp1, next);
 1706 
 1707     /* Get the inner hash of HMAC */
 1708     CVMX_MF_HSH_IV(tmp1, 0);
 1709     CVMX_MF_HSH_IV(tmp2, 1);
 1710 
 1711     /* Initialize hash unit */
 1712     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
 1713     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 1714 
 1715     CVMX_MT_HSH_DAT(tmp1, 0);
 1716     CVMX_MT_HSH_DAT(tmp2, 1);
 1717     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
 1718     CVMX_MT_HSH_DATZ(3);
 1719     CVMX_MT_HSH_DATZ(4);
 1720     CVMX_MT_HSH_DATZ(5);
 1721     CVMX_MT_HSH_DATZ(6);
 1722     CVMX_ES64(tmp1, ((64 + 16) << 3));
 1723     CVMX_MT_HSH_STARTMD5(tmp1);
 1724 
 1725     /* save the HMAC */
 1726     IOV_INIT(iov, data32, data_i, data_l);
 1727     while (icv_off > 0) {
 1728         IOV_CONSUME(iov, data32, data_i, data_l);
 1729         icv_off -= 4;
 1730     }
 1731     CVMX_MF_HSH_IV(tmp1, 0);
 1732     *data32 = (uint32_t) (tmp1 >> 32);
 1733     IOV_CONSUME(iov, data32, data_i, data_l);
 1734     *data32 = (uint32_t) tmp1;
 1735     IOV_CONSUME(iov, data32, data_i, data_l);
 1736     CVMX_MF_HSH_IV(tmp1, 1);
 1737     *data32 = (uint32_t) (tmp1 >> 32);
 1738 
 1739     return 0;
 1740 }
 1741 
 1742 /****************************************************************************/
 1743 /* AES SHA1 */
 1744 
 1745 int
 1746 octo_aes_cbc_sha1_encrypt(
 1747     struct octo_sess *od,
 1748     struct iovec *iov, size_t iovcnt, size_t iovlen,
 1749     int auth_off, int auth_len,
 1750     int crypt_off, int crypt_len,
 1751     int icv_off, uint8_t *ivp)
 1752 {
 1753     register int next = 0;
 1754     union {
 1755         uint32_t data32[2];
 1756         uint64_t data64[1];
 1757     } mydata[2];
 1758     uint64_t *pdata = &mydata[0].data64[0];
 1759     uint64_t *data =  &mydata[1].data64[0];
 1760     uint32_t *data32;
 1761     uint64_t tmp1, tmp2, tmp3;
 1762     int data_i, data_l, alen = auth_len;
 1763 
 1764     dprintf("%s()\n", __func__);
 1765 
 1766     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 1767             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 1768             (crypt_len  & 0x7) ||
 1769             (auth_len  & 0x7) ||
 1770             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 1771         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 1772                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
 1773                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
 1774                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
 1775         return -EINVAL;
 1776     }
 1777 
 1778     IOV_INIT(iov, data32, data_i, data_l);
 1779 
 1780     CVMX_PREFETCH0(ivp);
 1781     CVMX_PREFETCH0(od->octo_enckey);
 1782 
 1783     /* load AES Key */
 1784     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
 1785     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 1786 
 1787     if (od->octo_encklen == 16) {
 1788         CVMX_MT_AES_KEY(0x0, 2);
 1789         CVMX_MT_AES_KEY(0x0, 3);
 1790     } else if (od->octo_encklen == 24) {
 1791         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1792         CVMX_MT_AES_KEY(0x0, 3);
 1793     } else if (od->octo_encklen == 32) {
 1794         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1795         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
 1796     } else {
 1797         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 1798         return -EINVAL;
 1799     }
 1800     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 1801 
 1802     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
 1803     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 1804 
 1805     /* Load SHA IV */
 1806     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
 1807     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 1808     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 1809 
 1810     while (crypt_off > 0 && auth_off > 0) {
 1811         IOV_CONSUME(iov, data32, data_i, data_l);
 1812         crypt_off -= 4;
 1813         auth_off -= 4;
 1814     }
 1815 
 1816     while (crypt_len > 0 || auth_len > 0) {
 1817         uint32_t *pdata32[3];
 1818 
 1819         pdata32[0] = data32;
 1820         mydata[0].data32[0] = *data32;
 1821         IOV_CONSUME(iov, data32, data_i, data_l);
 1822         pdata32[1] = data32;
 1823         mydata[0].data32[1] = *data32;
 1824         IOV_CONSUME(iov, data32, data_i, data_l);
 1825         pdata32[2] = data32;
 1826         mydata[1].data32[0] = *data32;
 1827         IOV_CONSUME(iov, data32, data_i, data_l);
 1828         mydata[1].data32[1] = *data32;
 1829 
 1830 
 1831         if (crypt_off <= 0) {
 1832             if (crypt_len > 0) {
 1833                 CVMX_MT_AES_ENC_CBC0(*pdata);
 1834                 CVMX_MT_AES_ENC_CBC1(*data);
 1835                 CVMX_MF_AES_RESULT(*pdata, 0);
 1836                 CVMX_MF_AES_RESULT(*data, 1);
 1837                 crypt_len -= 16;
 1838             }
 1839         } else
 1840             crypt_off -= 16;
 1841 
 1842         if (auth_off <= 0) {
 1843             if (auth_len > 0) {
 1844                 CVM_LOAD_SHA_UNIT(*pdata, next);
 1845                 CVM_LOAD_SHA_UNIT(*data, next);
 1846                 auth_len -= 16;
 1847             }
 1848         } else
 1849             auth_off -= 16;
 1850 
 1851         *pdata32[0] = mydata[0].data32[0];
 1852         *pdata32[1] = mydata[0].data32[1];
 1853         *pdata32[2] = mydata[1].data32[0];
 1854         *data32     = mydata[1].data32[1];
 1855 
 1856         IOV_CONSUME(iov, data32, data_i, data_l);
 1857     }
 1858 
 1859     /* finish the hash */
 1860     CVMX_PREFETCH0(od->octo_hmouter);
 1861 #if 0
 1862     if (__predict_false(inplen)) {
 1863         uint64_t tmp = 0;
 1864         uint8_t *p = (uint8_t *) & tmp;
 1865         p[inplen] = 0x80;
 1866         do {
 1867             inplen--;
 1868             p[inplen] = ((uint8_t *) data)[inplen];
 1869         } while (inplen);
 1870         CVM_LOAD_SHA_UNIT(tmp, next);
 1871     } else {
 1872         CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 1873     }
 1874 #else
 1875     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 1876 #endif
 1877 
 1878     /* Finish Inner hash */
 1879     while (next != 7) {
 1880         CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
 1881     }
 1882         CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 1883 
 1884     /* Get the inner hash of HMAC */
 1885     CVMX_MF_HSH_IV(tmp1, 0);
 1886     CVMX_MF_HSH_IV(tmp2, 1);
 1887     tmp3 = 0;
 1888     CVMX_MF_HSH_IV(tmp3, 2);
 1889 
 1890     /* Initialize hash unit */
 1891     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
 1892     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 1893     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 1894 
 1895     CVMX_MT_HSH_DAT(tmp1, 0);
 1896     CVMX_MT_HSH_DAT(tmp2, 1);
 1897     tmp3 |= 0x0000000080000000;
 1898     CVMX_MT_HSH_DAT(tmp3, 2);
 1899     CVMX_MT_HSH_DATZ(3);
 1900     CVMX_MT_HSH_DATZ(4);
 1901     CVMX_MT_HSH_DATZ(5);
 1902     CVMX_MT_HSH_DATZ(6);
 1903     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 1904 
 1905     /* finish the hash */
 1906     CVMX_PREFETCH0(od->octo_hmouter);
 1907 #if 0
 1908     if (__predict_false(inplen)) {
 1909         uint64_t tmp = 0;
 1910         uint8_t *p = (uint8_t *) & tmp;
 1911         p[inplen] = 0x80;
 1912         do {
 1913             inplen--;
 1914             p[inplen] = ((uint8_t *) data)[inplen];
 1915         } while (inplen);
 1916         CVM_LOAD_MD5_UNIT(tmp, next);
 1917     } else {
 1918         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1919     }
 1920 #else
 1921     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 1922 #endif
 1923 
 1924     /* save the HMAC */
 1925     IOV_INIT(iov, data32, data_i, data_l);
 1926     while (icv_off > 0) {
 1927         IOV_CONSUME(iov, data32, data_i, data_l);
 1928         icv_off -= 4;
 1929     }
 1930     CVMX_MF_HSH_IV(tmp1, 0);
 1931     *data32 = (uint32_t) (tmp1 >> 32);
 1932     IOV_CONSUME(iov, data32, data_i, data_l);
 1933     *data32 = (uint32_t) tmp1;
 1934     IOV_CONSUME(iov, data32, data_i, data_l);
 1935     CVMX_MF_HSH_IV(tmp1, 1);
 1936     *data32 = (uint32_t) (tmp1 >> 32);
 1937 
 1938     return 0;
 1939 }
 1940 
 1941 int
 1942 octo_aes_cbc_sha1_decrypt(
 1943     struct octo_sess *od,
 1944     struct iovec *iov, size_t iovcnt, size_t iovlen,
 1945     int auth_off, int auth_len,
 1946     int crypt_off, int crypt_len,
 1947     int icv_off, uint8_t *ivp)
 1948 {
 1949     register int next = 0;
 1950     union {
 1951         uint32_t data32[2];
 1952         uint64_t data64[1];
 1953     } mydata[2];
 1954     uint64_t *pdata = &mydata[0].data64[0];
 1955     uint64_t *data =  &mydata[1].data64[0];
 1956     uint32_t *data32;
 1957     uint64_t tmp1, tmp2, tmp3;
 1958     int data_i, data_l, alen = auth_len;
 1959 
 1960     dprintf("%s()\n", __func__);
 1961 
 1962     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 1963             (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 1964             (crypt_len  & 0x7) ||
 1965             (auth_len  & 0x7) ||
 1966             (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 1967         dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 1968                 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
 1969                 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
 1970                 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
 1971         return -EINVAL;
 1972     }
 1973 
 1974     IOV_INIT(iov, data32, data_i, data_l);
 1975 
 1976     CVMX_PREFETCH0(ivp);
 1977     CVMX_PREFETCH0(od->octo_enckey);
 1978 
 1979     /* load AES Key */
 1980     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
 1981     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 1982 
 1983     if (od->octo_encklen == 16) {
 1984         CVMX_MT_AES_KEY(0x0, 2);
 1985         CVMX_MT_AES_KEY(0x0, 3);
 1986     } else if (od->octo_encklen == 24) {
 1987         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1988         CVMX_MT_AES_KEY(0x0, 3);
 1989     } else if (od->octo_encklen == 32) {
 1990         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 1991         CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
 1992     } else {
 1993         dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 1994         return -EINVAL;
 1995     }
 1996     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 1997 
 1998     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
 1999     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 2000 
 2001     /* Load MD5 IV */
 2002     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
 2003     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 2004     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 2005 
 2006     while (crypt_off > 0 && auth_off > 0) {
 2007         IOV_CONSUME(iov, data32, data_i, data_l);
 2008         crypt_off -= 4;
 2009         auth_off -= 4;
 2010     }
 2011 
 2012     while (crypt_len > 0 || auth_len > 0) {
 2013         uint32_t *pdata32[3];
 2014 
 2015         pdata32[0] = data32;
 2016         mydata[0].data32[0] = *data32;
 2017         IOV_CONSUME(iov, data32, data_i, data_l);
 2018         pdata32[1] = data32;
 2019         mydata[0].data32[1] = *data32;
 2020         IOV_CONSUME(iov, data32, data_i, data_l);
 2021         pdata32[2] = data32;
 2022         mydata[1].data32[0] = *data32;
 2023         IOV_CONSUME(iov, data32, data_i, data_l);
 2024         mydata[1].data32[1] = *data32;
 2025 
 2026         if (auth_off <= 0) {
 2027             if (auth_len > 0) {
 2028                 CVM_LOAD_SHA_UNIT(*pdata, next);
 2029                 CVM_LOAD_SHA_UNIT(*data, next);
 2030                 auth_len -= 16;
 2031             }
 2032         } else
 2033             auth_off -= 16;
 2034 
 2035         if (crypt_off <= 0) {
 2036             if (crypt_len > 0) {
 2037                 CVMX_MT_AES_DEC_CBC0(*pdata);
 2038                 CVMX_MT_AES_DEC_CBC1(*data);
 2039                 CVMX_MF_AES_RESULT(*pdata, 0);
 2040                 CVMX_MF_AES_RESULT(*data, 1);
 2041                 crypt_len -= 16;
 2042             }
 2043         } else
 2044             crypt_off -= 16;
 2045 
 2046         *pdata32[0] = mydata[0].data32[0];
 2047         *pdata32[1] = mydata[0].data32[1];
 2048         *pdata32[2] = mydata[1].data32[0];
 2049         *data32     = mydata[1].data32[1];
 2050 
 2051         IOV_CONSUME(iov, data32, data_i, data_l);
 2052     }
 2053 
 2054     /* finish the hash */
 2055     CVMX_PREFETCH0(od->octo_hmouter);
 2056 #if 0
 2057     if (__predict_false(inplen)) {
 2058         uint64_t tmp = 0;
 2059         uint8_t *p = (uint8_t *) & tmp;
 2060         p[inplen] = 0x80;
 2061         do {
 2062             inplen--;
 2063             p[inplen] = ((uint8_t *) data)[inplen];
 2064         } while (inplen);
 2065         CVM_LOAD_SHA_UNIT(tmp, next);
 2066     } else {
 2067         CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 2068     }
 2069 #else
 2070     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 2071 #endif
 2072 
 2073     /* Finish Inner hash */
 2074     while (next != 7) {
 2075         CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
 2076     }
 2077         CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 2078 
 2079     /* Get the inner hash of HMAC */
 2080     CVMX_MF_HSH_IV(tmp1, 0);
 2081     CVMX_MF_HSH_IV(tmp2, 1);
 2082     tmp3 = 0;
 2083     CVMX_MF_HSH_IV(tmp3, 2);
 2084 
 2085     /* Initialize hash unit */
 2086     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
 2087     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 2088     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 2089 
 2090     CVMX_MT_HSH_DAT(tmp1, 0);
 2091     CVMX_MT_HSH_DAT(tmp2, 1);
 2092     tmp3 |= 0x0000000080000000;
 2093     CVMX_MT_HSH_DAT(tmp3, 2);
 2094     CVMX_MT_HSH_DATZ(3);
 2095     CVMX_MT_HSH_DATZ(4);
 2096     CVMX_MT_HSH_DATZ(5);
 2097     CVMX_MT_HSH_DATZ(6);
 2098     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 2099 
 2100     /* finish the hash */
 2101     CVMX_PREFETCH0(od->octo_hmouter);
 2102 #if 0
 2103     if (__predict_false(inplen)) {
 2104         uint64_t tmp = 0;
 2105         uint8_t *p = (uint8_t *) & tmp;
 2106         p[inplen] = 0x80;
 2107         do {
 2108             inplen--;
 2109             p[inplen] = ((uint8_t *) data)[inplen];
 2110         } while (inplen);
 2111         CVM_LOAD_MD5_UNIT(tmp, next);
 2112     } else {
 2113         CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 2114     }
 2115 #else
 2116     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 2117 #endif
 2118 
 2119     /* save the HMAC */
 2120     IOV_INIT(iov, data32, data_i, data_l);
 2121     while (icv_off > 0) {
 2122         IOV_CONSUME(iov, data32, data_i, data_l);
 2123         icv_off -= 4;
 2124     }
 2125     CVMX_MF_HSH_IV(tmp1, 0);
 2126     *data32 = (uint32_t) (tmp1 >> 32);
 2127     IOV_CONSUME(iov, data32, data_i, data_l);
 2128     *data32 = (uint32_t) tmp1;
 2129     IOV_CONSUME(iov, data32, data_i, data_l);
 2130     CVMX_MF_HSH_IV(tmp1, 1);
 2131     *data32 = (uint32_t) (tmp1 >> 32);
 2132 
 2133     return 0;
 2134 }
 2135 
 2136 /****************************************************************************/

Cache object: c783af0d2da0c540746a9af9eb961dc9


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.