aesni.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
    3  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
    4  * Copyright (c) 2014 The FreeBSD Foundation
    5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
    6  * All rights reserved.
    7  *
    8  * Portions of this software were developed by John-Mark Gurney
    9  * under sponsorship of the FreeBSD Foundation and
   10  * Rubicon Communications, LLC (Netgate).
   11  *
   12  * Redistribution and use in source and binary forms, with or without
   13  * modification, are permitted provided that the following conditions
   14  * are met:
   15  * 1. Redistributions of source code must retain the above copyright
   16  *    notice, this list of conditions and the following disclaimer.
   17  * 2. Redistributions in binary form must reproduce the above copyright
   18  *    notice, this list of conditions and the following disclaimer in the
   19  *    documentation and/or other materials provided with the distribution.
   20  *
   21  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
   22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
   25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   31  * SUCH DAMAGE.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD$");
   36 
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/kernel.h>
   40 #include <sys/kobj.h>
   41 #include <sys/libkern.h>
   42 #include <sys/lock.h>
   43 #include <sys/module.h>
   44 #include <sys/malloc.h>
   45 #include <sys/bus.h>
   46 #include <sys/uio.h>
   47 #include <sys/mbuf.h>
   48 #include <sys/smp.h>
   49 
   50 #include <crypto/aesni/aesni.h>
   51 #include <crypto/aesni/sha_sse.h>
   52 #include <crypto/sha1.h>
   53 #include <crypto/sha2/sha224.h>
   54 #include <crypto/sha2/sha256.h>
   55 
   56 #include <opencrypto/cryptodev.h>
   57 #include <opencrypto/gmac.h>
   58 #include <cryptodev_if.h>
   59 
   60 #include <machine/md_var.h>
   61 #include <machine/specialreg.h>
   62 #if defined(__i386__)
   63 #include <machine/npx.h>
   64 #elif defined(__amd64__)
   65 #include <machine/fpu.h>
   66 #endif
   67 
   68 static struct mtx_padalign *ctx_mtx;
   69 static struct fpu_kern_ctx **ctx_fpu;
   70 
   71 struct aesni_softc {
   72         int32_t cid;
   73         bool    has_aes;
   74         bool    has_sha;
   75 };
   76 
   77 #define ACQUIRE_CTX(i, ctx)                                     \
   78         do {                                                    \
   79                 (i) = PCPU_GET(cpuid);                          \
   80                 mtx_lock(&ctx_mtx[(i)]);                        \
   81                 (ctx) = ctx_fpu[(i)];                           \
   82         } while (0)
   83 #define RELEASE_CTX(i, ctx)                                     \
   84         do {                                                    \
   85                 mtx_unlock(&ctx_mtx[(i)]);                      \
   86                 (i) = -1;                                       \
   87                 (ctx) = NULL;                                   \
   88         } while (0)
   89 
   90 static int aesni_newsession(device_t, crypto_session_t cses,
   91     struct cryptoini *cri);
   92 static int aesni_cipher_setup(struct aesni_session *ses,
   93     struct cryptoini *encini, struct cryptoini *authini);
   94 static int aesni_cipher_process(struct aesni_session *ses,
   95     struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
   96 static int aesni_cipher_crypt(struct aesni_session *ses,
   97     struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
   98 static int aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd,
   99     struct cryptop *crp);
  100 
  101 MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
  102 
  103 static void
  104 aesni_identify(driver_t *drv, device_t parent)
  105 {
  106 
  107         /* NB: order 10 is so we get attached after h/w devices */
  108         if (device_find_child(parent, "aesni", -1) == NULL &&
  109             BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
  110                 panic("aesni: could not attach");
  111 }
  112 
  113 static void
  114 detect_cpu_features(bool *has_aes, bool *has_sha)
  115 {
  116 
  117         *has_aes = ((cpu_feature2 & CPUID2_AESNI) != 0 &&
  118             (cpu_feature2 & CPUID2_SSE41) != 0);
  119         *has_sha = ((cpu_stdext_feature & CPUID_STDEXT_SHA) != 0 &&
  120             (cpu_feature2 & CPUID2_SSSE3) != 0);
  121 }
  122 
  123 static int
  124 aesni_probe(device_t dev)
  125 {
  126         bool has_aes, has_sha;
  127 
  128         detect_cpu_features(&has_aes, &has_sha);
  129         if (!has_aes && !has_sha) {
  130                 device_printf(dev, "No AES or SHA support.\n");
  131                 return (EINVAL);
  132         } else if (has_aes && has_sha)
  133                 device_set_desc(dev,
  134                     "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256");
  135         else if (has_aes)
  136                 device_set_desc(dev,
  137                     "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS");
  138         else
  139                 device_set_desc(dev, "SHA1,SHA256");
  140 
  141         return (0);
  142 }
  143 
  144 static void
  145 aesni_cleanctx(void)
  146 {
  147         int i;
  148 
  149         /* XXX - no way to return driverid */
  150         CPU_FOREACH(i) {
  151                 if (ctx_fpu[i] != NULL) {
  152                         mtx_destroy(&ctx_mtx[i]);
  153                         fpu_kern_free_ctx(ctx_fpu[i]);
  154                 }
  155                 ctx_fpu[i] = NULL;
  156         }
  157         free(ctx_mtx, M_AESNI);
  158         ctx_mtx = NULL;
  159         free(ctx_fpu, M_AESNI);
  160         ctx_fpu = NULL;
  161 }
  162 
  163 static int
  164 aesni_attach(device_t dev)
  165 {
  166         struct aesni_softc *sc;
  167         int i;
  168 
  169         sc = device_get_softc(dev);
  170 
  171         sc->cid = crypto_get_driverid(dev, sizeof(struct aesni_session),
  172             CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
  173         if (sc->cid < 0) {
  174                 device_printf(dev, "Could not get crypto driver id.\n");
  175                 return (ENOMEM);
  176         }
  177 
  178         ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI,
  179             M_WAITOK|M_ZERO);
  180         ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI,
  181             M_WAITOK|M_ZERO);
  182 
  183         CPU_FOREACH(i) {
  184                 ctx_fpu[i] = fpu_kern_alloc_ctx(0);
  185                 mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW);
  186         }
  187 
  188         detect_cpu_features(&sc->has_aes, &sc->has_sha);
  189         if (sc->has_aes) {
  190                 crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
  191                 crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
  192                 crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
  193                 crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
  194                 crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
  195                 crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
  196                 crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
  197                 crypto_register(sc->cid, CRYPTO_AES_CCM_16, 0, 0);
  198                 crypto_register(sc->cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0);
  199         }
  200         if (sc->has_sha) {
  201                 crypto_register(sc->cid, CRYPTO_SHA1, 0, 0);
  202                 crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
  203                 crypto_register(sc->cid, CRYPTO_SHA2_224, 0, 0);
  204                 crypto_register(sc->cid, CRYPTO_SHA2_224_HMAC, 0, 0);
  205                 crypto_register(sc->cid, CRYPTO_SHA2_256, 0, 0);
  206                 crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
  207         }
  208         return (0);
  209 }
  210 
  211 static int
  212 aesni_detach(device_t dev)
  213 {
  214         struct aesni_softc *sc;
  215 
  216         sc = device_get_softc(dev);
  217 
  218         crypto_unregister_all(sc->cid);
  219 
  220         aesni_cleanctx();
  221 
  222         return (0);
  223 }
  224 
  225 static int
  226 aesni_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
  227 {
  228         struct aesni_softc *sc;
  229         struct aesni_session *ses;
  230         struct cryptoini *encini, *authini;
  231         bool gcm_hash, gcm;
  232         bool cbc_hash, ccm;
  233         int error;
  234 
  235         KASSERT(cses != NULL, ("EDOOFUS"));
  236         if (cri == NULL) {
  237                 CRYPTDEB("no cri");
  238                 return (EINVAL);
  239         }
  240 
  241         sc = device_get_softc(dev);
  242 
  243         ses = crypto_get_driver_session(cses);
  244 
  245         authini = NULL;
  246         encini = NULL;
  247         gcm = false;
  248         gcm_hash = false;
  249         ccm = cbc_hash = false;
  250 
  251         for (; cri != NULL; cri = cri->cri_next) {
  252                 switch (cri->cri_alg) {
  253                 case CRYPTO_AES_NIST_GCM_16:
  254                 case CRYPTO_AES_CCM_16:
  255                         if (cri->cri_alg == CRYPTO_AES_NIST_GCM_16) {
  256                                 gcm = true;
  257                         } else if (cri->cri_alg == CRYPTO_AES_CCM_16) {
  258                                 ccm = true;
  259                         }
  260                         /* FALLTHROUGH */
  261                 case CRYPTO_AES_CBC:
  262                 case CRYPTO_AES_ICM:
  263                 case CRYPTO_AES_XTS:
  264                         if (!sc->has_aes)
  265                                 goto unhandled;
  266                         if (encini != NULL) {
  267                                 CRYPTDEB("encini already set");
  268                                 return (EINVAL);
  269                         }
  270                         encini = cri;
  271                         break;
  272                 case CRYPTO_AES_CCM_CBC_MAC:
  273                         cbc_hash = true;
  274                         authini = cri;
  275                         break;
  276                 case CRYPTO_AES_128_NIST_GMAC:
  277                 case CRYPTO_AES_192_NIST_GMAC:
  278                 case CRYPTO_AES_256_NIST_GMAC:
  279                         /*
  280                          * nothing to do here, maybe in the future cache some
  281                          * values for GHASH
  282                          */
  283                         if (authini != NULL) {
  284                                 CRYPTDEB("authini already set");
  285                                 return (EINVAL);
  286                         }
  287                         gcm_hash = true;
  288                         authini = cri;
  289                         break;
  290                 case CRYPTO_SHA1:
  291                 case CRYPTO_SHA1_HMAC:
  292                 case CRYPTO_SHA2_224:
  293                 case CRYPTO_SHA2_224_HMAC:
  294                 case CRYPTO_SHA2_256:
  295                 case CRYPTO_SHA2_256_HMAC:
  296                         if (!sc->has_sha)
  297                                 goto unhandled;
  298                         if (authini != NULL) {
  299                                 CRYPTDEB("authini already set");
  300                                 return (EINVAL);
  301                         }
  302                         authini = cri;
  303                         break;
  304                 default:
  305 unhandled:
  306                         CRYPTDEB("unhandled algorithm");
  307                         return (EINVAL);
  308                 }
  309         }
  310         if (encini == NULL && authini == NULL) {
  311                 CRYPTDEB("no cipher");
  312                 return (EINVAL);
  313         }
  314         /*
  315          * GMAC algorithms are only supported with simultaneous GCM.  Likewise
  316          * GCM is not supported without GMAC.
  317          */
  318         if (gcm_hash != gcm) {
  319                 CRYPTDEB("gcm_hash != gcm");
  320                 return (EINVAL);
  321         }
  322 
  323         if (cbc_hash != ccm) {
  324                 CRYPTDEB("cbc_hash != ccm");
  325                 return (EINVAL);
  326         }
  327 
  328         if (encini != NULL)
  329                 ses->algo = encini->cri_alg;
  330         if (authini != NULL)
  331                 ses->auth_algo = authini->cri_alg;
  332 
  333         error = aesni_cipher_setup(ses, encini, authini);
  334         if (error != 0) {
  335                 CRYPTDEB("setup failed");
  336                 return (error);
  337         }
  338 
  339         return (0);
  340 }
  341 
  342 static int
  343 aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
  344 {
  345         struct aesni_session *ses;
  346         struct cryptodesc *crd, *enccrd, *authcrd;
  347         int error, needauth;
  348 
  349         ses = NULL;
  350         error = 0;
  351         enccrd = NULL;
  352         authcrd = NULL;
  353         needauth = 0;
  354 
  355         /* Sanity check. */
  356         if (crp == NULL)
  357                 return (EINVAL);
  358 
  359         if (crp->crp_callback == NULL || crp->crp_desc == NULL ||
  360             crp->crp_session == NULL) {
  361                 error = EINVAL;
  362                 goto out;
  363         }
  364 
  365         for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
  366                 switch (crd->crd_alg) {
  367                 case CRYPTO_AES_NIST_GCM_16:
  368                 case CRYPTO_AES_CCM_16:
  369                         needauth = 1;
  370                         /* FALLTHROUGH */
  371                 case CRYPTO_AES_CBC:
  372                 case CRYPTO_AES_ICM:
  373                 case CRYPTO_AES_XTS:
  374                         if (enccrd != NULL) {
  375                                 error = EINVAL;
  376                                 goto out;
  377                         }
  378                         enccrd = crd;
  379                         break;
  380 
  381                 case CRYPTO_AES_128_NIST_GMAC:
  382                 case CRYPTO_AES_192_NIST_GMAC:
  383                 case CRYPTO_AES_256_NIST_GMAC:
  384                 case CRYPTO_AES_CCM_CBC_MAC:
  385                 case CRYPTO_SHA1:
  386                 case CRYPTO_SHA1_HMAC:
  387                 case CRYPTO_SHA2_224:
  388                 case CRYPTO_SHA2_224_HMAC:
  389                 case CRYPTO_SHA2_256:
  390                 case CRYPTO_SHA2_256_HMAC:
  391                         if (authcrd != NULL) {
  392                                 error = EINVAL;
  393                                 goto out;
  394                         }
  395                         authcrd = crd;
  396                         break;
  397 
  398                 default:
  399                         error = EINVAL;
  400                         goto out;
  401                 }
  402         }
  403 
  404         if ((enccrd == NULL && authcrd == NULL) ||
  405             (needauth && authcrd == NULL)) {
  406                 error = EINVAL;
  407                 goto out;
  408         }
  409 
  410         /* CBC & XTS can only handle full blocks for now */
  411         if (enccrd != NULL && (enccrd->crd_alg == CRYPTO_AES_CBC ||
  412             enccrd->crd_alg == CRYPTO_AES_XTS) &&
  413             (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
  414                 error = EINVAL;
  415                 goto out;
  416         }
  417 
  418         ses = crypto_get_driver_session(crp->crp_session);
  419         KASSERT(ses != NULL, ("EDOOFUS"));
  420 
  421         error = aesni_cipher_process(ses, enccrd, authcrd, crp);
  422         if (error != 0)
  423                 goto out;
  424 
  425 out:
  426         crp->crp_etype = error;
  427         crypto_done(crp);
  428         return (error);
  429 }
  430 
  431 static uint8_t *
  432 aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
  433     bool *allocated)
  434 {
  435         uint8_t *addr;
  436 
  437         addr = crypto_contiguous_subsegment(crp->crp_flags,
  438             crp->crp_buf, enccrd->crd_skip, enccrd->crd_len);
  439         if (addr != NULL) {
  440                 *allocated = false;
  441                 return (addr);
  442         }
  443         addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
  444         if (addr != NULL) {
  445                 *allocated = true;
  446                 crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
  447                     enccrd->crd_len, addr);
  448         } else
  449                 *allocated = false;
  450         return (addr);
  451 }
  452 
  453 static device_method_t aesni_methods[] = {
  454         DEVMETHOD(device_identify, aesni_identify),
  455         DEVMETHOD(device_probe, aesni_probe),
  456         DEVMETHOD(device_attach, aesni_attach),
  457         DEVMETHOD(device_detach, aesni_detach),
  458 
  459         DEVMETHOD(cryptodev_newsession, aesni_newsession),
  460         DEVMETHOD(cryptodev_process, aesni_process),
  461 
  462         DEVMETHOD_END
  463 };
  464 
  465 static driver_t aesni_driver = {
  466         "aesni",
  467         aesni_methods,
  468         sizeof(struct aesni_softc),
  469 };
  470 static devclass_t aesni_devclass;
  471 
  472 DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
  473 MODULE_VERSION(aesni, 1);
  474 MODULE_DEPEND(aesni, crypto, 1, 1, 1);
  475 
  476 static int
  477 aesni_authprepare(struct aesni_session *ses, int klen, const void *cri_key)
  478 {
  479         int keylen;
  480 
  481         if (klen % 8 != 0)
  482                 return (EINVAL);
  483         keylen = klen / 8;
  484         if (keylen > sizeof(ses->hmac_key))
  485                 return (EINVAL);
  486         if (ses->auth_algo == CRYPTO_SHA1 && keylen > 0)
  487                 return (EINVAL);
  488         memcpy(ses->hmac_key, cri_key, keylen);
  489         return (0);
  490 }
  491 
  492 static int
  493 aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini,
  494     struct cryptoini *authini)
  495 {
  496         struct fpu_kern_ctx *ctx;
  497         int kt, ctxidx, error;
  498 
  499         switch (ses->auth_algo) {
  500         case CRYPTO_SHA1:
  501         case CRYPTO_SHA1_HMAC:
  502         case CRYPTO_SHA2_224:
  503         case CRYPTO_SHA2_224_HMAC:
  504         case CRYPTO_SHA2_256:
  505         case CRYPTO_SHA2_256_HMAC:
  506                 error = aesni_authprepare(ses, authini->cri_klen,
  507                     authini->cri_key);
  508                 if (error != 0)
  509                         return (error);
  510                 ses->mlen = authini->cri_mlen;
  511         }
  512 
  513         kt = is_fpu_kern_thread(0) || (encini == NULL);
  514         if (!kt) {
  515                 ACQUIRE_CTX(ctxidx, ctx);
  516                 fpu_kern_enter(curthread, ctx,
  517                     FPU_KERN_NORMAL | FPU_KERN_KTHR);
  518         }
  519 
  520         error = 0;
  521         if (encini != NULL)
  522                 error = aesni_cipher_setup_common(ses, encini->cri_key,
  523                     encini->cri_klen);
  524 
  525         if (!kt) {
  526                 fpu_kern_leave(curthread, ctx);
  527                 RELEASE_CTX(ctxidx, ctx);
  528         }
  529         return (error);
  530 }
  531 
  532 static int
  533 intel_sha1_update(void *vctx, const void *vdata, u_int datalen)
  534 {
  535         struct sha1_ctxt *ctx = vctx;
  536         const char *data = vdata;
  537         size_t gaplen;
  538         size_t gapstart;
  539         size_t off;
  540         size_t copysiz;
  541         u_int blocks;
  542 
  543         off = 0;
  544         /* Do any aligned blocks without redundant copying. */
  545         if (datalen >= 64 && ctx->count % 64 == 0) {
  546                 blocks = datalen / 64;
  547                 ctx->c.b64[0] += blocks * 64 * 8;
  548                 intel_sha1_step(ctx->h.b32, data + off, blocks);
  549                 off += blocks * 64;
  550         }
  551 
  552         while (off < datalen) {
  553                 gapstart = ctx->count % 64;
  554                 gaplen = 64 - gapstart;
  555 
  556                 copysiz = (gaplen < datalen - off) ? gaplen : datalen - off;
  557                 bcopy(&data[off], &ctx->m.b8[gapstart], copysiz);
  558                 ctx->count += copysiz;
  559                 ctx->count %= 64;
  560                 ctx->c.b64[0] += copysiz * 8;
  561                 if (ctx->count % 64 == 0)
  562                         intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1);
  563                 off += copysiz;
  564         }
  565         return (0);
  566 }
  567 
  568 static void
  569 SHA1_Init_fn(void *ctx)
  570 {
  571         sha1_init(ctx);
  572 }
  573 
  574 static void
  575 SHA1_Finalize_fn(void *digest, void *ctx)
  576 {
  577         sha1_result(ctx, digest);
  578 }
  579 
  580 static int
  581 intel_sha256_update(void *vctx, const void *vdata, u_int len)
  582 {
  583         SHA256_CTX *ctx = vctx;
  584         uint64_t bitlen;
  585         uint32_t r;
  586         u_int blocks;
  587         const unsigned char *src = vdata;
  588 
  589         /* Number of bytes left in the buffer from previous updates */
  590         r = (ctx->count >> 3) & 0x3f;
  591 
  592         /* Convert the length into a number of bits */
  593         bitlen = len << 3;
  594 
  595         /* Update number of bits */
  596         ctx->count += bitlen;
  597 
  598         /* Handle the case where we don't need to perform any transforms */
  599         if (len < 64 - r) {
  600                 memcpy(&ctx->buf[r], src, len);
  601                 return (0);
  602         }
  603 
  604         /* Finish the current block */
  605         memcpy(&ctx->buf[r], src, 64 - r);
  606         intel_sha256_step(ctx->state, ctx->buf, 1);
  607         src += 64 - r;
  608         len -= 64 - r;
  609 
  610         /* Perform complete blocks */
  611         if (len >= 64) {
  612                 blocks = len / 64;
  613                 intel_sha256_step(ctx->state, src, blocks);
  614                 src += blocks * 64;
  615                 len -= blocks * 64;
  616         }
  617 
  618         /* Copy left over data into buffer */
  619         memcpy(ctx->buf, src, len);
  620         return (0);
  621 }
  622 
  623 static void
  624 SHA224_Init_fn(void *ctx)
  625 {
  626         SHA224_Init(ctx);
  627 }
  628 
  629 static void
  630 SHA224_Finalize_fn(void *digest, void *ctx)
  631 {
  632         SHA224_Final(digest, ctx);
  633 }
  634 
  635 static void
  636 SHA256_Init_fn(void *ctx)
  637 {
  638         SHA256_Init(ctx);
  639 }
  640 
  641 static void
  642 SHA256_Finalize_fn(void *digest, void *ctx)
  643 {
  644         SHA256_Final(digest, ctx);
  645 }
  646 
  647 /*
  648  * Compute the HASH( (key ^ xorbyte) || buf )
  649  */
  650 static void
  651 hmac_internal(void *ctx, uint32_t *res,
  652         int (*update)(void *, const void *, u_int),
  653         void (*finalize)(void *, void *), uint8_t *key, uint8_t xorbyte,
  654         const void *buf, size_t off, size_t buflen, int crpflags)
  655 {
  656         size_t i;
  657 
  658         for (i = 0; i < AESNI_SHA_BLOCK_LEN; i++)
  659                 key[i] ^= xorbyte;
  660         update(ctx, key, AESNI_SHA_BLOCK_LEN);
  661         for (i = 0; i < AESNI_SHA_BLOCK_LEN; i++)
  662                 key[i] ^= xorbyte;
  663 
  664         crypto_apply(crpflags, __DECONST(void *, buf), off, buflen,
  665             __DECONST(int (*)(void *, void *, u_int), update), ctx);
  666         finalize(res, ctx);
  667 }
  668 
  669 static int
  670 aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
  671     struct cryptodesc *authcrd, struct cryptop *crp)
  672 {
  673         struct fpu_kern_ctx *ctx;
  674         int error, ctxidx;
  675         bool kt;
  676 
  677         if (enccrd != NULL) {
  678                 if ((enccrd->crd_alg == CRYPTO_AES_ICM ||
  679                     enccrd->crd_alg == CRYPTO_AES_CCM_16 ||
  680                     enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) &&
  681                     (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
  682                         return (EINVAL);
  683         }
  684 
  685         ctx = NULL;
  686         ctxidx = 0;
  687         error = 0;
  688         kt = is_fpu_kern_thread(0);
  689         if (!kt) {
  690                 ACQUIRE_CTX(ctxidx, ctx);
  691                 fpu_kern_enter(curthread, ctx,
  692                     FPU_KERN_NORMAL | FPU_KERN_KTHR);
  693         }
  694 
  695         /* Do work */
  696         if (enccrd != NULL && authcrd != NULL) {
  697                 /* Perform the first operation */
  698                 if (crp->crp_desc == enccrd)
  699                         error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
  700                 else
  701                         error = aesni_cipher_mac(ses, authcrd, crp);
  702                 if (error != 0)
  703                         goto out;
  704                 /* Perform the second operation */
  705                 if (crp->crp_desc == enccrd)
  706                         error = aesni_cipher_mac(ses, authcrd, crp);
  707                 else
  708                         error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
  709         } else if (enccrd != NULL)
  710                 error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
  711         else
  712                 error = aesni_cipher_mac(ses, authcrd, crp);
  713 
  714         if (error != 0)
  715                 goto out;
  716 
  717 out:
  718         if (!kt) {
  719                 fpu_kern_leave(curthread, ctx);
  720                 RELEASE_CTX(ctxidx, ctx);
  721         }
  722         return (error);
  723 }
  724 
  725 static int
  726 aesni_cipher_crypt(struct aesni_session *ses, struct cryptodesc *enccrd,
  727         struct cryptodesc *authcrd, struct cryptop *crp)
  728 {
  729         uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN], *buf, *authbuf;
  730         int error, ivlen;
  731         bool encflag, allocated, authallocated;
  732 
  733         KASSERT((ses->algo != CRYPTO_AES_NIST_GCM_16 &&
  734                 ses->algo != CRYPTO_AES_CCM_16) || authcrd != NULL,
  735             ("AES_NIST_GCM_16/AES_CCM_16  must include MAC descriptor"));
  736 
  737         ivlen = 0;
  738         authbuf = NULL;
  739 
  740         buf = aesni_cipher_alloc(enccrd, crp, &allocated);
  741         if (buf == NULL)
  742                 return (ENOMEM);
  743 
  744         authallocated = false;
  745         if (ses->algo == CRYPTO_AES_NIST_GCM_16 ||
  746             ses->algo == CRYPTO_AES_CCM_16) {
  747                 authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
  748                 if (authbuf == NULL) {
  749                         error = ENOMEM;
  750                         goto out;
  751                 }
  752         }
  753 
  754         error = 0;
  755         encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
  756         if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
  757                 error = aesni_cipher_setup_common(ses, enccrd->crd_key,
  758                     enccrd->crd_klen);
  759                 if (error != 0)
  760                         goto out;
  761         }
  762 
  763         switch (enccrd->crd_alg) {
  764         case CRYPTO_AES_CBC:
  765         case CRYPTO_AES_ICM:
  766                 ivlen = AES_BLOCK_LEN;
  767                 break;
  768         case CRYPTO_AES_XTS:
  769                 ivlen = 8;
  770                 break;
  771         case CRYPTO_AES_NIST_GCM_16:
  772         case CRYPTO_AES_CCM_16:
  773                 ivlen = 12;     /* should support arbitarily larger */
  774                 break;
  775         }
  776 
  777         /* Setup iv */
  778         if (encflag) {
  779                 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
  780                         bcopy(enccrd->crd_iv, iv, ivlen);
  781                 else
  782                         arc4rand(iv, ivlen, 0);
  783                 
  784                 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
  785                         crypto_copyback(crp->crp_flags, crp->crp_buf,
  786                             enccrd->crd_inject, ivlen, iv);
  787         } else {
  788                 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
  789                         bcopy(enccrd->crd_iv, iv, ivlen);
  790                 else
  791                         crypto_copydata(crp->crp_flags, crp->crp_buf,
  792                             enccrd->crd_inject, ivlen, iv);
  793         }
  794 
  795         switch (ses->algo) {
  796         case CRYPTO_AES_CBC:
  797                 if (encflag)
  798                         aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
  799                             enccrd->crd_len, buf, buf, iv);
  800                 else
  801                         aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
  802                             enccrd->crd_len, buf, iv);
  803                 break;
  804         case CRYPTO_AES_ICM:
  805                 /* encryption & decryption are the same */
  806                 aesni_encrypt_icm(ses->rounds, ses->enc_schedule,
  807                     enccrd->crd_len, buf, buf, iv);
  808                 break;
  809         case CRYPTO_AES_XTS:
  810                 if (encflag)
  811                         aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
  812                             ses->xts_schedule, enccrd->crd_len, buf, buf,
  813                             iv);
  814                 else
  815                         aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
  816                             ses->xts_schedule, enccrd->crd_len, buf, buf,
  817                             iv);
  818                 break;
  819         case CRYPTO_AES_NIST_GCM_16:
  820                 if (!encflag)
  821                         crypto_copydata(crp->crp_flags, crp->crp_buf,
  822                             authcrd->crd_inject, sizeof(tag), tag);
  823                 else
  824                         bzero(tag, sizeof tag);
  825 
  826                 if (encflag) {
  827                         AES_GCM_encrypt(buf, buf, authbuf, iv, tag,
  828                             enccrd->crd_len, authcrd->crd_len, ivlen,
  829                             ses->enc_schedule, ses->rounds);
  830 
  831                         if (authcrd != NULL)
  832                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
  833                                     authcrd->crd_inject, sizeof(tag), tag);
  834                 } else {
  835                         if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag,
  836                             enccrd->crd_len, authcrd->crd_len, ivlen,
  837                             ses->enc_schedule, ses->rounds))
  838                                 error = EBADMSG;
  839                 }
  840                 break;
  841         case CRYPTO_AES_CCM_16:
  842                 if (!encflag)
  843                         crypto_copydata(crp->crp_flags, crp->crp_buf,
  844                             authcrd->crd_inject, sizeof(tag), tag);
  845                 else
  846                         bzero(tag, sizeof tag);
  847                 if (encflag) {
  848                         AES_CCM_encrypt(buf, buf, authbuf, iv, tag,
  849                             enccrd->crd_len, authcrd->crd_len, ivlen,
  850                             ses->enc_schedule, ses->rounds);
  851                         if (authcrd != NULL)
  852                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
  853                                     authcrd->crd_inject, sizeof(tag), tag);
  854                 } else {
  855                         if (!AES_CCM_decrypt(buf, buf, authbuf, iv, tag,
  856                             enccrd->crd_len, authcrd->crd_len, ivlen,
  857                             ses->enc_schedule, ses->rounds))
  858                                 error = EBADMSG;
  859                 }
  860                 break;
  861         }
  862         if (allocated && error == 0)
  863                 crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
  864                     enccrd->crd_len, buf);
  865 
  866 out:
  867         if (allocated) {
  868                 explicit_bzero(buf, enccrd->crd_len);
  869                 free(buf, M_AESNI);
  870         }
  871         if (authallocated) {
  872                 explicit_bzero(authbuf, authcrd->crd_len);
  873                 free(authbuf, M_AESNI);
  874         }
  875         return (error);
  876 }
  877 
  878 static int
  879 aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd,
  880     struct cryptop *crp)
  881 {
  882         union {
  883                 struct SHA256Context sha2 __aligned(16);
  884                 struct sha1_ctxt sha1 __aligned(16);
  885         } sctx;
  886         uint8_t hmac_key[AESNI_SHA_BLOCK_LEN] __aligned(16);
  887         uint32_t res[SHA2_256_HASH_LEN / sizeof(uint32_t)];
  888         int hashlen, error;
  889         void *ctx;
  890         void (*InitFn)(void *);
  891         int (*UpdateFn)(void *, const void *, unsigned);
  892         void (*FinalizeFn)(void *, void *);
  893 
  894         bool hmac;
  895 
  896         if ((crd->crd_flags & ~CRD_F_KEY_EXPLICIT) != 0) {
  897                 CRYPTDEB("%s: Unsupported MAC flags: 0x%x", __func__,
  898                     (crd->crd_flags & ~CRD_F_KEY_EXPLICIT));
  899                 return (EINVAL);
  900         }
  901         if ((crd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
  902                 error = aesni_authprepare(ses, crd->crd_klen, crd->crd_key);
  903                 if (error != 0)
  904                         return (error);
  905         }
  906 
  907         hmac = false;
  908         switch (ses->auth_algo) {
  909         case CRYPTO_SHA1_HMAC:
  910                 hmac = true;
  911                 /* FALLTHROUGH */
  912         case CRYPTO_SHA1:
  913                 hashlen = SHA1_HASH_LEN;
  914                 InitFn = SHA1_Init_fn;
  915                 UpdateFn = intel_sha1_update;
  916                 FinalizeFn = SHA1_Finalize_fn;
  917                 ctx = &sctx.sha1;
  918                 break;
  919 
  920         case CRYPTO_SHA2_256_HMAC:
  921                 hmac = true;
  922                 /* FALLTHROUGH */
  923         case CRYPTO_SHA2_256:
  924                 hashlen = SHA2_256_HASH_LEN;
  925                 InitFn = SHA256_Init_fn;
  926                 UpdateFn = intel_sha256_update;
  927                 FinalizeFn = SHA256_Finalize_fn;
  928                 ctx = &sctx.sha2;
  929                 break;
  930 
  931         case CRYPTO_SHA2_224_HMAC:
  932                 hmac = true;
  933                 /* FALLTHROUGH */
  934         case CRYPTO_SHA2_224:
  935                 hashlen = SHA2_224_HASH_LEN;
  936                 InitFn = SHA224_Init_fn;
  937                 UpdateFn = intel_sha256_update;
  938                 FinalizeFn = SHA224_Finalize_fn;
  939                 ctx = &sctx.sha2;
  940                 break;
  941         default:
  942                 /*
  943                  * AES-GMAC authentication is verified while processing the
  944                  * enccrd
  945                  */
  946                 return (0);
  947         }
  948 
  949         if (hmac) {
  950                 memcpy(hmac_key, ses->hmac_key, AESNI_SHA_BLOCK_LEN);
  951 
  952                 /* Inner hash: (K ^ IPAD) || data */
  953                 InitFn(ctx);
  954                 hmac_internal(ctx, res, UpdateFn, FinalizeFn, hmac_key, 0x36,
  955                     crp->crp_buf, crd->crd_skip, crd->crd_len, crp->crp_flags);
  956                 /* Outer hash: (K ^ OPAD) || inner hash */
  957                 InitFn(ctx);
  958                 hmac_internal(ctx, res, UpdateFn, FinalizeFn, hmac_key, 0x5C,
  959                     res, 0, hashlen, 0);
  960         } else {
  961                 InitFn(ctx);
  962                 crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
  963                     crd->crd_len, __DECONST(int (*)(void *, void *, u_int),
  964                     UpdateFn), ctx);
  965                 FinalizeFn(res, ctx);
  966         }
  967 
  968         if (ses->mlen != 0 && ses->mlen < hashlen)
  969                 hashlen = ses->mlen;
  970 
  971         crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
  972             (void *)res);
  973         return (0);
  974 }
Cache object: cb1aefeaac8392ab1ae5392b44b0ffa5
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/crypto/aesni/aesni.c

FreeBSD/Linux Kernel Cross Reference
sys/crypto/aesni/aesni.c
