aesni.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
    3  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
    4  * Copyright (c) 2014-2021 The FreeBSD Foundation
    5  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
    6  * All rights reserved.
    7  *
    8  * Portions of this software were developed by John-Mark Gurney
    9  * under sponsorship of the FreeBSD Foundation and
   10  * Rubicon Communications, LLC (Netgate).
   11  *
   12  * Portions of this software were developed by Ararat River
   13  * Consulting, LLC under sponsorship of the FreeBSD Foundation.
   14  *
   15  * Redistribution and use in source and binary forms, with or without
   16  * modification, are permitted provided that the following conditions
   17  * are met:
   18  * 1. Redistributions of source code must retain the above copyright
   19  *    notice, this list of conditions and the following disclaimer.
   20  * 2. Redistributions in binary form must reproduce the above copyright
   21  *    notice, this list of conditions and the following disclaimer in the
   22  *    documentation and/or other materials provided with the distribution.
   23  *
   24  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
   25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
   28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   34  * SUCH DAMAGE.
   35  */
   36 
   37 #include <sys/cdefs.h>
   38 __FBSDID("$FreeBSD$");
   39 
   40 #include <sys/param.h>
   41 #include <sys/bus.h>
   42 #include <sys/kernel.h>
   43 #include <sys/kobj.h>
   44 #include <sys/libkern.h>
   45 #include <sys/lock.h>
   46 #include <sys/malloc.h>
   47 #include <sys/mbuf.h>
   48 #include <sys/module.h>
   49 #include <sys/mutex.h>
   50 #include <sys/smp.h>
   51 #include <sys/systm.h>
   52 #include <sys/uio.h>
   53 
   54 #include <crypto/aesni/aesni.h>
   55 #include <crypto/aesni/sha_sse.h>
   56 #include <crypto/sha1.h>
   57 #include <crypto/sha2/sha224.h>
   58 #include <crypto/sha2/sha256.h>
   59 
   60 #include <opencrypto/cryptodev.h>
   61 #include <opencrypto/gmac.h>
   62 #include <cryptodev_if.h>
   63 
   64 #include <machine/md_var.h>
   65 #include <machine/specialreg.h>
   66 #include <machine/fpu.h>
   67 
   68 static struct mtx_padalign *ctx_mtx;
   69 static struct fpu_kern_ctx **ctx_fpu;
   70 
   71 struct aesni_softc {
   72         int32_t cid;
   73         bool    has_aes;
   74         bool    has_sha;
   75 };
   76 
   77 #define ACQUIRE_CTX(i, ctx)                                     \
   78         do {                                                    \
   79                 (i) = PCPU_GET(cpuid);                          \
   80                 mtx_lock(&ctx_mtx[(i)]);                        \
   81                 (ctx) = ctx_fpu[(i)];                           \
   82         } while (0)
   83 #define RELEASE_CTX(i, ctx)                                     \
   84         do {                                                    \
   85                 mtx_unlock(&ctx_mtx[(i)]);                      \
   86                 (i) = -1;                                       \
   87                 (ctx) = NULL;                                   \
   88         } while (0)
   89 
   90 static int aesni_cipher_setup(struct aesni_session *ses,
   91     const struct crypto_session_params *csp);
   92 static int aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp);
   93 static int aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
   94     const struct crypto_session_params *csp);
   95 static int aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
   96     const struct crypto_session_params *csp);
   97 
   98 MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
   99 
  100 static void
  101 aesni_identify(driver_t *drv, device_t parent)
  102 {
  103 
  104         /* NB: order 10 is so we get attached after h/w devices */
  105         if (device_find_child(parent, "aesni", -1) == NULL &&
  106             BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
  107                 panic("aesni: could not attach");
  108 }
  109 
  110 static void
  111 detect_cpu_features(bool *has_aes, bool *has_sha)
  112 {
  113 
  114         *has_aes = ((cpu_feature2 & CPUID2_AESNI) != 0 &&
  115             (cpu_feature2 & CPUID2_SSE41) != 0);
  116         *has_sha = ((cpu_stdext_feature & CPUID_STDEXT_SHA) != 0 &&
  117             (cpu_feature2 & CPUID2_SSSE3) != 0);
  118 }
  119 
  120 static int
  121 aesni_probe(device_t dev)
  122 {
  123         bool has_aes, has_sha;
  124 
  125         detect_cpu_features(&has_aes, &has_sha);
  126         if (!has_aes && !has_sha) {
  127                 device_printf(dev, "No AES or SHA support.\n");
  128                 return (EINVAL);
  129         } else if (has_aes && has_sha)
  130                 device_set_desc(dev,
  131                     "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256");
  132         else if (has_aes)
  133                 device_set_desc(dev,
  134                     "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS");
  135         else
  136                 device_set_desc(dev, "SHA1,SHA256");
  137 
  138         return (0);
  139 }
  140 
  141 static void
  142 aesni_cleanctx(void)
  143 {
  144         int i;
  145 
  146         /* XXX - no way to return driverid */
  147         CPU_FOREACH(i) {
  148                 if (ctx_fpu[i] != NULL) {
  149                         mtx_destroy(&ctx_mtx[i]);
  150                         fpu_kern_free_ctx(ctx_fpu[i]);
  151                 }
  152                 ctx_fpu[i] = NULL;
  153         }
  154         free(ctx_mtx, M_AESNI);
  155         ctx_mtx = NULL;
  156         free(ctx_fpu, M_AESNI);
  157         ctx_fpu = NULL;
  158 }
  159 
  160 static int
  161 aesni_attach(device_t dev)
  162 {
  163         struct aesni_softc *sc;
  164         int i;
  165 
  166         sc = device_get_softc(dev);
  167 
  168         sc->cid = crypto_get_driverid(dev, sizeof(struct aesni_session),
  169             CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
  170             CRYPTOCAP_F_ACCEL_SOFTWARE);
  171         if (sc->cid < 0) {
  172                 device_printf(dev, "Could not get crypto driver id.\n");
  173                 return (ENOMEM);
  174         }
  175 
  176         ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI,
  177             M_WAITOK|M_ZERO);
  178         ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI,
  179             M_WAITOK|M_ZERO);
  180 
  181         CPU_FOREACH(i) {
  182 #ifdef __amd64__
  183                 ctx_fpu[i] = fpu_kern_alloc_ctx_domain(
  184                     pcpu_find(i)->pc_domain, FPU_KERN_NORMAL);
  185 #else
  186                 ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
  187 #endif
  188                 mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW);
  189         }
  190 
  191         detect_cpu_features(&sc->has_aes, &sc->has_sha);
  192         return (0);
  193 }
  194 
  195 static int
  196 aesni_detach(device_t dev)
  197 {
  198         struct aesni_softc *sc;
  199 
  200         sc = device_get_softc(dev);
  201 
  202         crypto_unregister_all(sc->cid);
  203 
  204         aesni_cleanctx();
  205 
  206         return (0);
  207 }
  208 
  209 static bool
  210 aesni_auth_supported(struct aesni_softc *sc,
  211     const struct crypto_session_params *csp)
  212 {
  213 
  214         if (!sc->has_sha)
  215                 return (false);
  216 
  217         switch (csp->csp_auth_alg) {
  218         case CRYPTO_SHA1:
  219         case CRYPTO_SHA2_224:
  220         case CRYPTO_SHA2_256:
  221         case CRYPTO_SHA1_HMAC:
  222         case CRYPTO_SHA2_224_HMAC:
  223         case CRYPTO_SHA2_256_HMAC:
  224                 break;
  225         default:
  226                 return (false);
  227         }
  228 
  229         return (true);
  230 }
  231 
  232 static bool
  233 aesni_cipher_supported(struct aesni_softc *sc,
  234     const struct crypto_session_params *csp)
  235 {
  236 
  237         if (!sc->has_aes)
  238                 return (false);
  239 
  240         switch (csp->csp_cipher_alg) {
  241         case CRYPTO_AES_CBC:
  242         case CRYPTO_AES_ICM:
  243                 switch (csp->csp_cipher_klen * 8) {
  244                 case 128:
  245                 case 192:
  246                 case 256:
  247                         break;
  248                 default:
  249                         CRYPTDEB("invalid CBC/ICM key length");
  250                         return (false);
  251                 }
  252                 if (csp->csp_ivlen != AES_BLOCK_LEN)
  253                         return (false);
  254                 break;
  255         case CRYPTO_AES_XTS:
  256                 switch (csp->csp_cipher_klen * 8) {
  257                 case 256:
  258                 case 512:
  259                         break;
  260                 default:
  261                         CRYPTDEB("invalid XTS key length");
  262                         return (false);
  263                 }
  264                 if (csp->csp_ivlen != AES_XTS_IV_LEN)
  265                         return (false);
  266                 break;
  267         default:
  268                 return (false);
  269         }
  270 
  271         return (true);
  272 }
  273 
  274 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
  275 
  276 static int
  277 aesni_probesession(device_t dev, const struct crypto_session_params *csp)
  278 {
  279         struct aesni_softc *sc;
  280 
  281         sc = device_get_softc(dev);
  282         if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
  283                 return (EINVAL);
  284         switch (csp->csp_mode) {
  285         case CSP_MODE_DIGEST:
  286                 if (!aesni_auth_supported(sc, csp))
  287                         return (EINVAL);
  288                 break;
  289         case CSP_MODE_CIPHER:
  290                 if (!aesni_cipher_supported(sc, csp))
  291                         return (EINVAL);
  292                 break;
  293         case CSP_MODE_AEAD:
  294                 switch (csp->csp_cipher_alg) {
  295                 case CRYPTO_AES_NIST_GCM_16:
  296                         switch (csp->csp_cipher_klen * 8) {
  297                         case 128:
  298                         case 192:
  299                         case 256:
  300                                 break;
  301                         default:
  302                                 CRYPTDEB("invalid GCM key length");
  303                                 return (EINVAL);
  304                         }
  305                         if (csp->csp_auth_mlen != 0 &&
  306                             csp->csp_auth_mlen != GMAC_DIGEST_LEN)
  307                                 return (EINVAL);
  308                         if (!sc->has_aes)
  309                                 return (EINVAL);
  310                         break;
  311                 case CRYPTO_AES_CCM_16:
  312                         switch (csp->csp_cipher_klen * 8) {
  313                         case 128:
  314                         case 192:
  315                         case 256:
  316                                 break;
  317                         default:
  318                                 CRYPTDEB("invalid CCM key length");
  319                                 return (EINVAL);
  320                         }
  321                         if (!sc->has_aes)
  322                                 return (EINVAL);
  323                         break;
  324                 default:
  325                         return (EINVAL);
  326                 }
  327                 break;
  328         case CSP_MODE_ETA:
  329                 if (!aesni_auth_supported(sc, csp) ||
  330                     !aesni_cipher_supported(sc, csp))
  331                         return (EINVAL);
  332                 break;
  333         default:
  334                 return (EINVAL);
  335         }
  336 
  337         return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
  338 }
  339 
  340 static int
  341 aesni_newsession(device_t dev, crypto_session_t cses,
  342     const struct crypto_session_params *csp)
  343 {
  344         struct aesni_session *ses;
  345         int error;
  346 
  347         ses = crypto_get_driver_session(cses);
  348 
  349         switch (csp->csp_mode) {
  350         case CSP_MODE_DIGEST:
  351         case CSP_MODE_CIPHER:
  352         case CSP_MODE_AEAD:
  353         case CSP_MODE_ETA:
  354                 break;
  355         default:
  356                 return (EINVAL);
  357         }
  358         error = aesni_cipher_setup(ses, csp);
  359         if (error != 0) {
  360                 CRYPTDEB("setup failed");
  361                 return (error);
  362         }
  363 
  364         return (0);
  365 }
  366 
  367 static int
  368 aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
  369 {
  370         struct aesni_session *ses;
  371         int error;
  372 
  373         ses = crypto_get_driver_session(crp->crp_session);
  374 
  375         error = aesni_cipher_process(ses, crp);
  376 
  377         crp->crp_etype = error;
  378         crypto_done(crp);
  379         return (0);
  380 }
  381 
  382 static uint8_t *
  383 aesni_cipher_alloc(struct cryptop *crp, int start, int length, bool *allocated)
  384 {
  385         uint8_t *addr;
  386 
  387         addr = crypto_contiguous_subsegment(crp, start, length);
  388         if (addr != NULL) {
  389                 *allocated = false;
  390                 return (addr);
  391         }
  392         addr = malloc(length, M_AESNI, M_NOWAIT);
  393         if (addr != NULL) {
  394                 *allocated = true;
  395                 crypto_copydata(crp, start, length, addr);
  396         } else
  397                 *allocated = false;
  398         return (addr);
  399 }
  400 
  401 static device_method_t aesni_methods[] = {
  402         DEVMETHOD(device_identify, aesni_identify),
  403         DEVMETHOD(device_probe, aesni_probe),
  404         DEVMETHOD(device_attach, aesni_attach),
  405         DEVMETHOD(device_detach, aesni_detach),
  406 
  407         DEVMETHOD(cryptodev_probesession, aesni_probesession),
  408         DEVMETHOD(cryptodev_newsession, aesni_newsession),
  409         DEVMETHOD(cryptodev_process, aesni_process),
  410 
  411         DEVMETHOD_END
  412 };
  413 
  414 static driver_t aesni_driver = {
  415         "aesni",
  416         aesni_methods,
  417         sizeof(struct aesni_softc),
  418 };
  419 
  420 DRIVER_MODULE(aesni, nexus, aesni_driver, 0, 0);
  421 MODULE_VERSION(aesni, 1);
  422 MODULE_DEPEND(aesni, crypto, 1, 1, 1);
  423 
  424 static int
  425 intel_sha1_update(void *vctx, const void *vdata, u_int datalen)
  426 {
  427         struct sha1_ctxt *ctx = vctx;
  428         const char *data = vdata;
  429         size_t gaplen;
  430         size_t gapstart;
  431         size_t off;
  432         size_t copysiz;
  433         u_int blocks;
  434 
  435         off = 0;
  436         /* Do any aligned blocks without redundant copying. */
  437         if (datalen >= 64 && ctx->count % 64 == 0) {
  438                 blocks = datalen / 64;
  439                 ctx->c.b64[0] += blocks * 64 * 8;
  440                 intel_sha1_step(ctx->h.b32, data + off, blocks);
  441                 off += blocks * 64;
  442         }
  443 
  444         while (off < datalen) {
  445                 gapstart = ctx->count % 64;
  446                 gaplen = 64 - gapstart;
  447 
  448                 copysiz = (gaplen < datalen - off) ? gaplen : datalen - off;
  449                 bcopy(&data[off], &ctx->m.b8[gapstart], copysiz);
  450                 ctx->count += copysiz;
  451                 ctx->count %= 64;
  452                 ctx->c.b64[0] += copysiz * 8;
  453                 if (ctx->count % 64 == 0)
  454                         intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1);
  455                 off += copysiz;
  456         }
  457 
  458         return (0);
  459 }
  460 
  461 static void
  462 SHA1_Init_fn(void *ctx)
  463 {
  464         sha1_init(ctx);
  465 }
  466 
  467 static void
  468 SHA1_Finalize_fn(void *digest, void *ctx)
  469 {
  470         sha1_result(ctx, digest);
  471 }
  472 
  473 static int
  474 intel_sha256_update(void *vctx, const void *vdata, u_int len)
  475 {
  476         SHA256_CTX *ctx = vctx;
  477         uint64_t bitlen;
  478         uint32_t r;
  479         u_int blocks;
  480         const unsigned char *src = vdata;
  481 
  482         /* Number of bytes left in the buffer from previous updates */
  483         r = (ctx->count >> 3) & 0x3f;
  484 
  485         /* Convert the length into a number of bits */
  486         bitlen = len << 3;
  487 
  488         /* Update number of bits */
  489         ctx->count += bitlen;
  490 
  491         /* Handle the case where we don't need to perform any transforms */
  492         if (len < 64 - r) {
  493                 memcpy(&ctx->buf[r], src, len);
  494                 return (0);
  495         }
  496 
  497         /* Finish the current block */
  498         memcpy(&ctx->buf[r], src, 64 - r);
  499         intel_sha256_step(ctx->state, ctx->buf, 1);
  500         src += 64 - r;
  501         len -= 64 - r;
  502 
  503         /* Perform complete blocks */
  504         if (len >= 64) {
  505                 blocks = len / 64;
  506                 intel_sha256_step(ctx->state, src, blocks);
  507                 src += blocks * 64;
  508                 len -= blocks * 64;
  509         }
  510 
  511         /* Copy left over data into buffer */
  512         memcpy(ctx->buf, src, len);
  513 
  514         return (0);
  515 }
  516 
  517 static void
  518 SHA224_Init_fn(void *ctx)
  519 {
  520         SHA224_Init(ctx);
  521 }
  522 
  523 static void
  524 SHA224_Finalize_fn(void *digest, void *ctx)
  525 {
  526         SHA224_Final(digest, ctx);
  527 }
  528 
  529 static void
  530 SHA256_Init_fn(void *ctx)
  531 {
  532         SHA256_Init(ctx);
  533 }
  534 
  535 static void
  536 SHA256_Finalize_fn(void *digest, void *ctx)
  537 {
  538         SHA256_Final(digest, ctx);
  539 }
  540 
  541 static int
  542 aesni_authprepare(struct aesni_session *ses, int klen)
  543 {
  544 
  545         if (klen > SHA1_BLOCK_LEN)
  546                 return (EINVAL);
  547         if ((ses->hmac && klen == 0) || (!ses->hmac && klen != 0))
  548                 return (EINVAL);
  549         return (0);
  550 }
  551 
  552 static int
  553 aesni_cipher_setup(struct aesni_session *ses,
  554     const struct crypto_session_params *csp)
  555 {
  556         struct fpu_kern_ctx *ctx;
  557         uint8_t *schedbase;
  558         int kt, ctxidx, error;
  559 
  560         schedbase = (uint8_t *)roundup2((uintptr_t)ses->schedules,
  561             AES_SCHED_ALIGN);
  562         ses->enc_schedule = schedbase;
  563         ses->dec_schedule = schedbase + AES_SCHED_LEN;
  564         ses->xts_schedule = schedbase + AES_SCHED_LEN * 2;
  565 
  566         switch (csp->csp_auth_alg) {
  567         case CRYPTO_SHA1_HMAC:
  568                 ses->hmac = true;
  569                 /* FALLTHROUGH */
  570         case CRYPTO_SHA1:
  571                 ses->hash_len = SHA1_HASH_LEN;
  572                 ses->hash_init = SHA1_Init_fn;
  573                 ses->hash_update = intel_sha1_update;
  574                 ses->hash_finalize = SHA1_Finalize_fn;
  575                 break;
  576         case CRYPTO_SHA2_224_HMAC:
  577                 ses->hmac = true;
  578                 /* FALLTHROUGH */
  579         case CRYPTO_SHA2_224:
  580                 ses->hash_len = SHA2_224_HASH_LEN;
  581                 ses->hash_init = SHA224_Init_fn;
  582                 ses->hash_update = intel_sha256_update;
  583                 ses->hash_finalize = SHA224_Finalize_fn;
  584                 break;
  585         case CRYPTO_SHA2_256_HMAC:
  586                 ses->hmac = true;
  587                 /* FALLTHROUGH */
  588         case CRYPTO_SHA2_256:
  589                 ses->hash_len = SHA2_256_HASH_LEN;
  590                 ses->hash_init = SHA256_Init_fn;
  591                 ses->hash_update = intel_sha256_update;
  592                 ses->hash_finalize = SHA256_Finalize_fn;
  593                 break;
  594         }
  595 
  596         if (ses->hash_len != 0) {
  597                 if (csp->csp_auth_mlen == 0)
  598                         ses->mlen = ses->hash_len;
  599                 else
  600                         ses->mlen = csp->csp_auth_mlen;
  601 
  602                 error = aesni_authprepare(ses, csp->csp_auth_klen);
  603                 if (error != 0)
  604                         return (error);
  605         } else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
  606                 if (csp->csp_auth_mlen == 0)
  607                         ses->mlen = AES_CBC_MAC_HASH_LEN;
  608                 else
  609                         ses->mlen = csp->csp_auth_mlen;
  610         }
  611 
  612         kt = is_fpu_kern_thread(0) || (csp->csp_cipher_alg == 0);
  613         if (!kt) {
  614                 ACQUIRE_CTX(ctxidx, ctx);
  615                 fpu_kern_enter(curthread, ctx,
  616                     FPU_KERN_NORMAL | FPU_KERN_KTHR);
  617         }
  618 
  619         error = 0;
  620         if (csp->csp_cipher_key != NULL)
  621                 aesni_cipher_setup_common(ses, csp, csp->csp_cipher_key,
  622                     csp->csp_cipher_klen);
  623 
  624         if (!kt) {
  625                 fpu_kern_leave(curthread, ctx);
  626                 RELEASE_CTX(ctxidx, ctx);
  627         }
  628         return (error);
  629 }
  630 
  631 static int
  632 aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp)
  633 {
  634         const struct crypto_session_params *csp;
  635         struct fpu_kern_ctx *ctx;
  636         int error, ctxidx;
  637         bool kt;
  638 
  639         csp = crypto_get_params(crp->crp_session);
  640         switch (csp->csp_cipher_alg) {
  641         case CRYPTO_AES_CCM_16:
  642                 if (crp->crp_payload_length > ccm_max_payload_length(csp))
  643                         return (EMSGSIZE);
  644                 /* FALLTHROUGH */
  645         case CRYPTO_AES_ICM:
  646         case CRYPTO_AES_NIST_GCM_16:
  647                 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
  648                         return (EINVAL);
  649                 break;
  650         case CRYPTO_AES_CBC:
  651         case CRYPTO_AES_XTS:
  652                 /* CBC & XTS can only handle full blocks for now */
  653                 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0)
  654                         return (EINVAL);
  655                 break;
  656         }
  657 
  658         ctx = NULL;
  659         ctxidx = 0;
  660         error = 0;
  661         kt = is_fpu_kern_thread(0);
  662         if (!kt) {
  663                 ACQUIRE_CTX(ctxidx, ctx);
  664                 fpu_kern_enter(curthread, ctx,
  665                     FPU_KERN_NORMAL | FPU_KERN_KTHR);
  666         }
  667 
  668         /* Do work */
  669         if (csp->csp_mode == CSP_MODE_ETA) {
  670                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
  671                         error = aesni_cipher_crypt(ses, crp, csp);
  672                         if (error == 0)
  673                                 error = aesni_cipher_mac(ses, crp, csp);
  674                 } else {
  675                         error = aesni_cipher_mac(ses, crp, csp);
  676                         if (error == 0)
  677                                 error = aesni_cipher_crypt(ses, crp, csp);
  678                 }
  679         } else if (csp->csp_mode == CSP_MODE_DIGEST)
  680                 error = aesni_cipher_mac(ses, crp, csp);
  681         else
  682                 error = aesni_cipher_crypt(ses, crp, csp);
  683 
  684         if (!kt) {
  685                 fpu_kern_leave(curthread, ctx);
  686                 RELEASE_CTX(ctxidx, ctx);
  687         }
  688         return (error);
  689 }
  690 
  691 static int
  692 aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
  693     const struct crypto_session_params *csp)
  694 {
  695         uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
  696         uint8_t *authbuf, *buf, *outbuf;
  697         int error;
  698         bool encflag, allocated, authallocated, outallocated, outcopy;
  699 
  700         if (crp->crp_payload_length == 0) {
  701                 buf = NULL;
  702                 allocated = false;
  703         } else {
  704                 buf = aesni_cipher_alloc(crp, crp->crp_payload_start,
  705                     crp->crp_payload_length, &allocated);
  706                 if (buf == NULL)
  707                         return (ENOMEM);
  708         }
  709 
  710         outallocated = false;
  711         authallocated = false;
  712         authbuf = NULL;
  713         if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 ||
  714             csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
  715                 if (crp->crp_aad_length == 0) {
  716                         authbuf = NULL;
  717                 } else if (crp->crp_aad != NULL) {
  718                         authbuf = crp->crp_aad;
  719                 } else {
  720                         authbuf = aesni_cipher_alloc(crp, crp->crp_aad_start,
  721                             crp->crp_aad_length, &authallocated);
  722                         if (authbuf == NULL) {
  723                                 error = ENOMEM;
  724                                 goto out;
  725                         }
  726                 }
  727         }
  728 
  729         if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && crp->crp_payload_length > 0) {
  730                 outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf,
  731                     crp->crp_payload_output_start, crp->crp_payload_length);
  732                 if (outbuf == NULL) {
  733                         outcopy = true;
  734                         if (allocated)
  735                                 outbuf = buf;
  736                         else {
  737                                 outbuf = malloc(crp->crp_payload_length,
  738                                     M_AESNI, M_NOWAIT);
  739                                 if (outbuf == NULL) {
  740                                         error = ENOMEM;
  741                                         goto out;
  742                                 }
  743                                 outallocated = true;
  744                         }
  745                 } else
  746                         outcopy = false;
  747         } else {
  748                 outbuf = buf;
  749                 outcopy = allocated;
  750         }
  751 
  752         error = 0;
  753         encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
  754         if (crp->crp_cipher_key != NULL)
  755                 aesni_cipher_setup_common(ses, csp, crp->crp_cipher_key,
  756                     csp->csp_cipher_klen);
  757 
  758         crypto_read_iv(crp, iv);
  759 
  760         switch (csp->csp_cipher_alg) {
  761         case CRYPTO_AES_CBC:
  762                 if (encflag)
  763                         aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
  764                             crp->crp_payload_length, buf, outbuf, iv);
  765                 else {
  766                         if (buf != outbuf)
  767                                 memcpy(outbuf, buf, crp->crp_payload_length);
  768                         aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
  769                             crp->crp_payload_length, outbuf, iv);
  770                 }
  771                 break;
  772         case CRYPTO_AES_ICM:
  773                 /* encryption & decryption are the same */
  774                 aesni_encrypt_icm(ses->rounds, ses->enc_schedule,
  775                     crp->crp_payload_length, buf, outbuf, iv);
  776                 break;
  777         case CRYPTO_AES_XTS:
  778                 if (encflag)
  779                         aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
  780                             ses->xts_schedule, crp->crp_payload_length, buf,
  781                             outbuf, iv);
  782                 else
  783                         aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
  784                             ses->xts_schedule, crp->crp_payload_length, buf,
  785                             outbuf, iv);
  786                 break;
  787         case CRYPTO_AES_NIST_GCM_16:
  788                 if (encflag) {
  789                         memset(tag, 0, sizeof(tag));
  790                         AES_GCM_encrypt(buf, outbuf, authbuf, iv, tag,
  791                             crp->crp_payload_length, crp->crp_aad_length,
  792                             csp->csp_ivlen, ses->enc_schedule, ses->rounds);
  793                         crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
  794                             tag);
  795                 } else {
  796                         crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
  797                             tag);
  798                         if (!AES_GCM_decrypt(buf, outbuf, authbuf, iv, tag,
  799                             crp->crp_payload_length, crp->crp_aad_length,
  800                             csp->csp_ivlen, ses->enc_schedule, ses->rounds))
  801                                 error = EBADMSG;
  802                 }
  803                 break;
  804         case CRYPTO_AES_CCM_16:
  805                 if (encflag) {
  806                         memset(tag, 0, sizeof(tag));                    
  807                         AES_CCM_encrypt(buf, outbuf, authbuf, iv, tag,
  808                             crp->crp_payload_length, crp->crp_aad_length,
  809                             csp->csp_ivlen, ses->mlen, ses->enc_schedule,
  810                             ses->rounds);
  811                         crypto_copyback(crp, crp->crp_digest_start, ses->mlen,
  812                             tag);
  813                 } else {
  814                         crypto_copydata(crp, crp->crp_digest_start, ses->mlen,
  815                             tag);
  816                         if (!AES_CCM_decrypt(buf, outbuf, authbuf, iv, tag,
  817                             crp->crp_payload_length, crp->crp_aad_length,
  818                             csp->csp_ivlen, ses->mlen, ses->enc_schedule,
  819                             ses->rounds))
  820                                 error = EBADMSG;
  821                 }
  822                 break;
  823         }
  824         if (outcopy && error == 0)
  825                 crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ?
  826                     crp->crp_payload_output_start : crp->crp_payload_start,
  827                     crp->crp_payload_length, outbuf);
  828 
  829 out:
  830         if (allocated)
  831                 zfree(buf, M_AESNI);
  832         if (authallocated)
  833                 zfree(authbuf, M_AESNI);
  834         if (outallocated)
  835                 zfree(outbuf, M_AESNI);
  836         explicit_bzero(iv, sizeof(iv));
  837         explicit_bzero(tag, sizeof(tag));
  838         return (error);
  839 }
  840 
  841 static int
  842 aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
  843     const struct crypto_session_params *csp)
  844 {
  845         union {
  846                 struct SHA256Context sha2 __aligned(16);
  847                 struct sha1_ctxt sha1 __aligned(16);
  848         } sctx;
  849         uint32_t res[SHA2_256_HASH_LEN / sizeof(uint32_t)];
  850         const uint8_t *key;
  851         int i, keylen;
  852 
  853         if (crp->crp_auth_key != NULL)
  854                 key = crp->crp_auth_key;
  855         else
  856                 key = csp->csp_auth_key;
  857         keylen = csp->csp_auth_klen;
  858 
  859         if (ses->hmac) {
  860                 uint8_t hmac_key[SHA1_BLOCK_LEN] __aligned(16);
  861 
  862                 /* Inner hash: (K ^ IPAD) || data */
  863                 ses->hash_init(&sctx);
  864                 for (i = 0; i < keylen; i++)
  865                         hmac_key[i] = key[i] ^ HMAC_IPAD_VAL;
  866                 for (i = keylen; i < sizeof(hmac_key); i++)
  867                         hmac_key[i] = 0 ^ HMAC_IPAD_VAL;
  868                 ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
  869 
  870                 if (crp->crp_aad != NULL)
  871                         ses->hash_update(&sctx, crp->crp_aad,
  872                             crp->crp_aad_length);
  873                 else
  874                         crypto_apply(crp, crp->crp_aad_start,
  875                             crp->crp_aad_length, ses->hash_update, &sctx);
  876                 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) &&
  877                     CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
  878                         crypto_apply_buf(&crp->crp_obuf,
  879                             crp->crp_payload_output_start,
  880                             crp->crp_payload_length,
  881                             ses->hash_update, &sctx);
  882                 else
  883                         crypto_apply(crp, crp->crp_payload_start,
  884                             crp->crp_payload_length, ses->hash_update, &sctx);
  885 
  886                 if (csp->csp_flags & CSP_F_ESN)
  887                         ses->hash_update(&sctx, crp->crp_esn, 4);
  888 
  889                 ses->hash_finalize(res, &sctx);
  890 
  891                 /* Outer hash: (K ^ OPAD) || inner hash */
  892                 ses->hash_init(&sctx);
  893                 for (i = 0; i < keylen; i++)
  894                         hmac_key[i] = key[i] ^ HMAC_OPAD_VAL;
  895                 for (i = keylen; i < sizeof(hmac_key); i++)
  896                         hmac_key[i] = 0 ^ HMAC_OPAD_VAL;
  897                 ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
  898                 ses->hash_update(&sctx, res, ses->hash_len);
  899                 ses->hash_finalize(res, &sctx);
  900                 explicit_bzero(hmac_key, sizeof(hmac_key));
  901         } else {
  902                 ses->hash_init(&sctx);
  903 
  904                 if (crp->crp_aad != NULL)
  905                         ses->hash_update(&sctx, crp->crp_aad,
  906                             crp->crp_aad_length);
  907                 else
  908                         crypto_apply(crp, crp->crp_aad_start,
  909                             crp->crp_aad_length, ses->hash_update, &sctx);
  910                 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) &&
  911                     CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
  912                         crypto_apply_buf(&crp->crp_obuf,
  913                             crp->crp_payload_output_start,
  914                             crp->crp_payload_length,
  915                             ses->hash_update, &sctx);
  916                 else
  917                         crypto_apply(crp, crp->crp_payload_start,
  918                             crp->crp_payload_length,
  919                             ses->hash_update, &sctx);
  920 
  921                 ses->hash_finalize(res, &sctx);
  922         }
  923 
  924         if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
  925                 uint32_t res2[SHA2_256_HASH_LEN / sizeof(uint32_t)];
  926 
  927                 crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
  928                 if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
  929                         return (EBADMSG);
  930                 explicit_bzero(res2, sizeof(res2));
  931         } else
  932                 crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
  933         explicit_bzero(res, sizeof(res));
  934         return (0);
  935 }
Cache object: 04c0052654f06adc02b8ce33ab9c007a
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/crypto/aesni/aesni.c

FreeBSD/Linux Kernel Cross Reference
sys/crypto/aesni/aesni.c
