cryptosoft.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*      $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */
    2 
    3 /*-
    4  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
    5  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
    6  *
    7  * This code was written by Angelos D. Keromytis in Athens, Greece, in
    8  * February 2000. Network Security Technologies Inc. (NSTI) kindly
    9  * supported the development of this code.
   10  *
   11  * Copyright (c) 2000, 2001 Angelos D. Keromytis
   12  * Copyright (c) 2014-2021 The FreeBSD Foundation
   13  * All rights reserved.
   14  *
   15  * Portions of this software were developed by John-Mark Gurney
   16  * under sponsorship of the FreeBSD Foundation and
   17  * Rubicon Communications, LLC (Netgate).
   18  *
   19  * Portions of this software were developed by Ararat River
   20  * Consulting, LLC under sponsorship of the FreeBSD Foundation.
   21  *
   22  * Permission to use, copy, and modify this software with or without fee
   23  * is hereby granted, provided that this entire notice is included in
   24  * all source code copies of any software which is or includes a copy or
   25  * modification of this software.
   26  *
   27  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
   28  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
   29  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
   30  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
   31  * PURPOSE.
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD$");
   36 
   37 #include <sys/param.h>
   38 #include <sys/systm.h>
   39 #include <sys/malloc.h>
   40 #include <sys/mbuf.h>
   41 #include <sys/module.h>
   42 #include <sys/sysctl.h>
   43 #include <sys/errno.h>
   44 #include <sys/random.h>
   45 #include <sys/kernel.h>
   46 #include <sys/uio.h>
   47 #include <sys/endian.h>
   48 #include <sys/limits.h>
   49 
   50 #include <crypto/sha1.h>
   51 #include <opencrypto/rmd160.h>
   52 
   53 #include <opencrypto/cryptodev.h>
   54 #include <opencrypto/xform.h>
   55 
   56 #include <sys/kobj.h>
   57 #include <sys/bus.h>
   58 #include "cryptodev_if.h"
   59 
   60 struct swcr_auth {
   61         void            *sw_ictx;
   62         void            *sw_octx;
   63         const struct auth_hash *sw_axf;
   64         uint16_t        sw_mlen;
   65         bool            sw_hmac;
   66 };
   67 
   68 struct swcr_encdec {
   69         void            *sw_ctx;
   70         const struct enc_xform *sw_exf;
   71 };
   72 
   73 struct swcr_compdec {
   74         const struct comp_algo *sw_cxf;
   75 };
   76 
   77 struct swcr_session {
   78         int     (*swcr_process)(const struct swcr_session *, struct cryptop *);
   79 
   80         struct swcr_auth swcr_auth;
   81         struct swcr_encdec swcr_encdec;
   82         struct swcr_compdec swcr_compdec;
   83 };
   84 
   85 static  int32_t swcr_id;
   86 
   87 static  void swcr_freesession(device_t dev, crypto_session_t cses);
   88 
   89 /* Used for CRYPTO_NULL_CBC. */
   90 static int
   91 swcr_null(const struct swcr_session *ses, struct cryptop *crp)
   92 {
   93 
   94         return (0);
   95 }
   96 
   97 /*
   98  * Apply a symmetric encryption/decryption algorithm.
   99  */
  100 static int
  101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp)
  102 {
  103         unsigned char blk[EALG_MAX_BLOCK_LEN];
  104         const struct crypto_session_params *csp;
  105         const struct enc_xform *exf;
  106         const struct swcr_encdec *sw;
  107         void *ctx;
  108         size_t inlen, outlen, todo;
  109         int blksz, resid;
  110         struct crypto_buffer_cursor cc_in, cc_out;
  111         const unsigned char *inblk;
  112         unsigned char *outblk;
  113         int error;
  114         bool encrypting;
  115 
  116         error = 0;
  117 
  118         sw = &ses->swcr_encdec;
  119         exf = sw->sw_exf;
  120         csp = crypto_get_params(crp->crp_session);
  121 
  122         if (exf->native_blocksize == 0) {
  123                 /* Check for non-padded data */
  124                 if ((crp->crp_payload_length % exf->blocksize) != 0)
  125                         return (EINVAL);
  126 
  127                 blksz = exf->blocksize;
  128         } else
  129                 blksz = exf->native_blocksize;
  130 
  131         if (exf == &enc_xform_aes_icm &&
  132             (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
  133                 return (EINVAL);
  134 
  135         ctx = __builtin_alloca(exf->ctxsize);
  136         if (crp->crp_cipher_key != NULL) {
  137                 error = exf->setkey(ctx, crp->crp_cipher_key,
  138                     csp->csp_cipher_klen);
  139                 if (error)
  140                         return (error);
  141         } else
  142                 memcpy(ctx, sw->sw_ctx, exf->ctxsize);
  143 
  144         crypto_read_iv(crp, blk);
  145         exf->reinit(ctx, blk, csp->csp_ivlen);
  146 
  147         crypto_cursor_init(&cc_in, &crp->crp_buf);
  148         crypto_cursor_advance(&cc_in, crp->crp_payload_start);
  149         if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
  150                 crypto_cursor_init(&cc_out, &crp->crp_obuf);
  151                 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
  152         } else
  153                 cc_out = cc_in;
  154 
  155         encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
  156 
  157         /*
  158          * Loop through encrypting blocks.  'inlen' is the remaining
  159          * length of the current segment in the input buffer.
  160          * 'outlen' is the remaining length of current segment in the
  161          * output buffer.
  162          */
  163         inlen = outlen = 0;
  164         for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
  165                 if (inlen == 0)
  166                         inblk = crypto_cursor_segment(&cc_in, &inlen);
  167                 if (outlen == 0)
  168                         outblk = crypto_cursor_segment(&cc_out, &outlen);
  169 
  170                 /*
  171                  * If the current block is not contained within the
  172                  * current input/output segment, use 'blk' as a local
  173                  * buffer.
  174                  */
  175                 if (inlen < blksz) {
  176                         crypto_cursor_copydata(&cc_in, blksz, blk);
  177                         inblk = blk;
  178                         inlen = blksz;
  179                 }
  180                 if (outlen < blksz) {
  181                         outblk = blk;
  182                         outlen = blksz;
  183                 }
  184 
  185                 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz);
  186 
  187                 if (encrypting)
  188                         exf->encrypt_multi(ctx, inblk, outblk, todo);
  189                 else
  190                         exf->decrypt_multi(ctx, inblk, outblk, todo);
  191 
  192                 if (inblk == blk) {
  193                         inblk = crypto_cursor_segment(&cc_in, &inlen);
  194                 } else {
  195                         crypto_cursor_advance(&cc_in, todo);
  196                         inlen -= todo;
  197                         inblk += todo;
  198                 }
  199 
  200                 if (outblk == blk) {
  201                         crypto_cursor_copyback(&cc_out, blksz, blk);
  202                         outblk = crypto_cursor_segment(&cc_out, &outlen);
  203                 } else {
  204                         crypto_cursor_advance(&cc_out, todo);
  205                         outlen -= todo;
  206                         outblk += todo;
  207                 }
  208         }
  209 
  210         /* Handle trailing partial block for stream ciphers. */
  211         if (resid > 0) {
  212                 KASSERT(exf->native_blocksize != 0,
  213                     ("%s: partial block of %d bytes for cipher %s",
  214                     __func__, resid, exf->name));
  215                 KASSERT(resid < blksz, ("%s: partial block too big", __func__));
  216 
  217                 inblk = crypto_cursor_segment(&cc_in, &inlen);
  218                 outblk = crypto_cursor_segment(&cc_out, &outlen);
  219                 if (inlen < resid) {
  220                         crypto_cursor_copydata(&cc_in, resid, blk);
  221                         inblk = blk;
  222                 }
  223                 if (outlen < resid)
  224                         outblk = blk;
  225                 if (encrypting)
  226                         exf->encrypt_last(ctx, inblk, outblk,
  227                             resid);
  228                 else
  229                         exf->decrypt_last(ctx, inblk, outblk,
  230                             resid);
  231                 if (outlen < resid)
  232                         crypto_cursor_copyback(&cc_out, resid, blk);
  233         }
  234 
  235         explicit_bzero(ctx, exf->ctxsize);
  236         explicit_bzero(blk, sizeof(blk));
  237         return (0);
  238 }
  239 
  240 /*
  241  * Compute or verify hash.
  242  */
  243 static int
  244 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp)
  245 {
  246         struct {
  247                 union authctx ctx;
  248                 u_char aalg[HASH_MAX_LEN];
  249                 u_char uaalg[HASH_MAX_LEN];
  250         } s;
  251         const struct crypto_session_params *csp;
  252         const struct swcr_auth *sw;
  253         const struct auth_hash *axf;
  254         int err;
  255 
  256         sw = &ses->swcr_auth;
  257 
  258         axf = sw->sw_axf;
  259 
  260         csp = crypto_get_params(crp->crp_session);
  261         if (crp->crp_auth_key != NULL) {
  262                 if (sw->sw_hmac) {
  263                         hmac_init_ipad(axf, crp->crp_auth_key,
  264                             csp->csp_auth_klen, &s.ctx);
  265                 } else {
  266                         axf->Init(&s.ctx);
  267                         axf->Setkey(&s.ctx, crp->crp_auth_key,
  268                             csp->csp_auth_klen);
  269                 }
  270         } else
  271                 memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize);
  272 
  273         if (crp->crp_aad != NULL)
  274                 err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length);
  275         else
  276                 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
  277                     axf->Update, &s.ctx);
  278         if (err)
  279                 goto out;
  280 
  281         if (CRYPTO_HAS_OUTPUT_BUFFER(crp) &&
  282             CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
  283                 err = crypto_apply_buf(&crp->crp_obuf,
  284                     crp->crp_payload_output_start, crp->crp_payload_length,
  285                     axf->Update, &s.ctx);
  286         else
  287                 err = crypto_apply(crp, crp->crp_payload_start,
  288                     crp->crp_payload_length, axf->Update, &s.ctx);
  289         if (err)
  290                 goto out;
  291 
  292         if (csp->csp_flags & CSP_F_ESN)
  293                 axf->Update(&s.ctx, crp->crp_esn, 4);
  294 
  295         axf->Final(s.aalg, &s.ctx);
  296         if (sw->sw_hmac) {
  297                 if (crp->crp_auth_key != NULL)
  298                         hmac_init_opad(axf, crp->crp_auth_key,
  299                             csp->csp_auth_klen, &s.ctx);
  300                 else
  301                         memcpy(&s.ctx, sw->sw_octx, axf->ctxsize);
  302                 axf->Update(&s.ctx, s.aalg, axf->hashsize);
  303                 axf->Final(s.aalg, &s.ctx);
  304         }
  305 
  306         if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
  307                 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg);
  308                 if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0)
  309                         err = EBADMSG;
  310         } else {
  311                 /* Inject the authentication data */
  312                 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg);
  313         }
  314 out:
  315         explicit_bzero(&s, sizeof(s));
  316         return (err);
  317 }
  318 
  319 CTASSERT(INT_MAX <= (1ll<<39) - 256);   /* GCM: plain text < 2^39-256 */
  320 CTASSERT(INT_MAX <= (uint64_t)-1);      /* GCM: associated data <= 2^64-1 */
  321 
  322 static int
  323 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp)
  324 {
  325         struct {
  326                 union authctx ctx;
  327                 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))];
  328                 u_char tag[GMAC_DIGEST_LEN];
  329                 u_char tag2[GMAC_DIGEST_LEN];
  330         } s;
  331         u_char *blk = (u_char *)s.blkbuf;
  332         struct crypto_buffer_cursor cc;
  333         const u_char *inblk;
  334         const struct swcr_auth *swa;
  335         const struct auth_hash *axf;
  336         uint32_t *blkp;
  337         size_t len;
  338         int blksz, error, ivlen, resid;
  339 
  340         swa = &ses->swcr_auth;
  341         axf = swa->sw_axf;
  342         blksz = GMAC_BLOCK_LEN;
  343         KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch",
  344             __func__));
  345 
  346         if (crp->crp_auth_key != NULL) {
  347                 axf->Init(&s.ctx);
  348                 axf->Setkey(&s.ctx, crp->crp_auth_key,
  349                     crypto_get_params(crp->crp_session)->csp_auth_klen);
  350         } else
  351                 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize);
  352 
  353         /* Initialize the IV */
  354         ivlen = AES_GCM_IV_LEN;
  355         crypto_read_iv(crp, blk);
  356 
  357         axf->Reinit(&s.ctx, blk, ivlen);
  358         crypto_cursor_init(&cc, &crp->crp_buf);
  359         crypto_cursor_advance(&cc, crp->crp_payload_start);
  360         for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) {
  361                 inblk = crypto_cursor_segment(&cc, &len);
  362                 if (len >= blksz) {
  363                         len = rounddown(MIN(len, resid), blksz);
  364                         crypto_cursor_advance(&cc, len);
  365                 } else {
  366                         len = blksz;
  367                         crypto_cursor_copydata(&cc, len, blk);
  368                         inblk = blk;
  369                 }
  370                 axf->Update(&s.ctx, inblk, len);
  371         }
  372         if (resid > 0) {
  373                 memset(blk, 0, blksz);
  374                 crypto_cursor_copydata(&cc, resid, blk);
  375                 axf->Update(&s.ctx, blk, blksz);
  376         }
  377 
  378         /* length block */
  379         memset(blk, 0, blksz);
  380         blkp = (uint32_t *)blk + 1;
  381         *blkp = htobe32(crp->crp_payload_length * 8);
  382         axf->Update(&s.ctx, blk, blksz);
  383 
  384         /* Finalize MAC */
  385         axf->Final(s.tag, &s.ctx);
  386 
  387         error = 0;
  388         if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
  389                 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
  390                     s.tag2);
  391                 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0)
  392                         error = EBADMSG;
  393         } else {
  394                 /* Inject the authentication data */
  395                 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag);
  396         }
  397         explicit_bzero(&s, sizeof(s));
  398         return (error);
  399 }
  400 
  401 static int
  402 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp)
  403 {
  404         struct {
  405                 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))];
  406                 u_char tag[GMAC_DIGEST_LEN];
  407                 u_char tag2[GMAC_DIGEST_LEN];
  408         } s;
  409         u_char *blk = (u_char *)s.blkbuf;
  410         struct crypto_buffer_cursor cc_in, cc_out;
  411         const u_char *inblk;
  412         u_char *outblk;
  413         size_t inlen, outlen, todo;
  414         const struct swcr_auth *swa;
  415         const struct swcr_encdec *swe;
  416         const struct enc_xform *exf;
  417         void *ctx;
  418         uint32_t *blkp;
  419         int blksz, error, ivlen, r, resid;
  420 
  421         swa = &ses->swcr_auth;
  422         swe = &ses->swcr_encdec;
  423         exf = swe->sw_exf;
  424         blksz = GMAC_BLOCK_LEN;
  425         KASSERT(blksz == exf->native_blocksize,
  426             ("%s: blocksize mismatch", __func__));
  427 
  428         if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
  429                 return (EINVAL);
  430 
  431         ivlen = AES_GCM_IV_LEN;
  432 
  433         ctx = __builtin_alloca(exf->ctxsize);
  434         if (crp->crp_cipher_key != NULL)
  435                 exf->setkey(ctx, crp->crp_cipher_key,
  436                     crypto_get_params(crp->crp_session)->csp_cipher_klen);
  437         else
  438                 memcpy(ctx, swe->sw_ctx, exf->ctxsize);
  439         exf->reinit(ctx, crp->crp_iv, ivlen);
  440 
  441         /* Supply MAC with AAD */
  442         if (crp->crp_aad != NULL) {
  443                 inlen = rounddown2(crp->crp_aad_length, blksz);
  444                 if (inlen != 0)
  445                         exf->update(ctx, crp->crp_aad, inlen);
  446                 if (crp->crp_aad_length != inlen) {
  447                         memset(blk, 0, blksz);
  448                         memcpy(blk, (char *)crp->crp_aad + inlen,
  449                             crp->crp_aad_length - inlen);
  450                         exf->update(ctx, blk, blksz);
  451                 }
  452         } else {
  453                 crypto_cursor_init(&cc_in, &crp->crp_buf);
  454                 crypto_cursor_advance(&cc_in, crp->crp_aad_start);
  455                 for (resid = crp->crp_aad_length; resid >= blksz;
  456                      resid -= inlen) {
  457                         inblk = crypto_cursor_segment(&cc_in, &inlen);
  458                         if (inlen >= blksz) {
  459                                 inlen = rounddown2(MIN(inlen, resid), blksz);
  460                                 crypto_cursor_advance(&cc_in, inlen);
  461                         } else {
  462                                 inlen = blksz;
  463                                 crypto_cursor_copydata(&cc_in, inlen, blk);
  464                                 inblk = blk;
  465                         }
  466                         exf->update(ctx, inblk, inlen);
  467                 }
  468                 if (resid > 0) {
  469                         memset(blk, 0, blksz);
  470                         crypto_cursor_copydata(&cc_in, resid, blk);
  471                         exf->update(ctx, blk, blksz);
  472                 }
  473         }
  474 
  475         /* Do encryption with MAC */
  476         crypto_cursor_init(&cc_in, &crp->crp_buf);
  477         crypto_cursor_advance(&cc_in, crp->crp_payload_start);
  478         if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
  479                 crypto_cursor_init(&cc_out, &crp->crp_obuf);
  480                 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
  481         } else
  482                 cc_out = cc_in;
  483 
  484         inlen = outlen = 0;
  485         for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
  486                 if (inlen == 0)
  487                         inblk = crypto_cursor_segment(&cc_in, &inlen);
  488                 if (outlen == 0)
  489                         outblk = crypto_cursor_segment(&cc_out, &outlen);
  490 
  491                 if (inlen < blksz) {
  492                         crypto_cursor_copydata(&cc_in, blksz, blk);
  493                         inblk = blk;
  494                         inlen = blksz;
  495                 }
  496 
  497                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
  498                         if (outlen < blksz) {
  499                                 outblk = blk;
  500                                 outlen = blksz;
  501                         }
  502 
  503                         todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
  504                             blksz);
  505 
  506                         exf->encrypt_multi(ctx, inblk, outblk, todo);
  507                         exf->update(ctx, outblk, todo);
  508 
  509                         if (outblk == blk) {
  510                                 crypto_cursor_copyback(&cc_out, blksz, blk);
  511                                 outblk = crypto_cursor_segment(&cc_out, &outlen);
  512                         } else {
  513                                 crypto_cursor_advance(&cc_out, todo);
  514                                 outlen -= todo;
  515                                 outblk += todo;
  516                         }
  517                 } else {
  518                         todo = rounddown2(MIN(resid, inlen), blksz);
  519                         exf->update(ctx, inblk, todo);
  520                 }
  521 
  522                 if (inblk == blk) {
  523                         inblk = crypto_cursor_segment(&cc_in, &inlen);
  524                 } else {
  525                         crypto_cursor_advance(&cc_in, todo);
  526                         inlen -= todo;
  527                         inblk += todo;
  528                 }
  529         }
  530         if (resid > 0) {
  531                 crypto_cursor_copydata(&cc_in, resid, blk);
  532                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
  533                         exf->encrypt_last(ctx, blk, blk, resid);
  534                         crypto_cursor_copyback(&cc_out, resid, blk);
  535                 }
  536                 exf->update(ctx, blk, resid);
  537         }
  538 
  539         /* length block */
  540         memset(blk, 0, blksz);
  541         blkp = (uint32_t *)blk + 1;
  542         *blkp = htobe32(crp->crp_aad_length * 8);
  543         blkp = (uint32_t *)blk + 3;
  544         *blkp = htobe32(crp->crp_payload_length * 8);
  545         exf->update(ctx, blk, blksz);
  546 
  547         /* Finalize MAC */
  548         exf->final(s.tag, ctx);
  549 
  550         /* Validate tag */
  551         error = 0;
  552         if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
  553                 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
  554                     s.tag2);
  555                 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen);
  556                 if (r != 0) {
  557                         error = EBADMSG;
  558                         goto out;
  559                 }
  560 
  561                 /* tag matches, decrypt data */
  562                 crypto_cursor_init(&cc_in, &crp->crp_buf);
  563                 crypto_cursor_advance(&cc_in, crp->crp_payload_start);
  564 
  565                 inlen = 0;
  566                 for (resid = crp->crp_payload_length; resid > blksz;
  567                      resid -= todo) {
  568                         if (inlen == 0)
  569                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
  570                         if (outlen == 0)
  571                                 outblk = crypto_cursor_segment(&cc_out, &outlen);
  572                         if (inlen < blksz) {
  573                                 crypto_cursor_copydata(&cc_in, blksz, blk);
  574                                 inblk = blk;
  575                                 inlen = blksz;
  576                         }
  577                         if (outlen < blksz) {
  578                                 outblk = blk;
  579                                 outlen = blksz;
  580                         }
  581 
  582                         todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
  583                             blksz);
  584 
  585                         exf->decrypt_multi(ctx, inblk, outblk, todo);
  586 
  587                         if (inblk == blk) {
  588                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
  589                         } else {
  590                                 crypto_cursor_advance(&cc_in, todo);
  591                                 inlen -= todo;
  592                                 inblk += todo;
  593                         }
  594 
  595                         if (outblk == blk) {
  596                                 crypto_cursor_copyback(&cc_out, blksz, blk);
  597                                 outblk = crypto_cursor_segment(&cc_out,
  598                                     &outlen);
  599                         } else {
  600                                 crypto_cursor_advance(&cc_out, todo);
  601                                 outlen -= todo;
  602                                 outblk += todo;
  603                         }
  604                 }
  605                 if (resid > 0) {
  606                         crypto_cursor_copydata(&cc_in, resid, blk);
  607                         exf->decrypt_last(ctx, blk, blk, resid);
  608                         crypto_cursor_copyback(&cc_out, resid, blk);
  609                 }
  610         } else {
  611                 /* Inject the authentication data */
  612                 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
  613                     s.tag);
  614         }
  615 
  616 out:
  617         explicit_bzero(ctx, exf->ctxsize);
  618         explicit_bzero(&s, sizeof(s));
  619 
  620         return (error);
  621 }
  622 
  623 static void
  624 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length,
  625     u_int data_length, u_int tag_length, uint8_t *b0)
  626 {
  627         uint8_t *bp;
  628         uint8_t flags, L;
  629 
  630         KASSERT(nonce_length >= 7 && nonce_length <= 13,
  631             ("nonce_length must be between 7 and 13 bytes"));
  632 
  633         /*
  634          * Need to determine the L field value.  This is the number of
  635          * bytes needed to specify the length of the message; the length
  636          * is whatever is left in the 16 bytes after specifying flags and
  637          * the nonce.
  638          */
  639         L = 15 - nonce_length;
  640 
  641         flags = ((aad_length > 0) << 6) +
  642             (((tag_length - 2) / 2) << 3) +
  643             L - 1;
  644 
  645         /*
  646          * Now we need to set up the first block, which has flags, nonce,
  647          * and the message length.
  648          */
  649         b0[0] = flags;
  650         memcpy(b0 + 1, nonce, nonce_length);
  651         bp = b0 + 1 + nonce_length;
  652 
  653         /* Need to copy L' [aka L-1] bytes of data_length */
  654         for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) {
  655                 *dst = data_length;
  656                 data_length >>= 8;
  657         }
  658 }
  659 
  660 /* NB: OCF only supports AAD lengths < 2^32. */
  661 static int
  662 build_ccm_aad_length(u_int aad_length, uint8_t *blk)
  663 {
  664         if (aad_length < ((1 << 16) - (1 << 8))) {
  665                 be16enc(blk, aad_length);
  666                 return (sizeof(uint16_t));
  667         } else {
  668                 blk[0] = 0xff;
  669                 blk[1] = 0xfe;
  670                 be32enc(blk + 2, aad_length);
  671                 return (2 + sizeof(uint32_t));
  672         }
  673 }
  674 
  675 static int
  676 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp)
  677 {
  678         struct {
  679                 union authctx ctx;
  680                 u_char blk[CCM_CBC_BLOCK_LEN];
  681                 u_char tag[AES_CBC_MAC_HASH_LEN];
  682                 u_char tag2[AES_CBC_MAC_HASH_LEN];
  683         } s;
  684         const struct crypto_session_params *csp;
  685         const struct swcr_auth *swa;
  686         const struct auth_hash *axf;
  687         int error, ivlen, len;
  688 
  689         csp = crypto_get_params(crp->crp_session);
  690         swa = &ses->swcr_auth;
  691         axf = swa->sw_axf;
  692 
  693         if (crp->crp_auth_key != NULL) {
  694                 axf->Init(&s.ctx);
  695                 axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen);
  696         } else
  697                 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize);
  698 
  699         /* Initialize the IV */
  700         ivlen = csp->csp_ivlen;
  701 
  702         /* Supply MAC with IV */
  703         axf->Reinit(&s.ctx, crp->crp_iv, ivlen);
  704 
  705         /* Supply MAC with b0. */
  706         build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0,
  707             swa->sw_mlen, s.blk);
  708         axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN);
  709 
  710         len = build_ccm_aad_length(crp->crp_payload_length, s.blk);
  711         axf->Update(&s.ctx, s.blk, len);
  712 
  713         crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length,
  714             axf->Update, &s.ctx);
  715 
  716         /* Finalize MAC */
  717         axf->Final(s.tag, &s.ctx);
  718 
  719         error = 0;
  720         if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
  721                 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
  722                     s.tag2);
  723                 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0)
  724                         error = EBADMSG;
  725         } else {
  726                 /* Inject the authentication data */
  727                 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
  728                     s.tag);
  729         }
  730         explicit_bzero(&s, sizeof(s));
  731         return (error);
  732 }
  733 
  734 static int
  735 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp)
  736 {
  737         const struct crypto_session_params *csp;
  738         struct {
  739                 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))];
  740                 u_char tag[AES_CBC_MAC_HASH_LEN];
  741                 u_char tag2[AES_CBC_MAC_HASH_LEN];
  742         } s;
  743         u_char *blk = (u_char *)s.blkbuf;
  744         struct crypto_buffer_cursor cc_in, cc_out;
  745         const u_char *inblk;
  746         u_char *outblk;
  747         size_t inlen, outlen, todo;
  748         const struct swcr_auth *swa;
  749         const struct swcr_encdec *swe;
  750         const struct enc_xform *exf;
  751         void *ctx;
  752         size_t len;
  753         int blksz, error, ivlen, r, resid;
  754 
  755         csp = crypto_get_params(crp->crp_session);
  756         swa = &ses->swcr_auth;
  757         swe = &ses->swcr_encdec;
  758         exf = swe->sw_exf;
  759         blksz = AES_BLOCK_LEN;
  760         KASSERT(blksz == exf->native_blocksize,
  761             ("%s: blocksize mismatch", __func__));
  762 
  763         if (crp->crp_payload_length > ccm_max_payload_length(csp))
  764                 return (EMSGSIZE);
  765 
  766         if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
  767                 return (EINVAL);
  768 
  769         ivlen = csp->csp_ivlen;
  770 
  771         ctx = __builtin_alloca(exf->ctxsize);
  772         if (crp->crp_cipher_key != NULL)
  773                 exf->setkey(ctx, crp->crp_cipher_key,
  774                     crypto_get_params(crp->crp_session)->csp_cipher_klen);
  775         else
  776                 memcpy(ctx, swe->sw_ctx, exf->ctxsize);
  777         exf->reinit(ctx, crp->crp_iv, ivlen);
  778 
  779         /* Supply MAC with b0. */
  780         _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN,
  781             "blkbuf too small for b0");
  782         build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length,
  783             crp->crp_payload_length, swa->sw_mlen, blk);
  784         exf->update(ctx, blk, CCM_CBC_BLOCK_LEN);
  785 
  786         /* Supply MAC with AAD */
  787         if (crp->crp_aad_length != 0) {
  788                 len = build_ccm_aad_length(crp->crp_aad_length, blk);
  789                 exf->update(ctx, blk, len);
  790                 if (crp->crp_aad != NULL)
  791                         exf->update(ctx, crp->crp_aad, crp->crp_aad_length);
  792                 else
  793                         crypto_apply(crp, crp->crp_aad_start,
  794                             crp->crp_aad_length, exf->update, ctx);
  795 
  796                 /* Pad the AAD (including length field) to a full block. */
  797                 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN;
  798                 if (len != 0) {
  799                         len = CCM_CBC_BLOCK_LEN - len;
  800                         memset(blk, 0, CCM_CBC_BLOCK_LEN);
  801                         exf->update(ctx, blk, len);
  802                 }
  803         }
  804 
  805         /* Do encryption/decryption with MAC */
  806         crypto_cursor_init(&cc_in, &crp->crp_buf);
  807         crypto_cursor_advance(&cc_in, crp->crp_payload_start);
  808         if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
  809                 crypto_cursor_init(&cc_out, &crp->crp_obuf);
  810                 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
  811         } else
  812                 cc_out = cc_in;
  813 
  814         inlen = outlen = 0;
  815         for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) {
  816                 if (inlen == 0)
  817                         inblk = crypto_cursor_segment(&cc_in, &inlen);
  818                 if (outlen == 0)
  819                         outblk = crypto_cursor_segment(&cc_out, &outlen);
  820 
  821                 if (inlen < blksz) {
  822                         crypto_cursor_copydata(&cc_in, blksz, blk);
  823                         inblk = blk;
  824                         inlen = blksz;
  825                 }
  826 
  827                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
  828                         if (outlen < blksz) {
  829                                 outblk = blk;
  830                                 outlen = blksz;
  831                         }
  832 
  833                         todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
  834                             blksz);
  835 
  836                         exf->update(ctx, inblk, todo);
  837                         exf->encrypt_multi(ctx, inblk, outblk, todo);
  838 
  839                         if (outblk == blk) {
  840                                 crypto_cursor_copyback(&cc_out, blksz, blk);
  841                                 outblk = crypto_cursor_segment(&cc_out, &outlen);
  842                         } else {
  843                                 crypto_cursor_advance(&cc_out, todo);
  844                                 outlen -= todo;
  845                                 outblk += todo;
  846                         }
  847                 } else {
  848                         /*
  849                          * One of the problems with CCM+CBC is that
  850                          * the authentication is done on the
  851                          * unencrypted data.  As a result, we have to
  852                          * decrypt the data twice: once to generate
  853                          * the tag and a second time after the tag is
  854                          * verified.
  855                          */
  856                         todo = blksz;
  857                         exf->decrypt(ctx, inblk, blk);
  858                         exf->update(ctx, blk, todo);
  859                 }
  860 
  861                 if (inblk == blk) {
  862                         inblk = crypto_cursor_segment(&cc_in, &inlen);
  863                 } else {
  864                         crypto_cursor_advance(&cc_in, todo);
  865                         inlen -= todo;
  866                         inblk += todo;
  867                 }
  868         }
  869         if (resid > 0) {
  870                 crypto_cursor_copydata(&cc_in, resid, blk);
  871                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
  872                         exf->update(ctx, blk, resid);
  873                         exf->encrypt_last(ctx, blk, blk, resid);
  874                         crypto_cursor_copyback(&cc_out, resid, blk);
  875                 } else {
  876                         exf->decrypt_last(ctx, blk, blk, resid);
  877                         exf->update(ctx, blk, resid);
  878                 }
  879         }
  880 
  881         /* Finalize MAC */
  882         exf->final(s.tag, ctx);
  883 
  884         /* Validate tag */
  885         error = 0;
  886         if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
  887                 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
  888                     s.tag2);
  889                 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen);
  890                 if (r != 0) {
  891                         error = EBADMSG;
  892                         goto out;
  893                 }
  894 
  895                 /* tag matches, decrypt data */
  896                 exf->reinit(ctx, crp->crp_iv, ivlen);
  897                 crypto_cursor_init(&cc_in, &crp->crp_buf);
  898                 crypto_cursor_advance(&cc_in, crp->crp_payload_start);
  899 
  900                 inlen = 0;
  901                 for (resid = crp->crp_payload_length; resid >= blksz;
  902                      resid -= todo) {
  903                         if (inlen == 0)
  904                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
  905                         if (outlen == 0)
  906                                 outblk = crypto_cursor_segment(&cc_out,
  907                                     &outlen);
  908 
  909                         if (inlen < blksz) {
  910                                 crypto_cursor_copydata(&cc_in, blksz, blk);
  911                                 inblk = blk;
  912                                 inlen = blksz;
  913                         }
  914                         if (outlen < blksz) {
  915                                 outblk = blk;
  916                                 outlen = blksz;
  917                         }
  918 
  919                         todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
  920                             blksz);
  921 
  922                         exf->decrypt_multi(ctx, inblk, outblk, todo);
  923 
  924                         if (inblk == blk) {
  925                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
  926                         } else {
  927                                 crypto_cursor_advance(&cc_in, todo);
  928                                 inlen -= todo;
  929                                 inblk += todo;
  930                         }
  931 
  932                         if (outblk == blk) {
  933                                 crypto_cursor_copyback(&cc_out, blksz, blk);
  934                                 outblk = crypto_cursor_segment(&cc_out,
  935                                     &outlen);
  936                         } else {
  937                                 crypto_cursor_advance(&cc_out, todo);
  938                                 outlen -= todo;
  939                                 outblk += todo;
  940                         }
  941                 }
  942                 if (resid > 0) {
  943                         crypto_cursor_copydata(&cc_in, resid, blk);
  944                         exf->decrypt_last(ctx, blk, blk, resid);
  945                         crypto_cursor_copyback(&cc_out, resid, blk);
  946                 }
  947         } else {
  948                 /* Inject the authentication data */
  949                 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
  950                     s.tag);
  951         }
  952 
  953 out:
  954         explicit_bzero(ctx, exf->ctxsize);
  955         explicit_bzero(&s, sizeof(s));
  956         return (error);
  957 }
  958 
  959 static int
  960 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp)
  961 {
  962         const struct crypto_session_params *csp;
  963         struct {
  964                 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))];
  965                 u_char tag[POLY1305_HASH_LEN];
  966                 u_char tag2[POLY1305_HASH_LEN];
  967         } s;
  968         u_char *blk = (u_char *)s.blkbuf;
  969         struct crypto_buffer_cursor cc_in, cc_out;
  970         const u_char *inblk;
  971         u_char *outblk;
  972         size_t inlen, outlen, todo;
  973         uint64_t *blkp;
  974         const struct swcr_auth *swa;
  975         const struct swcr_encdec *swe;
  976         const struct enc_xform *exf;
  977         void *ctx;
  978         int blksz, error, r, resid;
  979 
  980         swa = &ses->swcr_auth;
  981         swe = &ses->swcr_encdec;
  982         exf = swe->sw_exf;
  983         blksz = exf->native_blocksize;
  984         KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__));
  985 
  986         if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
  987                 return (EINVAL);
  988 
  989         csp = crypto_get_params(crp->crp_session);
  990 
  991         ctx = __builtin_alloca(exf->ctxsize);
  992         if (crp->crp_cipher_key != NULL)
  993                 exf->setkey(ctx, crp->crp_cipher_key,
  994                     csp->csp_cipher_klen);
  995         else
  996                 memcpy(ctx, swe->sw_ctx, exf->ctxsize);
  997         exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen);
  998 
  999         /* Supply MAC with AAD */
 1000         if (crp->crp_aad != NULL)
 1001                 exf->update(ctx, crp->crp_aad, crp->crp_aad_length);
 1002         else
 1003                 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
 1004                     exf->update, ctx);
 1005         if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) {
 1006                 /* padding1 */
 1007                 memset(blk, 0, POLY1305_BLOCK_LEN);
 1008                 exf->update(ctx, blk, POLY1305_BLOCK_LEN -
 1009                     crp->crp_aad_length % POLY1305_BLOCK_LEN);
 1010         }
 1011 
 1012         /* Do encryption with MAC */
 1013         crypto_cursor_init(&cc_in, &crp->crp_buf);
 1014         crypto_cursor_advance(&cc_in, crp->crp_payload_start);
 1015         if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
 1016                 crypto_cursor_init(&cc_out, &crp->crp_obuf);
 1017                 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start);
 1018         } else
 1019                 cc_out = cc_in;
 1020 
 1021         inlen = outlen = 0;
 1022         if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 1023                 for (resid = crp->crp_payload_length; resid >= blksz;
 1024                      resid -= todo) {
 1025                         if (inlen == 0)
 1026                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
 1027                         if (outlen == 0)
 1028                                 outblk = crypto_cursor_segment(&cc_out,
 1029                                     &outlen);
 1030 
 1031                         if (inlen < blksz) {
 1032                                 crypto_cursor_copydata(&cc_in, blksz, blk);
 1033                                 inblk = blk;
 1034                                 inlen = blksz;
 1035                         }
 1036 
 1037                         if (outlen < blksz) {
 1038                                 outblk = blk;
 1039                                 outlen = blksz;
 1040                         }
 1041 
 1042                         todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
 1043                             blksz);
 1044 
 1045                         exf->encrypt_multi(ctx, inblk, outblk, todo);
 1046                         exf->update(ctx, outblk, todo);
 1047 
 1048                         if (inblk == blk) {
 1049                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
 1050                         } else {
 1051                                 crypto_cursor_advance(&cc_in, todo);
 1052                                 inlen -= todo;
 1053                                 inblk += todo;
 1054                         }
 1055 
 1056                         if (outblk == blk) {
 1057                                 crypto_cursor_copyback(&cc_out, blksz, blk);
 1058                                 outblk = crypto_cursor_segment(&cc_out, &outlen);
 1059                         } else {
 1060                                 crypto_cursor_advance(&cc_out, todo);
 1061                                 outlen -= todo;
 1062                                 outblk += todo;
 1063                         }
 1064                 }
 1065                 if (resid > 0) {
 1066                         crypto_cursor_copydata(&cc_in, resid, blk);
 1067                         exf->encrypt_last(ctx, blk, blk, resid);
 1068                         crypto_cursor_copyback(&cc_out, resid, blk);
 1069                         exf->update(ctx, blk, resid);
 1070                 }
 1071         } else
 1072                 crypto_apply(crp, crp->crp_payload_start,
 1073                     crp->crp_payload_length, exf->update, ctx);
 1074         if (crp->crp_payload_length % POLY1305_BLOCK_LEN != 0) {
 1075                 /* padding2 */
 1076                 memset(blk, 0, POLY1305_BLOCK_LEN);
 1077                 exf->update(ctx, blk, POLY1305_BLOCK_LEN -
 1078                     crp->crp_payload_length % POLY1305_BLOCK_LEN);
 1079         }
 1080 
 1081         /* lengths */
 1082         blkp = (uint64_t *)blk;
 1083         blkp[0] = htole64(crp->crp_aad_length);
 1084         blkp[1] = htole64(crp->crp_payload_length);
 1085         exf->update(ctx, blk, sizeof(uint64_t) * 2);
 1086 
 1087         /* Finalize MAC */
 1088         exf->final(s.tag, ctx);
 1089 
 1090         /* Validate tag */
 1091         error = 0;
 1092         if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 1093                 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
 1094                     s.tag2);
 1095                 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen);
 1096                 if (r != 0) {
 1097                         error = EBADMSG;
 1098                         goto out;
 1099                 }
 1100 
 1101                 /* tag matches, decrypt data */
 1102                 crypto_cursor_init(&cc_in, &crp->crp_buf);
 1103                 crypto_cursor_advance(&cc_in, crp->crp_payload_start);
 1104 
 1105                 inlen = 0;
 1106                 for (resid = crp->crp_payload_length; resid > blksz;
 1107                      resid -= todo) {
 1108                         if (inlen == 0)
 1109                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
 1110                         if (outlen == 0)
 1111                                 outblk = crypto_cursor_segment(&cc_out,
 1112                                     &outlen);
 1113                         if (inlen < blksz) {
 1114                                 crypto_cursor_copydata(&cc_in, blksz, blk);
 1115                                 inblk = blk;
 1116                                 inlen = blksz;
 1117                         }
 1118                         if (outlen < blksz) {
 1119                                 outblk = blk;
 1120                                 outlen = blksz;
 1121                         }
 1122 
 1123                         todo = rounddown2(MIN(resid, MIN(inlen, outlen)),
 1124                             blksz);
 1125 
 1126                         exf->decrypt_multi(ctx, inblk, outblk, todo);
 1127 
 1128                         if (inblk == blk) {
 1129                                 inblk = crypto_cursor_segment(&cc_in, &inlen);
 1130                         } else {
 1131                                 crypto_cursor_advance(&cc_in, todo);
 1132                                 inlen -= todo;
 1133                                 inblk += todo;
 1134                         }
 1135 
 1136                         if (outblk == blk) {
 1137                                 crypto_cursor_copyback(&cc_out, blksz, blk);
 1138                                 outblk = crypto_cursor_segment(&cc_out,
 1139                                     &outlen);
 1140                         } else {
 1141                                 crypto_cursor_advance(&cc_out, todo);
 1142                                 outlen -= todo;
 1143                                 outblk += todo;
 1144                         }
 1145                 }
 1146                 if (resid > 0) {
 1147                         crypto_cursor_copydata(&cc_in, resid, blk);
 1148                         exf->decrypt_last(ctx, blk, blk, resid);
 1149                         crypto_cursor_copyback(&cc_out, resid, blk);
 1150                 }
 1151         } else {
 1152                 /* Inject the authentication data */
 1153                 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
 1154                     s.tag);
 1155         }
 1156 
 1157 out:
 1158         explicit_bzero(ctx, exf->ctxsize);
 1159         explicit_bzero(&s, sizeof(s));
 1160         return (error);
 1161 }
 1162 
 1163 /*
 1164  * Apply a cipher and a digest to perform EtA.
 1165  */
 1166 static int
 1167 swcr_eta(const struct swcr_session *ses, struct cryptop *crp)
 1168 {
 1169         int error;
 1170 
 1171         if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 1172                 error = swcr_encdec(ses, crp);
 1173                 if (error == 0)
 1174                         error = swcr_authcompute(ses, crp);
 1175         } else {
 1176                 error = swcr_authcompute(ses, crp);
 1177                 if (error == 0)
 1178                         error = swcr_encdec(ses, crp);
 1179         }
 1180         return (error);
 1181 }
 1182 
 1183 /*
 1184  * Apply a compression/decompression algorithm
 1185  */
 1186 static int
 1187 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp)
 1188 {
 1189         const struct comp_algo *cxf;
 1190         uint8_t *data, *out;
 1191         int adj;
 1192         uint32_t result;
 1193 
 1194         cxf = ses->swcr_compdec.sw_cxf;
 1195 
 1196         /* We must handle the whole buffer of data in one time
 1197          * then if there is not all the data in the mbuf, we must
 1198          * copy in a buffer.
 1199          */
 1200 
 1201         data = malloc(crp->crp_payload_length, M_CRYPTO_DATA,  M_NOWAIT);
 1202         if (data == NULL)
 1203                 return (EINVAL);
 1204         crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length,
 1205             data);
 1206 
 1207         if (CRYPTO_OP_IS_COMPRESS(crp->crp_op))
 1208                 result = cxf->compress(data, crp->crp_payload_length, &out);
 1209         else
 1210                 result = cxf->decompress(data, crp->crp_payload_length, &out);
 1211 
 1212         free(data, M_CRYPTO_DATA);
 1213         if (result == 0)
 1214                 return (EINVAL);
 1215         crp->crp_olen = result;
 1216 
 1217         /* Check the compressed size when doing compression */
 1218         if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) {
 1219                 if (result >= crp->crp_payload_length) {
 1220                         /* Compression was useless, we lost time */
 1221                         free(out, M_CRYPTO_DATA);
 1222                         return (0);
 1223                 }
 1224         }
 1225 
 1226         /* Copy back the (de)compressed data. m_copyback is
 1227          * extending the mbuf as necessary.
 1228          */
 1229         crypto_copyback(crp, crp->crp_payload_start, result, out);
 1230         if (result < crp->crp_payload_length) {
 1231                 switch (crp->crp_buf.cb_type) {
 1232                 case CRYPTO_BUF_MBUF:
 1233                 case CRYPTO_BUF_SINGLE_MBUF:
 1234                         adj = result - crp->crp_payload_length;
 1235                         m_adj(crp->crp_buf.cb_mbuf, adj);
 1236                         break;
 1237                 case CRYPTO_BUF_UIO: {
 1238                         struct uio *uio = crp->crp_buf.cb_uio;
 1239                         int ind;
 1240 
 1241                         adj = crp->crp_payload_length - result;
 1242                         ind = uio->uio_iovcnt - 1;
 1243 
 1244                         while (adj > 0 && ind >= 0) {
 1245                                 if (adj < uio->uio_iov[ind].iov_len) {
 1246                                         uio->uio_iov[ind].iov_len -= adj;
 1247                                         break;
 1248                                 }
 1249 
 1250                                 adj -= uio->uio_iov[ind].iov_len;
 1251                                 uio->uio_iov[ind].iov_len = 0;
 1252                                 ind--;
 1253                                 uio->uio_iovcnt--;
 1254                         }
 1255                         }
 1256                         break;
 1257                 case CRYPTO_BUF_VMPAGE:
 1258                         adj = crp->crp_payload_length - result;
 1259                         crp->crp_buf.cb_vm_page_len -= adj;
 1260                         break;
 1261                 default:
 1262                         break;
 1263                 }
 1264         }
 1265         free(out, M_CRYPTO_DATA);
 1266         return 0;
 1267 }
 1268 
 1269 static int
 1270 swcr_setup_cipher(struct swcr_session *ses,
 1271     const struct crypto_session_params *csp)
 1272 {
 1273         struct swcr_encdec *swe;
 1274         const struct enc_xform *txf;
 1275         int error;
 1276 
 1277         swe = &ses->swcr_encdec;
 1278         txf = crypto_cipher(csp);
 1279         if (csp->csp_cipher_key != NULL) {
 1280                 if (txf->ctxsize != 0) {
 1281                         swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA,
 1282                             M_NOWAIT);
 1283                         if (swe->sw_ctx == NULL)
 1284                                 return (ENOMEM);
 1285                 }
 1286                 error = txf->setkey(swe->sw_ctx,
 1287                     csp->csp_cipher_key, csp->csp_cipher_klen);
 1288                 if (error)
 1289                         return (error);
 1290         }
 1291         swe->sw_exf = txf;
 1292         return (0);
 1293 }
 1294 
 1295 static int
 1296 swcr_setup_auth(struct swcr_session *ses,
 1297     const struct crypto_session_params *csp)
 1298 {
 1299         struct swcr_auth *swa;
 1300         const struct auth_hash *axf;
 1301 
 1302         swa = &ses->swcr_auth;
 1303 
 1304         axf = crypto_auth_hash(csp);
 1305         swa->sw_axf = axf;
 1306         if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
 1307                 return (EINVAL);
 1308         if (csp->csp_auth_mlen == 0)
 1309                 swa->sw_mlen = axf->hashsize;
 1310         else
 1311                 swa->sw_mlen = csp->csp_auth_mlen;
 1312         if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) {
 1313                 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
 1314                     M_NOWAIT);
 1315                 if (swa->sw_ictx == NULL)
 1316                         return (ENOBUFS);
 1317         }
 1318 
 1319         switch (csp->csp_auth_alg) {
 1320         case CRYPTO_SHA1_HMAC:
 1321         case CRYPTO_SHA2_224_HMAC:
 1322         case CRYPTO_SHA2_256_HMAC:
 1323         case CRYPTO_SHA2_384_HMAC:
 1324         case CRYPTO_SHA2_512_HMAC:
 1325         case CRYPTO_RIPEMD160_HMAC:
 1326                 swa->sw_hmac = true;
 1327                 if (csp->csp_auth_key != NULL) {
 1328                         swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
 1329                             M_NOWAIT);
 1330                         if (swa->sw_octx == NULL)
 1331                                 return (ENOBUFS);
 1332                         hmac_init_ipad(axf, csp->csp_auth_key,
 1333                             csp->csp_auth_klen, swa->sw_ictx);
 1334                         hmac_init_opad(axf, csp->csp_auth_key,
 1335                             csp->csp_auth_klen, swa->sw_octx);
 1336                 }
 1337                 break;
 1338         case CRYPTO_RIPEMD160:
 1339         case CRYPTO_SHA1:
 1340         case CRYPTO_SHA2_224:
 1341         case CRYPTO_SHA2_256:
 1342         case CRYPTO_SHA2_384:
 1343         case CRYPTO_SHA2_512:
 1344         case CRYPTO_NULL_HMAC:
 1345                 axf->Init(swa->sw_ictx);
 1346                 break;
 1347         case CRYPTO_AES_NIST_GMAC:
 1348         case CRYPTO_AES_CCM_CBC_MAC:
 1349         case CRYPTO_POLY1305:
 1350                 if (csp->csp_auth_key != NULL) {
 1351                         axf->Init(swa->sw_ictx);
 1352                         axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
 1353                             csp->csp_auth_klen);
 1354                 }
 1355                 break;
 1356         case CRYPTO_BLAKE2B:
 1357         case CRYPTO_BLAKE2S:
 1358                 /*
 1359                  * Blake2b and Blake2s support an optional key but do
 1360                  * not require one.
 1361                  */
 1362                 if (csp->csp_auth_klen == 0)
 1363                         axf->Init(swa->sw_ictx);
 1364                 else if (csp->csp_auth_key != NULL)
 1365                         axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
 1366                             csp->csp_auth_klen);
 1367                 break;
 1368         }
 1369 
 1370         if (csp->csp_mode == CSP_MODE_DIGEST) {
 1371                 switch (csp->csp_auth_alg) {
 1372                 case CRYPTO_AES_NIST_GMAC:
 1373                         ses->swcr_process = swcr_gmac;
 1374                         break;
 1375                 case CRYPTO_AES_CCM_CBC_MAC:
 1376                         ses->swcr_process = swcr_ccm_cbc_mac;
 1377                         break;
 1378                 default:
 1379                         ses->swcr_process = swcr_authcompute;
 1380                 }
 1381         }
 1382 
 1383         return (0);
 1384 }
 1385 
 1386 static int
 1387 swcr_setup_aead(struct swcr_session *ses,
 1388     const struct crypto_session_params *csp)
 1389 {
 1390         struct swcr_auth *swa;
 1391         int error;
 1392 
 1393         error = swcr_setup_cipher(ses, csp);
 1394         if (error)
 1395                 return (error);
 1396 
 1397         swa = &ses->swcr_auth;
 1398         if (csp->csp_auth_mlen == 0)
 1399                 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize;
 1400         else
 1401                 swa->sw_mlen = csp->csp_auth_mlen;
 1402         return (0);
 1403 }
 1404 
 1405 static bool
 1406 swcr_auth_supported(const struct crypto_session_params *csp)
 1407 {
 1408         const struct auth_hash *axf;
 1409 
 1410         axf = crypto_auth_hash(csp);
 1411         if (axf == NULL)
 1412                 return (false);
 1413         switch (csp->csp_auth_alg) {
 1414         case CRYPTO_SHA1_HMAC:
 1415         case CRYPTO_SHA2_224_HMAC:
 1416         case CRYPTO_SHA2_256_HMAC:
 1417         case CRYPTO_SHA2_384_HMAC:
 1418         case CRYPTO_SHA2_512_HMAC:
 1419         case CRYPTO_NULL_HMAC:
 1420         case CRYPTO_RIPEMD160_HMAC:
 1421                 break;
 1422         case CRYPTO_AES_NIST_GMAC:
 1423                 switch (csp->csp_auth_klen * 8) {
 1424                 case 128:
 1425                 case 192:
 1426                 case 256:
 1427                         break;
 1428                 default:
 1429                         return (false);
 1430                 }
 1431                 if (csp->csp_auth_key == NULL)
 1432                         return (false);
 1433                 if (csp->csp_ivlen != AES_GCM_IV_LEN)
 1434                         return (false);
 1435                 break;
 1436         case CRYPTO_POLY1305:
 1437                 if (csp->csp_auth_klen != POLY1305_KEY_LEN)
 1438                         return (false);
 1439                 break;
 1440         case CRYPTO_AES_CCM_CBC_MAC:
 1441                 switch (csp->csp_auth_klen * 8) {
 1442                 case 128:
 1443                 case 192:
 1444                 case 256:
 1445                         break;
 1446                 default:
 1447                         return (false);
 1448                 }
 1449                 if (csp->csp_auth_key == NULL)
 1450                         return (false);
 1451                 break;
 1452         }
 1453         return (true);
 1454 }
 1455 
 1456 static bool
 1457 swcr_cipher_supported(const struct crypto_session_params *csp)
 1458 {
 1459         const struct enc_xform *txf;
 1460 
 1461         txf = crypto_cipher(csp);
 1462         if (txf == NULL)
 1463                 return (false);
 1464         if (csp->csp_cipher_alg != CRYPTO_NULL_CBC &&
 1465             txf->ivsize != csp->csp_ivlen)
 1466                 return (false);
 1467         return (true);
 1468 }
 1469 
 1470 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN)
 1471 
 1472 static int
 1473 swcr_probesession(device_t dev, const struct crypto_session_params *csp)
 1474 {
 1475         if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
 1476                 return (EINVAL);
 1477         switch (csp->csp_mode) {
 1478         case CSP_MODE_COMPRESS:
 1479                 switch (csp->csp_cipher_alg) {
 1480                 case CRYPTO_DEFLATE_COMP:
 1481                         break;
 1482                 default:
 1483                         return (EINVAL);
 1484                 }
 1485                 break;
 1486         case CSP_MODE_CIPHER:
 1487                 switch (csp->csp_cipher_alg) {
 1488                 case CRYPTO_AES_NIST_GCM_16:
 1489                 case CRYPTO_AES_CCM_16:
 1490                 case CRYPTO_CHACHA20_POLY1305:
 1491                 case CRYPTO_XCHACHA20_POLY1305:
 1492                         return (EINVAL);
 1493                 default:
 1494                         if (!swcr_cipher_supported(csp))
 1495                                 return (EINVAL);
 1496                         break;
 1497                 }
 1498                 break;
 1499         case CSP_MODE_DIGEST:
 1500                 if (!swcr_auth_supported(csp))
 1501                         return (EINVAL);
 1502                 break;
 1503         case CSP_MODE_AEAD:
 1504                 switch (csp->csp_cipher_alg) {
 1505                 case CRYPTO_AES_NIST_GCM_16:
 1506                 case CRYPTO_AES_CCM_16:
 1507                         switch (csp->csp_cipher_klen * 8) {
 1508                         case 128:
 1509                         case 192:
 1510                         case 256:
 1511                                 break;
 1512                         default:
 1513                                 return (EINVAL);
 1514                         }
 1515                         break;
 1516                 case CRYPTO_CHACHA20_POLY1305:
 1517                 case CRYPTO_XCHACHA20_POLY1305:
 1518                         break;
 1519                 default:
 1520                         return (EINVAL);
 1521                 }
 1522                 break;
 1523         case CSP_MODE_ETA:
 1524                 /* AEAD algorithms cannot be used for EtA. */
 1525                 switch (csp->csp_cipher_alg) {
 1526                 case CRYPTO_AES_NIST_GCM_16:
 1527                 case CRYPTO_AES_CCM_16:
 1528                 case CRYPTO_CHACHA20_POLY1305:
 1529                 case CRYPTO_XCHACHA20_POLY1305:
 1530                         return (EINVAL);
 1531                 }
 1532                 switch (csp->csp_auth_alg) {
 1533                 case CRYPTO_AES_NIST_GMAC:
 1534                 case CRYPTO_AES_CCM_CBC_MAC:
 1535                         return (EINVAL);
 1536                 }
 1537 
 1538                 if (!swcr_cipher_supported(csp) ||
 1539                     !swcr_auth_supported(csp))
 1540                         return (EINVAL);
 1541                 break;
 1542         default:
 1543                 return (EINVAL);
 1544         }
 1545 
 1546         return (CRYPTODEV_PROBE_SOFTWARE);
 1547 }
 1548 
 1549 /*
 1550  * Generate a new software session.
 1551  */
 1552 static int
 1553 swcr_newsession(device_t dev, crypto_session_t cses,
 1554     const struct crypto_session_params *csp)
 1555 {
 1556         struct swcr_session *ses;
 1557         const struct comp_algo *cxf;
 1558         int error;
 1559 
 1560         ses = crypto_get_driver_session(cses);
 1561 
 1562         error = 0;
 1563         switch (csp->csp_mode) {
 1564         case CSP_MODE_COMPRESS:
 1565                 switch (csp->csp_cipher_alg) {
 1566                 case CRYPTO_DEFLATE_COMP:
 1567                         cxf = &comp_algo_deflate;
 1568                         break;
 1569 #ifdef INVARIANTS
 1570                 default:
 1571                         panic("bad compression algo");
 1572 #endif
 1573                 }
 1574                 ses->swcr_compdec.sw_cxf = cxf;
 1575                 ses->swcr_process = swcr_compdec;
 1576                 break;
 1577         case CSP_MODE_CIPHER:
 1578                 switch (csp->csp_cipher_alg) {
 1579                 case CRYPTO_NULL_CBC:
 1580                         ses->swcr_process = swcr_null;
 1581                         break;
 1582 #ifdef INVARIANTS
 1583                 case CRYPTO_AES_NIST_GCM_16:
 1584                 case CRYPTO_AES_CCM_16:
 1585                 case CRYPTO_CHACHA20_POLY1305:
 1586                 case CRYPTO_XCHACHA20_POLY1305:
 1587                         panic("bad cipher algo");
 1588 #endif
 1589                 default:
 1590                         error = swcr_setup_cipher(ses, csp);
 1591                         if (error == 0)
 1592                                 ses->swcr_process = swcr_encdec;
 1593                 }
 1594                 break;
 1595         case CSP_MODE_DIGEST:
 1596                 error = swcr_setup_auth(ses, csp);
 1597                 break;
 1598         case CSP_MODE_AEAD:
 1599                 switch (csp->csp_cipher_alg) {
 1600                 case CRYPTO_AES_NIST_GCM_16:
 1601                         error = swcr_setup_aead(ses, csp);
 1602                         if (error == 0)
 1603                                 ses->swcr_process = swcr_gcm;
 1604                         break;
 1605                 case CRYPTO_AES_CCM_16:
 1606                         error = swcr_setup_aead(ses, csp);
 1607                         if (error == 0)
 1608                                 ses->swcr_process = swcr_ccm;
 1609                         break;
 1610                 case CRYPTO_CHACHA20_POLY1305:
 1611                 case CRYPTO_XCHACHA20_POLY1305:
 1612                         error = swcr_setup_aead(ses, csp);
 1613                         if (error == 0)
 1614                                 ses->swcr_process = swcr_chacha20_poly1305;
 1615                         break;
 1616 #ifdef INVARIANTS
 1617                 default:
 1618                         panic("bad aead algo");
 1619 #endif
 1620                 }
 1621                 break;
 1622         case CSP_MODE_ETA:
 1623 #ifdef INVARIANTS
 1624                 switch (csp->csp_cipher_alg) {
 1625                 case CRYPTO_AES_NIST_GCM_16:
 1626                 case CRYPTO_AES_CCM_16:
 1627                 case CRYPTO_CHACHA20_POLY1305:
 1628                 case CRYPTO_XCHACHA20_POLY1305:
 1629                         panic("bad eta cipher algo");
 1630                 }
 1631                 switch (csp->csp_auth_alg) {
 1632                 case CRYPTO_AES_NIST_GMAC:
 1633                 case CRYPTO_AES_CCM_CBC_MAC:
 1634                         panic("bad eta auth algo");
 1635                 }
 1636 #endif
 1637 
 1638                 error = swcr_setup_auth(ses, csp);
 1639                 if (error)
 1640                         break;
 1641                 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) {
 1642                         /* Effectively degrade to digest mode. */
 1643                         ses->swcr_process = swcr_authcompute;
 1644                         break;
 1645                 }
 1646 
 1647                 error = swcr_setup_cipher(ses, csp);
 1648                 if (error == 0)
 1649                         ses->swcr_process = swcr_eta;
 1650                 break;
 1651         default:
 1652                 error = EINVAL;
 1653         }
 1654 
 1655         if (error)
 1656                 swcr_freesession(dev, cses);
 1657         return (error);
 1658 }
 1659 
 1660 static void
 1661 swcr_freesession(device_t dev, crypto_session_t cses)
 1662 {
 1663         struct swcr_session *ses;
 1664 
 1665         ses = crypto_get_driver_session(cses);
 1666 
 1667         zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA);
 1668         zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA);
 1669         zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA);
 1670 }
 1671 
 1672 /*
 1673  * Process a software request.
 1674  */
 1675 static int
 1676 swcr_process(device_t dev, struct cryptop *crp, int hint)
 1677 {
 1678         struct swcr_session *ses;
 1679 
 1680         ses = crypto_get_driver_session(crp->crp_session);
 1681 
 1682         crp->crp_etype = ses->swcr_process(ses, crp);
 1683 
 1684         crypto_done(crp);
 1685         return (0);
 1686 }
 1687 
 1688 static void
 1689 swcr_identify(driver_t *drv, device_t parent)
 1690 {
 1691         /* NB: order 10 is so we get attached after h/w devices */
 1692         if (device_find_child(parent, "cryptosoft", -1) == NULL &&
 1693             BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0)
 1694                 panic("cryptosoft: could not attach");
 1695 }
 1696 
 1697 static int
 1698 swcr_probe(device_t dev)
 1699 {
 1700         device_set_desc(dev, "software crypto");
 1701         device_quiet(dev);
 1702         return (BUS_PROBE_NOWILDCARD);
 1703 }
 1704 
 1705 static int
 1706 swcr_attach(device_t dev)
 1707 {
 1708 
 1709         swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session),
 1710                         CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
 1711         if (swcr_id < 0) {
 1712                 device_printf(dev, "cannot initialize!");
 1713                 return (ENXIO);
 1714         }
 1715 
 1716         return (0);
 1717 }
 1718 
 1719 static int
 1720 swcr_detach(device_t dev)
 1721 {
 1722         crypto_unregister_all(swcr_id);
 1723         return 0;
 1724 }
 1725 
 1726 static device_method_t swcr_methods[] = {
 1727         DEVMETHOD(device_identify,      swcr_identify),
 1728         DEVMETHOD(device_probe,         swcr_probe),
 1729         DEVMETHOD(device_attach,        swcr_attach),
 1730         DEVMETHOD(device_detach,        swcr_detach),
 1731 
 1732         DEVMETHOD(cryptodev_probesession, swcr_probesession),
 1733         DEVMETHOD(cryptodev_newsession, swcr_newsession),
 1734         DEVMETHOD(cryptodev_freesession,swcr_freesession),
 1735         DEVMETHOD(cryptodev_process,    swcr_process),
 1736 
 1737         {0, 0},
 1738 };
 1739 
 1740 static driver_t swcr_driver = {
 1741         "cryptosoft",
 1742         swcr_methods,
 1743         0,              /* NB: no softc */
 1744 };
 1745 
 1746 /*
 1747  * NB: We explicitly reference the crypto module so we
 1748  * get the necessary ordering when built as a loadable
 1749  * module.  This is required because we bundle the crypto
 1750  * module code together with the cryptosoft driver (otherwise
 1751  * normal module dependencies would handle things).
 1752  */
 1753 extern int crypto_modevent(struct module *, int, void *);
 1754 /* XXX where to attach */
 1755 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, crypto_modevent, NULL);
 1756 MODULE_VERSION(cryptosoft, 1);
 1757 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);
Cache object: 8a99b222a1065739196dabd25fbbae98
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/opencrypto/cryptosoft.c

FreeBSD/Linux Kernel Cross Reference
sys/opencrypto/cryptosoft.c
