qat_crypt.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*
    2  * CDDL HEADER START
    3  *
    4  * The contents of this file are subject to the terms of the
    5  * Common Development and Distribution License (the "License").
    6  * You may not use this file except in compliance with the License.
    7  *
    8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
    9  * or https://opensource.org/licenses/CDDL-1.0.
   10  * See the License for the specific language governing permissions
   11  * and limitations under the License.
   12  *
   13  * When distributing Covered Code, include this CDDL HEADER in each
   14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   15  * If applicable, add the following below this CDDL HEADER, with the
   16  * fields enclosed by brackets "[]" replaced with your own identifying
   17  * information: Portions Copyright [yyyy] [name of copyright owner]
   18  *
   19  * CDDL HEADER END
   20  */
   21 
   22 /*
   23  * This file represents the QAT implementation of checksums and encryption.
   24  * Internally, QAT shares the same cryptographic instances for both of these
   25  * operations, so the code has been combined here. QAT data compression uses
   26  * compression instances, so that code is separated into qat_compress.c
   27  */
   28 
   29 #if defined(_KERNEL) && defined(HAVE_QAT)
   30 #include <linux/slab.h>
   31 #include <linux/vmalloc.h>
   32 #include <linux/pagemap.h>
   33 #include <linux/completion.h>
   34 #include <sys/zfs_context.h>
   35 #include <sys/zio_crypt.h>
   36 #include "lac/cpa_cy_im.h"
   37 #include "lac/cpa_cy_common.h"
   38 #include <sys/qat.h>
   39 
   40 /*
   41  * Max instances in a QAT device, each instance is a channel to submit
   42  * jobs to QAT hardware, this is only for pre-allocating instances
   43  * and session arrays; the actual number of instances are defined in
   44  * the QAT driver's configure file.
   45  */
   46 #define QAT_CRYPT_MAX_INSTANCES         48
   47 
   48 #define MAX_PAGE_NUM                    1024
   49 
   50 static Cpa32U inst_num = 0;
   51 static Cpa16U num_inst = 0;
   52 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
   53 static boolean_t qat_cy_init_done = B_FALSE;
   54 int zfs_qat_encrypt_disable = 0;
   55 int zfs_qat_checksum_disable = 0;
   56 
   57 typedef struct cy_callback {
   58         CpaBoolean verify_result;
   59         struct completion complete;
   60 } cy_callback_t;
   61 
   62 static void
   63 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
   64     void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
   65 {
   66         cy_callback_t *cb = p_callback;
   67 
   68         if (cb != NULL) {
   69                 /* indicate that the function has been called */
   70                 cb->verify_result = verify;
   71                 complete(&cb->complete);
   72         }
   73 }
   74 
   75 boolean_t
   76 qat_crypt_use_accel(size_t s_len)
   77 {
   78         return (!zfs_qat_encrypt_disable &&
   79             qat_cy_init_done &&
   80             s_len >= QAT_MIN_BUF_SIZE &&
   81             s_len <= QAT_MAX_BUF_SIZE);
   82 }
   83 
   84 boolean_t
   85 qat_checksum_use_accel(size_t s_len)
   86 {
   87         return (!zfs_qat_checksum_disable &&
   88             qat_cy_init_done &&
   89             s_len >= QAT_MIN_BUF_SIZE &&
   90             s_len <= QAT_MAX_BUF_SIZE);
   91 }
   92 
   93 void
   94 qat_cy_clean(void)
   95 {
   96         for (Cpa16U i = 0; i < num_inst; i++)
   97                 cpaCyStopInstance(cy_inst_handles[i]);
   98 
   99         num_inst = 0;
  100         qat_cy_init_done = B_FALSE;
  101 }
  102 
  103 int
  104 qat_cy_init(void)
  105 {
  106         CpaStatus status = CPA_STATUS_FAIL;
  107 
  108         if (qat_cy_init_done)
  109                 return (0);
  110 
  111         status = cpaCyGetNumInstances(&num_inst);
  112         if (status != CPA_STATUS_SUCCESS)
  113                 return (-1);
  114 
  115         /* if the user has configured no QAT encryption units just return */
  116         if (num_inst == 0)
  117                 return (0);
  118 
  119         if (num_inst > QAT_CRYPT_MAX_INSTANCES)
  120                 num_inst = QAT_CRYPT_MAX_INSTANCES;
  121 
  122         status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
  123         if (status != CPA_STATUS_SUCCESS)
  124                 return (-1);
  125 
  126         for (Cpa16U i = 0; i < num_inst; i++) {
  127                 status = cpaCySetAddressTranslation(cy_inst_handles[i],
  128                     (void *)virt_to_phys);
  129                 if (status != CPA_STATUS_SUCCESS)
  130                         goto error;
  131 
  132                 status = cpaCyStartInstance(cy_inst_handles[i]);
  133                 if (status != CPA_STATUS_SUCCESS)
  134                         goto error;
  135         }
  136 
  137         qat_cy_init_done = B_TRUE;
  138         return (0);
  139 
  140 error:
  141         qat_cy_clean();
  142         return (-1);
  143 }
  144 
  145 void
  146 qat_cy_fini(void)
  147 {
  148         if (!qat_cy_init_done)
  149                 return;
  150 
  151         qat_cy_clean();
  152 }
  153 
  154 static CpaStatus
  155 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
  156     CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
  157     Cpa64U crypt, Cpa32U aad_len)
  158 {
  159         CpaStatus status = CPA_STATUS_SUCCESS;
  160         Cpa32U ctx_size;
  161         Cpa32U ciper_algorithm;
  162         Cpa32U hash_algorithm;
  163         CpaCySymSessionSetupData sd = { 0 };
  164 
  165         if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
  166                 return (CPA_STATUS_FAIL);
  167         } else {
  168                 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
  169                 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
  170         }
  171 
  172         sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
  173         sd.cipherSetupData.pCipherKey = key->ck_data;
  174         sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
  175         sd.hashSetupData.hashAlgorithm = hash_algorithm;
  176         sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
  177         sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
  178         sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
  179         sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
  180         sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
  181         sd.digestIsAppended = CPA_FALSE;
  182         sd.verifyDigest = CPA_FALSE;
  183 
  184         if (dir == QAT_ENCRYPT) {
  185                 sd.cipherSetupData.cipherDirection =
  186                     CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
  187                 sd.algChainOrder =
  188                     CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
  189         } else {
  190                 ASSERT3U(dir, ==, QAT_DECRYPT);
  191                 sd.cipherSetupData.cipherDirection =
  192                     CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
  193                 sd.algChainOrder =
  194                     CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
  195         }
  196 
  197         status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
  198         if (status != CPA_STATUS_SUCCESS)
  199                 return (status);
  200 
  201         status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
  202         if (status != CPA_STATUS_SUCCESS)
  203                 return (status);
  204 
  205         status = cpaCySymInitSession(inst_handle, symcallback, &sd,
  206             *cy_session_ctx);
  207         if (status != CPA_STATUS_SUCCESS) {
  208                 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
  209                 return (status);
  210         }
  211 
  212         return (CPA_STATUS_SUCCESS);
  213 }
  214 
  215 static CpaStatus
  216 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
  217     CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
  218 {
  219         CpaStatus status = CPA_STATUS_SUCCESS;
  220         Cpa32U ctx_size;
  221         Cpa32U hash_algorithm;
  222         CpaCySymSessionSetupData sd = { 0 };
  223 
  224         /*
  225          * ZFS's SHA512 checksum is actually SHA512/256, which uses
  226          * a different IV from standard SHA512. QAT does not support
  227          * SHA512/256, so we can only support SHA256.
  228          */
  229         if (cksum == ZIO_CHECKSUM_SHA256)
  230                 hash_algorithm = CPA_CY_SYM_HASH_SHA256;
  231         else
  232                 return (CPA_STATUS_FAIL);
  233 
  234         sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
  235         sd.symOperation = CPA_CY_SYM_OP_HASH;
  236         sd.hashSetupData.hashAlgorithm = hash_algorithm;
  237         sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
  238         sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
  239         sd.digestIsAppended = CPA_FALSE;
  240         sd.verifyDigest = CPA_FALSE;
  241 
  242         status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
  243         if (status != CPA_STATUS_SUCCESS)
  244                 return (status);
  245 
  246         status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
  247         if (status != CPA_STATUS_SUCCESS)
  248                 return (status);
  249 
  250         status = cpaCySymInitSession(inst_handle, symcallback, &sd,
  251             *cy_session_ctx);
  252         if (status != CPA_STATUS_SUCCESS) {
  253                 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
  254                 return (status);
  255         }
  256 
  257         return (CPA_STATUS_SUCCESS);
  258 }
  259 
  260 static CpaStatus
  261 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
  262     CpaBufferList *src, CpaBufferList *dst)
  263 {
  264         CpaStatus status = CPA_STATUS_SUCCESS;
  265         Cpa32U meta_size = 0;
  266 
  267         status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
  268         if (status != CPA_STATUS_SUCCESS)
  269                 return (status);
  270 
  271         status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
  272         if (status != CPA_STATUS_SUCCESS)
  273                 goto error;
  274 
  275         if (src != dst) {
  276                 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
  277                     meta_size);
  278                 if (status != CPA_STATUS_SUCCESS)
  279                         goto error;
  280         }
  281 
  282         return (CPA_STATUS_SUCCESS);
  283 
  284 error:
  285         QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
  286         if (src != dst)
  287                 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
  288 
  289         return (status);
  290 }
  291 
  292 int
  293 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
  294     uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
  295     crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
  296 {
  297         CpaStatus status = CPA_STATUS_SUCCESS;
  298         Cpa16U i;
  299         CpaInstanceHandle cy_inst_handle;
  300         Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
  301         Cpa32U bytes_left = 0;
  302         Cpa8S *data = NULL;
  303         CpaCySymSessionCtx *cy_session_ctx = NULL;
  304         cy_callback_t cb;
  305         CpaCySymOpData op_data = { 0 };
  306         CpaBufferList src_buffer_list = { 0 };
  307         CpaBufferList dst_buffer_list = { 0 };
  308         CpaFlatBuffer *flat_src_buf_array = NULL;
  309         CpaFlatBuffer *flat_src_buf = NULL;
  310         CpaFlatBuffer *flat_dst_buf_array = NULL;
  311         CpaFlatBuffer *flat_dst_buf = NULL;
  312         struct page *in_pages[MAX_PAGE_NUM];
  313         struct page *out_pages[MAX_PAGE_NUM];
  314         Cpa32U in_page_num = 0;
  315         Cpa32U out_page_num = 0;
  316         Cpa32U in_page_off = 0;
  317         Cpa32U out_page_off = 0;
  318 
  319         if (dir == QAT_ENCRYPT) {
  320                 QAT_STAT_BUMP(encrypt_requests);
  321                 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
  322         } else {
  323                 QAT_STAT_BUMP(decrypt_requests);
  324                 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
  325         }
  326 
  327         i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
  328         cy_inst_handle = cy_inst_handles[i];
  329 
  330         status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
  331             &cy_session_ctx, key, crypt, aad_len);
  332         if (status != CPA_STATUS_SUCCESS) {
  333                 /* don't count CCM as a failure since it's not supported */
  334                 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
  335                         QAT_STAT_BUMP(crypt_fails);
  336                 return (status);
  337         }
  338 
  339         /*
  340          * We increment nr_bufs by 2 to allow us to handle non
  341          * page-aligned buffer addresses and buffers whose sizes
  342          * are not divisible by PAGE_SIZE.
  343          */
  344         status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
  345             &src_buffer_list, &dst_buffer_list);
  346         if (status != CPA_STATUS_SUCCESS)
  347                 goto fail;
  348 
  349         status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
  350             nr_bufs * sizeof (CpaFlatBuffer));
  351         if (status != CPA_STATUS_SUCCESS)
  352                 goto fail;
  353         status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
  354             nr_bufs * sizeof (CpaFlatBuffer));
  355         if (status != CPA_STATUS_SUCCESS)
  356                 goto fail;
  357         status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
  358             ZIO_DATA_MAC_LEN);
  359         if (status != CPA_STATUS_SUCCESS)
  360                 goto fail;
  361         status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
  362             ZIO_DATA_IV_LEN);
  363         if (status != CPA_STATUS_SUCCESS)
  364                 goto fail;
  365         if (aad_len > 0) {
  366                 status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
  367                     aad_len);
  368                 if (status != CPA_STATUS_SUCCESS)
  369                         goto fail;
  370                 memcpy(op_data.pAdditionalAuthData, aad_buf, aad_len);
  371         }
  372 
  373         bytes_left = enc_len;
  374         data = src_buf;
  375         flat_src_buf = flat_src_buf_array;
  376         while (bytes_left > 0) {
  377                 in_page_off = ((long)data & ~PAGE_MASK);
  378                 in_pages[in_page_num] = qat_mem_to_page(data);
  379                 flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
  380                 flat_src_buf->dataLenInBytes =
  381                     min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
  382                 data += flat_src_buf->dataLenInBytes;
  383                 bytes_left -= flat_src_buf->dataLenInBytes;
  384                 flat_src_buf++;
  385                 in_page_num++;
  386         }
  387         src_buffer_list.pBuffers = flat_src_buf_array;
  388         src_buffer_list.numBuffers = in_page_num;
  389 
  390         bytes_left = enc_len;
  391         data = dst_buf;
  392         flat_dst_buf = flat_dst_buf_array;
  393         while (bytes_left > 0) {
  394                 out_page_off = ((long)data & ~PAGE_MASK);
  395                 out_pages[out_page_num] = qat_mem_to_page(data);
  396                 flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
  397                     out_page_off;
  398                 flat_dst_buf->dataLenInBytes =
  399                     min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
  400                 data += flat_dst_buf->dataLenInBytes;
  401                 bytes_left -= flat_dst_buf->dataLenInBytes;
  402                 flat_dst_buf++;
  403                 out_page_num++;
  404         }
  405         dst_buffer_list.pBuffers = flat_dst_buf_array;
  406         dst_buffer_list.numBuffers = out_page_num;
  407 
  408         op_data.sessionCtx = cy_session_ctx;
  409         op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
  410         op_data.cryptoStartSrcOffsetInBytes = 0;
  411         op_data.messageLenToCipherInBytes = 0;
  412         op_data.hashStartSrcOffsetInBytes = 0;
  413         op_data.messageLenToHashInBytes = 0;
  414         op_data.messageLenToCipherInBytes = enc_len;
  415         op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
  416         memcpy(op_data.pIv, iv_buf, ZIO_DATA_IV_LEN);
  417         /* if dir is QAT_DECRYPT, copy digest_buf to pDigestResult */
  418         if (dir == QAT_DECRYPT)
  419                 memcpy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
  420 
  421         cb.verify_result = CPA_FALSE;
  422         init_completion(&cb.complete);
  423         status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
  424             &src_buffer_list, &dst_buffer_list, NULL);
  425         if (status != CPA_STATUS_SUCCESS)
  426                 goto fail;
  427 
  428         /* we now wait until the completion of the operation. */
  429         wait_for_completion(&cb.complete);
  430 
  431         if (cb.verify_result == CPA_FALSE) {
  432                 status = CPA_STATUS_FAIL;
  433                 goto fail;
  434         }
  435 
  436         if (dir == QAT_ENCRYPT) {
  437                 /* if dir is QAT_ENCRYPT, save pDigestResult to digest_buf */
  438                 memcpy(digest_buf, op_data.pDigestResult, ZIO_DATA_MAC_LEN);
  439                 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
  440         } else {
  441                 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
  442         }
  443 
  444 fail:
  445         if (status != CPA_STATUS_SUCCESS)
  446                 QAT_STAT_BUMP(crypt_fails);
  447 
  448         for (i = 0; i < in_page_num; i++)
  449                 kunmap(in_pages[i]);
  450         for (i = 0; i < out_page_num; i++)
  451                 kunmap(out_pages[i]);
  452 
  453         cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
  454         if (aad_len > 0)
  455                 QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
  456         QAT_PHYS_CONTIG_FREE(op_data.pIv);
  457         QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
  458         QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
  459         QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
  460         QAT_PHYS_CONTIG_FREE(cy_session_ctx);
  461         QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
  462         QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
  463 
  464         return (status);
  465 }
  466 
  467 int
  468 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
  469 {
  470         CpaStatus status;
  471         Cpa16U i;
  472         CpaInstanceHandle cy_inst_handle;
  473         Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
  474         Cpa32U bytes_left = 0;
  475         Cpa8S *data = NULL;
  476         CpaCySymSessionCtx *cy_session_ctx = NULL;
  477         cy_callback_t cb;
  478         Cpa8U *digest_buffer = NULL;
  479         CpaCySymOpData op_data = { 0 };
  480         CpaBufferList src_buffer_list = { 0 };
  481         CpaFlatBuffer *flat_src_buf_array = NULL;
  482         CpaFlatBuffer *flat_src_buf = NULL;
  483         struct page *in_pages[MAX_PAGE_NUM];
  484         Cpa32U page_num = 0;
  485         Cpa32U page_off = 0;
  486 
  487         QAT_STAT_BUMP(cksum_requests);
  488         QAT_STAT_INCR(cksum_total_in_bytes, size);
  489 
  490         i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
  491         cy_inst_handle = cy_inst_handles[i];
  492 
  493         status = qat_init_checksum_session_ctx(cy_inst_handle,
  494             &cy_session_ctx, cksum);
  495         if (status != CPA_STATUS_SUCCESS) {
  496                 /* don't count unsupported checksums as a failure */
  497                 if (cksum == ZIO_CHECKSUM_SHA256 ||
  498                     cksum == ZIO_CHECKSUM_SHA512)
  499                         QAT_STAT_BUMP(cksum_fails);
  500                 return (status);
  501         }
  502 
  503         /*
  504          * We increment nr_bufs by 2 to allow us to handle non
  505          * page-aligned buffer addresses and buffers whose sizes
  506          * are not divisible by PAGE_SIZE.
  507          */
  508         status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
  509             &src_buffer_list, &src_buffer_list);
  510         if (status != CPA_STATUS_SUCCESS)
  511                 goto fail;
  512 
  513         status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
  514             nr_bufs * sizeof (CpaFlatBuffer));
  515         if (status != CPA_STATUS_SUCCESS)
  516                 goto fail;
  517         status = QAT_PHYS_CONTIG_ALLOC(&digest_buffer,
  518             sizeof (zio_cksum_t));
  519         if (status != CPA_STATUS_SUCCESS)
  520                 goto fail;
  521 
  522         bytes_left = size;
  523         data = buf;
  524         flat_src_buf = flat_src_buf_array;
  525         while (bytes_left > 0) {
  526                 page_off = ((long)data & ~PAGE_MASK);
  527                 in_pages[page_num] = qat_mem_to_page(data);
  528                 flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
  529                 flat_src_buf->dataLenInBytes =
  530                     min((long)PAGE_SIZE - page_off, (long)bytes_left);
  531                 data += flat_src_buf->dataLenInBytes;
  532                 bytes_left -= flat_src_buf->dataLenInBytes;
  533                 flat_src_buf++;
  534                 page_num++;
  535         }
  536         src_buffer_list.pBuffers = flat_src_buf_array;
  537         src_buffer_list.numBuffers = page_num;
  538 
  539         op_data.sessionCtx = cy_session_ctx;
  540         op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
  541         op_data.hashStartSrcOffsetInBytes = 0;
  542         op_data.messageLenToHashInBytes = size;
  543         op_data.pDigestResult = digest_buffer;
  544 
  545         cb.verify_result = CPA_FALSE;
  546         init_completion(&cb.complete);
  547         status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
  548             &src_buffer_list, &src_buffer_list, NULL);
  549         if (status != CPA_STATUS_SUCCESS)
  550                 goto fail;
  551 
  552         /* we now wait until the completion of the operation. */
  553         wait_for_completion(&cb.complete);
  554 
  555         if (cb.verify_result == CPA_FALSE) {
  556                 status = CPA_STATUS_FAIL;
  557                 goto fail;
  558         }
  559 
  560         memcpy(zcp, digest_buffer, sizeof (zio_cksum_t));
  561 
  562 fail:
  563         if (status != CPA_STATUS_SUCCESS)
  564                 QAT_STAT_BUMP(cksum_fails);
  565 
  566         for (i = 0; i < page_num; i++)
  567                 kunmap(in_pages[i]);
  568 
  569         cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
  570         QAT_PHYS_CONTIG_FREE(digest_buffer);
  571         QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
  572         QAT_PHYS_CONTIG_FREE(cy_session_ctx);
  573         QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
  574 
  575         return (status);
  576 }
  577 
  578 static int
  579 param_set_qat_encrypt(const char *val, zfs_kernel_param_t *kp)
  580 {
  581         int ret;
  582         int *pvalue = kp->arg;
  583         ret = param_set_int(val, kp);
  584         if (ret)
  585                 return (ret);
  586         /*
  587          * zfs_qat_encrypt_disable = 0: enable qat encrypt
  588          * try to initialize qat instance if it has not been done
  589          */
  590         if (*pvalue == 0 && !qat_cy_init_done) {
  591                 ret = qat_cy_init();
  592                 if (ret != 0) {
  593                         zfs_qat_encrypt_disable = 1;
  594                         return (ret);
  595                 }
  596         }
  597         return (ret);
  598 }
  599 
  600 static int
  601 param_set_qat_checksum(const char *val, zfs_kernel_param_t *kp)
  602 {
  603         int ret;
  604         int *pvalue = kp->arg;
  605         ret = param_set_int(val, kp);
  606         if (ret)
  607                 return (ret);
  608         /*
  609          * set_checksum_param_ops = 0: enable qat checksum
  610          * try to initialize qat instance if it has not been done
  611          */
  612         if (*pvalue == 0 && !qat_cy_init_done) {
  613                 ret = qat_cy_init();
  614                 if (ret != 0) {
  615                         zfs_qat_checksum_disable = 1;
  616                         return (ret);
  617                 }
  618         }
  619         return (ret);
  620 }
  621 
  622 module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
  623     param_get_int, &zfs_qat_encrypt_disable, 0644);
  624 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");
  625 
  626 module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
  627     param_get_int, &zfs_qat_checksum_disable, 0644);
  628 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");
  629 
  630 #endif
Cache object: 5744766d1a65dc76f7272dbbd762fd97
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/contrib/openzfs/module/os/linux/zfs/qat_crypt.c

FreeBSD/Linux Kernel Cross Reference
sys/contrib/openzfs/module/os/linux/zfs/qat_crypt.c
