dc_datapath.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /* SPDX-License-Identifier: BSD-3-Clause */
    2 /* Copyright(c) 2007-2022 Intel Corporation */
    3 /* $FreeBSD$ */
    4 /**
    5  *****************************************************************************
    6  * @file dc_datapath.c
    7  *
    8  * @defgroup Dc_DataCompression DC Data Compression
    9  *
   10  * @ingroup Dc_DataCompression
   11  *
   12  * @description
   13  *      Implementation of the Data Compression datapath operations.
   14  *
   15  *****************************************************************************/
   16 
   17 /*
   18 *******************************************************************************
   19 * Include public/global header files
   20 *******************************************************************************
   21 */
   22 #include "cpa.h"
   23 #include "cpa_dc.h"
   24 #include "cpa_dc_dp.h"
   25 
   26 /*
   27 *******************************************************************************
   28 * Include private header files
   29 *******************************************************************************
   30 */
   31 #include "dc_session.h"
   32 #include "dc_datapath.h"
   33 #include "sal_statistics.h"
   34 #include "lac_common.h"
   35 #include "lac_mem.h"
   36 #include "lac_mem_pools.h"
   37 #include "sal_types_compression.h"
   38 #include "dc_stats.h"
   39 #include "lac_buffer_desc.h"
   40 #include "lac_sal.h"
   41 #include "lac_log.h"
   42 #include "lac_sync.h"
   43 #include "sal_service_state.h"
   44 #include "sal_qat_cmn_msg.h"
   45 #include "sal_hw_gen.h"
   46 #include "dc_error_counter.h"
   47 #define DC_COMP_MAX_BUFF_SIZE (1024 * 64)
   48 
   49 static QatUtilsAtomic dcErrorCount[MAX_DC_ERROR_TYPE];
   50 
   51 void
   52 dcErrorLog(CpaDcReqStatus dcError)
   53 {
   54         Cpa32U absError = 0;
   55 
   56         absError = abs(dcError);
   57         if ((dcError < CPA_DC_OK) && (absError < MAX_DC_ERROR_TYPE)) {
   58                 qatUtilsAtomicInc(&(dcErrorCount[absError]));
   59         }
   60 }
   61 
   62 Cpa64U
   63 getDcErrorCounter(CpaDcReqStatus dcError)
   64 {
   65         Cpa32U absError = 0;
   66 
   67         absError = abs(dcError);
   68         if (!(dcError >= CPA_DC_OK || dcError < CPA_DC_EMPTY_DYM_BLK)) {
   69                 return (Cpa64U)qatUtilsAtomicGet(&dcErrorCount[absError]);
   70         }
   71 
   72         return 0;
   73 }
   74 
   75 static inline void
   76 dcUpdateXltOverflowChecksumsGen4(const dc_compression_cookie_t *pCookie,
   77                                  const icp_qat_fw_resp_comp_pars_t *pRespPars,
   78                                  CpaDcRqResults *pDcResults)
   79 {
   80         dc_session_desc_t *pSessionDesc =
   81             DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle);
   82 
   83         /* Recompute CRC checksum when either the checksum type
   84          * is CPA_DC_CRC32 or when the integrity CRCs are enabled.
   85          */
   86         if (CPA_DC_CRC32 == pSessionDesc->checksumType) {
   87                 pDcResults->checksum = pRespPars->crc.legacy.curr_crc32;
   88 
   89                 /* No need to recalculate the swCrc64I here as this will get
   90                  * handled later in dcHandleIntegrityChecksumsGen4.
   91                  */
   92         } else if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
   93                 pDcResults->checksum = pRespPars->crc.legacy.curr_adler_32;
   94         }
   95 }
   96 
   97 void
   98 dcCompression_ProcessCallback(void *pRespMsg)
   99 {
  100         CpaStatus status = CPA_STATUS_SUCCESS;
  101         icp_qat_fw_comp_resp_t *pCompRespMsg = NULL;
  102         void *callbackTag = NULL;
  103         Cpa64U *pReqData = NULL;
  104         CpaDcDpOpData *pResponse = NULL;
  105         CpaDcRqResults *pResults = NULL;
  106         CpaDcCallbackFn pCbFunc = NULL;
  107         dc_session_desc_t *pSessionDesc = NULL;
  108         sal_compression_service_t *pService = NULL;
  109         dc_compression_cookie_t *pCookie = NULL;
  110         CpaDcOpData *pOpData = NULL;
  111         CpaBoolean cmpPass = CPA_TRUE, xlatPass = CPA_TRUE;
  112         CpaBoolean isDcDp = CPA_FALSE;
  113         CpaBoolean integrityCrcCheck = CPA_FALSE;
  114         CpaBoolean verifyHwIntegrityCrcs = CPA_FALSE;
  115         Cpa8U cmpErr = ERR_CODE_NO_ERROR, xlatErr = ERR_CODE_NO_ERROR;
  116         dc_request_dir_t compDecomp = DC_COMPRESSION_REQUEST;
  117         Cpa8U opStatus = ICP_QAT_FW_COMN_STATUS_FLAG_OK;
  118         Cpa8U hdrFlags = 0;
  119 
  120         /* Cast response message to compression response message type */
  121         pCompRespMsg = (icp_qat_fw_comp_resp_t *)pRespMsg;
  122 
  123         /* Extract request data pointer from the opaque data */
  124         LAC_MEM_SHARED_READ_TO_PTR(pCompRespMsg->opaque_data, pReqData);
  125 
  126         /* Extract fields from the request data structure */
  127         pCookie = (dc_compression_cookie_t *)pReqData;
  128         if (!pCookie)
  129                 return;
  130 
  131         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pCookie->pSessionHandle);
  132         pService = (sal_compression_service_t *)(pCookie->dcInstance);
  133 
  134         isDcDp = pSessionDesc->isDcDp;
  135         if (CPA_TRUE == isDcDp) {
  136                 pResponse = (CpaDcDpOpData *)pReqData;
  137                 pResults = &(pResponse->results);
  138 
  139                 if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) {
  140                         compDecomp = DC_DECOMPRESSION_REQUEST;
  141                 }
  142                 pCookie = NULL;
  143         } else {
  144                 pResults = pCookie->pResults;
  145                 callbackTag = pCookie->callbackTag;
  146                 pCbFunc = pCookie->pSessionDesc->pCompressionCb;
  147                 compDecomp = pCookie->compDecomp;
  148                 pOpData = pCookie->pDcOpData;
  149         }
  150 
  151         opStatus = pCompRespMsg->comn_resp.comn_status;
  152 
  153         if (NULL != pOpData) {
  154                 verifyHwIntegrityCrcs = pOpData->verifyHwIntegrityCrcs;
  155         }
  156 
  157         hdrFlags = pCompRespMsg->comn_resp.hdr_flags;
  158 
  159         /* Get the cmp error code */
  160         cmpErr = pCompRespMsg->comn_resp.comn_error.s1.cmp_err_code;
  161         if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET(opStatus)) {
  162                 /* Compression not supported by firmware, set produced/consumed
  163                    to zero
  164                    and call the cb function with status CPA_STATUS_UNSUPPORTED
  165                    */
  166                 QAT_UTILS_LOG("Compression feature not supported\n");
  167                 status = CPA_STATUS_UNSUPPORTED;
  168                 pResults->status = (Cpa8S)cmpErr;
  169                 pResults->consumed = 0;
  170                 pResults->produced = 0;
  171                 if (CPA_TRUE == isDcDp) {
  172                         if (pResponse)
  173                                 pResponse->responseStatus =
  174                                     CPA_STATUS_UNSUPPORTED;
  175                         (pService->pDcDpCb)(pResponse);
  176                 } else {
  177                         /* Free the memory pool */
  178                         if (NULL != pCookie) {
  179                                 Lac_MemPoolEntryFree(pCookie);
  180                                 pCookie = NULL;
  181                         }
  182                         if (NULL != pCbFunc) {
  183                                 pCbFunc(callbackTag, status);
  184                         }
  185                 }
  186                 if (DC_COMPRESSION_REQUEST == compDecomp) {
  187                         COMPRESSION_STAT_INC(numCompCompletedErrors, pService);
  188                 } else {
  189                         COMPRESSION_STAT_INC(numDecompCompletedErrors,
  190                                              pService);
  191                 }
  192                 return;
  193         } else {
  194                 /* Check compression response status */
  195                 cmpPass =
  196                     (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK ==
  197                                  ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(opStatus));
  198         }
  199 
  200         if (isDcGen2x(pService)) {
  201                 /* QAT1.7 and QAT 1.8 hardware */
  202                 if (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr) {
  203                         cmpPass = CPA_TRUE;
  204                         cmpErr = ERR_CODE_NO_ERROR;
  205                 }
  206         } else {
  207                 /* QAT2.0 hardware cancels the incomplete file errors
  208                  * only for DEFLATE algorithm.
  209                  * Decompression direction is not tested in the callback as
  210                  * the request does not allow it.
  211                  */
  212                 if ((pSessionDesc->compType == CPA_DC_DEFLATE) &&
  213                     (CPA_DC_INCOMPLETE_FILE_ERR == (Cpa8S)cmpErr)) {
  214                         cmpPass = CPA_TRUE;
  215                         cmpErr = ERR_CODE_NO_ERROR;
  216                 }
  217         }
  218         /* log the slice hang and endpoint push/pull error inside the response
  219          */
  220         if (ERR_CODE_SSM_ERROR == (Cpa8S)cmpErr) {
  221                 QAT_UTILS_LOG(
  222                     "Slice hang detected on the compression slice.\n");
  223         } else if (ERR_CODE_ENDPOINT_ERROR == (Cpa8S)cmpErr) {
  224                 QAT_UTILS_LOG(
  225                     "PCIe End Point Push/Pull or TI/RI Parity error detected.\n");
  226         }
  227 
  228         /* We return the compression error code for now. We would need to update
  229          * the API if we decide to return both error codes */
  230         pResults->status = (Cpa8S)cmpErr;
  231 
  232         /* Check the translator status */
  233         if ((DC_COMPRESSION_REQUEST == compDecomp) &&
  234             (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) {
  235                 /* Check translator response status */
  236                 xlatPass =
  237                     (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK ==
  238                                  ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(opStatus));
  239 
  240                 /* Get the translator error code */
  241                 xlatErr = pCompRespMsg->comn_resp.comn_error.s1.xlat_err_code;
  242 
  243                 /* Return a fatal error or a potential error in the translator
  244                  * slice if the compression slice did not return any error */
  245                 if ((CPA_DC_OK == pResults->status) ||
  246                     (CPA_DC_FATALERR == (Cpa8S)xlatErr)) {
  247                         pResults->status = (Cpa8S)xlatErr;
  248                 }
  249         }
  250         /* Update dc error counter */
  251         dcErrorLog(pResults->status);
  252 
  253         if (CPA_FALSE == isDcDp) {
  254                 /* In case of any error for an end of packet request, we need to
  255                  * update
  256                  * the request type for the following request */
  257                 if (CPA_DC_FLUSH_FINAL == pCookie->flushFlag && cmpPass &&
  258                     xlatPass) {
  259                         pSessionDesc->requestType = DC_REQUEST_FIRST;
  260                 } else {
  261                         pSessionDesc->requestType = DC_REQUEST_SUBSEQUENT;
  262                 }
  263                 if ((CPA_DC_STATEFUL == pSessionDesc->sessState) ||
  264                     ((CPA_DC_STATELESS == pSessionDesc->sessState) &&
  265                      (DC_COMPRESSION_REQUEST == compDecomp))) {
  266                         /* Overflow is a valid use case for Traditional API
  267                          * only. Stateful Overflow is supported in both
  268                          * compression and decompression direction. Stateless
  269                          * Overflow is supported only in compression direction.
  270                          */
  271                         if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr)
  272                                 cmpPass = CPA_TRUE;
  273 
  274                         if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) {
  275                                 if (isDcGen4x(pService) &&
  276                                     (CPA_TRUE ==
  277                                      pService->comp_device_data
  278                                          .translatorOverflow)) {
  279                                         pResults->consumed =
  280                                             pCompRespMsg->comp_resp_pars
  281                                                 .input_byte_counter;
  282 
  283                                         dcUpdateXltOverflowChecksumsGen4(
  284                                             pCookie,
  285                                             &pCompRespMsg->comp_resp_pars,
  286                                             pResults);
  287                                 }
  288                                 xlatPass = CPA_TRUE;
  289                         }
  290                 }
  291         } else {
  292                 if (CPA_DC_OVERFLOW == (Cpa8S)cmpErr) {
  293                         cmpPass = CPA_FALSE;
  294                 }
  295                 if (CPA_DC_OVERFLOW == (Cpa8S)xlatErr) {
  296                         /* XLT overflow is not valid for Data Plane requests */
  297                         xlatPass = CPA_FALSE;
  298                 }
  299         }
  300 
  301         if ((CPA_TRUE == cmpPass) && (CPA_TRUE == xlatPass)) {
  302                 /* Extract the response from the firmware */
  303                 pResults->consumed =
  304                     pCompRespMsg->comp_resp_pars.input_byte_counter;
  305                 pResults->produced =
  306                     pCompRespMsg->comp_resp_pars.output_byte_counter;
  307                 pSessionDesc->cumulativeConsumedBytes += pResults->consumed;
  308 
  309                 /* Handle Checksum for end to end data integrity. */
  310                 if (CPA_TRUE ==
  311                         pService->generic_service_info.integrityCrcCheck &&
  312                     CPA_TRUE == integrityCrcCheck) {
  313                         pSessionDesc->previousChecksum =
  314                             pSessionDesc->seedSwCrc.swCrc32I;
  315                 } else if (CPA_DC_OVERFLOW != (Cpa8S)xlatErr) {
  316                         if (CPA_DC_CRC32 == pSessionDesc->checksumType) {
  317                                 pResults->checksum =
  318                                     pCompRespMsg->comp_resp_pars.crc.legacy
  319                                         .curr_crc32;
  320                         } else if (CPA_DC_ADLER32 ==
  321                                    pSessionDesc->checksumType) {
  322                                 pResults->checksum =
  323                                     pCompRespMsg->comp_resp_pars.crc.legacy
  324                                         .curr_adler_32;
  325                         }
  326                         pSessionDesc->previousChecksum = pResults->checksum;
  327                 }
  328 
  329                 if (DC_DECOMPRESSION_REQUEST == compDecomp) {
  330                         pResults->endOfLastBlock =
  331                             (ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET ==
  332                              ICP_QAT_FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(
  333                                  opStatus));
  334                 }
  335 
  336                 /* Save the checksum for the next request */
  337                 if ((CPA_DC_OVERFLOW != (Cpa8S)xlatErr) &&
  338                     (CPA_TRUE == verifyHwIntegrityCrcs)) {
  339                         pSessionDesc->previousChecksum =
  340                             pSessionDesc->seedSwCrc.swCrc32I;
  341                 }
  342 
  343                 /* Check if a CNV recovery happened and
  344                  * increase stats counter
  345                  */
  346                 if ((DC_COMPRESSION_REQUEST == compDecomp) &&
  347                     ICP_QAT_FW_COMN_HDR_CNV_FLAG_GET(hdrFlags) &&
  348                     ICP_QAT_FW_COMN_HDR_CNVNR_FLAG_GET(hdrFlags)) {
  349                         COMPRESSION_STAT_INC(numCompCnvErrorsRecovered,
  350                                              pService);
  351                 }
  352 
  353                 if (CPA_TRUE == isDcDp) {
  354                         if (pResponse)
  355                                 pResponse->responseStatus = CPA_STATUS_SUCCESS;
  356                 } else {
  357                         if (DC_COMPRESSION_REQUEST == compDecomp) {
  358                                 COMPRESSION_STAT_INC(numCompCompleted,
  359                                                      pService);
  360                         } else {
  361                                 COMPRESSION_STAT_INC(numDecompCompleted,
  362                                                      pService);
  363                         }
  364                 }
  365         } else {
  366 #ifdef ICP_DC_RETURN_COUNTERS_ON_ERROR
  367                 /* Extract the response from the firmware */
  368                 pResults->consumed =
  369                     pCompRespMsg->comp_resp_pars.input_byte_counter;
  370                 pResults->produced =
  371                     pCompRespMsg->comp_resp_pars.output_byte_counter;
  372 
  373                 if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
  374                         pSessionDesc->cumulativeConsumedBytes +=
  375                             pResults->consumed;
  376                 } else {
  377                         /* In the stateless case all requests have both SOP and
  378                          * EOP set */
  379                         pSessionDesc->cumulativeConsumedBytes =
  380                             pResults->consumed;
  381                 }
  382 #else
  383                 pResults->consumed = 0;
  384                 pResults->produced = 0;
  385 #endif
  386                 if (CPA_DC_OVERFLOW == pResults->status &&
  387                     CPA_DC_STATELESS == pSessionDesc->sessState) {
  388                         /* This error message will be returned by Data Plane API
  389                          * in both
  390                          * compression and decompression direction. With
  391                          * Traditional API
  392                          * this error message will be returned only in stateless
  393                          * decompression direction */
  394                         QAT_UTILS_LOG(
  395                             "Unrecoverable error: stateless overflow. You may need to increase the size of your destination buffer.\n");
  396                 }
  397 
  398                 if (CPA_TRUE == isDcDp) {
  399                         if (pResponse)
  400                                 pResponse->responseStatus = CPA_STATUS_FAIL;
  401                 } else {
  402                         if (CPA_DC_OK != pResults->status &&
  403                             CPA_DC_INCOMPLETE_FILE_ERR != pResults->status) {
  404                                 status = CPA_STATUS_FAIL;
  405                         }
  406 
  407                         if (DC_COMPRESSION_REQUEST == compDecomp) {
  408                                 COMPRESSION_STAT_INC(numCompCompletedErrors,
  409                                                      pService);
  410                         } else {
  411                                 COMPRESSION_STAT_INC(numDecompCompletedErrors,
  412                                                      pService);
  413                         }
  414                 }
  415         }
  416 
  417         if (CPA_TRUE == isDcDp) {
  418                 /* Decrement number of stateless pending callbacks for session
  419                  */
  420                 pSessionDesc->pendingDpStatelessCbCount--;
  421                 (pService->pDcDpCb)(pResponse);
  422         } else {
  423                 /* Decrement number of pending callbacks for session */
  424                 if (CPA_DC_STATELESS == pSessionDesc->sessState) {
  425                         qatUtilsAtomicDec(
  426                             &(pCookie->pSessionDesc->pendingStatelessCbCount));
  427                 } else if (0 !=
  428                            qatUtilsAtomicGet(&pCookie->pSessionDesc
  429                                                   ->pendingStatefulCbCount)) {
  430                         qatUtilsAtomicDec(
  431                             &(pCookie->pSessionDesc->pendingStatefulCbCount));
  432                 }
  433 
  434                 /* Free the memory pool */
  435                 if (NULL != pCookie) {
  436                         Lac_MemPoolEntryFree(pCookie);
  437                         pCookie = NULL;
  438                 }
  439 
  440                 if (NULL != pCbFunc) {
  441                         pCbFunc(callbackTag, status);
  442                 }
  443         }
  444 }
  445 
  446 /**
  447  *****************************************************************************
  448  * @ingroup Dc_DataCompression
  449  *      Check that all the parameters in the pOpData structure are valid
  450  *
  451  * @description
  452  *      Check that all the parameters in the pOpData structure are valid
  453  *
  454  * @param[in]   pService              Pointer to the compression service
  455  * @param[in]   pOpData               Pointer to request information structure
  456  *                                    holding parameters for cpaDcCompress2 and
  457  *                                    CpaDcDecompressData2
  458  * @retval CPA_STATUS_SUCCESS         Function executed successfully
  459  * @retval CPA_STATUS_INVALID_PARAM   Invalid parameter passed in
  460  *
  461  *****************************************************************************/
  462 CpaStatus
  463 dcCheckOpData(sal_compression_service_t *pService, CpaDcOpData *pOpData)
  464 {
  465         CpaDcSkipMode skipMode = 0;
  466 
  467         if ((pOpData->flushFlag < CPA_DC_FLUSH_NONE) ||
  468             (pOpData->flushFlag > CPA_DC_FLUSH_FULL)) {
  469                 LAC_INVALID_PARAM_LOG("Invalid flushFlag value");
  470                 return CPA_STATUS_INVALID_PARAM;
  471         }
  472 
  473         skipMode = pOpData->inputSkipData.skipMode;
  474         if ((skipMode < CPA_DC_SKIP_DISABLED) ||
  475             (skipMode > CPA_DC_SKIP_STRIDE)) {
  476                 LAC_INVALID_PARAM_LOG("Invalid input skip mode value");
  477                 return CPA_STATUS_INVALID_PARAM;
  478         }
  479 
  480         skipMode = pOpData->outputSkipData.skipMode;
  481         if ((skipMode < CPA_DC_SKIP_DISABLED) ||
  482             (skipMode > CPA_DC_SKIP_STRIDE)) {
  483                 LAC_INVALID_PARAM_LOG("Invalid output skip mode value");
  484                 return CPA_STATUS_INVALID_PARAM;
  485         }
  486 
  487         if (pOpData->integrityCrcCheck == CPA_FALSE &&
  488             pOpData->verifyHwIntegrityCrcs == CPA_TRUE) {
  489                 LAC_INVALID_PARAM_LOG(
  490                     "integrityCrcCheck must be set to true"
  491                     "in order to enable verifyHwIntegrityCrcs");
  492                 return CPA_STATUS_INVALID_PARAM;
  493         }
  494 
  495         if (pOpData->integrityCrcCheck != CPA_TRUE &&
  496             pOpData->integrityCrcCheck != CPA_FALSE) {
  497                 LAC_INVALID_PARAM_LOG("Invalid integrityCrcCheck value");
  498                 return CPA_STATUS_INVALID_PARAM;
  499         }
  500 
  501         if (pOpData->verifyHwIntegrityCrcs != CPA_TRUE &&
  502             pOpData->verifyHwIntegrityCrcs != CPA_FALSE) {
  503                 LAC_INVALID_PARAM_LOG("Invalid verifyHwIntegrityCrcs value");
  504                 return CPA_STATUS_INVALID_PARAM;
  505         }
  506 
  507         if (pOpData->compressAndVerify != CPA_TRUE &&
  508             pOpData->compressAndVerify != CPA_FALSE) {
  509                 LAC_INVALID_PARAM_LOG("Invalid cnv decompress check value");
  510                 return CPA_STATUS_INVALID_PARAM;
  511         }
  512 
  513         if (CPA_TRUE == pOpData->integrityCrcCheck &&
  514             CPA_FALSE == pService->generic_service_info.integrityCrcCheck) {
  515                 LAC_INVALID_PARAM_LOG("Integrity CRC check is not "
  516                                       "supported on this device");
  517                 return CPA_STATUS_INVALID_PARAM;
  518         }
  519 
  520         if (CPA_TRUE == pOpData->integrityCrcCheck &&
  521             NULL == pOpData->pCrcData) {
  522                 LAC_INVALID_PARAM_LOG("Integrity CRC data structure "
  523                                       "not intialized in CpaDcOpData");
  524                 return CPA_STATUS_INVALID_PARAM;
  525         }
  526 
  527         return CPA_STATUS_SUCCESS;
  528 }
  529 
  530 /**
  531  *****************************************************************************
  532  * @ingroup Dc_DataCompression
  533  *      Check the compression source buffer for Batch and Pack API.
  534  *
  535  * @description
  536  *      Check that all the parameters used for Pack compression
  537  *      request are valid. This function essentially checks the source buffer
  538  *      parameters and results structure parameters.
  539  *
  540  * @param[in]   pSessionHandle        Session handle
  541  * @param[in]   pSrcBuff              Pointer to data buffer for compression
  542  * @param[in]   pDestBuff             Pointer to buffer space allocated for
  543  *                                    output data
  544  * @param[in]   pResults              Pointer to results structure
  545  * @param[in]   flushFlag             Indicates the type of flush to be
  546  *                                    performed
  547  * @param[in]   srcBuffSize           Size of the source buffer
  548  *
  549  * @retval CPA_STATUS_SUCCESS         Function executed successfully
  550  * @retval CPA_STATUS_INVALID_PARAM   Invalid parameter passed in
  551  *
  552  *****************************************************************************/
  553 static CpaStatus
  554 dcCheckSourceData(CpaDcSessionHandle pSessionHandle,
  555                   CpaBufferList *pSrcBuff,
  556                   CpaBufferList *pDestBuff,
  557                   CpaDcRqResults *pResults,
  558                   CpaDcFlush flushFlag,
  559                   Cpa64U srcBuffSize,
  560                   CpaDcSkipData *skipData)
  561 {
  562         dc_session_desc_t *pSessionDesc = NULL;
  563 
  564         LAC_CHECK_NULL_PARAM(pSessionHandle);
  565         LAC_CHECK_NULL_PARAM(pSrcBuff);
  566         LAC_CHECK_NULL_PARAM(pDestBuff);
  567         LAC_CHECK_NULL_PARAM(pResults);
  568 
  569         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
  570         if (NULL == pSessionDesc) {
  571                 LAC_INVALID_PARAM_LOG("Session handle not as expected");
  572                 return CPA_STATUS_INVALID_PARAM;
  573         }
  574 
  575         if ((flushFlag < CPA_DC_FLUSH_NONE) ||
  576             (flushFlag > CPA_DC_FLUSH_FULL)) {
  577                 LAC_INVALID_PARAM_LOG("Invalid flushFlag value");
  578                 return CPA_STATUS_INVALID_PARAM;
  579         }
  580 
  581         if (pSrcBuff == pDestBuff) {
  582                 LAC_INVALID_PARAM_LOG("In place operation not supported");
  583                 return CPA_STATUS_INVALID_PARAM;
  584         }
  585 
  586         /* Compressing zero bytes is not supported for stateless sessions
  587          * for non Batch and Pack requests */
  588         if ((CPA_DC_STATELESS == pSessionDesc->sessState) &&
  589             (0 == srcBuffSize) && (NULL == skipData)) {
  590                 LAC_INVALID_PARAM_LOG(
  591                     "The source buffer size needs to be greater than "
  592                     "zero bytes for stateless sessions");
  593                 return CPA_STATUS_INVALID_PARAM;
  594         }
  595 
  596         if (srcBuffSize > DC_BUFFER_MAX_SIZE) {
  597                 LAC_INVALID_PARAM_LOG(
  598                     "The source buffer size needs to be less than or "
  599                     "equal to 2^32-1 bytes");
  600                 return CPA_STATUS_INVALID_PARAM;
  601         }
  602 
  603         return CPA_STATUS_SUCCESS;
  604 }
  605 
  606 /**
  607  *****************************************************************************
  608  * @ingroup Dc_DataCompression
  609  *      Check the compression or decompression function parameters.
  610  *
  611  * @description
  612  *      Check that all the parameters used for a Batch and Pack compression
  613  *      request are valid. This function essentially checks the destination
  614  *      buffer parameters and intermediate buffer parameters.
  615  *
  616  * @param[in]   pService              Pointer to the compression service
  617  * @param[in]   pSessionHandle        Session handle
  618  * @param[in]   pDestBuff             Pointer to buffer space allocated for
  619  *                                    output data
  620  * @param[in]   compDecomp            Direction of the operation
  621  *
  622  * @retval CPA_STATUS_SUCCESS         Function executed successfully
  623  * @retval CPA_STATUS_INVALID_PARAM   Invalid parameter passed in
  624  *
  625  *****************************************************************************/
  626 static CpaStatus
  627 dcCheckDestinationData(sal_compression_service_t *pService,
  628                        CpaDcSessionHandle pSessionHandle,
  629                        CpaBufferList *pDestBuff,
  630                        dc_request_dir_t compDecomp)
  631 {
  632         dc_session_desc_t *pSessionDesc = NULL;
  633         Cpa64U destBuffSize = 0;
  634 
  635         LAC_CHECK_NULL_PARAM(pSessionHandle);
  636         LAC_CHECK_NULL_PARAM(pDestBuff);
  637 
  638         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
  639         if (NULL == pSessionDesc) {
  640                 LAC_INVALID_PARAM_LOG("Session handle not as expected");
  641                 return CPA_STATUS_INVALID_PARAM;
  642         }
  643 
  644         if (LacBuffDesc_BufferListVerify(pDestBuff,
  645                                          &destBuffSize,
  646                                          LAC_NO_ALIGNMENT_SHIFT) !=
  647             CPA_STATUS_SUCCESS) {
  648                 LAC_INVALID_PARAM_LOG(
  649                     "Invalid destination buffer list parameter");
  650                 return CPA_STATUS_INVALID_PARAM;
  651         }
  652 
  653         if (destBuffSize > DC_BUFFER_MAX_SIZE) {
  654                 LAC_INVALID_PARAM_LOG(
  655                     "The destination buffer size needs to be less "
  656                     "than or equal to 2^32-1 bytes");
  657                 return CPA_STATUS_INVALID_PARAM;
  658         }
  659 
  660         if (CPA_TRUE == pSessionDesc->isDcDp) {
  661                 LAC_INVALID_PARAM_LOG(
  662                     "The session type should not be data plane");
  663                 return CPA_STATUS_INVALID_PARAM;
  664         }
  665 
  666         if (DC_COMPRESSION_REQUEST == compDecomp) {
  667                 if (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType) {
  668 
  669                         /* Check if intermediate buffers are supported */
  670                         if ((isDcGen2x(pService)) &&
  671                             ((0 == pService->pInterBuffPtrsArrayPhyAddr) ||
  672                              (NULL == pService->pInterBuffPtrsArray))) {
  673                                 LAC_LOG_ERROR(
  674                                     "No intermediate buffer defined for this instance "
  675                                     "- see cpaDcStartInstance");
  676                                 return CPA_STATUS_INVALID_PARAM;
  677                         }
  678 
  679                         /* Ensure that the destination buffer size is greater or
  680                          * equal to 128B */
  681                         if (destBuffSize < DC_DEST_BUFFER_DYN_MIN_SIZE) {
  682                                 LAC_INVALID_PARAM_LOG(
  683                                     "Destination buffer size should be "
  684                                     "greater or equal to 128B");
  685                                 return CPA_STATUS_INVALID_PARAM;
  686                         }
  687                 } else
  688                 {
  689                         /* Ensure that the destination buffer size is greater or
  690                          * equal to devices min output buff size */
  691                         if (destBuffSize <
  692                             pService->comp_device_data.minOutputBuffSize) {
  693                                 LAC_INVALID_PARAM_LOG1(
  694                                     "Destination buffer size should be "
  695                                     "greater or equal to %d bytes",
  696                                     pService->comp_device_data
  697                                         .minOutputBuffSize);
  698                                 return CPA_STATUS_INVALID_PARAM;
  699                         }
  700                 }
  701         } else {
  702                 /* Ensure that the destination buffer size is greater than
  703                  * 0 bytes */
  704                 if (destBuffSize < DC_DEST_BUFFER_DEC_MIN_SIZE) {
  705                         LAC_INVALID_PARAM_LOG(
  706                             "Destination buffer size should be "
  707                             "greater than 0 bytes");
  708                         return CPA_STATUS_INVALID_PARAM;
  709                 }
  710         }
  711         return CPA_STATUS_SUCCESS;
  712 }
  713 
  714 /**
  715  *****************************************************************************
  716  * @ingroup Dc_DataCompression
  717  *      Populate the compression request parameters
  718  *
  719  * @description
  720  *      This function will populate the compression request parameters
  721  *
  722  * @param[out]  pCompReqParams   Pointer to the compression request parameters
  723  * @param[in]   pCookie          Pointer to the compression cookie
  724  *
  725  *****************************************************************************/
  726 static void
  727 dcCompRequestParamsPopulate(icp_qat_fw_comp_req_params_t *pCompReqParams,
  728                             dc_compression_cookie_t *pCookie)
  729 {
  730         pCompReqParams->comp_len = pCookie->srcTotalDataLenInBytes;
  731         pCompReqParams->out_buffer_sz = pCookie->dstTotalDataLenInBytes;
  732 }
  733 
  734 /**
  735  *****************************************************************************
  736  * @ingroup Dc_DataCompression
  737  *      Create the requests for compression or decompression
  738  *
  739  * @description
  740  *      Create the requests for compression or decompression. This function
  741  *      will update the cookie will all required information.
  742  *
  743  * @param{out]  pCookie             Pointer to the compression cookie
  744  * @param[in]   pService            Pointer to the compression service
  745  * @param[in]   pSessionDesc        Pointer to the session descriptor
  746  * @param[in    pSessionHandle      Session handle
  747  * @param[in]   pSrcBuff            Pointer to data buffer for compression
  748  * @param[in]   pDestBuff           Pointer to buffer space for data after
  749  *                                  compression
  750  * @param[in]   pResults            Pointer to results structure
  751  * @param[in]   flushFlag           Indicates the type of flush to be
  752  *                                  performed
  753  * @param[in]   pOpData             Pointer to request information structure
  754  *                                  holding parameters for cpaDcCompress2
  755  *                                  and CpaDcDecompressData2
  756  * @param[in]   callbackTag         Pointer to the callback tag
  757  * @param[in]   compDecomp          Direction of the operation
  758  * @param[in]   compressAndVerify   Compress and Verify
  759  *
  760  * @retval CPA_STATUS_SUCCESS       Function executed successfully
  761  * @retval CPA_STATUS_INVALID_PARAM Invalid parameter passed in
  762  *
  763  *****************************************************************************/
  764 static CpaStatus
  765 dcCreateRequest(dc_compression_cookie_t *pCookie,
  766                 sal_compression_service_t *pService,
  767                 dc_session_desc_t *pSessionDesc,
  768                 CpaDcSessionHandle pSessionHandle,
  769                 CpaBufferList *pSrcBuff,
  770                 CpaBufferList *pDestBuff,
  771                 CpaDcRqResults *pResults,
  772                 CpaDcFlush flushFlag,
  773                 CpaDcOpData *pOpData,
  774                 void *callbackTag,
  775                 dc_request_dir_t compDecomp,
  776                 dc_cnv_mode_t cnvMode)
  777 {
  778         icp_qat_fw_comp_req_t *pMsg = NULL;
  779         icp_qat_fw_comp_req_params_t *pCompReqParams = NULL;
  780         Cpa64U srcAddrPhys = 0, dstAddrPhys = 0;
  781         Cpa64U srcTotalDataLenInBytes = 0, dstTotalDataLenInBytes = 0;
  782 
  783         Cpa32U rpCmdFlags = 0;
  784         Cpa8U sop = ICP_QAT_FW_COMP_SOP;
  785         Cpa8U eop = ICP_QAT_FW_COMP_EOP;
  786         Cpa8U bFinal = ICP_QAT_FW_COMP_NOT_BFINAL;
  787         Cpa8U crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY;
  788         Cpa8U cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV;
  789         Cpa8U cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY;
  790         CpaBoolean cnvErrorInjection = ICP_QAT_FW_COMP_NO_CNV_DFX;
  791         CpaBoolean integrityCrcCheck = CPA_FALSE;
  792         CpaStatus status = CPA_STATUS_SUCCESS;
  793         CpaDcFlush flush = CPA_DC_FLUSH_NONE;
  794         Cpa32U initial_adler = 1;
  795         Cpa32U initial_crc32 = 0;
  796         icp_qat_fw_comp_req_t *pReqCache = NULL;
  797 
  798         /* Write the buffer descriptors */
  799         status = LacBuffDesc_BufferListDescWriteAndGetSize(
  800             pSrcBuff,
  801             &srcAddrPhys,
  802             CPA_FALSE,
  803             &srcTotalDataLenInBytes,
  804             &(pService->generic_service_info));
  805         if (status != CPA_STATUS_SUCCESS) {
  806                 return status;
  807         }
  808 
  809         status = LacBuffDesc_BufferListDescWriteAndGetSize(
  810             pDestBuff,
  811             &dstAddrPhys,
  812             CPA_FALSE,
  813             &dstTotalDataLenInBytes,
  814             &(pService->generic_service_info));
  815         if (status != CPA_STATUS_SUCCESS) {
  816                 return status;
  817         }
  818 
  819         /* Populate the compression cookie */
  820         pCookie->dcInstance = pService;
  821         pCookie->pSessionHandle = pSessionHandle;
  822         pCookie->callbackTag = callbackTag;
  823         pCookie->pSessionDesc = pSessionDesc;
  824         pCookie->pDcOpData = pOpData;
  825         pCookie->pResults = pResults;
  826         pCookie->compDecomp = compDecomp;
  827         pCookie->pUserSrcBuff = NULL;
  828         pCookie->pUserDestBuff = NULL;
  829 
  830         /* Extract flush flag from either the opData or from the
  831          * parameter. Opdata have been introduce with APIs
  832          * cpaDcCompressData2 and cpaDcDecompressData2 */
  833         if (NULL != pOpData) {
  834                 flush = pOpData->flushFlag;
  835                 integrityCrcCheck = pOpData->integrityCrcCheck;
  836         } else {
  837                 flush = flushFlag;
  838         }
  839         pCookie->flushFlag = flush;
  840 
  841         /* The firmware expects the length in bytes for source and destination
  842          * to be Cpa32U parameters. However the total data length could be
  843          * bigger as allocated by the user. We ensure that this is not the case
  844          * in dcCheckSourceData and cast the values to Cpa32U here */
  845         pCookie->srcTotalDataLenInBytes = (Cpa32U)srcTotalDataLenInBytes;
  846         if ((isDcGen2x(pService)) && (DC_COMPRESSION_REQUEST == compDecomp) &&
  847             (CPA_DC_HT_FULL_DYNAMIC == pSessionDesc->huffType)) {
  848                 if (pService->minInterBuffSizeInBytes <
  849                     (Cpa32U)dstTotalDataLenInBytes) {
  850                         pCookie->dstTotalDataLenInBytes =
  851                             (Cpa32U)(pService->minInterBuffSizeInBytes);
  852                 } else {
  853                         pCookie->dstTotalDataLenInBytes =
  854                             (Cpa32U)dstTotalDataLenInBytes;
  855                 }
  856         } else
  857         {
  858                 pCookie->dstTotalDataLenInBytes =
  859                     (Cpa32U)dstTotalDataLenInBytes;
  860         }
  861 
  862         /* Device can not decompress an odd byte decompression request
  863          * if bFinal is not set
  864          */
  865         if (CPA_TRUE != pService->comp_device_data.oddByteDecompNobFinal) {
  866                 if ((CPA_DC_STATEFUL == pSessionDesc->sessState) &&
  867                     (CPA_DC_FLUSH_FINAL != flushFlag) &&
  868                     (DC_DECOMPRESSION_REQUEST == compDecomp) &&
  869                     (pCookie->srcTotalDataLenInBytes & 0x1)) {
  870                         pCookie->srcTotalDataLenInBytes--;
  871                 }
  872         }
  873         /* Device can not decompress odd byte interim requests */
  874         if (CPA_TRUE != pService->comp_device_data.oddByteDecompInterim) {
  875                 if ((CPA_DC_STATEFUL == pSessionDesc->sessState) &&
  876                     (CPA_DC_FLUSH_FINAL != flushFlag) &&
  877                     (CPA_DC_FLUSH_FULL != flushFlag) &&
  878                     (DC_DECOMPRESSION_REQUEST == compDecomp) &&
  879                     (pCookie->srcTotalDataLenInBytes & 0x1)) {
  880                         pCookie->srcTotalDataLenInBytes--;
  881                 }
  882         }
  883 
  884         pMsg = (icp_qat_fw_comp_req_t *)&pCookie->request;
  885 
  886         if (DC_COMPRESSION_REQUEST == compDecomp) {
  887                 pReqCache = &(pSessionDesc->reqCacheComp);
  888         } else {
  889                 pReqCache = &(pSessionDesc->reqCacheDecomp);
  890         }
  891 
  892         /* Fills the msg from the template cached in the session descriptor */
  893         memcpy((void *)pMsg,
  894                (void *)(pReqCache),
  895                LAC_QAT_DC_REQ_SZ_LW * LAC_LONG_WORD_IN_BYTES);
  896 
  897         if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
  898                 initial_adler = 1;
  899                 initial_crc32 = 0;
  900 
  901                 if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
  902                         pSessionDesc->previousChecksum = initial_adler;
  903                 } else {
  904                         pSessionDesc->previousChecksum = initial_crc32;
  905                 }
  906         } else if (CPA_DC_STATELESS == pSessionDesc->sessState) {
  907                 pSessionDesc->previousChecksum = pResults->checksum;
  908 
  909                 if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
  910                         initial_adler = pSessionDesc->previousChecksum;
  911                 } else {
  912                         initial_crc32 = pSessionDesc->previousChecksum;
  913                 }
  914         }
  915 
  916         /* Backup source and destination buffer addresses,
  917          * CRC calculations both for CNV and translator overflow
  918          * will be performed on them in the callback function.
  919          */
  920         pCookie->pUserSrcBuff = pSrcBuff;
  921         pCookie->pUserDestBuff = pDestBuff;
  922 
  923         /*
  924          * Due to implementation of CNV support and need for backwards
  925          * compatibility certain fields in the request and response structs had
  926          * been changed, moved or placed in unions cnvMode flag signifies fields
  927          * to be selected from req/res
  928          *
  929          * Doing extended crc checks makes sense only when we want to do the
  930          * actual CNV
  931          */
  932         if (CPA_TRUE == pService->generic_service_info.integrityCrcCheck &&
  933             CPA_TRUE == integrityCrcCheck) {
  934                 pMsg->comp_pars.crc.crc_data_addr =
  935                     pSessionDesc->physDataIntegrityCrcs;
  936                 crcMode = ICP_QAT_FW_COMP_CRC_MODE_E2E;
  937         } else {
  938                 /* Legacy request structure */
  939                 pMsg->comp_pars.crc.legacy.initial_adler = initial_adler;
  940                 pMsg->comp_pars.crc.legacy.initial_crc32 = initial_crc32;
  941                 crcMode = ICP_QAT_FW_COMP_CRC_MODE_LEGACY;
  942         }
  943 
  944         /* Populate the cmdFlags */
  945         if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
  946                 pSessionDesc->previousRequestType = pSessionDesc->requestType;
  947 
  948                 if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
  949                         /* Update the request type for following requests */
  950                         pSessionDesc->requestType = DC_REQUEST_SUBSEQUENT;
  951 
  952                         /* Reinitialise the cumulative amount of consumed bytes
  953                          */
  954                         pSessionDesc->cumulativeConsumedBytes = 0;
  955 
  956                         if (DC_COMPRESSION_REQUEST == compDecomp) {
  957                                 pSessionDesc->isSopForCompressionProcessed =
  958                                     CPA_TRUE;
  959                         } else if (DC_DECOMPRESSION_REQUEST == compDecomp) {
  960                                 pSessionDesc->isSopForDecompressionProcessed =
  961                                     CPA_TRUE;
  962                         }
  963                 } else {
  964                         if (DC_COMPRESSION_REQUEST == compDecomp) {
  965                                 if (CPA_TRUE ==
  966                                     pSessionDesc
  967                                         ->isSopForCompressionProcessed) {
  968                                         sop = ICP_QAT_FW_COMP_NOT_SOP;
  969                                 } else {
  970                                         pSessionDesc
  971                                             ->isSopForCompressionProcessed =
  972                                             CPA_TRUE;
  973                                 }
  974                         } else if (DC_DECOMPRESSION_REQUEST == compDecomp) {
  975                                 if (CPA_TRUE ==
  976                                     pSessionDesc
  977                                         ->isSopForDecompressionProcessed) {
  978                                         sop = ICP_QAT_FW_COMP_NOT_SOP;
  979                                 } else {
  980                                         pSessionDesc
  981                                             ->isSopForDecompressionProcessed =
  982                                             CPA_TRUE;
  983                                 }
  984                         }
  985                 }
  986 
  987                 if ((CPA_DC_FLUSH_FINAL == flush) ||
  988                     (CPA_DC_FLUSH_FULL == flush)) {
  989                         /* Update the request type for following requests */
  990                         pSessionDesc->requestType = DC_REQUEST_FIRST;
  991                 } else {
  992                         eop = ICP_QAT_FW_COMP_NOT_EOP;
  993                 }
  994         } else {
  995                 if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
  996                         /* Reinitialise the cumulative amount of consumed bytes
  997                          */
  998                         pSessionDesc->cumulativeConsumedBytes = 0;
  999                 }
 1000         }
 1001 
 1002         /* (LW 14 - 15) */
 1003         pCompReqParams = &(pMsg->comp_pars);
 1004         dcCompRequestParamsPopulate(pCompReqParams, pCookie);
 1005         if (CPA_DC_FLUSH_FINAL == flush) {
 1006                 bFinal = ICP_QAT_FW_COMP_BFINAL;
 1007         }
 1008 
 1009         switch (cnvMode) {
 1010         case DC_CNVNR:
 1011                 cnvRecovery = ICP_QAT_FW_COMP_CNV_RECOVERY;
 1012         /* Fall through is intended here, because for CNVNR
 1013          * cnvDecompReq also needs to be set */
 1014         case DC_CNV:
 1015                 cnvDecompReq = ICP_QAT_FW_COMP_CNV;
 1016                 if (isDcGen4x(pService)) {
 1017                         cnvErrorInjection = pSessionDesc->cnvErrorInjection;
 1018                 }
 1019                 break;
 1020         case DC_NO_CNV:
 1021                 cnvDecompReq = ICP_QAT_FW_COMP_NO_CNV;
 1022                 cnvRecovery = ICP_QAT_FW_COMP_NO_CNV_RECOVERY;
 1023                 break;
 1024         }
 1025 
 1026         /* LW 18 */
 1027         rpCmdFlags = ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(sop,
 1028                                                            eop,
 1029                                                            bFinal,
 1030                                                            cnvDecompReq,
 1031                                                            cnvRecovery,
 1032                                                            cnvErrorInjection,
 1033                                                            crcMode);
 1034 
 1035         pMsg->comp_pars.req_par_flags = rpCmdFlags;
 1036 
 1037         /* Populates the QAT common request middle part of the message
 1038          * (LW 6 to 11) */
 1039         SalQatMsg_CmnMidWrite((icp_qat_fw_la_bulk_req_t *)pMsg,
 1040                               pCookie,
 1041                               DC_DEFAULT_QAT_PTR_TYPE,
 1042                               srcAddrPhys,
 1043                               dstAddrPhys,
 1044                               0,
 1045                               0);
 1046 
 1047         return CPA_STATUS_SUCCESS;
 1048 }
 1049 
 1050 /**
 1051  *****************************************************************************
 1052  * @ingroup Dc_DataCompression
 1053  *      Send a compression request to QAT
 1054  *
 1055  * @description
 1056  *      Send the requests for compression or decompression to QAT
 1057  *
 1058  * @param{in]   pCookie               Pointer to the compression cookie
 1059  * @param[in]   pService              Pointer to the compression service
 1060  * @param[in]   pSessionDesc          Pointer to the session descriptor
 1061  * @param[in]   compDecomp            Direction of the operation
 1062  *
 1063  * @retval CPA_STATUS_SUCCESS         Function executed successfully
 1064  * @retval CPA_STATUS_INVALID_PARAM   Invalid parameter passed in
 1065  *
 1066  *****************************************************************************/
 1067 static CpaStatus
 1068 dcSendRequest(dc_compression_cookie_t *pCookie,
 1069               sal_compression_service_t *pService,
 1070               dc_session_desc_t *pSessionDesc,
 1071               dc_request_dir_t compDecomp)
 1072 {
 1073         CpaStatus status = CPA_STATUS_SUCCESS;
 1074 
 1075         /* Send to QAT */
 1076         status = icp_adf_transPutMsg(pService->trans_handle_compression_tx,
 1077                                      (void *)&(pCookie->request),
 1078                                      LAC_QAT_DC_REQ_SZ_LW);
 1079 
 1080         if ((CPA_DC_STATEFUL == pSessionDesc->sessState) &&
 1081             (CPA_STATUS_RETRY == status)) {
 1082                 /* reset requestType after receiving an retry on
 1083                  * the stateful request */
 1084                 pSessionDesc->requestType = pSessionDesc->previousRequestType;
 1085         }
 1086 
 1087         return status;
 1088 }
 1089 
 1090 /**
 1091  *****************************************************************************
 1092  * @ingroup Dc_DataCompression
 1093  *      Process the synchronous and asynchronous case for compression or
 1094  *      decompression
 1095  *
 1096  * @description
 1097  *      Process the synchronous and asynchronous case for compression or
 1098  *      decompression. This function will then create and send the request to
 1099  *      the firmware.
 1100  *
 1101  * @param[in]   pService            Pointer to the compression service
 1102  * @param[in]   pSessionDesc        Pointer to the session descriptor
 1103  * @param[in]   dcInstance          Instance handle derived from discovery
 1104  *                                  functions
 1105  * @param[in]   pSessionHandle      Session handle
 1106  * @param[in]   numRequests         Number of operations in the batch request
 1107  * @param[in]   pBatchOpData        Address of the list of jobs to be processed
 1108  * @param[in]   pSrcBuff            Pointer to data buffer for compression
 1109  * @param[in]   pDestBuff           Pointer to buffer space for data after
 1110  *                                  compression
 1111  * @param[in]   pResults            Pointer to results structure
 1112  * @param[in]   flushFlag           Indicates the type of flush to be
 1113  *                                  performed
 1114  * @param[in]   pOpData             Pointer to request information structure
 1115  *                                  holding parameters for cpaDcCompress2 and
 1116  *                                  CpaDcDecompressData2
 1117  * @param[in]   callbackTag         Pointer to the callback tag
 1118  * @param[in]   compDecomp          Direction of the operation
 1119  * @param[in]   isAsyncMode         Used to know if synchronous or asynchronous
 1120  *                                  mode
 1121  * @param[in]   cnvMode             CNV Mode
 1122  *
 1123  * @retval CPA_STATUS_SUCCESS       Function executed successfully
 1124  * @retval CPA_STATUS_RETRY         Retry operation
 1125  * @retval CPA_STATUS_FAIL          Function failed
 1126  * @retval CPA_STATUS_RESOURCE      Resource error
 1127  *
 1128  *****************************************************************************/
 1129 static CpaStatus
 1130 dcCompDecompData(sal_compression_service_t *pService,
 1131                  dc_session_desc_t *pSessionDesc,
 1132                  CpaInstanceHandle dcInstance,
 1133                  CpaDcSessionHandle pSessionHandle,
 1134                  CpaBufferList *pSrcBuff,
 1135                  CpaBufferList *pDestBuff,
 1136                  CpaDcRqResults *pResults,
 1137                  CpaDcFlush flushFlag,
 1138                  CpaDcOpData *pOpData,
 1139                  void *callbackTag,
 1140                  dc_request_dir_t compDecomp,
 1141                  CpaBoolean isAsyncMode,
 1142                  dc_cnv_mode_t cnvMode)
 1143 {
 1144         CpaStatus status = CPA_STATUS_SUCCESS;
 1145         dc_compression_cookie_t *pCookie = NULL;
 1146 
 1147         if ((LacSync_GenWakeupSyncCaller == pSessionDesc->pCompressionCb) &&
 1148             isAsyncMode == CPA_TRUE) {
 1149                 lac_sync_op_data_t *pSyncCallbackData = NULL;
 1150 
 1151                 status = LacSync_CreateSyncCookie(&pSyncCallbackData);
 1152 
 1153                 if (CPA_STATUS_SUCCESS == status) {
 1154                         status = dcCompDecompData(pService,
 1155                                                   pSessionDesc,
 1156                                                   dcInstance,
 1157                                                   pSessionHandle,
 1158                                                   pSrcBuff,
 1159                                                   pDestBuff,
 1160                                                   pResults,
 1161                                                   flushFlag,
 1162                                                   pOpData,
 1163                                                   pSyncCallbackData,
 1164                                                   compDecomp,
 1165                                                   CPA_FALSE,
 1166                                                   cnvMode);
 1167                 } else {
 1168                         return status;
 1169                 }
 1170 
 1171                 if (CPA_STATUS_SUCCESS == status) {
 1172                         CpaStatus syncStatus = CPA_STATUS_SUCCESS;
 1173 
 1174                         syncStatus =
 1175                             LacSync_WaitForCallback(pSyncCallbackData,
 1176                                                     DC_SYNC_CALLBACK_TIMEOUT,
 1177                                                     &status,
 1178                                                     NULL);
 1179 
 1180                         /* If callback doesn't come back */
 1181                         if (CPA_STATUS_SUCCESS != syncStatus) {
 1182                                 if (DC_COMPRESSION_REQUEST == compDecomp) {
 1183                                         COMPRESSION_STAT_INC(
 1184                                             numCompCompletedErrors, pService);
 1185                                 } else {
 1186                                         COMPRESSION_STAT_INC(
 1187                                             numDecompCompletedErrors, pService);
 1188                                 }
 1189                                 LAC_LOG_ERROR("Callback timed out");
 1190                                 status = syncStatus;
 1191                         }
 1192                 } else {
 1193                         /* As the Request was not sent the Callback will never
 1194                          * be called, so need to indicate that we're finished
 1195                          * with cookie so it can be destroyed. */
 1196                         LacSync_SetSyncCookieComplete(pSyncCallbackData);
 1197                 }
 1198 
 1199                 LacSync_DestroySyncCookie(&pSyncCallbackData);
 1200                 return status;
 1201         }
 1202 
 1203         /* Allocate the compression cookie
 1204          * The memory is freed in callback or in sendRequest if an error occurs
 1205          */
 1206         pCookie = (dc_compression_cookie_t *)Lac_MemPoolEntryAlloc(
 1207             pService->compression_mem_pool);
 1208         if (NULL == pCookie) {
 1209                 LAC_LOG_ERROR("Cannot get mem pool entry for compression");
 1210                 status = CPA_STATUS_RESOURCE;
 1211         } else if ((void *)CPA_STATUS_RETRY == pCookie) {
 1212                 pCookie = NULL;
 1213                 status = CPA_STATUS_RETRY;
 1214         }
 1215 
 1216         if (CPA_STATUS_SUCCESS == status) {
 1217                 status = dcCreateRequest(pCookie,
 1218                                          pService,
 1219                                          pSessionDesc,
 1220                                          pSessionHandle,
 1221                                          pSrcBuff,
 1222                                          pDestBuff,
 1223                                          pResults,
 1224                                          flushFlag,
 1225                                          pOpData,
 1226                                          callbackTag,
 1227                                          compDecomp,
 1228                                          cnvMode);
 1229         }
 1230 
 1231         if (CPA_STATUS_SUCCESS == status) {
 1232                 /* Increment number of pending callbacks for session */
 1233                 if (CPA_DC_STATELESS == pSessionDesc->sessState) {
 1234                         qatUtilsAtomicInc(
 1235                             &(pSessionDesc->pendingStatelessCbCount));
 1236                 }
 1237                 status =
 1238                     dcSendRequest(pCookie, pService, pSessionDesc, compDecomp);
 1239         }
 1240 
 1241         if (CPA_STATUS_SUCCESS == status) {
 1242                 if (DC_COMPRESSION_REQUEST == compDecomp) {
 1243                         COMPRESSION_STAT_INC(numCompRequests, pService);
 1244                 } else {
 1245                         COMPRESSION_STAT_INC(numDecompRequests, pService);
 1246                 }
 1247         } else {
 1248                 if (DC_COMPRESSION_REQUEST == compDecomp) {
 1249                         COMPRESSION_STAT_INC(numCompRequestsErrors, pService);
 1250                 } else {
 1251                         COMPRESSION_STAT_INC(numDecompRequestsErrors, pService);
 1252                 }
 1253 
 1254                 /* Decrement number of pending callbacks for session */
 1255                 if (CPA_DC_STATELESS == pSessionDesc->sessState) {
 1256                         qatUtilsAtomicDec(
 1257                             &(pSessionDesc->pendingStatelessCbCount));
 1258                 } else {
 1259                         qatUtilsAtomicDec(
 1260                             &(pSessionDesc->pendingStatefulCbCount));
 1261                 }
 1262 
 1263                 /* Free the memory pool */
 1264                 if (NULL != pCookie) {
 1265                         if (status != CPA_STATUS_UNSUPPORTED) {
 1266                                 /* Free the memory pool */
 1267                                 Lac_MemPoolEntryFree(pCookie);
 1268                                 pCookie = NULL;
 1269                         }
 1270                 }
 1271         }
 1272 
 1273         return status;
 1274 }
 1275 
 1276 /**
 1277  *****************************************************************************
 1278  * @ingroup Dc_DataCompression
 1279  *      Handle zero length compression or decompression requests
 1280  *
 1281  * @description
 1282  *      Handle zero length compression or decompression requests
 1283  *
 1284  * @param[in]   pService              Pointer to the compression service
 1285  * @param[in]   pSessionDesc          Pointer to the session descriptor
 1286  * @param[in]   pResults              Pointer to results structure
 1287  * @param[in]   flushFlag             Indicates the type of flush to be
 1288  *                                    performed
 1289  * @param[in]   callbackTag           User supplied value to help correlate
 1290  *                                    the callback with its associated request
 1291  * @param[in]   compDecomp            Direction of the operation
 1292  *
 1293  * @retval CPA_TRUE                   Zero length SOP or MOP processed
 1294  * @retval CPA_FALSE                  Zero length EOP
 1295  *
 1296  *****************************************************************************/
 1297 static CpaStatus
 1298 dcZeroLengthRequests(sal_compression_service_t *pService,
 1299                      dc_session_desc_t *pSessionDesc,
 1300                      CpaDcRqResults *pResults,
 1301                      CpaDcFlush flushFlag,
 1302                      void *callbackTag,
 1303                      dc_request_dir_t compDecomp)
 1304 {
 1305         CpaBoolean status = CPA_FALSE;
 1306         CpaDcCallbackFn pCbFunc = pSessionDesc->pCompressionCb;
 1307 
 1308         if (DC_REQUEST_FIRST == pSessionDesc->requestType) {
 1309                 /* Reinitialise the cumulative amount of consumed bytes */
 1310                 pSessionDesc->cumulativeConsumedBytes = 0;
 1311 
 1312                 /* Zero length SOP */
 1313                 if (CPA_DC_ADLER32 == pSessionDesc->checksumType) {
 1314                         pResults->checksum = 1;
 1315                 } else {
 1316                         pResults->checksum = 0;
 1317                 }
 1318 
 1319                 status = CPA_TRUE;
 1320         } else if ((CPA_DC_FLUSH_NONE == flushFlag) ||
 1321                    (CPA_DC_FLUSH_SYNC == flushFlag)) {
 1322                 /* Zero length MOP */
 1323                 pResults->checksum = pSessionDesc->previousChecksum;
 1324                 status = CPA_TRUE;
 1325         }
 1326 
 1327         if (CPA_TRUE == status) {
 1328                 pResults->status = CPA_DC_OK;
 1329                 pResults->produced = 0;
 1330                 pResults->consumed = 0;
 1331 
 1332                 /* Increment statistics */
 1333                 if (DC_COMPRESSION_REQUEST == compDecomp) {
 1334                         COMPRESSION_STAT_INC(numCompRequests, pService);
 1335                         COMPRESSION_STAT_INC(numCompCompleted, pService);
 1336                 } else {
 1337                         COMPRESSION_STAT_INC(numDecompRequests, pService);
 1338                         COMPRESSION_STAT_INC(numDecompCompleted, pService);
 1339                 }
 1340 
 1341                 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
 1342 
 1343                 if ((NULL != pCbFunc) &&
 1344                     (LacSync_GenWakeupSyncCaller != pCbFunc)) {
 1345                         pCbFunc(callbackTag, CPA_STATUS_SUCCESS);
 1346                 }
 1347 
 1348                 return CPA_TRUE;
 1349         }
 1350 
 1351         return CPA_FALSE;
 1352 }
 1353 
 1354 static CpaStatus
 1355 dcParamCheck(CpaInstanceHandle dcInstance,
 1356              CpaDcSessionHandle pSessionHandle,
 1357              sal_compression_service_t *pService,
 1358              CpaBufferList *pSrcBuff,
 1359              CpaBufferList *pDestBuff,
 1360              CpaDcRqResults *pResults,
 1361              dc_session_desc_t *pSessionDesc,
 1362              CpaDcFlush flushFlag,
 1363              Cpa64U srcBuffSize)
 1364 {
 1365 
 1366         if (dcCheckSourceData(pSessionHandle,
 1367                               pSrcBuff,
 1368                               pDestBuff,
 1369                               pResults,
 1370                               flushFlag,
 1371                               srcBuffSize,
 1372                               NULL) != CPA_STATUS_SUCCESS) {
 1373                 return CPA_STATUS_INVALID_PARAM;
 1374         }
 1375         if (dcCheckDestinationData(
 1376                 pService, pSessionHandle, pDestBuff, DC_COMPRESSION_REQUEST) !=
 1377             CPA_STATUS_SUCCESS) {
 1378                 return CPA_STATUS_INVALID_PARAM;
 1379         }
 1380         if (CPA_DC_DIR_DECOMPRESS == pSessionDesc->sessDirection) {
 1381                 LAC_INVALID_PARAM_LOG("Invalid sessDirection value");
 1382                 return CPA_STATUS_INVALID_PARAM;
 1383         }
 1384         return CPA_STATUS_SUCCESS;
 1385 }
 1386 
 1387 CpaStatus
 1388 cpaDcCompressData(CpaInstanceHandle dcInstance,
 1389                   CpaDcSessionHandle pSessionHandle,
 1390                   CpaBufferList *pSrcBuff,
 1391                   CpaBufferList *pDestBuff,
 1392                   CpaDcRqResults *pResults,
 1393                   CpaDcFlush flushFlag,
 1394                   void *callbackTag)
 1395 {
 1396         sal_compression_service_t *pService = NULL;
 1397         dc_session_desc_t *pSessionDesc = NULL;
 1398         CpaInstanceHandle insHandle = NULL;
 1399         Cpa64U srcBuffSize = 0;
 1400 
 1401 
 1402         if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
 1403                 insHandle = dcGetFirstHandle();
 1404         } else {
 1405                 insHandle = dcInstance;
 1406         }
 1407 
 1408         pService = (sal_compression_service_t *)insHandle;
 1409 
 1410         LAC_CHECK_NULL_PARAM(insHandle);
 1411         LAC_CHECK_NULL_PARAM(pSessionHandle);
 1412 
 1413         /* Check if SAL is initialised otherwise return an error */
 1414         SAL_RUNNING_CHECK(insHandle);
 1415 
 1416         /* This check is outside the parameter checking as it is needed to
 1417          * manage zero length requests */
 1418         if (LacBuffDesc_BufferListVerifyNull(pSrcBuff,
 1419                                              &srcBuffSize,
 1420                                              LAC_NO_ALIGNMENT_SHIFT) !=
 1421             CPA_STATUS_SUCCESS) {
 1422                 LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter");
 1423                 return CPA_STATUS_INVALID_PARAM;
 1424         }
 1425 
 1426         /* Ensure this is a compression instance */
 1427         SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
 1428 
 1429         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
 1430         if (CPA_STATUS_SUCCESS !=
 1431             dcParamCheck(insHandle,
 1432                          pSessionHandle,
 1433                          pService,
 1434                          pSrcBuff,
 1435                          pDestBuff,
 1436                          pResults,
 1437                          pSessionDesc,
 1438                          flushFlag,
 1439                          srcBuffSize)) {
 1440                 return CPA_STATUS_INVALID_PARAM;
 1441         }
 1442         if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
 1443                 LAC_INVALID_PARAM_LOG(
 1444                     "Invalid session state, stateful sessions "
 1445                     "are not supported");
 1446                 return CPA_STATUS_UNSUPPORTED;
 1447         }
 1448 
 1449         if (!(pService->generic_service_info.dcExtendedFeatures &
 1450               DC_CNV_EXTENDED_CAPABILITY)) {
 1451                 LAC_INVALID_PARAM_LOG(
 1452                     "CompressAndVerify feature not supported");
 1453                 return CPA_STATUS_UNSUPPORTED;
 1454         }
 1455 
 1456         if (!(pService->generic_service_info.dcExtendedFeatures &
 1457               DC_CNVNR_EXTENDED_CAPABILITY)) {
 1458                 LAC_INVALID_PARAM_LOG(
 1459                     "CompressAndVerifyAndRecovery feature not supported");
 1460                 return CPA_STATUS_UNSUPPORTED;
 1461         }
 1462 
 1463         return dcCompDecompData(pService,
 1464                                 pSessionDesc,
 1465                                 insHandle,
 1466                                 pSessionHandle,
 1467                                 pSrcBuff,
 1468                                 pDestBuff,
 1469                                 pResults,
 1470                                 flushFlag,
 1471                                 NULL,
 1472                                 callbackTag,
 1473                                 DC_COMPRESSION_REQUEST,
 1474                                 CPA_TRUE,
 1475                                 DC_CNVNR);
 1476 }
 1477 
 1478 CpaStatus
 1479 cpaDcCompressData2(CpaInstanceHandle dcInstance,
 1480                    CpaDcSessionHandle pSessionHandle,
 1481                    CpaBufferList *pSrcBuff,
 1482                    CpaBufferList *pDestBuff,
 1483                    CpaDcOpData *pOpData,
 1484                    CpaDcRqResults *pResults,
 1485                    void *callbackTag)
 1486 {
 1487         sal_compression_service_t *pService = NULL;
 1488         dc_session_desc_t *pSessionDesc = NULL;
 1489         CpaInstanceHandle insHandle = NULL;
 1490         Cpa64U srcBuffSize = 0;
 1491         dc_cnv_mode_t cnvMode = DC_NO_CNV;
 1492 
 1493         LAC_CHECK_NULL_PARAM(pOpData);
 1494 
 1495         if (((CPA_TRUE != pOpData->compressAndVerify) &&
 1496              (CPA_FALSE != pOpData->compressAndVerify)) ||
 1497             ((CPA_FALSE != pOpData->compressAndVerifyAndRecover) &&
 1498              (CPA_TRUE != pOpData->compressAndVerifyAndRecover))) {
 1499                 return CPA_STATUS_INVALID_PARAM;
 1500         }
 1501 
 1502         if ((CPA_FALSE == pOpData->compressAndVerify) &&
 1503             (CPA_TRUE == pOpData->compressAndVerifyAndRecover)) {
 1504                 return CPA_STATUS_INVALID_PARAM;
 1505         }
 1506 
 1507 
 1508         if ((CPA_TRUE == pOpData->compressAndVerify) &&
 1509             (CPA_TRUE == pOpData->compressAndVerifyAndRecover) &&
 1510             (CPA_FALSE == pOpData->integrityCrcCheck)) {
 1511                 return cpaDcCompressData(dcInstance,
 1512                                          pSessionHandle,
 1513                                          pSrcBuff,
 1514                                          pDestBuff,
 1515                                          pResults,
 1516                                          pOpData->flushFlag,
 1517                                          callbackTag);
 1518         }
 1519 
 1520         if (CPA_FALSE == pOpData->compressAndVerify) {
 1521                 LAC_INVALID_PARAM_LOG(
 1522                     "Data compression without verification not allowed");
 1523                 return CPA_STATUS_UNSUPPORTED;
 1524         }
 1525 
 1526 
 1527         if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
 1528                 insHandle = dcGetFirstHandle();
 1529         } else {
 1530                 insHandle = dcInstance;
 1531         }
 1532 
 1533         pService = (sal_compression_service_t *)insHandle;
 1534 
 1535         LAC_CHECK_NULL_PARAM(insHandle);
 1536         LAC_CHECK_NULL_PARAM(pSessionHandle);
 1537         LAC_CHECK_NULL_PARAM(pOpData);
 1538 
 1539         /* Check if SAL is initialised otherwise return an error */
 1540         SAL_RUNNING_CHECK(insHandle);
 1541 
 1542         /* This check is outside the parameter checking as it is needed to
 1543          * manage zero length requests */
 1544         if (LacBuffDesc_BufferListVerifyNull(pSrcBuff,
 1545                                              &srcBuffSize,
 1546                                              LAC_NO_ALIGNMENT_SHIFT) !=
 1547             CPA_STATUS_SUCCESS) {
 1548                 LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter");
 1549                 return CPA_STATUS_INVALID_PARAM;
 1550         }
 1551 
 1552         /* Ensure this is a compression instance */
 1553         SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
 1554 
 1555         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
 1556 
 1557         if (CPA_TRUE == pOpData->compressAndVerify &&
 1558             CPA_DC_STATEFUL == pSessionDesc->sessState) {
 1559                 LAC_INVALID_PARAM_LOG(
 1560                     "Invalid session state, stateful sessions "
 1561                     "not supported with CNV");
 1562                 return CPA_STATUS_UNSUPPORTED;
 1563         }
 1564 
 1565         if (!(pService->generic_service_info.dcExtendedFeatures &
 1566               DC_CNV_EXTENDED_CAPABILITY) &&
 1567             (CPA_TRUE == pOpData->compressAndVerify)) {
 1568                 LAC_INVALID_PARAM_LOG(
 1569                     "CompressAndVerify feature not supported");
 1570                 return CPA_STATUS_UNSUPPORTED;
 1571         }
 1572 
 1573         if (CPA_STATUS_SUCCESS !=
 1574             dcParamCheck(insHandle,
 1575                          pSessionHandle,
 1576                          pService,
 1577                          pSrcBuff,
 1578                          pDestBuff,
 1579                          pResults,
 1580                          pSessionDesc,
 1581                          pOpData->flushFlag,
 1582                          srcBuffSize)) {
 1583                 return CPA_STATUS_INVALID_PARAM;
 1584         }
 1585         if (CPA_STATUS_SUCCESS != dcCheckOpData(pService, pOpData)) {
 1586                 return CPA_STATUS_INVALID_PARAM;
 1587         }
 1588         if (CPA_TRUE != pOpData->compressAndVerify) {
 1589                 if (srcBuffSize > DC_COMP_MAX_BUFF_SIZE) {
 1590                         LAC_LOG_ERROR(
 1591                             "Compression payload greater than 64KB is "
 1592                             "unsupported, when CnV is disabled\n");
 1593                         return CPA_STATUS_UNSUPPORTED;
 1594                 }
 1595         }
 1596 
 1597         if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
 1598                 /* Lock the session to check if there are in-flight stateful
 1599                  * requests */
 1600                 LAC_SPINLOCK(&(pSessionDesc->sessionLock));
 1601 
 1602                 /* Check if there is already one in-flight stateful request */
 1603                 if (0 !=
 1604                     qatUtilsAtomicGet(
 1605                         &(pSessionDesc->pendingStatefulCbCount))) {
 1606                         LAC_LOG_ERROR(
 1607                             "Only one in-flight stateful request supported");
 1608                         LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
 1609                         return CPA_STATUS_RETRY;
 1610                 }
 1611 
 1612                 if (0 == srcBuffSize) {
 1613                         if (CPA_TRUE ==
 1614                             dcZeroLengthRequests(pService,
 1615                                                  pSessionDesc,
 1616                                                  pResults,
 1617                                                  pOpData->flushFlag,
 1618                                                  callbackTag,
 1619                                                  DC_COMPRESSION_REQUEST)) {
 1620                                 return CPA_STATUS_SUCCESS;
 1621                         }
 1622                 }
 1623 
 1624                 qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
 1625                 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
 1626         }
 1627 
 1628         if (CPA_TRUE == pOpData->compressAndVerify) {
 1629                 cnvMode = DC_CNV;
 1630         }
 1631 
 1632         return dcCompDecompData(pService,
 1633                                 pSessionDesc,
 1634                                 insHandle,
 1635                                 pSessionHandle,
 1636                                 pSrcBuff,
 1637                                 pDestBuff,
 1638                                 pResults,
 1639                                 pOpData->flushFlag,
 1640                                 pOpData,
 1641                                 callbackTag,
 1642                                 DC_COMPRESSION_REQUEST,
 1643                                 CPA_TRUE,
 1644                                 cnvMode);
 1645 }
 1646 
 1647 static CpaStatus
 1648 dcDecompressDataCheck(CpaInstanceHandle insHandle,
 1649                       CpaDcSessionHandle pSessionHandle,
 1650                       CpaBufferList *pSrcBuff,
 1651                       CpaBufferList *pDestBuff,
 1652                       CpaDcRqResults *pResults,
 1653                       CpaDcFlush flushFlag,
 1654                       Cpa64U *srcBufferSize)
 1655 {
 1656         sal_compression_service_t *pService = NULL;
 1657         dc_session_desc_t *pSessionDesc = NULL;
 1658         Cpa64U srcBuffSize = 0;
 1659 
 1660         pService = (sal_compression_service_t *)insHandle;
 1661 
 1662         LAC_CHECK_NULL_PARAM(insHandle);
 1663 
 1664         /* Check if SAL is initialised otherwise return an error */
 1665         SAL_RUNNING_CHECK(insHandle);
 1666 
 1667         /* This check is outside the parameter checking as it is needed to
 1668          * manage zero length requests */
 1669         if (LacBuffDesc_BufferListVerifyNull(pSrcBuff,
 1670                                              &srcBuffSize,
 1671                                              LAC_NO_ALIGNMENT_SHIFT) !=
 1672             CPA_STATUS_SUCCESS) {
 1673                 LAC_INVALID_PARAM_LOG("Invalid source buffer list parameter");
 1674                 return CPA_STATUS_INVALID_PARAM;
 1675         }
 1676 
 1677         /* Ensure this is a compression instance */
 1678         SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
 1679 
 1680         if (dcCheckSourceData(pSessionHandle,
 1681                               pSrcBuff,
 1682                               pDestBuff,
 1683                               pResults,
 1684                               flushFlag,
 1685                               srcBuffSize,
 1686                               NULL) != CPA_STATUS_SUCCESS) {
 1687                 return CPA_STATUS_INVALID_PARAM;
 1688         }
 1689         if (dcCheckDestinationData(pService,
 1690                                    pSessionHandle,
 1691                                    pDestBuff,
 1692                                    DC_DECOMPRESSION_REQUEST) !=
 1693             CPA_STATUS_SUCCESS) {
 1694                 return CPA_STATUS_INVALID_PARAM;
 1695         }
 1696         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
 1697 
 1698         if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
 1699                 LAC_INVALID_PARAM_LOG("Invalid sessDirection value");
 1700                 return CPA_STATUS_INVALID_PARAM;
 1701         }
 1702 
 1703 
 1704         *srcBufferSize = srcBuffSize;
 1705 
 1706         return CPA_STATUS_SUCCESS;
 1707 }
 1708 
 1709 CpaStatus
 1710 cpaDcDecompressData(CpaInstanceHandle dcInstance,
 1711                     CpaDcSessionHandle pSessionHandle,
 1712                     CpaBufferList *pSrcBuff,
 1713                     CpaBufferList *pDestBuff,
 1714                     CpaDcRqResults *pResults,
 1715                     CpaDcFlush flushFlag,
 1716                     void *callbackTag)
 1717 {
 1718         sal_compression_service_t *pService = NULL;
 1719         dc_session_desc_t *pSessionDesc = NULL;
 1720         CpaInstanceHandle insHandle = NULL;
 1721         Cpa64U srcBuffSize = 0;
 1722         CpaStatus status = CPA_STATUS_SUCCESS;
 1723 
 1724 
 1725         if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
 1726                 insHandle = dcGetFirstHandle();
 1727         } else {
 1728                 insHandle = dcInstance;
 1729         }
 1730 
 1731         status = dcDecompressDataCheck(insHandle,
 1732                                        pSessionHandle,
 1733                                        pSrcBuff,
 1734                                        pDestBuff,
 1735                                        pResults,
 1736                                        flushFlag,
 1737                                        &srcBuffSize);
 1738         if (CPA_STATUS_SUCCESS != status) {
 1739                 return status;
 1740         }
 1741 
 1742         pService = (sal_compression_service_t *)insHandle;
 1743 
 1744         /* Check if SAL is initialised otherwise return an error */
 1745         SAL_RUNNING_CHECK(insHandle);
 1746 
 1747         /* This check is outside the parameter checking as it is needed to
 1748          * manage zero length requests */
 1749         if (CPA_STATUS_SUCCESS !=
 1750             LacBuffDesc_BufferListVerifyNull(pSrcBuff,
 1751                                              &srcBuffSize,
 1752                                              LAC_NO_ALIGNMENT_SHIFT)) {
 1753                 QAT_UTILS_LOG("Invalid source buffer list parameter");
 1754                 return CPA_STATUS_INVALID_PARAM;
 1755         }
 1756 
 1757         /* Ensure this is a compression instance */
 1758         SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
 1759 
 1760         if (dcCheckSourceData(pSessionHandle,
 1761                               pSrcBuff,
 1762                               pDestBuff,
 1763                               pResults,
 1764                               flushFlag,
 1765                               srcBuffSize,
 1766                               NULL) != CPA_STATUS_SUCCESS) {
 1767                 return CPA_STATUS_INVALID_PARAM;
 1768         }
 1769         if (dcCheckDestinationData(pService,
 1770                                    pSessionHandle,
 1771                                    pDestBuff,
 1772                                    DC_DECOMPRESSION_REQUEST) !=
 1773             CPA_STATUS_SUCCESS) {
 1774                 return CPA_STATUS_INVALID_PARAM;
 1775         }
 1776         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
 1777 
 1778         if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
 1779                 QAT_UTILS_LOG("Invalid sessDirection value");
 1780                 return CPA_STATUS_INVALID_PARAM;
 1781         }
 1782 
 1783 
 1784         if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
 1785                 /* Lock the session to check if there are in-flight stateful
 1786                  * requests */
 1787                 LAC_SPINLOCK(&(pSessionDesc->sessionLock));
 1788 
 1789                 /* Check if there is already one in-flight stateful request */
 1790                 if (0 !=
 1791                     qatUtilsAtomicGet(
 1792                         &(pSessionDesc->pendingStatefulCbCount))) {
 1793                         LAC_LOG_ERROR(
 1794                             "Only one in-flight stateful request supported");
 1795                         LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
 1796                         return CPA_STATUS_RETRY;
 1797                 }
 1798 
 1799                 /* Gen 4 handle 0 len requests in FW */
 1800                 if (isDcGen2x(pService)) {
 1801                         if ((0 == srcBuffSize) ||
 1802                             ((1 == srcBuffSize) &&
 1803                              (CPA_DC_FLUSH_FINAL != flushFlag) &&
 1804                              (CPA_DC_FLUSH_FULL != flushFlag))) {
 1805                                 if (CPA_TRUE ==
 1806                                     dcZeroLengthRequests(
 1807                                         pService,
 1808                                         pSessionDesc,
 1809                                         pResults,
 1810                                         flushFlag,
 1811                                         callbackTag,
 1812                                         DC_DECOMPRESSION_REQUEST)) {
 1813                                         return CPA_STATUS_SUCCESS;
 1814                                 }
 1815                         }
 1816                 }
 1817 
 1818                 qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
 1819                 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
 1820         }
 1821 
 1822         return dcCompDecompData(pService,
 1823                                 pSessionDesc,
 1824                                 insHandle,
 1825                                 pSessionHandle,
 1826                                 pSrcBuff,
 1827                                 pDestBuff,
 1828                                 pResults,
 1829                                 flushFlag,
 1830                                 NULL,
 1831                                 callbackTag,
 1832                                 DC_DECOMPRESSION_REQUEST,
 1833                                 CPA_TRUE,
 1834                                 DC_NO_CNV);
 1835 }
 1836 
 1837 CpaStatus
 1838 cpaDcDecompressData2(CpaInstanceHandle dcInstance,
 1839                      CpaDcSessionHandle pSessionHandle,
 1840                      CpaBufferList *pSrcBuff,
 1841                      CpaBufferList *pDestBuff,
 1842                      CpaDcOpData *pOpData,
 1843                      CpaDcRqResults *pResults,
 1844                      void *callbackTag)
 1845 {
 1846         sal_compression_service_t *pService = NULL;
 1847         dc_session_desc_t *pSessionDesc = NULL;
 1848         CpaInstanceHandle insHandle = NULL;
 1849         CpaStatus status = CPA_STATUS_SUCCESS;
 1850         Cpa64U srcBuffSize = 0;
 1851         LAC_CHECK_NULL_PARAM(pOpData);
 1852 
 1853         if (CPA_FALSE == pOpData->integrityCrcCheck) {
 1854 
 1855                 return cpaDcDecompressData(dcInstance,
 1856                                            pSessionHandle,
 1857                                            pSrcBuff,
 1858                                            pDestBuff,
 1859                                            pResults,
 1860                                            pOpData->flushFlag,
 1861                                            callbackTag);
 1862         }
 1863 
 1864 
 1865         if (CPA_INSTANCE_HANDLE_SINGLE == dcInstance) {
 1866                 insHandle = dcGetFirstHandle();
 1867         } else {
 1868                 insHandle = dcInstance;
 1869         }
 1870 
 1871         status = dcDecompressDataCheck(insHandle,
 1872                                        pSessionHandle,
 1873                                        pSrcBuff,
 1874                                        pDestBuff,
 1875                                        pResults,
 1876                                        pOpData->flushFlag,
 1877                                        &srcBuffSize);
 1878         if (CPA_STATUS_SUCCESS != status) {
 1879                 return status;
 1880         }
 1881 
 1882         pService = (sal_compression_service_t *)insHandle;
 1883 
 1884         pSessionDesc = DC_SESSION_DESC_FROM_CTX_GET(pSessionHandle);
 1885 
 1886         LAC_CHECK_NULL_PARAM(insHandle);
 1887 
 1888         /* Check if SAL is initialised otherwise return an error */
 1889         SAL_RUNNING_CHECK(insHandle);
 1890 
 1891         /* This check is outside the parameter checking as it is needed to
 1892          * manage zero length requests */
 1893         if (CPA_STATUS_SUCCESS !=
 1894             LacBuffDesc_BufferListVerifyNull(pSrcBuff,
 1895                                              &srcBuffSize,
 1896                                              LAC_NO_ALIGNMENT_SHIFT)) {
 1897                 QAT_UTILS_LOG("Invalid source buffer list parameter");
 1898                 return CPA_STATUS_INVALID_PARAM;
 1899         }
 1900 
 1901         /* Ensure this is a compression instance */
 1902         SAL_CHECK_INSTANCE_TYPE(insHandle, SAL_SERVICE_TYPE_COMPRESSION);
 1903 
 1904         if (CPA_STATUS_SUCCESS !=
 1905             dcCheckSourceData(pSessionHandle,
 1906                               pSrcBuff,
 1907                               pDestBuff,
 1908                               pResults,
 1909                               CPA_DC_FLUSH_NONE,
 1910                               srcBuffSize,
 1911                               NULL)) {
 1912                 return CPA_STATUS_INVALID_PARAM;
 1913         }
 1914         if (CPA_STATUS_SUCCESS !=
 1915             dcCheckDestinationData(pService,
 1916                                    pSessionHandle,
 1917                                    pDestBuff,
 1918                                    DC_DECOMPRESSION_REQUEST)) {
 1919                 return CPA_STATUS_INVALID_PARAM;
 1920         }
 1921 
 1922         if (CPA_STATUS_SUCCESS != dcCheckOpData(pService, pOpData)) {
 1923                 return CPA_STATUS_INVALID_PARAM;
 1924         }
 1925 
 1926         if (CPA_DC_DIR_COMPRESS == pSessionDesc->sessDirection) {
 1927                 QAT_UTILS_LOG("Invalid sessDirection value");
 1928                 return CPA_STATUS_INVALID_PARAM;
 1929         }
 1930 
 1931 
 1932         if (CPA_DC_STATEFUL == pSessionDesc->sessState) {
 1933                 /* Lock the session to check if there are in-flight stateful
 1934                  * requests */
 1935                 LAC_SPINLOCK(&(pSessionDesc->sessionLock));
 1936 
 1937                 /* Check if there is already one in-flight stateful request */
 1938                 if (0 !=
 1939                     qatUtilsAtomicGet(
 1940                         &(pSessionDesc->pendingStatefulCbCount))) {
 1941                         LAC_LOG_ERROR(
 1942                             "Only one in-flight stateful request supported");
 1943                         LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
 1944                         return CPA_STATUS_RETRY;
 1945                 }
 1946 
 1947                 /* Gen 4 handle 0 len requests in FW */
 1948                 if (isDcGen2x(pService)) {
 1949                         if ((0 == srcBuffSize) ||
 1950                             ((1 == srcBuffSize) &&
 1951                              (CPA_DC_FLUSH_FINAL != pOpData->flushFlag) &&
 1952                              (CPA_DC_FLUSH_FULL != pOpData->flushFlag))) {
 1953                                 if (CPA_TRUE ==
 1954                                     dcZeroLengthRequests(
 1955                                         pService,
 1956                                         pSessionDesc,
 1957                                         pResults,
 1958                                         pOpData->flushFlag,
 1959                                         callbackTag,
 1960                                         DC_DECOMPRESSION_REQUEST)) {
 1961                                         return CPA_STATUS_SUCCESS;
 1962                                 }
 1963                         }
 1964                 }
 1965                 qatUtilsAtomicInc(&(pSessionDesc->pendingStatefulCbCount));
 1966                 LAC_SPINUNLOCK(&(pSessionDesc->sessionLock));
 1967         }
 1968 
 1969         return dcCompDecompData(pService,
 1970                                 pSessionDesc,
 1971                                 insHandle,
 1972                                 pSessionHandle,
 1973                                 pSrcBuff,
 1974                                 pDestBuff,
 1975                                 pResults,
 1976                                 pOpData->flushFlag,
 1977                                 pOpData,
 1978                                 callbackTag,
 1979                                 DC_DECOMPRESSION_REQUEST,
 1980                                 CPA_TRUE,
 1981                                 DC_NO_CNV);
 1982 }
Cache object: bed64caf2b94cde4656d00d0df0c63c5
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/qat/qat_api/common/compression/dc_datapath.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/qat/qat_api/common/compression/dc_datapath.c
