Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/dev/qat/qat_api/common/crypto/sym/lac_sym_cb.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 5 /** 6 *************************************************************************** 7 * @file lac_sym_cb.c Callback handler functions for symmetric components 8 * 9 * @ingroup LacSym 10 * 11 ***************************************************************************/ 12 13 /* 14 ******************************************************************************* 15 * Include public/global header files 16 ******************************************************************************* 17 */ 18 19 #include "cpa.h" 20 #include "cpa_cy_sym.h" 21 22 #include "icp_accel_devices.h" 23 #include "icp_adf_init.h" 24 #include "icp_qat_fw_la.h" 25 #include "icp_adf_transport.h" 26 #include "icp_adf_debug.h" 27 28 #include "lac_sym.h" 29 #include "lac_sym_cipher.h" 30 #include "lac_common.h" 31 #include "lac_list.h" 32 #include "lac_sal_types_crypto.h" 33 #include "lac_sal.h" 34 #include "lac_sal_ctrl.h" 35 #include "lac_session.h" 36 #include "lac_sym_stats.h" 37 #include "lac_log.h" 38 #include "lac_sym_cb.h" 39 #include "lac_sym_hash.h" 40 #include "lac_sym_qat_cipher.h" 41 #include "lac_sym_qat.h" 42 43 #define DEQUEUE_MSGPUT_MAX_RETRIES 10000 44 45 /* 46 ******************************************************************************* 47 * Define static function definitions 48 ******************************************************************************* 49 */ 50 51 /** 52 ***************************************************************************** 53 * @ingroup LacSymCb 54 * Function to clean computed data. 55 * 56 * @description 57 * This function cleans GCM or CCM data in the case of a failure. 58 * 59 * @param[in] pSessionDesc pointer to the session descriptor 60 * @param[out] pBufferList pointer to the bufferlist to clean 61 * @param[in] pOpData pointer to operation data 62 * @param[in] isCCM is it a CCM operation boolean 63 * 64 * @return None 65 *****************************************************************************/ 66 static void 67 LacSymCb_CleanUserData(const lac_session_desc_t *pSessionDesc, 68 CpaBufferList *pBufferList, 69 const CpaCySymOpData *pOpData, 70 CpaBoolean isCCM) 71 { 72 Cpa32U authTagLen = 0; 73 74 /* Retrieve authTagLen */ 75 authTagLen = pSessionDesc->hashResultSize; 76 77 /* Cleaning */ 78 if (isCCM) { 79 /* for CCM the digest is inside the buffer list */ 80 LacBuffDesc_BufferListZeroFromOffset( 81 pBufferList, 82 pOpData->cryptoStartSrcOffsetInBytes, 83 pOpData->messageLenToCipherInBytes + authTagLen); 84 } else { 85 /* clean buffer list */ 86 LacBuffDesc_BufferListZeroFromOffset( 87 pBufferList, 88 pOpData->cryptoStartSrcOffsetInBytes, 89 pOpData->messageLenToCipherInBytes); 90 } 91 if ((CPA_TRUE != pSessionDesc->digestIsAppended) && 92 (NULL != pOpData->pDigestResult)) { 93 /* clean digest */ 94 memset(pOpData->pDigestResult, 0, authTagLen); 95 } 96 } 97 98 /** 99 ***************************************************************************** 100 * @ingroup LacSymCb 101 * Definition of callback function for processing symmetric responses 102 * 103 * @description 104 * This callback is invoked to process symmetric response messages from 105 * the QAT. It will extract some details from the message and invoke 106 * the user's callback to complete a symmetric operation. 107 * 108 * @param[in] pCookie Pointer to cookie associated with this request 109 * @param[in] qatRespStatusOkFlag Boolean indicating ok/fail status from QAT 110 * @param[in] status Status variable indicating an error occurred 111 * in sending the message (e.g. when dequeueing) 112 * @param[in] pSessionDesc Session descriptor 113 * 114 * @return None 115 *****************************************************************************/ 116 static void 117 LacSymCb_ProcessCallbackInternal(lac_sym_bulk_cookie_t *pCookie, 118 CpaBoolean qatRespStatusOkFlag, 119 CpaStatus status, 120 lac_session_desc_t *pSessionDesc) 121 { 122 CpaCySymCbFunc pSymCb = NULL; 123 void *pCallbackTag = NULL; 124 CpaCySymOpData *pOpData = NULL; 125 CpaBufferList *pDstBuffer = NULL; 126 CpaCySymOp operationType = CPA_CY_SYM_OP_NONE; 127 CpaStatus dequeueStatus = CPA_STATUS_SUCCESS; 128 129 CpaInstanceHandle instanceHandle = CPA_INSTANCE_HANDLE_SINGLE; 130 /* NOTE: cookie pointer validated in previous function */ 131 instanceHandle = pCookie->instanceHandle; 132 133 pOpData = (CpaCySymOpData *)LAC_CONST_PTR_CAST(pCookie->pOpData); 134 operationType = pSessionDesc->symOperation; 135 136 /* Set the destination pointer to the one supplied in the cookie. */ 137 pDstBuffer = pCookie->pDstBuffer; 138 139 /* For a digest verify operation - for full packet and final partial 140 * only, perform a comparison with the digest generated and with the one 141 * supplied in the packet. In case of AES_GCM in SPC mode, destination 142 * buffer needs to be cleared if digest verify operation fails */ 143 144 if (((SPC == pSessionDesc->singlePassState) || 145 (CPA_CY_SYM_OP_CIPHER != operationType)) && 146 (CPA_TRUE == pSessionDesc->digestVerify) && 147 ((CPA_CY_SYM_PACKET_TYPE_FULL == pOpData->packetType) || 148 (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType))) { 149 if (CPA_FALSE == qatRespStatusOkFlag) { 150 LAC_SYM_STAT_INC(numSymOpVerifyFailures, 151 instanceHandle); 152 153 /* The comparison has failed at this point (status is 154 * fail), need to clean any sensitive calculated data up 155 * to this point. The data calculated is no longer 156 * useful to the end result and does not need to be 157 * returned to the user so setting buffers to zero. 158 */ 159 if (pSessionDesc->cipherAlgorithm == 160 CPA_CY_SYM_CIPHER_AES_CCM) { 161 LacSymCb_CleanUserData(pSessionDesc, 162 pDstBuffer, 163 pOpData, 164 CPA_TRUE); 165 } else if (pSessionDesc->cipherAlgorithm == 166 CPA_CY_SYM_CIPHER_AES_GCM) { 167 LacSymCb_CleanUserData(pSessionDesc, 168 pDstBuffer, 169 pOpData, 170 CPA_FALSE); 171 } 172 } 173 } else { 174 /* Most commands have no point of failure and always return 175 * success. This is the default response from the QAT. 176 * If status is already set to an error value, don't overwrite 177 * it 178 */ 179 if ((CPA_STATUS_SUCCESS == status) && 180 (CPA_TRUE != qatRespStatusOkFlag)) { 181 LAC_LOG_ERROR("Response status value not as expected"); 182 status = CPA_STATUS_FAIL; 183 } 184 } 185 186 pSymCb = pSessionDesc->pSymCb; 187 pCallbackTag = pCookie->pCallbackTag; 188 189 /* State returned to the client for intermediate partials packets 190 * for hash only and cipher only partial packets. Cipher update 191 * allow next partial through */ 192 if (CPA_CY_SYM_PACKET_TYPE_PARTIAL == pOpData->packetType) { 193 if ((CPA_CY_SYM_OP_CIPHER == operationType) || 194 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) { 195 if (CPA_TRUE == pCookie->updateUserIvOnRecieve) { 196 /* Update the user's IV buffer 197 * Very important to do this BEFORE dequeuing 198 * subsequent partial requests, as the state 199 * buffer may get overwritten 200 */ 201 memcpy(pCookie->pOpData->pIv, 202 pSessionDesc->cipherPartialOpState, 203 pCookie->pOpData->ivLenInBytes); 204 } 205 if (CPA_TRUE == pCookie->updateKeySizeOnRecieve && 206 LAC_CIPHER_IS_XTS_MODE( 207 pSessionDesc->cipherAlgorithm)) { 208 LacSymQat_CipherXTSModeUpdateKeyLen( 209 pSessionDesc, 210 pSessionDesc->cipherKeyLenInBytes / 2); 211 } 212 } 213 } else if (CPA_CY_SYM_PACKET_TYPE_LAST_PARTIAL == pOpData->packetType) { 214 if ((CPA_CY_SYM_OP_CIPHER == operationType) || 215 (CPA_CY_SYM_OP_ALGORITHM_CHAINING == operationType)) { 216 if (CPA_TRUE == 217 LAC_CIPHER_IS_XTS_MODE( 218 pSessionDesc->cipherAlgorithm)) { 219 /* 220 * For XTS mode, we replace the updated key with 221 * the original key - for subsequent partial 222 * requests 223 * 224 */ 225 LacSymQat_CipherXTSModeUpdateKeyLen( 226 pSessionDesc, 227 pSessionDesc->cipherKeyLenInBytes); 228 } 229 } 230 } 231 232 if ((CPA_CY_SYM_PACKET_TYPE_FULL != pOpData->packetType) && 233 (qatRespStatusOkFlag != CPA_FALSE)) { 234 /* There may be requests blocked pending the completion of this 235 * operation 236 */ 237 238 dequeueStatus = LacSymCb_PendingReqsDequeue(pSessionDesc); 239 if (CPA_STATUS_SUCCESS != dequeueStatus) { 240 LAC_SYM_STAT_INC(numSymOpCompletedErrors, 241 instanceHandle); 242 qatRespStatusOkFlag = CPA_FALSE; 243 if (CPA_STATUS_SUCCESS == status) { 244 status = dequeueStatus; 245 } 246 } 247 } 248 249 if (CPA_STATUS_SUCCESS == status) { 250 /* update stats */ 251 if (pSessionDesc->internalSession == CPA_FALSE) { 252 LAC_SYM_STAT_INC(numSymOpCompleted, instanceHandle); 253 if (CPA_STATUS_SUCCESS != status) { 254 LAC_SYM_STAT_INC(numSymOpCompletedErrors, 255 instanceHandle); 256 } 257 } 258 } 259 260 qatUtilsAtomicDec(&(pSessionDesc->u.pendingCbCount)); 261 262 /* deallocate the memory for the internal callback cookie */ 263 Lac_MemPoolEntryFree(pCookie); 264 265 /* user callback function is the last thing to be called */ 266 pSymCb(pCallbackTag, 267 status, 268 operationType, 269 pOpData, 270 pDstBuffer, 271 qatRespStatusOkFlag); 272 } 273 274 /** 275 ****************************************************************************** 276 * @ingroup LacSymCb 277 * Definition of callback function for processing symmetric Data Plane 278 * responses 279 * 280 * @description 281 * This callback checks the status, decrements the number of operations 282 * pending and calls the user callback 283 * 284 * @param[in/out] pResponse pointer to the response structure 285 * @param[in] qatRespStatusOkFlag status 286 * @param[in] pSessionDesc pointer to the session descriptor 287 * 288 * @return None 289 ******************************************************************************/ 290 static void 291 LacSymCb_ProcessDpCallback(CpaCySymDpOpData *pResponse, 292 CpaBoolean qatRespStatusOkFlag, 293 CpaStatus status, 294 lac_session_desc_t *pSessionDesc) 295 { 296 CpaCySymDpCbFunc pSymDpCb = NULL; 297 298 /* For CCM and GCM, if qatRespStatusOkFlag is false, the data has to be 299 * cleaned as stated in RFC 3610; in DP mode, it is the user 300 * responsability to do so */ 301 302 if (((CPA_CY_SYM_OP_CIPHER == pSessionDesc->symOperation) && 303 SPC != pSessionDesc->singlePassState) || 304 (CPA_FALSE == pSessionDesc->digestVerify)) { 305 /* If not doing digest compare and qatRespStatusOkFlag != 306 CPA_TRUE then there is something very wrong */ 307 if ((CPA_FALSE == qatRespStatusOkFlag) && 308 (status != CPA_STATUS_UNSUPPORTED)) { 309 LAC_LOG_ERROR("Response status value not as expected"); 310 status = CPA_STATUS_FAIL; 311 } 312 } 313 314 pSymDpCb = 315 ((sal_crypto_service_t *)pResponse->instanceHandle)->pSymDpCb; 316 317 pSymDpCb(pResponse, status, qatRespStatusOkFlag); 318 319 /* 320 * Decrement the number of pending CB. 321 * 322 * If the @pendingDpCbCount becomes zero, we may remove the session, 323 * please read more information in the cpaCySymRemoveSession(). 324 * 325 * But there is a field in the @pResponse to store the session, 326 * the "sessionCtx". In another word, in the above @->pSymDpCb() 327 * callback, it may use the session again. If we decrease the 328 * @pendingDpCbCount before the @->pSymDpCb(), there is a _risk_ the 329 * @->pSymDpCb() may reference to a deleted session. 330 * 331 * So in order to avoid the risk, we decrease the @pendingDpCbCount 332 * after the @->pSymDpCb() callback. 333 */ 334 qatUtilsAtomicDec(&pSessionDesc->u.pendingDpCbCount); 335 } 336 337 /** 338 ****************************************************************************** 339 * @ingroup LacSymCb 340 * Definition of callback function for processing symmetric responses 341 * 342 * @description 343 * This callback, which is registered with the common symmetric response 344 * message handler, is invoked to process symmetric response messages from 345 * the QAT. It will extract the response status from the cmnRespFlags set 346 * by the QAT, and then will pass it to @ref 347 * LacSymCb_ProcessCallbackInternal to complete the response processing. 348 * 349 * @param[in] lacCmdId ID of the symmetric QAT command of the request 350 * message 351 * @param[in] pOpaqueData pointer to opaque data in the request message 352 * @param[in] cmnRespFlags Flags set by QAT to indicate response status 353 * 354 * @return None 355 ******************************************************************************/ 356 static void 357 LacSymCb_ProcessCallback(icp_qat_fw_la_cmd_id_t lacCmdId, 358 void *pOpaqueData, 359 icp_qat_fw_comn_flags cmnRespFlags) 360 { 361 CpaStatus status = CPA_STATUS_SUCCESS; 362 CpaCySymDpOpData *pDpOpData = (CpaCySymDpOpData *)pOpaqueData; 363 lac_session_desc_t *pSessionDesc = 364 LAC_SYM_SESSION_DESC_FROM_CTX_GET(pDpOpData->sessionCtx); 365 CpaBoolean qatRespStatusOkFlag = 366 (CpaBoolean)(ICP_QAT_FW_COMN_STATUS_FLAG_OK == 367 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(cmnRespFlags)); 368 369 if (CPA_TRUE == pSessionDesc->isDPSession) { 370 /* DP session */ 371 if (ICP_QAT_FW_COMN_RESP_UNSUPPORTED_REQUEST_STAT_GET( 372 cmnRespFlags)) { 373 status = CPA_STATUS_UNSUPPORTED; 374 } 375 LacSymCb_ProcessDpCallback(pDpOpData, 376 qatRespStatusOkFlag, 377 status, 378 pSessionDesc); 379 } else { 380 /* Trad session */ 381 LacSymCb_ProcessCallbackInternal((lac_sym_bulk_cookie_t *) 382 pOpaqueData, 383 qatRespStatusOkFlag, 384 CPA_STATUS_SUCCESS, 385 pSessionDesc); 386 } 387 } 388 389 /* 390 ******************************************************************************* 391 * Define public/global function definitions 392 ******************************************************************************* 393 */ 394 395 /** 396 * @ingroup LacSymCb 397 * 398 * @return CpaStatus 399 * value returned will be the result of icp_adf_transPutMsg 400 */ 401 CpaStatus 402 LacSymCb_PendingReqsDequeue(lac_session_desc_t *pSessionDesc) 403 { 404 CpaStatus status = CPA_STATUS_SUCCESS; 405 sal_crypto_service_t *pService = NULL; 406 Cpa32U retries = 0; 407 408 pService = (sal_crypto_service_t *)pSessionDesc->pInstance; 409 410 /* Need to protect access to queue head and tail pointers, which may 411 * be accessed by multiple contexts simultaneously for enqueue and 412 * dequeue operations 413 */ 414 LAC_SPINLOCK(&pSessionDesc->requestQueueLock); 415 416 /* Clear the blocking flag in the session descriptor */ 417 pSessionDesc->nonBlockingOpsInProgress = CPA_TRUE; 418 419 while ((NULL != pSessionDesc->pRequestQueueHead) && 420 (CPA_TRUE == pSessionDesc->nonBlockingOpsInProgress)) { 421 422 /* If we send a partial packet request, set the 423 * blockingOpsInProgress flag for the session to indicate that 424 * subsequent requests must be queued up until this request 425 * completes 426 */ 427 if (CPA_CY_SYM_PACKET_TYPE_FULL != 428 pSessionDesc->pRequestQueueHead->pOpData->packetType) { 429 pSessionDesc->nonBlockingOpsInProgress = CPA_FALSE; 430 } 431 432 /* At this point, we're clear to send the request. For cipher 433 * requests, we need to check if the session IV needs to be 434 * updated. This can only be done when no other partials are in 435 * flight for this session, to ensure the cipherPartialOpState 436 * buffer in the session descriptor is not currently in use 437 */ 438 if (CPA_TRUE == 439 pSessionDesc->pRequestQueueHead->updateSessionIvOnSend) { 440 if (LAC_CIPHER_IS_ARC4(pSessionDesc->cipherAlgorithm)) { 441 memcpy(pSessionDesc->cipherPartialOpState, 442 pSessionDesc->cipherARC4InitialState, 443 LAC_CIPHER_ARC4_STATE_LEN_BYTES); 444 } else { 445 memcpy(pSessionDesc->cipherPartialOpState, 446 pSessionDesc->pRequestQueueHead->pOpData 447 ->pIv, 448 pSessionDesc->pRequestQueueHead->pOpData 449 ->ivLenInBytes); 450 } 451 } 452 453 /* 454 * Now we'll attempt to send the message directly to QAT. We'll 455 * keep looing until it succeeds (or at least a very high number 456 * of retries), as the failure only happens when the ring is 457 * full, and this is only a temporary situation. After a few 458 * retries, space will become availble, allowing the putMsg to 459 * succeed. 460 */ 461 retries = 0; 462 do { 463 /* Send to QAT */ 464 status = icp_adf_transPutMsg( 465 pService->trans_handle_sym_tx, 466 (void *)&(pSessionDesc->pRequestQueueHead->qatMsg), 467 LAC_QAT_SYM_REQ_SZ_LW); 468 469 retries++; 470 /* 471 * Yield to allow other threads that may be on this 472 * session to poll and make some space on the ring 473 */ 474 if (CPA_STATUS_SUCCESS != status) { 475 qatUtilsYield(); 476 } 477 } while ((CPA_STATUS_SUCCESS != status) && 478 (retries < DEQUEUE_MSGPUT_MAX_RETRIES)); 479 480 if ((CPA_STATUS_SUCCESS != status) || 481 (retries >= DEQUEUE_MSGPUT_MAX_RETRIES)) { 482 LAC_LOG_ERROR( 483 "Failed to SalQatMsg_transPutMsg, maximum retries exceeded."); 484 goto cleanup; 485 } 486 487 pSessionDesc->pRequestQueueHead = 488 pSessionDesc->pRequestQueueHead->pNext; 489 } 490 491 /* If we've drained the queue, ensure the tail pointer is set to NULL */ 492 if (NULL == pSessionDesc->pRequestQueueHead) { 493 pSessionDesc->pRequestQueueTail = NULL; 494 } 495 496 cleanup: 497 LAC_SPINUNLOCK(&pSessionDesc->requestQueueLock); 498 return status; 499 } 500 501 /** 502 * @ingroup LacSymCb 503 */ 504 void 505 LacSymCb_CallbacksRegister(void) 506 { 507 /*** HASH ***/ 508 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_AUTH, 509 LacSymCb_ProcessCallback); 510 511 /*** ALGORITHM-CHAINING CIPHER_HASH***/ 512 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER_HASH, 513 LacSymCb_ProcessCallback); 514 515 /*** ALGORITHM-CHAINING HASH_CIPHER***/ 516 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_HASH_CIPHER, 517 LacSymCb_ProcessCallback); 518 519 /*** CIPHER ***/ 520 LacSymQat_RespHandlerRegister(ICP_QAT_FW_LA_CMD_CIPHER, 521 LacSymCb_ProcessCallback); 522 523 /* Call compile time param check function to ensure it is included 524 in the build by the compiler - this compile time check 525 ensures callbacks run as expected */ 526 LacSym_CompileTimeAssertions(); 527 }
Cache object: 06825755787f94731b16d6ddc1d9b15b
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]