lac_mem_pools.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /* SPDX-License-Identifier: BSD-3-Clause */
    2 /* Copyright(c) 2007-2022 Intel Corporation */
    3 /* $FreeBSD$ */
    4 /**
    5  ***************************************************************************
    6  * @file lac_mem_pools.c
    7  *
    8  * @ingroup LacMemPool
    9  *
   10  * Memory Pool creation and mgmt function implementations
   11  *
   12  ***************************************************************************/
   13 
   14 #include "cpa.h"
   15 #include "qat_utils.h"
   16 #include "icp_accel_devices.h"
   17 #include "icp_adf_init.h"
   18 #include "icp_adf_transport.h"
   19 #include "icp_adf_debug.h"
   20 #include "lac_lock_free_stack.h"
   21 #include "lac_mem_pools.h"
   22 #include "lac_mem.h"
   23 #include "lac_common.h"
   24 #include "cpa_dc.h"
   25 #include "dc_session.h"
   26 #include "dc_datapath.h"
   27 #include "icp_qat_fw_comp.h"
   28 #include "icp_buffer_desc.h"
   29 #include "lac_sym.h"
   30 
   31 #define LAC_MEM_POOLS_NUM_SUPPORTED 32000
   32 /**< @ingroup LacMemPool
   33  * Number of mem pools supported */
   34 
   35 #define LAC_MEM_POOLS_NAME_SIZE 17
   36 /**< @ingroup LacMemPool
   37  * 16 bytes plus '\\' terminator */
   38 
   39 /**< @ingroup LacMemPool
   40  *     This structure is used to manage each pool created using this utility
   41  * feature. The client will maintain a pointer (identifier) to the created
   42  * structure per pool.
   43  */
   44 typedef struct lac_mem_pool_hdr_s {
   45         lock_free_stack_t stack;
   46         char poolName[LAC_MEM_POOLS_NAME_SIZE]; /*16 bytes of a pool name */
   47         /**< up to 16 bytes of a pool name */
   48         unsigned int numElementsInPool;
   49         /**< number of elements in the Pool */
   50         unsigned int blkSizeInBytes;
   51         /**< Block size in bytes */
   52         unsigned int blkAlignmentInBytes;
   53         /**< block alignment in bytes */
   54         lac_mem_blk_t **trackBlks;
   55         /* An array of mem block pointers to track the allocated entries in pool
   56          */
   57         volatile size_t availBlks;
   58         /* Number of blocks available for allocation in this pool */
   59 } lac_mem_pool_hdr_t;
   60 
   61 static lac_mem_pool_hdr_t *lac_mem_pools[LAC_MEM_POOLS_NUM_SUPPORTED] = {
   62         NULL
   63 };
   64 /**< @ingroup LacMemPool
   65  * Array of pointers to the mem pool header structure
   66  */
   67 
   68 LAC_DECLARE_HIGHEST_BIT_OF(lac_mem_blk_t);
   69 /**< @ingroup LacMemPool
   70  * local constant for quickening computation of additional space allocated
   71  * for holding lac_mem_blk_t container-structure
   72  */
   73 
   74 /**
   75  *******************************************************************************
   76  * @ingroup LacMemPool
   77  * This function cleans up a mem pool.
   78  ******************************************************************************/
   79 void Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID);
   80 
   81 static inline Cpa32U
   82 Lac_MemPoolGetElementRealSize(Cpa32U blkSizeInBytes, Cpa32U blkAlignmentInBytes)
   83 {
   84         Cpa32U addSize = (blkAlignmentInBytes >= sizeof(lac_mem_blk_t) ?
   85                               blkAlignmentInBytes :
   86                               1 << (highest_bit_of_lac_mem_blk_t + 1));
   87         return blkSizeInBytes + addSize;
   88 }
   89 
   90 CpaStatus
   91 Lac_MemPoolCreate(lac_memory_pool_id_t *pPoolID,
   92                   char *poolName,
   93                   unsigned int numElementsInPool,   /*Number of elements*/
   94                   unsigned int blkSizeInBytes,      /*Block Size in bytes*/
   95                   unsigned int blkAlignmentInBytes, /*Block alignment (bytes)*/
   96                   CpaBoolean trackMemory,
   97                   Cpa32U node)
   98 {
   99         unsigned int poolSearch = 0;
  100         unsigned int counter = 0;
  101         lac_mem_blk_t *pMemBlkCurrent = NULL;
  102 
  103         void *pMemBlk = NULL;
  104 
  105         if (pPoolID == NULL) {
  106                 QAT_UTILS_LOG("Invalid Pool ID param\n");
  107                 return CPA_STATUS_INVALID_PARAM; /*Error*/
  108         }
  109 
  110         /* Find First available Pool return error otherwise */
  111         while (lac_mem_pools[poolSearch] != NULL) {
  112                 poolSearch++;
  113                 if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) {
  114                         QAT_UTILS_LOG(
  115                             "No more memory pools available for allocation.\n");
  116                         return CPA_STATUS_FAIL;
  117                 }
  118         }
  119 
  120         /* Allocate a Pool header */
  121         lac_mem_pools[poolSearch] = LAC_OS_MALLOC(sizeof(lac_mem_pool_hdr_t));
  122         if (NULL == lac_mem_pools[poolSearch]) {
  123                 QAT_UTILS_LOG(
  124                     "Unable to allocate memory for creation of the pool.\n");
  125                 return CPA_STATUS_RESOURCE; /*Error*/
  126         }
  127         memset(lac_mem_pools[poolSearch], 0, sizeof(lac_mem_pool_hdr_t));
  128 
  129         /* Copy in Pool Name */
  130         if (poolName != NULL) {
  131                 snprintf(lac_mem_pools[poolSearch]->poolName,
  132                          LAC_MEM_POOLS_NAME_SIZE,
  133                          "%s",
  134                          poolName);
  135         } else {
  136                 LAC_OS_FREE(lac_mem_pools[poolSearch]);
  137                 lac_mem_pools[poolSearch] = NULL;
  138                 QAT_UTILS_LOG("Invalid Pool Name pointer\n");
  139                 return CPA_STATUS_INVALID_PARAM; /*Error*/
  140         }
  141 
  142         /* Allocate table for tracking memory blocks */
  143         if (CPA_TRUE == trackMemory) {
  144                 lac_mem_pools[poolSearch]->trackBlks = LAC_OS_MALLOC(
  145                     (sizeof(lac_mem_blk_t *) * numElementsInPool));
  146                 if (NULL == lac_mem_pools[poolSearch]->trackBlks) {
  147                         LAC_OS_FREE(lac_mem_pools[poolSearch]);
  148                         lac_mem_pools[poolSearch] = NULL;
  149                         QAT_UTILS_LOG(
  150                             "Unable to allocate memory for tracking memory blocks.\n");
  151                         return CPA_STATUS_RESOURCE; /*Error*/
  152                 }
  153         } else {
  154                 lac_mem_pools[poolSearch]->trackBlks = NULL;
  155         }
  156 
  157         lac_mem_pools[poolSearch]->availBlks = 0;
  158         lac_mem_pools[poolSearch]->stack = _init_stack();
  159 
  160         /* Calculate alignment needed for allocation   */
  161         for (counter = 0; counter < numElementsInPool; counter++) {
  162                 CpaPhysicalAddr physAddr = 0;
  163                 /* realSize is computed for allocation of  blkSize bytes +
  164                    additional
  165                    capacity for lac_mem_blk_t structure storage due to the some
  166                    OSes
  167                    (BSD) limitations for memory alignment to be power of 2;
  168                    sizeof(lac_mem_blk_t) is being round up to the closest power
  169                    of 2 -
  170                    optimised towards the least CPU overhead but at additional
  171                    memory
  172                    cost
  173                  */
  174                 Cpa32U realSize =
  175                     Lac_MemPoolGetElementRealSize(blkSizeInBytes,
  176                                                   blkAlignmentInBytes);
  177                 Cpa32U addSize = realSize - blkSizeInBytes;
  178 
  179                 if (CPA_STATUS_SUCCESS != LAC_OS_CAMALLOC(&pMemBlk,
  180                                                           realSize,
  181                                                           blkAlignmentInBytes,
  182                                                           node)) {
  183                         Lac_MemPoolCleanUpInternal(lac_mem_pools[poolSearch]);
  184                         lac_mem_pools[poolSearch] = NULL;
  185                         QAT_UTILS_LOG(
  186                             "Unable to allocate contiguous chunk of memory.\n");
  187                         return CPA_STATUS_RESOURCE;
  188                 }
  189 
  190                 /* Calcaulate various offsets */
  191                 physAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
  192                     (void *)((LAC_ARCH_UINT)pMemBlk + addSize));
  193 
  194                 /* physAddr is now already aligned to the greater power of 2:
  195                     blkAlignmentInBytes or sizeof(lac_mem_blk_t) round up
  196                     We safely put the structure right before the blkSize
  197                     real data block
  198                  */
  199                 pMemBlkCurrent =
  200                     (lac_mem_blk_t *)(((LAC_ARCH_UINT)(pMemBlk)) + addSize -
  201                                       sizeof(lac_mem_blk_t));
  202 
  203                 pMemBlkCurrent->physDataPtr = physAddr;
  204                 pMemBlkCurrent->pMemAllocPtr = pMemBlk;
  205                 pMemBlkCurrent->pPoolID = lac_mem_pools[poolSearch];
  206                 pMemBlkCurrent->isInUse = CPA_FALSE;
  207                 pMemBlkCurrent->pNext = NULL;
  208 
  209                 push(&lac_mem_pools[poolSearch]->stack, pMemBlkCurrent);
  210 
  211                 /* Store allocated memory pointer */
  212                 if (lac_mem_pools[poolSearch]->trackBlks != NULL) {
  213                         (lac_mem_pools[poolSearch]->trackBlks[counter]) =
  214                             (lac_mem_blk_t *)pMemBlkCurrent;
  215                 }
  216                 __sync_add_and_fetch(&lac_mem_pools[poolSearch]->availBlks, 1);
  217                 (lac_mem_pools[poolSearch])->numElementsInPool = counter + 1;
  218         }
  219 
  220         /* Set Pool details in the header */
  221         (lac_mem_pools[poolSearch])->blkSizeInBytes = blkSizeInBytes;
  222         (lac_mem_pools[poolSearch])->blkAlignmentInBytes = blkAlignmentInBytes;
  223         /* Set the Pool ID output parameter */
  224         *pPoolID = (LAC_ARCH_UINT)(lac_mem_pools[poolSearch]);
  225         /* Success */
  226         return CPA_STATUS_SUCCESS;
  227 }
  228 
  229 void *
  230 Lac_MemPoolEntryAlloc(lac_memory_pool_id_t poolID)
  231 {
  232         lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
  233         lac_mem_blk_t *pMemBlkCurrent = NULL;
  234 
  235         /* Explicitly removing NULL PoolID check for speed */
  236         if (pPoolID == NULL) {
  237                 QAT_UTILS_LOG("Invalid Pool ID");
  238                 return NULL;
  239         }
  240 
  241         /* Remove block from pool */
  242         pMemBlkCurrent = pop(&pPoolID->stack);
  243         if (NULL == pMemBlkCurrent) {
  244                 return (void *)CPA_STATUS_RETRY;
  245         }
  246         __sync_sub_and_fetch(&pPoolID->availBlks, 1);
  247         pMemBlkCurrent->isInUse = CPA_TRUE;
  248         return (void *)((LAC_ARCH_UINT)(pMemBlkCurrent) +
  249                         sizeof(lac_mem_blk_t));
  250 }
  251 
  252 void
  253 Lac_MemPoolEntryFree(void *pEntry)
  254 {
  255         lac_mem_blk_t *pMemBlk = NULL;
  256 
  257         /* Explicitly NULL pointer check */
  258         if (pEntry == NULL) {
  259                 QAT_UTILS_LOG("Memory Handle NULL");
  260                 return;
  261         }
  262 
  263         pMemBlk =
  264             (lac_mem_blk_t *)((LAC_ARCH_UINT)pEntry - sizeof(lac_mem_blk_t));
  265         pMemBlk->isInUse = CPA_FALSE;
  266 
  267         push(&pMemBlk->pPoolID->stack, pMemBlk);
  268         __sync_add_and_fetch(&pMemBlk->pPoolID->availBlks, 1);
  269 }
  270 
  271 void
  272 Lac_MemPoolDestroy(lac_memory_pool_id_t poolID)
  273 {
  274         unsigned int poolSearch = 0;
  275         lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
  276 
  277         if (pPoolID != NULL) {
  278                 /*Remove entry from table*/
  279                 while (lac_mem_pools[poolSearch] != pPoolID) {
  280                         poolSearch++;
  281 
  282                         if (LAC_MEM_POOLS_NUM_SUPPORTED == poolSearch) {
  283                                 QAT_UTILS_LOG("Invalid Pool ID submitted.\n");
  284                                 return;
  285                         }
  286                 }
  287 
  288                 lac_mem_pools[poolSearch] = NULL; /*Remove handle from pool*/
  289 
  290                 Lac_MemPoolCleanUpInternal(pPoolID);
  291         }
  292 }
  293 
  294 void
  295 Lac_MemPoolCleanUpInternal(lac_mem_pool_hdr_t *pPoolID)
  296 {
  297         lac_mem_blk_t *pCurrentBlk = NULL;
  298         void *pFreePtr = NULL;
  299         Cpa32U count = 0;
  300 
  301         if (pPoolID->trackBlks == NULL) {
  302                 pCurrentBlk = pop(&pPoolID->stack);
  303 
  304                 while (pCurrentBlk != NULL) {
  305                         /* Free Data Blocks */
  306                         pFreePtr = pCurrentBlk->pMemAllocPtr;
  307                         pCurrentBlk = pop(&pPoolID->stack);
  308                         LAC_OS_CAFREE(pFreePtr);
  309                 }
  310         } else {
  311                 for (count = 0; count < pPoolID->numElementsInPool; count++) {
  312                         pFreePtr = (pPoolID->trackBlks[count])->pMemAllocPtr;
  313                         LAC_OS_CAFREE(pFreePtr);
  314                 }
  315                 LAC_OS_FREE(pPoolID->trackBlks);
  316         }
  317         LAC_OS_FREE(pPoolID);
  318 }
  319 
  320 unsigned int
  321 Lac_MemPoolAvailableEntries(lac_memory_pool_id_t poolID)
  322 {
  323         lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
  324         if (pPoolID == NULL) {
  325                 QAT_UTILS_LOG("Invalid Pool ID\n");
  326                 return 0;
  327         }
  328         return pPoolID->availBlks;
  329 }
  330 
  331 void
  332 Lac_MemPoolStatsShow(void)
  333 {
  334         unsigned int index = 0;
  335         QAT_UTILS_LOG(SEPARATOR BORDER
  336                       "           Memory Pools Stats\n" SEPARATOR);
  337 
  338         while (index < LAC_MEM_POOLS_NUM_SUPPORTED) {
  339                 if (lac_mem_pools[index] != NULL) {
  340                         QAT_UTILS_LOG(
  341                             BORDER " Pool Name:             %s \n" BORDER
  342                                    " No. Elements in Pool:  %10u \n" BORDER
  343                                    " Element Size in Bytes: %10u \n" BORDER
  344                                    " Alignment in Bytes:    %10u \n" BORDER
  345                                    " No. Available Blocks:  %10zu \n" SEPARATOR,
  346                             lac_mem_pools[index]->poolName,
  347                             lac_mem_pools[index]->numElementsInPool,
  348                             lac_mem_pools[index]->blkSizeInBytes,
  349                             lac_mem_pools[index]->blkAlignmentInBytes,
  350                             lac_mem_pools[index]->availBlks);
  351                 }
  352                 index++;
  353         }
  354 }
  355 
  356 static void
  357 Lac_MemPoolInitSymCookies(lac_sym_cookie_t *pSymCookie)
  358 {
  359         pSymCookie->keyContentDescPhyAddr =
  360             LAC_OS_VIRT_TO_PHYS_INTERNAL(pSymCookie->u.keyCookie.contentDesc);
  361         pSymCookie->keyHashStateBufferPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
  362             pSymCookie->u.keyCookie.hashStateBuffer);
  363         pSymCookie->keySslKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
  364             &(pSymCookie->u.keyCookie.u.sslKeyInput));
  365         pSymCookie->keyTlsKeyInputPhyAddr = LAC_OS_VIRT_TO_PHYS_INTERNAL(
  366             &(pSymCookie->u.keyCookie.u.tlsKeyInput));
  367 }
  368 
  369 CpaStatus
  370 Lac_MemPoolInitSymCookiesPhyAddr(lac_memory_pool_id_t poolID)
  371 {
  372         lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
  373         lac_sym_cookie_t *pSymCookie = NULL;
  374         lac_mem_blk_t *pCurrentBlk = NULL;
  375 
  376         if (NULL == pPoolID) {
  377                 QAT_UTILS_LOG("Invalid Pool ID\n");
  378                 return CPA_STATUS_FAIL;
  379         }
  380 
  381         if (pPoolID->trackBlks == NULL) {
  382                 pCurrentBlk = top(&pPoolID->stack);
  383 
  384                 while (pCurrentBlk != NULL) {
  385                         pSymCookie =
  386                             (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) +
  387                                                  sizeof(lac_mem_blk_t));
  388                         pCurrentBlk = pCurrentBlk->pNext;
  389                         Lac_MemPoolInitSymCookies(pSymCookie);
  390                 }
  391         } else {
  392                 Cpa32U count = 0;
  393 
  394                 for (count = 0; count < pPoolID->numElementsInPool; count++) {
  395                         pCurrentBlk = pPoolID->trackBlks[count];
  396                         pSymCookie =
  397                             (lac_sym_cookie_t *)((LAC_ARCH_UINT)(pCurrentBlk) +
  398                                                  sizeof(lac_mem_blk_t));
  399                         Lac_MemPoolInitSymCookies(pSymCookie);
  400                 }
  401         }
  402         return CPA_STATUS_SUCCESS;
  403 }
  404 
  405 CpaStatus
  406 Lac_MemPoolInitDcCookiePhyAddr(lac_memory_pool_id_t poolID)
  407 {
  408         lac_mem_pool_hdr_t *pPoolID = (lac_mem_pool_hdr_t *)poolID;
  409         lac_mem_blk_t *pCurrentBlk = NULL;
  410 
  411         if (NULL == pPoolID) {
  412                 QAT_UTILS_LOG("Invalid Pool ID\n");
  413                 return CPA_STATUS_FAIL;
  414         }
  415 
  416         if (NULL == pPoolID->trackBlks) {
  417                 pCurrentBlk = top(&pPoolID->stack);
  418 
  419                 while (pCurrentBlk != NULL) {
  420                         pCurrentBlk = pCurrentBlk->pNext;
  421                 }
  422         } else {
  423                 Cpa32U count = 0;
  424 
  425                 for (count = 0; count < pPoolID->numElementsInPool; count++) {
  426                         pCurrentBlk = pPoolID->trackBlks[count];
  427                 }
  428         }
  429         return CPA_STATUS_SUCCESS;
  430 }
Cache object: b942ef8d0884688ea8de455102b9d99e
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/qat/qat_api/common/utils/lac_mem_pools.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/qat/qat_api/common/utils/lac_mem_pools.c
