adf_4xxx_hw_data.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /* SPDX-License-Identifier: BSD-3-Clause */
    2 /* Copyright(c) 2007-2022 Intel Corporation */
    3 /* $FreeBSD$ */
    4 #include <linux/iopoll.h>
    5 #include <adf_accel_devices.h>
    6 #include <adf_cfg.h>
    7 #include <adf_common_drv.h>
    8 #include <adf_dev_err.h>
    9 #include <adf_pf2vf_msg.h>
   10 #include <adf_gen4_hw_data.h>
   11 #include "adf_4xxx_hw_data.h"
   12 #include "adf_heartbeat.h"
   13 #include "icp_qat_fw_init_admin.h"
   14 #include "icp_qat_hw.h"
   15 
   16 #define ADF_CONST_TABLE_SIZE 1024
   17 
   18 struct adf_fw_config {
   19         u32 ae_mask;
   20         char *obj_name;
   21 };
   22 
   23 /* Accel unit information */
   24 static const struct adf_accel_unit adf_4xxx_au_a_ae[] = {
   25         { 0x1, 0x1, 0xF, 0x1B, 4, ADF_ACCEL_SERVICE_NULL },
   26         { 0x2, 0x1, 0xF0, 0x6C0, 4, ADF_ACCEL_SERVICE_NULL },
   27         { 0x4, 0x1, 0x100, 0xF000, 1, ADF_ACCEL_ADMIN },
   28 };
   29 
   30 /* Worker thread to service arbiter mappings */
   31 static u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { 0x5555555, 0x5555555,
   32                                                           0x5555555, 0x5555555,
   33                                                           0xAAAAAAA, 0xAAAAAAA,
   34                                                           0xAAAAAAA, 0xAAAAAAA,
   35                                                           0x0 };
   36 
   37 /* Masks representing ME thread-service mappings.
   38  * Thread 7 carries out Admin work and is thus
   39  * left out.
   40  */
   41 static u8 default_active_thd_mask = 0x7F;
   42 static u8 dc_me_active_thd_mask = 0x03;
   43 
   44 static u32 thrd_to_arb_map_gen[ADF_4XXX_MAX_ACCELENGINES] = { 0 };
   45 
   46 #define ADF_4XXX_ASYM_SYM                                                      \
   47         (ASYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                        \
   48          ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                              \
   49          SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
   50 
   51 #define ADF_4XXX_DC                                                            \
   52         (COMP | COMP << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                       \
   53          COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                              \
   54          COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
   55 
   56 #define ADF_4XXX_SYM                                                           \
   57         (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                         \
   58          SYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                               \
   59          SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
   60 
   61 #define ADF_4XXX_ASYM                                                          \
   62         (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                       \
   63          ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                              \
   64          ASYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
   65 
   66 #define ADF_4XXX_ASYM_DC                                                       \
   67         (ASYM | ASYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                       \
   68          COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                              \
   69          COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
   70 
   71 #define ADF_4XXX_SYM_DC                                                        \
   72         (SYM | SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                         \
   73          COMP << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                              \
   74          COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
   75 
   76 #define ADF_4XXX_NA                                                            \
   77         (NA | NA << ADF_CFG_SERV_RING_PAIR_1_SHIFT |                           \
   78          NA << ADF_CFG_SERV_RING_PAIR_2_SHIFT |                                \
   79          NA << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
   80 
   81 #define ADF_4XXX_DEFAULT_RING_TO_SRV_MAP ADF_4XXX_ASYM_SYM
   82 
   83 struct adf_enabled_services {
   84         const char svcs_enabled[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
   85         u16 rng_to_svc_msk;
   86 };
   87 
   88 static struct adf_enabled_services adf_4xxx_svcs[] = {
   89         { "dc", ADF_4XXX_DC },
   90         { "sym", ADF_4XXX_SYM },
   91         { "asym", ADF_4XXX_ASYM },
   92         { "dc;asym", ADF_4XXX_ASYM_DC },
   93         { "asym;dc", ADF_4XXX_ASYM_DC },
   94         { "sym;dc", ADF_4XXX_SYM_DC },
   95         { "dc;sym", ADF_4XXX_SYM_DC },
   96         { "asym;sym", ADF_4XXX_ASYM_SYM },
   97         { "sym;asym", ADF_4XXX_ASYM_SYM },
   98 };
   99 
  100 static struct adf_hw_device_class adf_4xxx_class = {
  101         .name = ADF_4XXX_DEVICE_NAME,
  102         .type = DEV_4XXX,
  103         .instances = 0,
  104 };
  105 
  106 static u32
  107 get_accel_mask(struct adf_accel_dev *accel_dev)
  108 {
  109         return ADF_4XXX_ACCELERATORS_MASK;
  110 }
  111 
  112 static u32
  113 get_ae_mask(struct adf_accel_dev *accel_dev)
  114 {
  115         u32 fusectl4 = accel_dev->hw_device->fuses;
  116 
  117         return ~fusectl4 & ADF_4XXX_ACCELENGINES_MASK;
  118 }
  119 
  120 static int
  121 get_ring_to_svc_map(struct adf_accel_dev *accel_dev, u16 *ring_to_svc_map)
  122 {
  123         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
  124         char val[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
  125         u32 i = 0;
  126 
  127         *ring_to_svc_map = 0;
  128         /* Get the services enabled by user */
  129         snprintf(key, sizeof(key), ADF_SERVICES_ENABLED);
  130         if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val))
  131                 return EFAULT;
  132 
  133         for (i = 0; i < ARRAY_SIZE(adf_4xxx_svcs); i++) {
  134                 if (!strncmp(val,
  135                              adf_4xxx_svcs[i].svcs_enabled,
  136                              ADF_CFG_MAX_KEY_LEN_IN_BYTES)) {
  137                         *ring_to_svc_map = adf_4xxx_svcs[i].rng_to_svc_msk;
  138                         return 0;
  139                 }
  140         }
  141 
  142         device_printf(GET_DEV(accel_dev),
  143                       "Invalid services enabled: %s\n",
  144                       val);
  145         return EFAULT;
  146 }
  147 
  148 static u32
  149 get_num_accels(struct adf_hw_device_data *self)
  150 {
  151         return ADF_4XXX_MAX_ACCELERATORS;
  152 }
  153 
  154 static u32
  155 get_num_aes(struct adf_hw_device_data *self)
  156 {
  157         if (!self || !self->ae_mask)
  158                 return 0;
  159 
  160         return hweight32(self->ae_mask);
  161 }
  162 
  163 static u32
  164 get_misc_bar_id(struct adf_hw_device_data *self)
  165 {
  166         return ADF_4XXX_PMISC_BAR;
  167 }
  168 
  169 static u32
  170 get_etr_bar_id(struct adf_hw_device_data *self)
  171 {
  172         return ADF_4XXX_ETR_BAR;
  173 }
  174 
  175 static u32
  176 get_sram_bar_id(struct adf_hw_device_data *self)
  177 {
  178         return ADF_4XXX_SRAM_BAR;
  179 }
  180 
  181 /*
  182  * The vector routing table is used to select the MSI-X entry to use for each
  183  * interrupt source.
  184  * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts.
  185  * The final entry corresponds to VF2PF or error interrupts.
  186  * This vector table could be used to configure one MSI-X entry to be shared
  187  * between multiple interrupt sources.
  188  *
  189  * The default routing is set to have a one to one correspondence between the
  190  * interrupt source and the MSI-X entry used.
  191  */
  192 static void
  193 set_msix_default_rttable(struct adf_accel_dev *accel_dev)
  194 {
  195         struct resource *csr;
  196         int i;
  197 
  198         csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr;
  199         for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++)
  200                 ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i);
  201 }
  202 
  203 static u32
  204 adf_4xxx_get_hw_cap(struct adf_accel_dev *accel_dev)
  205 {
  206         device_t pdev = accel_dev->accel_pci_dev.pci_dev;
  207         u32 fusectl1;
  208         u32 capabilities;
  209 
  210         /* Read accelerator capabilities mask */
  211         fusectl1 = pci_read_config(pdev, ADF_4XXX_FUSECTL1_OFFSET, 4);
  212         capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
  213             ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
  214             ICP_ACCEL_CAPABILITIES_CIPHER |
  215             ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
  216             ICP_ACCEL_CAPABILITIES_COMPRESSION |
  217             ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
  218             ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
  219             ICP_ACCEL_CAPABILITIES_HKDF | ICP_ACCEL_CAPABILITIES_SHA3_EXT |
  220             ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 |
  221             ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
  222             ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
  223             ICP_ACCEL_CAPABILITIES_AES_V2 | ICP_ACCEL_CAPABILITIES_RL;
  224 
  225         if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) {
  226                 capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
  227                 capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
  228         }
  229         if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE)
  230                 capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
  231         if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE)
  232                 capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
  233         if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) {
  234                 capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
  235                 capabilities &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
  236         }
  237         if (fusectl1 & ICP_ACCEL_4XXX_MASK_SMX_SLICE) {
  238                 capabilities &= ~ICP_ACCEL_CAPABILITIES_SM3;
  239                 capabilities &= ~ICP_ACCEL_CAPABILITIES_SM4;
  240         }
  241         return capabilities;
  242 }
  243 
  244 static u32
  245 get_hb_clock(struct adf_hw_device_data *self)
  246 {
  247         /*
  248          * 4XXX uses KPT counter for HB
  249          */
  250         return ADF_4XXX_KPT_COUNTER_FREQ;
  251 }
  252 
  253 static u32
  254 get_ae_clock(struct adf_hw_device_data *self)
  255 {
  256         /*
  257          * Clock update interval is <16> ticks for qat_4xxx.
  258          */
  259         return self->clock_frequency / 16;
  260 }
  261 
  262 static int
  263 measure_clock(struct adf_accel_dev *accel_dev)
  264 {
  265         u32 frequency;
  266         int ret = 0;
  267 
  268         ret = adf_dev_measure_clock(accel_dev,
  269                                     &frequency,
  270                                     ADF_4XXX_MIN_AE_FREQ,
  271                                     ADF_4XXX_MAX_AE_FREQ);
  272         if (ret)
  273                 return ret;
  274 
  275         accel_dev->hw_device->clock_frequency = frequency;
  276         return 0;
  277 }
  278 
  279 static int
  280 adf_4xxx_configure_accel_units(struct adf_accel_dev *accel_dev)
  281 {
  282         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES] = { 0 };
  283         char val_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
  284 
  285         if (adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC))
  286                 goto err;
  287 
  288         snprintf(key, sizeof(key), ADF_SERVICES_ENABLED);
  289         snprintf(val_str,
  290                  sizeof(val_str),
  291                  ADF_CFG_ASYM ADF_SERVICES_SEPARATOR ADF_CFG_SYM);
  292 
  293         if (adf_cfg_add_key_value_param(
  294                 accel_dev, ADF_GENERAL_SEC, key, (void *)val_str, ADF_STR))
  295                 goto err;
  296 
  297         return 0;
  298 err:
  299         device_printf(GET_DEV(accel_dev), "Failed to configure accel units\n");
  300         return EINVAL;
  301 }
  302 
  303 static u32
  304 get_num_accel_units(struct adf_hw_device_data *self)
  305 {
  306         return ADF_4XXX_MAX_ACCELUNITS;
  307 }
  308 
  309 static void
  310 get_accel_unit(struct adf_hw_device_data *self,
  311                struct adf_accel_unit **accel_unit)
  312 {
  313         memcpy(*accel_unit, adf_4xxx_au_a_ae, sizeof(adf_4xxx_au_a_ae));
  314 }
  315 
  316 static void
  317 adf_exit_accel_unit_services(struct adf_accel_dev *accel_dev)
  318 {
  319         if (accel_dev->au_info) {
  320                 kfree(accel_dev->au_info->au);
  321                 accel_dev->au_info->au = NULL;
  322                 kfree(accel_dev->au_info);
  323                 accel_dev->au_info = NULL;
  324         }
  325 }
  326 
  327 static int
  328 get_accel_unit_config(struct adf_accel_dev *accel_dev,
  329                       u8 *num_sym_au,
  330                       u8 *num_dc_au,
  331                       u8 *num_asym_au)
  332 {
  333         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  334         char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
  335         char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
  336         u32 num_au = hw_data->get_num_accel_units(hw_data);
  337         /* One AU will be allocated by default if a service enabled */
  338         u32 alloc_au = 1;
  339         /* There's always one AU that is used for Admin AE */
  340         u32 service_mask = ADF_ACCEL_ADMIN;
  341         char *token, *cur_str;
  342         u32 disabled_caps = 0;
  343 
  344         /* Get the services enabled by user */
  345         snprintf(key, sizeof(key), ADF_SERVICES_ENABLED);
  346         if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val))
  347                 return EFAULT;
  348         cur_str = val;
  349         token = strsep(&cur_str, ADF_SERVICES_SEPARATOR);
  350         while (token) {
  351                 if (!strncmp(token, ADF_CFG_SYM, strlen(ADF_CFG_SYM)))
  352                         service_mask |= ADF_ACCEL_CRYPTO;
  353                 if (!strncmp(token, ADF_CFG_ASYM, strlen(ADF_CFG_ASYM)))
  354                         service_mask |= ADF_ACCEL_ASYM;
  355 
  356                 /* cy means both asym & crypto should be enabled
  357                  * Hardware resources allocation check will be done later
  358                  */
  359                 if (!strncmp(token, ADF_CFG_CY, strlen(ADF_CFG_CY)))
  360                         service_mask |= ADF_ACCEL_ASYM | ADF_ACCEL_CRYPTO;
  361                 if (!strncmp(token, ADF_SERVICE_DC, strlen(ADF_SERVICE_DC)))
  362                         service_mask |= ADF_ACCEL_COMPRESSION;
  363 
  364                 token = strsep(&cur_str, ADF_SERVICES_SEPARATOR);
  365         }
  366 
  367         /* Ensure the user won't enable more services than it can support */
  368         if (hweight32(service_mask) > num_au) {
  369                 device_printf(GET_DEV(accel_dev),
  370                               "Can't enable more services than ");
  371                 device_printf(GET_DEV(accel_dev), "%d!\n", num_au);
  372                 return EFAULT;
  373         } else if (hweight32(service_mask) == 2) {
  374                 /* Due to limitation, besides AU for Admin AE
  375                  * only 2 more AUs can be allocated
  376                  */
  377                 alloc_au = 2;
  378         }
  379 
  380         if (service_mask & ADF_ACCEL_CRYPTO)
  381                 *num_sym_au = alloc_au;
  382         if (service_mask & ADF_ACCEL_ASYM)
  383                 *num_asym_au = alloc_au;
  384         if (service_mask & ADF_ACCEL_COMPRESSION)
  385                 *num_dc_au = alloc_au;
  386 
  387         /*update capability*/
  388         if (!*num_sym_au || !(service_mask & ADF_ACCEL_CRYPTO)) {
  389                 disabled_caps = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
  390                     ICP_ACCEL_CAPABILITIES_CIPHER |
  391                     ICP_ACCEL_CAPABILITIES_SHA3_EXT |
  392                     ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 |
  393                     ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
  394                     ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
  395                     ICP_ACCEL_CAPABILITIES_AES_V2;
  396         }
  397         if (!*num_asym_au || !(service_mask & ADF_ACCEL_ASYM)) {
  398                 disabled_caps |= ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
  399                     ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
  400         }
  401         if (!*num_dc_au || !(service_mask & ADF_ACCEL_COMPRESSION)) {
  402                 disabled_caps |= ICP_ACCEL_CAPABILITIES_COMPRESSION |
  403                     ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
  404                     ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
  405                     ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
  406                 accel_dev->hw_device->extended_dc_capabilities = 0;
  407         }
  408         accel_dev->hw_device->accel_capabilities_mask =
  409             adf_4xxx_get_hw_cap(accel_dev) & ~disabled_caps;
  410 
  411         hw_data->service_mask = service_mask;
  412         hw_data->service_to_load_mask = service_mask;
  413 
  414         return 0;
  415 }
  416 
  417 static int
  418 adf_init_accel_unit_services(struct adf_accel_dev *accel_dev)
  419 {
  420         u8 num_sym_au = 0, num_dc_au = 0, num_asym_au = 0;
  421         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  422         u32 num_au = hw_data->get_num_accel_units(hw_data);
  423         u32 au_size = num_au * sizeof(struct adf_accel_unit);
  424         u8 i;
  425 
  426         if (get_accel_unit_config(
  427                 accel_dev, &num_sym_au, &num_dc_au, &num_asym_au))
  428                 return EFAULT;
  429 
  430         accel_dev->au_info = kzalloc(sizeof(*accel_dev->au_info), GFP_KERNEL);
  431         if (!accel_dev->au_info)
  432                 return ENOMEM;
  433 
  434         accel_dev->au_info->au = kzalloc(au_size, GFP_KERNEL);
  435         if (!accel_dev->au_info->au) {
  436                 kfree(accel_dev->au_info);
  437                 accel_dev->au_info = NULL;
  438                 return ENOMEM;
  439         }
  440 
  441         accel_dev->au_info->num_cy_au = num_sym_au;
  442         accel_dev->au_info->num_dc_au = num_dc_au;
  443         accel_dev->au_info->num_asym_au = num_asym_au;
  444 
  445         get_accel_unit(hw_data, &accel_dev->au_info->au);
  446 
  447         /* Enable ASYM accel units */
  448         for (i = 0; i < num_au && num_asym_au > 0; i++) {
  449                 if (accel_dev->au_info->au[i].services ==
  450                     ADF_ACCEL_SERVICE_NULL) {
  451                         accel_dev->au_info->au[i].services = ADF_ACCEL_ASYM;
  452                         num_asym_au--;
  453                 }
  454         }
  455         /* Enable SYM accel units */
  456         for (i = 0; i < num_au && num_sym_au > 0; i++) {
  457                 if (accel_dev->au_info->au[i].services ==
  458                     ADF_ACCEL_SERVICE_NULL) {
  459                         accel_dev->au_info->au[i].services = ADF_ACCEL_CRYPTO;
  460                         num_sym_au--;
  461                 }
  462         }
  463         /* Enable compression accel units */
  464         for (i = 0; i < num_au && num_dc_au > 0; i++) {
  465                 if (accel_dev->au_info->au[i].services ==
  466                     ADF_ACCEL_SERVICE_NULL) {
  467                         accel_dev->au_info->au[i].services =
  468                             ADF_ACCEL_COMPRESSION;
  469                         num_dc_au--;
  470                 }
  471         }
  472         accel_dev->au_info->dc_ae_msk |=
  473             hw_data->get_obj_cfg_ae_mask(accel_dev, ADF_ACCEL_COMPRESSION);
  474 
  475         return 0;
  476 }
  477 
  478 static int
  479 adf_init_accel_units(struct adf_accel_dev *accel_dev)
  480 {
  481         return adf_init_accel_unit_services(accel_dev);
  482 }
  483 
  484 static void
  485 adf_exit_accel_units(struct adf_accel_dev *accel_dev)
  486 {
  487         /* reset the AU service */
  488         adf_exit_accel_unit_services(accel_dev);
  489 }
  490 
  491 static const char *
  492 get_obj_name(struct adf_accel_dev *accel_dev,
  493              enum adf_accel_unit_services service)
  494 {
  495         switch (service) {
  496         case ADF_ACCEL_ASYM:
  497                 return ADF_4XXX_ASYM_OBJ;
  498         case ADF_ACCEL_CRYPTO:
  499                 return ADF_4XXX_SYM_OBJ;
  500         case ADF_ACCEL_COMPRESSION:
  501                 return ADF_4XXX_DC_OBJ;
  502         case ADF_ACCEL_ADMIN:
  503                 return ADF_4XXX_ADMIN_OBJ;
  504         default:
  505                 return NULL;
  506         }
  507 }
  508 
  509 static uint32_t
  510 get_objs_num(struct adf_accel_dev *accel_dev)
  511 {
  512         return ADF_4XXX_MAX_OBJ;
  513 }
  514 
  515 static uint32_t
  516 get_obj_cfg_ae_mask(struct adf_accel_dev *accel_dev,
  517                     enum adf_accel_unit_services service)
  518 {
  519         u32 ae_mask = 0;
  520         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  521         u32 num_au = hw_data->get_num_accel_units(hw_data);
  522         struct adf_accel_unit *accel_unit = accel_dev->au_info->au;
  523         u32 i = 0;
  524 
  525         if (service == ADF_ACCEL_SERVICE_NULL)
  526                 return 0;
  527 
  528         for (i = 0; i < num_au; i++) {
  529                 if (accel_unit[i].services == service)
  530                         ae_mask |= accel_unit[i].ae_mask;
  531         }
  532 
  533         return ae_mask;
  534 }
  535 
  536 static enum adf_accel_unit_services
  537 adf_4xxx_get_service_type(struct adf_accel_dev *accel_dev, s32 obj_num)
  538 {
  539         struct adf_accel_unit *accel_unit;
  540         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  541         u8 num_au = hw_data->get_num_accel_units(hw_data);
  542         int i;
  543 
  544         if (!hw_data->service_to_load_mask)
  545                 return ADF_ACCEL_SERVICE_NULL;
  546 
  547         if (accel_dev->au_info && accel_dev->au_info->au)
  548                 accel_unit = accel_dev->au_info->au;
  549         else
  550                 return ADF_ACCEL_SERVICE_NULL;
  551 
  552         for (i = num_au - 2; i >= 0; i--) {
  553                 if (hw_data->service_to_load_mask & accel_unit[i].services) {
  554                         hw_data->service_to_load_mask &=
  555                             ~accel_unit[i].services;
  556                         return accel_unit[i].services;
  557                 }
  558         }
  559 
  560         /* admin AE should be loaded last */
  561         if (hw_data->service_to_load_mask & accel_unit[num_au - 1].services) {
  562                 hw_data->service_to_load_mask &=
  563                     ~accel_unit[num_au - 1].services;
  564                 return accel_unit[num_au - 1].services;
  565         }
  566 
  567         return ADF_ACCEL_SERVICE_NULL;
  568 }
  569 
  570 static void
  571 get_ring_svc_map_data(int ring_pair_index,
  572                       u16 ring_to_svc_map,
  573                       u8 *serv_type,
  574                       int *ring_index,
  575                       int *num_rings_per_srv,
  576                       int bundle_num)
  577 {
  578         *serv_type =
  579             GET_SRV_TYPE(ring_to_svc_map, bundle_num % ADF_CFG_NUM_SERVICES);
  580         *ring_index = 0;
  581         *num_rings_per_srv = ADF_4XXX_NUM_RINGS_PER_BANK / 2;
  582 }
  583 
  584 static int
  585 adf_get_dc_extcapabilities(struct adf_accel_dev *accel_dev, u32 *capabilities)
  586 {
  587         struct icp_qat_fw_init_admin_req req;
  588         struct icp_qat_fw_init_admin_resp resp;
  589         u8 i;
  590         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  591         u8 num_au = hw_data->get_num_accel_units(hw_data);
  592         u32 first_dc_ae = 0;
  593 
  594         for (i = 0; i < num_au; i++) {
  595                 if (accel_dev->au_info->au[i].services &
  596                     ADF_ACCEL_COMPRESSION) {
  597                         first_dc_ae = accel_dev->au_info->au[i].ae_mask;
  598                         first_dc_ae &= ~(first_dc_ae - 1);
  599                 }
  600         }
  601 
  602         memset(&req, 0, sizeof(req));
  603         memset(&resp, 0, sizeof(resp));
  604         req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET;
  605 
  606         if (likely(first_dc_ae)) {
  607                 if (adf_send_admin(accel_dev, &req, &resp, first_dc_ae) ||
  608                     resp.status) {
  609                         *capabilities = 0;
  610                         return EFAULT;
  611                 }
  612 
  613                 *capabilities = resp.extended_features;
  614         }
  615 
  616         return 0;
  617 }
  618 
  619 static int
  620 adf_get_fw_status(struct adf_accel_dev *accel_dev,
  621                   u8 *major,
  622                   u8 *minor,
  623                   u8 *patch)
  624 {
  625         struct icp_qat_fw_init_admin_req req;
  626         struct icp_qat_fw_init_admin_resp resp;
  627         u32 ae_mask = 1;
  628 
  629         memset(&req, 0, sizeof(req));
  630         memset(&resp, 0, sizeof(resp));
  631         req.cmd_id = ICP_QAT_FW_STATUS_GET;
  632 
  633         if (adf_send_admin(accel_dev, &req, &resp, ae_mask))
  634                 return EFAULT;
  635 
  636         *major = resp.version_major_num;
  637         *minor = resp.version_minor_num;
  638         *patch = resp.version_patch_num;
  639 
  640         return 0;
  641 }
  642 
  643 static int
  644 adf_4xxx_send_admin_init(struct adf_accel_dev *accel_dev)
  645 {
  646         int ret = 0;
  647         struct icp_qat_fw_init_admin_req req;
  648         struct icp_qat_fw_init_admin_resp resp;
  649         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
  650         u32 ae_mask = hw_data->ae_mask;
  651         u32 admin_ae_mask = hw_data->admin_ae_mask;
  652         u8 num_au = hw_data->get_num_accel_units(hw_data);
  653         u8 i;
  654         u32 dc_capabilities = 0;
  655 
  656         for (i = 0; i < num_au; i++) {
  657                 if (accel_dev->au_info->au[i].services ==
  658                     ADF_ACCEL_SERVICE_NULL)
  659                         ae_mask &= ~accel_dev->au_info->au[i].ae_mask;
  660 
  661                 if (accel_dev->au_info->au[i].services != ADF_ACCEL_ADMIN)
  662                         admin_ae_mask &= ~accel_dev->au_info->au[i].ae_mask;
  663         }
  664 
  665         if (!accel_dev->admin) {
  666                 device_printf(GET_DEV(accel_dev), "adf_admin not available\n");
  667                 return EFAULT;
  668         }
  669 
  670         memset(&req, 0, sizeof(req));
  671         memset(&resp, 0, sizeof(resp));
  672 
  673         req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG;
  674         req.init_cfg_sz = ADF_CONST_TABLE_SIZE;
  675         req.init_cfg_ptr = accel_dev->admin->const_tbl_addr;
  676         if (adf_send_admin(accel_dev, &req, &resp, admin_ae_mask)) {
  677                 device_printf(GET_DEV(accel_dev),
  678                               "Error sending constants config message\n");
  679                 return EFAULT;
  680         }
  681 
  682         memset(&req, 0, sizeof(req));
  683         memset(&resp, 0, sizeof(resp));
  684         req.cmd_id = ICP_QAT_FW_INIT_ME;
  685         if (adf_send_admin(accel_dev, &req, &resp, ae_mask)) {
  686                 device_printf(GET_DEV(accel_dev),
  687                               "Error sending init message\n");
  688                 return EFAULT;
  689         }
  690 
  691         memset(&req, 0, sizeof(req));
  692         memset(&resp, 0, sizeof(resp));
  693         req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET;
  694         req.init_cfg_ptr = accel_dev->admin->phy_hb_addr;
  695         if (adf_get_hb_timer(accel_dev, &req.heartbeat_ticks))
  696                 return EINVAL;
  697 
  698         if (adf_send_admin(accel_dev, &req, &resp, ae_mask))
  699                 device_printf(GET_DEV(accel_dev),
  700                               "Heartbeat is not supported\n");
  701 
  702         ret = adf_get_dc_extcapabilities(accel_dev, &dc_capabilities);
  703         if (unlikely(ret)) {
  704                 device_printf(GET_DEV(accel_dev),
  705                               "Could not get FW ext. capabilities\n");
  706         }
  707 
  708         accel_dev->hw_device->extended_dc_capabilities = dc_capabilities;
  709 
  710         adf_get_fw_status(accel_dev,
  711                           &accel_dev->fw_versions.fw_version_major,
  712                           &accel_dev->fw_versions.fw_version_minor,
  713                           &accel_dev->fw_versions.fw_version_patch);
  714 
  715         device_printf(GET_DEV(accel_dev),
  716                       "FW version: %d.%d.%d\n",
  717                       accel_dev->fw_versions.fw_version_major,
  718                       accel_dev->fw_versions.fw_version_minor,
  719                       accel_dev->fw_versions.fw_version_patch);
  720 
  721         return ret;
  722 }
  723 
  724 static enum dev_sku_info
  725 get_sku(struct adf_hw_device_data *self)
  726 {
  727         return DEV_SKU_1;
  728 }
  729 
  730 static struct adf_accel_unit *
  731 get_au_by_ae(struct adf_accel_dev *accel_dev, int ae_num)
  732 {
  733         int i = 0;
  734         struct adf_accel_unit *accel_unit = accel_dev->au_info->au;
  735 
  736         if (!accel_unit)
  737                 return NULL;
  738 
  739         for (i = 0; i < ADF_4XXX_MAX_ACCELUNITS; i++)
  740                 if (accel_unit[i].ae_mask & BIT(ae_num))
  741                         return &accel_unit[i];
  742 
  743         return NULL;
  744 }
  745 
  746 static bool
  747 check_accel_unit_service(enum adf_accel_unit_services au_srv,
  748                          enum adf_cfg_service_type ring_srv)
  749 {
  750         if ((au_srv & ADF_ACCEL_SERVICE_NULL) && ring_srv == NA)
  751                 return true;
  752         if ((au_srv & ADF_ACCEL_COMPRESSION) && ring_srv == COMP)
  753                 return true;
  754         if ((au_srv & ADF_ACCEL_ASYM) && ring_srv == ASYM)
  755                 return true;
  756         if ((au_srv & ADF_ACCEL_CRYPTO) && ring_srv == SYM)
  757                 return true;
  758 
  759         return false;
  760 }
  761 
  762 static void
  763 adf_4xxx_cfg_gen_dispatch_arbiter(struct adf_accel_dev *accel_dev,
  764                                   u32 *thrd_to_arb_map_gen)
  765 {
  766         struct adf_accel_unit *au = NULL;
  767         int engine = 0;
  768         int thread = 0;
  769         int service;
  770         u16 ena_srv_mask;
  771         u16 service_type;
  772         u32 service_mask;
  773         unsigned long thd_srv_mask = default_active_thd_mask;
  774 
  775         ena_srv_mask = accel_dev->hw_device->ring_to_svc_map;
  776         /* If ring_to_svc_map is not changed, return default arbiter value */
  777         if (ena_srv_mask == ADF_4XXX_DEFAULT_RING_TO_SRV_MAP) {
  778                 memcpy(thrd_to_arb_map_gen,
  779                        thrd_to_arb_map,
  780                        sizeof(thrd_to_arb_map_gen[0]) *
  781                            ADF_4XXX_MAX_ACCELENGINES);
  782                 return;
  783         }
  784 
  785         for (engine = 0; engine < ADF_4XXX_MAX_ACCELENGINES - 1; engine++) {
  786                 thrd_to_arb_map_gen[engine] = 0;
  787                 service_mask = 0;
  788                 au = get_au_by_ae(accel_dev, engine);
  789                 if (!au)
  790                         continue;
  791 
  792                 for (service = 0; service < ADF_CFG_MAX_SERVICES; service++) {
  793                         service_type = GET_SRV_TYPE(ena_srv_mask, service);
  794                         if (check_accel_unit_service(au->services,
  795                                                      service_type))
  796                                 service_mask |= BIT(service);
  797                 }
  798 
  799                 if (au->services == ADF_ACCEL_COMPRESSION)
  800                         thd_srv_mask = dc_me_active_thd_mask;
  801                 else
  802                         thd_srv_mask = default_active_thd_mask;
  803 
  804                 for_each_set_bit(thread, &thd_srv_mask, 8)
  805                 {
  806                         thrd_to_arb_map_gen[engine] |=
  807                             (service_mask << (ADF_CFG_MAX_SERVICES * thread));
  808                 }
  809         }
  810 }
  811 
  812 static void
  813 adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev,
  814                         u32 const **arb_map_config)
  815 {
  816         int i;
  817         struct adf_hw_device_data *hw_device = accel_dev->hw_device;
  818 
  819         for (i = 1; i < ADF_4XXX_MAX_ACCELENGINES; i++) {
  820                 if (~hw_device->ae_mask & (1 << i))
  821                         thrd_to_arb_map[i] = 0;
  822         }
  823         adf_4xxx_cfg_gen_dispatch_arbiter(accel_dev, thrd_to_arb_map_gen);
  824         *arb_map_config = thrd_to_arb_map_gen;
  825 }
  826 
  827 static void
  828 get_arb_info(struct arb_info *arb_info)
  829 {
  830         arb_info->wrk_cfg_offset = ADF_4XXX_ARB_CONFIG;
  831         arb_info->arbiter_offset = ADF_4XXX_ARB_OFFSET;
  832         arb_info->wrk_thd_2_srv_arb_map = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET;
  833 }
  834 
  835 static void
  836 get_admin_info(struct admin_info *admin_csrs_info)
  837 {
  838         admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET;
  839         admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET;
  840         admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET;
  841 }
  842 
  843 static void
  844 adf_enable_error_correction(struct adf_accel_dev *accel_dev)
  845 {
  846         struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR];
  847         struct resource *csr = misc_bar->virt_addr;
  848 
  849         /* Enable all in errsou3 except VFLR notification on host */
  850         ADF_CSR_WR(csr, ADF_4XXX_ERRMSK3, ADF_4XXX_VFLNOTIFY);
  851 }
  852 
  853 static void
  854 adf_enable_ints(struct adf_accel_dev *accel_dev)
  855 {
  856         struct resource *addr;
  857 
  858         addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr;
  859 
  860         /* Enable bundle interrupts */
  861         ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0);
  862         ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0);
  863 
  864         /* Enable misc interrupts */
  865         ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0);
  866 }
  867 
  868 static int
  869 adf_init_device(struct adf_accel_dev *accel_dev)
  870 {
  871         struct resource *addr;
  872         u32 status;
  873         u32 csr;
  874         int ret;
  875 
  876         addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr;
  877 
  878         /* Temporarily mask PM interrupt */
  879         csr = ADF_CSR_RD(addr, ADF_4XXX_ERRMSK2);
  880         csr |= ADF_4XXX_PM_SOU;
  881         ADF_CSR_WR(addr, ADF_4XXX_ERRMSK2, csr);
  882 
  883         /* Set DRV_ACTIVE bit to power up the device */
  884         ADF_CSR_WR(addr, ADF_4XXX_PM_INTERRUPT, ADF_4XXX_PM_DRV_ACTIVE);
  885 
  886         /* Poll status register to make sure the device is powered up */
  887         status = 0;
  888         ret = read_poll_timeout(ADF_CSR_RD,
  889                                 status,
  890                                 status & ADF_4XXX_PM_INIT_STATE,
  891                                 ADF_4XXX_PM_POLL_DELAY_US,
  892                                 ADF_4XXX_PM_POLL_TIMEOUT_US,
  893                                 true,
  894                                 addr,
  895                                 ADF_4XXX_PM_STATUS);
  896         if (ret)
  897                 device_printf(GET_DEV(accel_dev),
  898                               "Failed to power up the device\n");
  899 
  900         return ret;
  901 }
  902 
  903 void
  904 adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
  905 {
  906         hw_data->dev_class = &adf_4xxx_class;
  907         hw_data->instance_id = adf_4xxx_class.instances++;
  908         hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS;
  909         hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK;
  910         hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS;
  911         hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES;
  912         hw_data->num_logical_accel = 1;
  913         hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET;
  914         hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK;
  915         hw_data->alloc_irq = adf_isr_resource_alloc;
  916         hw_data->free_irq = adf_isr_resource_free;
  917         hw_data->enable_error_correction = adf_enable_error_correction;
  918         hw_data->get_accel_mask = get_accel_mask;
  919         hw_data->get_ae_mask = get_ae_mask;
  920         hw_data->get_num_accels = get_num_accels;
  921         hw_data->get_num_aes = get_num_aes;
  922         hw_data->get_sram_bar_id = get_sram_bar_id;
  923         hw_data->get_etr_bar_id = get_etr_bar_id;
  924         hw_data->get_misc_bar_id = get_misc_bar_id;
  925         hw_data->get_arb_info = get_arb_info;
  926         hw_data->get_admin_info = get_admin_info;
  927         hw_data->get_accel_cap = adf_4xxx_get_hw_cap;
  928         hw_data->clock_frequency = ADF_4XXX_AE_FREQ;
  929         hw_data->get_sku = get_sku;
  930         hw_data->heartbeat_ctr_num = ADF_NUM_HB_CNT_PER_AE;
  931         hw_data->fw_name = ADF_4XXX_FW;
  932         hw_data->fw_mmp_name = ADF_4XXX_MMP;
  933         hw_data->init_admin_comms = adf_init_admin_comms;
  934         hw_data->exit_admin_comms = adf_exit_admin_comms;
  935         hw_data->send_admin_init = adf_4xxx_send_admin_init;
  936         hw_data->init_arb = adf_init_gen2_arb;
  937         hw_data->exit_arb = adf_exit_arb;
  938         hw_data->get_arb_mapping = adf_get_arbiter_mapping;
  939         hw_data->enable_ints = adf_enable_ints;
  940         hw_data->init_device = adf_init_device;
  941         hw_data->reset_device = adf_reset_flr;
  942         hw_data->restore_device = adf_dev_restore;
  943         hw_data->init_accel_units = adf_init_accel_units;
  944         hw_data->exit_accel_units = adf_exit_accel_units;
  945         hw_data->get_num_accel_units = get_num_accel_units;
  946         hw_data->configure_accel_units = adf_4xxx_configure_accel_units;
  947         hw_data->get_ring_to_svc_map = get_ring_to_svc_map;
  948         hw_data->get_ring_svc_map_data = get_ring_svc_map_data;
  949         hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK;
  950         hw_data->get_objs_num = get_objs_num;
  951         hw_data->get_obj_name = get_obj_name;
  952         hw_data->get_obj_cfg_ae_mask = get_obj_cfg_ae_mask;
  953         hw_data->get_service_type = adf_4xxx_get_service_type;
  954         hw_data->set_msix_rttable = set_msix_default_rttable;
  955         hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer;
  956         hw_data->disable_iov = adf_disable_sriov;
  957         hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION;
  958         hw_data->config_device = adf_config_device;
  959         hw_data->set_asym_rings_mask = adf_cfg_set_asym_rings_mask;
  960         hw_data->get_hb_clock = get_hb_clock;
  961         hw_data->get_heartbeat_status = adf_get_heartbeat_status;
  962         hw_data->get_ae_clock = get_ae_clock;
  963         hw_data->measure_clock = measure_clock;
  964         hw_data->query_storage_cap = 1;
  965 
  966         adf_gen4_init_hw_csr_info(&hw_data->csr_info);
  967 }
  968 
  969 void
  970 adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data)
  971 {
  972         hw_data->dev_class->instances--;
  973 }
Cache object: b9f048ef9f49b3eaac6913823b021008
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c
