qat_uclo.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /* SPDX-License-Identifier: BSD-3-Clause */
    2 /* Copyright(c) 2007-2022 Intel Corporation */
    3 /* $FreeBSD$ */
    4 #include "qat_freebsd.h"
    5 #include "adf_cfg.h"
    6 #include "adf_common_drv.h"
    7 #include "adf_accel_devices.h"
    8 #include "icp_qat_uclo.h"
    9 #include "icp_qat_fw.h"
   10 #include "icp_qat_fw_init_admin.h"
   11 #include "adf_cfg_strings.h"
   12 #include "adf_transport_access_macros.h"
   13 #include "adf_transport_internal.h"
   14 #include <sys/ctype.h>
   15 #include <sys/kernel.h>
   16 #include <linux/delay.h>
   17 #include "adf_accel_devices.h"
   18 #include "adf_common_drv.h"
   19 #include "icp_qat_uclo.h"
   20 #include "icp_qat_hal.h"
   21 #include "icp_qat_fw_loader_handle.h"
   22 
   23 #define UWORD_CPYBUF_SIZE 1024
   24 #define INVLD_UWORD 0xffffffffffull
   25 #define PID_MINOR_REV 0xf
   26 #define PID_MAJOR_REV (0xf << 4)
   27 #define MAX_UINT32_VAL 0xfffffffful
   28 
   29 static int
   30 qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle,
   31                       unsigned int ae,
   32                       unsigned int image_num)
   33 {
   34         struct icp_qat_uclo_aedata *ae_data;
   35         struct icp_qat_uclo_encapme *encap_image;
   36         struct icp_qat_uclo_page *page = NULL;
   37         struct icp_qat_uclo_aeslice *ae_slice = NULL;
   38 
   39         ae_data = &obj_handle->ae_data[ae];
   40         encap_image = &obj_handle->ae_uimage[image_num];
   41         ae_slice = &ae_data->ae_slices[ae_data->slice_num];
   42         ae_slice->encap_image = encap_image;
   43 
   44         if (encap_image->img_ptr) {
   45                 ae_slice->ctx_mask_assigned =
   46                     encap_image->img_ptr->ctx_assigned;
   47                 ae_data->shareable_ustore =
   48                     ICP_QAT_SHARED_USTORE_MODE(encap_image->img_ptr->ae_mode);
   49                 if (obj_handle->prod_type == ICP_QAT_AC_4XXX_A_DEV_TYPE)
   50                         ae_data->eff_ustore_size = obj_handle->ustore_phy_size;
   51                 else {
   52                         ae_data->eff_ustore_size = ae_data->shareable_ustore ?
   53                             (obj_handle->ustore_phy_size << 1) :
   54                             obj_handle->ustore_phy_size;
   55                 }
   56         } else {
   57                 ae_slice->ctx_mask_assigned = 0;
   58         }
   59         ae_slice->region =
   60             malloc(sizeof(*ae_slice->region), M_QAT, M_WAITOK | M_ZERO);
   61         ae_slice->page =
   62             malloc(sizeof(*ae_slice->page), M_QAT, M_WAITOK | M_ZERO);
   63         page = ae_slice->page;
   64         page->encap_page = encap_image->page;
   65         ae_slice->page->region = ae_slice->region;
   66         ae_data->slice_num++;
   67         return 0;
   68 }
   69 
   70 static int
   71 qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data)
   72 {
   73         unsigned int i;
   74 
   75         if (!ae_data) {
   76                 pr_err("QAT: bad argument, ae_data is NULL\n ");
   77                 return EINVAL;
   78         }
   79 
   80         for (i = 0; i < ae_data->slice_num; i++) {
   81                 free(ae_data->ae_slices[i].region, M_QAT);
   82                 ae_data->ae_slices[i].region = NULL;
   83                 free(ae_data->ae_slices[i].page, M_QAT);
   84                 ae_data->ae_slices[i].page = NULL;
   85         }
   86         return 0;
   87 }
   88 
   89 static char *
   90 qat_uclo_get_string(struct icp_qat_uof_strtable *str_table,
   91                     unsigned int str_offset)
   92 {
   93         if (!str_table->table_len || str_offset > str_table->table_len)
   94                 return NULL;
   95         return (char *)(((uintptr_t)(str_table->strings)) + str_offset);
   96 }
   97 
   98 static int
   99 qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr)
  100 {
  101         int maj = hdr->maj_ver & 0xff;
  102         int min = hdr->min_ver & 0xff;
  103 
  104         if (hdr->file_id != ICP_QAT_UOF_FID) {
  105                 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id);
  106                 return EINVAL;
  107         }
  108         if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) {
  109                 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n",
  110                        maj,
  111                        min);
  112                 return EINVAL;
  113         }
  114         return 0;
  115 }
  116 
  117 static int
  118 qat_uclo_check_suof_format(const struct icp_qat_suof_filehdr *suof_hdr)
  119 {
  120         int maj = suof_hdr->maj_ver & 0xff;
  121         int min = suof_hdr->min_ver & 0xff;
  122 
  123         if (suof_hdr->file_id != ICP_QAT_SUOF_FID) {
  124                 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id);
  125                 return EINVAL;
  126         }
  127         if (suof_hdr->fw_type != 0) {
  128                 pr_err("QAT: unsupported firmware type\n");
  129                 return EINVAL;
  130         }
  131         if (suof_hdr->num_chunks <= 0x1) {
  132                 pr_err("QAT: SUOF chunk amount is incorrect\n");
  133                 return EINVAL;
  134         }
  135         if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) {
  136                 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n",
  137                        maj,
  138                        min);
  139                 return EINVAL;
  140         }
  141         return 0;
  142 }
  143 
  144 static int
  145 qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle,
  146                           unsigned int addr,
  147                           const unsigned int *val,
  148                           unsigned int num_in_bytes)
  149 {
  150         unsigned int outval;
  151         const unsigned char *ptr = (const unsigned char *)val;
  152 
  153         if (num_in_bytes > handle->hal_sram_size) {
  154                 pr_err("QAT: error, mmp size overflow %d\n", num_in_bytes);
  155                 return EINVAL;
  156         }
  157         while (num_in_bytes) {
  158                 memcpy(&outval, ptr, 4);
  159                 SRAM_WRITE(handle, addr, outval);
  160                 num_in_bytes -= 4;
  161                 ptr += 4;
  162                 addr += 4;
  163         }
  164         return 0;
  165 }
  166 
  167 static void
  168 qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle,
  169                           unsigned char ae,
  170                           unsigned int addr,
  171                           unsigned int *val,
  172                           unsigned int num_in_bytes)
  173 {
  174         unsigned int outval;
  175         unsigned char *ptr = (unsigned char *)val;
  176 
  177         addr >>= 0x2; /* convert to uword address */
  178 
  179         while (num_in_bytes) {
  180                 memcpy(&outval, ptr, 4);
  181                 qat_hal_wr_umem(handle, ae, addr++, 1, &outval);
  182                 num_in_bytes -= 4;
  183                 ptr += 4;
  184         }
  185 }
  186 
  187 static void
  188 qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle,
  189                        unsigned char ae,
  190                        struct icp_qat_uof_batch_init *umem_init_header)
  191 {
  192         struct icp_qat_uof_batch_init *umem_init;
  193 
  194         if (!umem_init_header)
  195                 return;
  196         umem_init = umem_init_header->next;
  197         while (umem_init) {
  198                 unsigned int addr, *value, size;
  199 
  200                 ae = umem_init->ae;
  201                 addr = umem_init->addr;
  202                 value = umem_init->value;
  203                 size = umem_init->size;
  204                 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size);
  205                 umem_init = umem_init->next;
  206         }
  207 }
  208 
  209 static void
  210 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle,
  211                                  struct icp_qat_uof_batch_init **base)
  212 {
  213         struct icp_qat_uof_batch_init *umem_init;
  214 
  215         umem_init = *base;
  216         while (umem_init) {
  217                 struct icp_qat_uof_batch_init *pre;
  218 
  219                 pre = umem_init;
  220                 umem_init = umem_init->next;
  221                 free(pre, M_QAT);
  222         }
  223         *base = NULL;
  224 }
  225 
  226 static int
  227 qat_uclo_parse_num(char *str, unsigned int *num)
  228 {
  229         char buf[16] = { 0 };
  230         unsigned long ae = 0;
  231         int i;
  232 
  233         strncpy(buf, str, 15);
  234         for (i = 0; i < 16; i++) {
  235                 if (!isdigit(buf[i])) {
  236                         buf[i] = '\0';
  237                         break;
  238                 }
  239         }
  240         if ((compat_strtoul(buf, 10, &ae)))
  241                 return EFAULT;
  242 
  243         if (ae > MAX_UINT32_VAL)
  244                 return EFAULT;
  245 
  246         *num = (unsigned int)ae;
  247         return 0;
  248 }
  249 
  250 static int
  251 qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle,
  252                           struct icp_qat_uof_initmem *init_mem,
  253                           unsigned int size_range,
  254                           unsigned int *ae)
  255 {
  256         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
  257         char *str;
  258 
  259         if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) {
  260                 pr_err("QAT: initmem is out of range");
  261                 return EINVAL;
  262         }
  263         if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) {
  264                 pr_err("QAT: Memory scope for init_mem error\n");
  265                 return EINVAL;
  266         }
  267         str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name);
  268         if (!str) {
  269                 pr_err("QAT: AE name assigned in UOF init table is NULL\n");
  270                 return EINVAL;
  271         }
  272         if (qat_uclo_parse_num(str, ae)) {
  273                 pr_err("QAT: Parse num for AE number failed\n");
  274                 return EINVAL;
  275         }
  276         if (*ae >= ICP_QAT_UCLO_MAX_AE) {
  277                 pr_err("QAT: ae %d out of range\n", *ae);
  278                 return EINVAL;
  279         }
  280         return 0;
  281 }
  282 
  283 static int
  284 qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle *handle,
  285                                 struct icp_qat_uof_initmem *init_mem,
  286                                 unsigned int ae,
  287                                 struct icp_qat_uof_batch_init **init_tab_base)
  288 {
  289         struct icp_qat_uof_batch_init *init_header, *tail;
  290         struct icp_qat_uof_batch_init *mem_init, *tail_old;
  291         struct icp_qat_uof_memvar_attr *mem_val_attr;
  292         unsigned int i = 0;
  293 
  294         mem_val_attr =
  295             (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem +
  296                                                sizeof(
  297                                                    struct icp_qat_uof_initmem));
  298 
  299         init_header = *init_tab_base;
  300         if (!init_header) {
  301                 init_header =
  302                     malloc(sizeof(*init_header), M_QAT, M_WAITOK | M_ZERO);
  303                 init_header->size = 1;
  304                 *init_tab_base = init_header;
  305         }
  306         tail_old = init_header;
  307         while (tail_old->next)
  308                 tail_old = tail_old->next;
  309         tail = tail_old;
  310         for (i = 0; i < init_mem->val_attr_num; i++) {
  311                 mem_init = malloc(sizeof(*mem_init), M_QAT, M_WAITOK | M_ZERO);
  312                 mem_init->ae = ae;
  313                 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte;
  314                 mem_init->value = &mem_val_attr->value;
  315                 mem_init->size = 4;
  316                 mem_init->next = NULL;
  317                 tail->next = mem_init;
  318                 tail = mem_init;
  319                 init_header->size += qat_hal_get_ins_num();
  320                 mem_val_attr++;
  321         }
  322         return 0;
  323 }
  324 
  325 static int
  326 qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle,
  327                        struct icp_qat_uof_initmem *init_mem)
  328 {
  329         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
  330         unsigned int ae;
  331         unsigned int lmem;
  332 
  333         lmem = IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))) ?
  334             ICP_QAT_UCLO_MAX_LMEM_REG_2X :
  335             ICP_QAT_UCLO_MAX_LMEM_REG;
  336 
  337         if (qat_uclo_fetch_initmem_ae(handle, init_mem, lmem, &ae))
  338                 return EINVAL;
  339         if (qat_uclo_create_batch_init_list(
  340                 handle, init_mem, ae, &obj_handle->lm_init_tab[ae]))
  341                 return EINVAL;
  342         return 0;
  343 }
  344 
  345 static int
  346 qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle,
  347                        struct icp_qat_uof_initmem *init_mem)
  348 {
  349         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
  350         unsigned int ae, ustore_size, uaddr, i;
  351         struct icp_qat_uclo_aedata *aed;
  352 
  353         ustore_size = obj_handle->ustore_phy_size;
  354         if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae))
  355                 return EINVAL;
  356         if (qat_uclo_create_batch_init_list(
  357                 handle, init_mem, ae, &obj_handle->umem_init_tab[ae]))
  358                 return EINVAL;
  359         /* set the highest ustore address referenced */
  360         uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2;
  361         aed = &obj_handle->ae_data[ae];
  362         for (i = 0; i < aed->slice_num; i++) {
  363                 if (aed->ae_slices[i].encap_image->uwords_num < uaddr)
  364                         aed->ae_slices[i].encap_image->uwords_num = uaddr;
  365         }
  366         return 0;
  367 }
  368 
  369 #define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000
  370 static int
  371 qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle,
  372                         struct icp_qat_uof_initmem *init_mem)
  373 {
  374         switch (init_mem->region) {
  375         case ICP_QAT_UOF_LMEM_REGION:
  376                 if (qat_uclo_init_lmem_seg(handle, init_mem))
  377                         return EINVAL;
  378                 break;
  379         case ICP_QAT_UOF_UMEM_REGION:
  380                 if (qat_uclo_init_umem_seg(handle, init_mem))
  381                         return EINVAL;
  382                 break;
  383         default:
  384                 pr_err("QAT: initmem region error. region type=0x%x\n",
  385                        init_mem->region);
  386                 return EINVAL;
  387         }
  388         return 0;
  389 }
  390 
  391 static int
  392 qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle,
  393                      struct icp_qat_uclo_encapme *image)
  394 {
  395         unsigned int i;
  396         struct icp_qat_uclo_encap_page *page;
  397         struct icp_qat_uof_image *uof_image;
  398         unsigned char ae = 0;
  399         unsigned char neigh_ae;
  400         unsigned int ustore_size;
  401         unsigned int patt_pos;
  402         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
  403         uint64_t *fill_data;
  404         static unsigned int init[32] = { 0 };
  405         unsigned long ae_mask = handle->hal_handle->ae_mask;
  406 
  407         uof_image = image->img_ptr;
  408         /*if shared CS mode, the ustore size should be 2*ustore_phy_size*/
  409         fill_data = malloc(obj_handle->ustore_phy_size * 2 * sizeof(uint64_t),
  410                            M_QAT,
  411                            M_WAITOK | M_ZERO);
  412         for (i = 0; i < obj_handle->ustore_phy_size * 2; i++)
  413                 memcpy(&fill_data[i],
  414                        &uof_image->fill_pattern,
  415                        sizeof(uint64_t));
  416         page = image->page;
  417 
  418         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
  419         {
  420                 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
  421                 unsigned long ae_assigned = uof_image->ae_assigned;
  422                 const bool gen4 =
  423                     IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)));
  424 
  425                 if (!test_bit(ae, &cfg_ae_mask))
  426                         continue;
  427 
  428                 if (!test_bit(ae, &ae_assigned))
  429                         continue;
  430 
  431                 if (obj_handle->ae_data[ae].shareable_ustore && (ae & 1) &&
  432                     !gen4) {
  433                         qat_hal_get_scs_neigh_ae(ae, &neigh_ae);
  434 
  435                         if (test_bit(neigh_ae, &ae_assigned))
  436                                 continue;
  437                 }
  438 
  439                 ustore_size = obj_handle->ae_data[ae].eff_ustore_size;
  440                 patt_pos = page->beg_addr_p + page->micro_words_num;
  441                 if (obj_handle->ae_data[ae].shareable_ustore && !gen4) {
  442                         qat_hal_get_scs_neigh_ae(ae, &neigh_ae);
  443                         if (init[ae] == 0 && page->beg_addr_p != 0) {
  444                                 qat_hal_wr_coalesce_uwords(handle,
  445                                                            (unsigned char)ae,
  446                                                            0,
  447                                                            page->beg_addr_p,
  448                                                            &fill_data[0]);
  449                         }
  450                         qat_hal_wr_coalesce_uwords(
  451                             handle,
  452                             (unsigned char)ae,
  453                             patt_pos,
  454                             ustore_size - patt_pos,
  455                             &fill_data[page->beg_addr_p]);
  456                         init[ae] = 1;
  457                         init[neigh_ae] = 1;
  458                 } else {
  459                         if (gen4 && (ae % 4 != 0))
  460                                 continue;
  461 
  462                         qat_hal_wr_uwords(handle,
  463                                           (unsigned char)ae,
  464                                           0,
  465                                           page->beg_addr_p,
  466                                           &fill_data[0]);
  467                         qat_hal_wr_uwords(handle,
  468                                           (unsigned char)ae,
  469                                           patt_pos,
  470                                           ustore_size - patt_pos + 1,
  471                                           &fill_data[page->beg_addr_p]);
  472                 }
  473         }
  474         free(fill_data, M_QAT);
  475         return 0;
  476 }
  477 
  478 static int
  479 qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle)
  480 {
  481         int i;
  482         int ae = 0;
  483         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
  484         struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem;
  485         unsigned long ae_mask = handle->hal_handle->ae_mask;
  486 
  487         for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) {
  488                 if (initmem->num_in_bytes) {
  489                         if (qat_uclo_init_ae_memory(handle, initmem))
  490                                 return EINVAL;
  491                 }
  492                 initmem =
  493                     (struct icp_qat_uof_initmem
  494                          *)((uintptr_t)((uintptr_t)initmem +
  495                                         sizeof(struct icp_qat_uof_initmem)) +
  496                             (sizeof(struct icp_qat_uof_memvar_attr) *
  497                              initmem->val_attr_num));
  498         }
  499 
  500         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
  501         {
  502                 if (qat_hal_batch_wr_lm(handle,
  503                                         ae,
  504                                         obj_handle->lm_init_tab[ae])) {
  505                         pr_err("QAT: fail to batch init lmem for AE %d\n", ae);
  506                         return EINVAL;
  507                 }
  508                 qat_uclo_cleanup_batch_init_list(handle,
  509                                                  &obj_handle->lm_init_tab[ae]);
  510                 qat_uclo_batch_wr_umem(handle,
  511                                        ae,
  512                                        obj_handle->umem_init_tab[ae]);
  513                 qat_uclo_cleanup_batch_init_list(
  514                     handle, &obj_handle->umem_init_tab[ae]);
  515         }
  516         return 0;
  517 }
  518 
  519 static void *
  520 qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr,
  521                     char *chunk_id,
  522                     void *cur)
  523 {
  524         int i;
  525         struct icp_qat_uof_chunkhdr *chunk_hdr =
  526             (struct icp_qat_uof_chunkhdr *)((uintptr_t)obj_hdr +
  527                                             sizeof(struct icp_qat_uof_objhdr));
  528 
  529         for (i = 0; i < obj_hdr->num_chunks; i++) {
  530                 if ((cur < (void *)&chunk_hdr[i]) &&
  531                     !strncmp(chunk_hdr[i].chunk_id,
  532                              chunk_id,
  533                              ICP_QAT_UOF_OBJID_LEN)) {
  534                         return &chunk_hdr[i];
  535                 }
  536         }
  537         return NULL;
  538 }
  539 
  540 static unsigned int
  541 qat_uclo_calc_checksum(unsigned int reg, int ch)
  542 {
  543         int i;
  544         unsigned int topbit = 1 << 0xF;
  545         unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch);
  546 
  547         reg ^= inbyte << 0x8;
  548         for (i = 0; i < 0x8; i++) {
  549                 if (reg & topbit)
  550                         reg = (reg << 1) ^ 0x1021;
  551                 else
  552                         reg <<= 1;
  553         }
  554         return reg & 0xFFFF;
  555 }
  556 
  557 static unsigned int
  558 qat_uclo_calc_str_checksum(const char *ptr, int num)
  559 {
  560         unsigned int chksum = 0;
  561 
  562         if (ptr)
  563                 while (num--)
  564                         chksum = qat_uclo_calc_checksum(chksum, *ptr++);
  565         return chksum;
  566 }
  567 
  568 static struct icp_qat_uclo_objhdr *
  569 qat_uclo_map_chunk(char *buf,
  570                    struct icp_qat_uof_filehdr *file_hdr,
  571                    char *chunk_id)
  572 {
  573         struct icp_qat_uof_filechunkhdr *file_chunk;
  574         struct icp_qat_uclo_objhdr *obj_hdr;
  575         char *chunk;
  576         int i;
  577 
  578         file_chunk = (struct icp_qat_uof_filechunkhdr
  579                           *)(buf + sizeof(struct icp_qat_uof_filehdr));
  580         for (i = 0; i < file_hdr->num_chunks; i++) {
  581                 if (!strncmp(file_chunk->chunk_id,
  582                              chunk_id,
  583                              ICP_QAT_UOF_OBJID_LEN)) {
  584                         chunk = buf + file_chunk->offset;
  585                         if (file_chunk->checksum !=
  586                             qat_uclo_calc_str_checksum(chunk, file_chunk->size))
  587                                 break;
  588                         obj_hdr =
  589                             malloc(sizeof(*obj_hdr), M_QAT, M_WAITOK | M_ZERO);
  590                         obj_hdr->file_buff = chunk;
  591                         obj_hdr->checksum = file_chunk->checksum;
  592                         obj_hdr->size = file_chunk->size;
  593                         return obj_hdr;
  594                 }
  595                 file_chunk++;
  596         }
  597         return NULL;
  598 }
  599 
  600 static unsigned int
  601 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj,
  602                             struct icp_qat_uof_image *image)
  603 {
  604         struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab;
  605         struct icp_qat_uof_objtable *neigh_reg_tab;
  606         struct icp_qat_uof_code_page *code_page;
  607 
  608         code_page =
  609             (struct icp_qat_uof_code_page *)((char *)image +
  610                                              sizeof(struct icp_qat_uof_image));
  611         uc_var_tab =
  612             (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
  613                                             code_page->uc_var_tab_offset);
  614         imp_var_tab =
  615             (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
  616                                             code_page->imp_var_tab_offset);
  617         imp_expr_tab =
  618             (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
  619                                             code_page->imp_expr_tab_offset);
  620         if (uc_var_tab->entry_num || imp_var_tab->entry_num ||
  621             imp_expr_tab->entry_num) {
  622                 pr_err("QAT: UOF can't contain imported variable to be parsed");
  623                 return EINVAL;
  624         }
  625         neigh_reg_tab =
  626             (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
  627                                             code_page->neigh_reg_tab_offset);
  628         if (neigh_reg_tab->entry_num) {
  629                 pr_err("QAT: UOF can't contain neighbor register table\n");
  630                 return EINVAL;
  631         }
  632         if (image->numpages > 1) {
  633                 pr_err("QAT: UOF can't contain multiple pages\n");
  634                 return EINVAL;
  635         }
  636         if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) {
  637                 pr_err("QAT: UOF can't use reloadable feature\n");
  638                 return EFAULT;
  639         }
  640         return 0;
  641 }
  642 
  643 static void
  644 qat_uclo_map_image_page(struct icp_qat_uof_encap_obj *encap_uof_obj,
  645                         struct icp_qat_uof_image *img,
  646                         struct icp_qat_uclo_encap_page *page)
  647 {
  648         struct icp_qat_uof_code_page *code_page;
  649         struct icp_qat_uof_code_area *code_area;
  650         struct icp_qat_uof_objtable *uword_block_tab;
  651         struct icp_qat_uof_uword_block *uwblock;
  652         int i;
  653 
  654         code_page =
  655             (struct icp_qat_uof_code_page *)((char *)img +
  656                                              sizeof(struct icp_qat_uof_image));
  657         page->def_page = code_page->def_page;
  658         page->page_region = code_page->page_region;
  659         page->beg_addr_v = code_page->beg_addr_v;
  660         page->beg_addr_p = code_page->beg_addr_p;
  661         code_area =
  662             (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof +
  663                                              code_page->code_area_offset);
  664         page->micro_words_num = code_area->micro_words_num;
  665         uword_block_tab =
  666             (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
  667                                             code_area->uword_block_tab);
  668         page->uwblock_num = uword_block_tab->entry_num;
  669         uwblock = (struct icp_qat_uof_uword_block
  670                        *)((char *)uword_block_tab +
  671                           sizeof(struct icp_qat_uof_objtable));
  672         page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock;
  673         for (i = 0; i < uword_block_tab->entry_num; i++)
  674                 page->uwblock[i].micro_words =
  675                     (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset;
  676 }
  677 
  678 static int
  679 qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle,
  680                     struct icp_qat_uclo_encapme *ae_uimage,
  681                     int max_image)
  682 {
  683         int i, j;
  684         struct icp_qat_uof_chunkhdr *chunk_hdr = NULL;
  685         struct icp_qat_uof_image *image;
  686         struct icp_qat_uof_objtable *ae_regtab;
  687         struct icp_qat_uof_objtable *init_reg_sym_tab;
  688         struct icp_qat_uof_objtable *sbreak_tab;
  689         struct icp_qat_uof_encap_obj *encap_uof_obj =
  690             &obj_handle->encap_uof_obj;
  691 
  692         for (j = 0; j < max_image; j++) {
  693                 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
  694                                                 ICP_QAT_UOF_IMAG,
  695                                                 chunk_hdr);
  696                 if (!chunk_hdr)
  697                         break;
  698                 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof +
  699                                                      chunk_hdr->offset);
  700                 ae_regtab =
  701                     (struct icp_qat_uof_objtable *)(image->reg_tab_offset +
  702                                                     obj_handle->obj_hdr
  703                                                         ->file_buff);
  704                 ae_uimage[j].ae_reg_num = ae_regtab->entry_num;
  705                 ae_uimage[j].ae_reg =
  706                     (struct icp_qat_uof_ae_reg
  707                          *)(((char *)ae_regtab) +
  708                             sizeof(struct icp_qat_uof_objtable));
  709                 init_reg_sym_tab =
  710                     (struct icp_qat_uof_objtable *)(image->init_reg_sym_tab +
  711                                                     obj_handle->obj_hdr
  712                                                         ->file_buff);
  713                 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num;
  714                 ae_uimage[j].init_regsym =
  715                     (struct icp_qat_uof_init_regsym
  716                          *)(((char *)init_reg_sym_tab) +
  717                             sizeof(struct icp_qat_uof_objtable));
  718                 sbreak_tab = (struct icp_qat_uof_objtable *)(image->sbreak_tab +
  719                                                              obj_handle->obj_hdr
  720                                                                  ->file_buff);
  721                 ae_uimage[j].sbreak_num = sbreak_tab->entry_num;
  722                 ae_uimage[j].sbreak =
  723                     (struct icp_qat_uof_sbreak
  724                          *)(((char *)sbreak_tab) +
  725                             sizeof(struct icp_qat_uof_objtable));
  726                 ae_uimage[j].img_ptr = image;
  727                 if (qat_uclo_check_image_compat(encap_uof_obj, image))
  728                         goto out_err;
  729                 ae_uimage[j].page =
  730                     malloc(sizeof(struct icp_qat_uclo_encap_page),
  731                            M_QAT,
  732                            M_WAITOK | M_ZERO);
  733                 qat_uclo_map_image_page(encap_uof_obj,
  734                                         image,
  735                                         ae_uimage[j].page);
  736         }
  737         return j;
  738 out_err:
  739         for (i = 0; i < j; i++)
  740                 free(ae_uimage[i].page, M_QAT);
  741         return 0;
  742 }
  743 
  744 static int
  745 UcLo_checkTGroupList2X(struct icp_qat_fw_loader_handle *handle)
  746 {
  747         int i;
  748         unsigned int swAe = 0;
  749         unsigned int ii, jj;
  750         struct icp_qat_uclo_aedata *ae_data0, *ae_datax;
  751         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
  752 
  753         for (i = 0; i < obj_handle->uimage_num; i++) {
  754                 struct icp_qat_uof_image *image =
  755                     obj_handle->ae_uimage[i].img_ptr;
  756                 if (image->numpages > 1) {
  757                         pr_err(
  758                             "Only 1 page is allowed in a UOF for CPM2X; We found %d in %s\n",
  759                             image->numpages,
  760                             qat_uclo_get_string(&obj_handle->str_table,
  761                                                 image->img_name));
  762                         return EINVAL;
  763                 }
  764         }
  765 
  766         for (swAe = 0;
  767              (swAe < obj_handle->ae_num) && (swAe < ICP_QAT_UCLO_MAX_AE);
  768              swAe += AE_TG_NUM_CPM2X) {
  769                 if (!qat_hal_check_ae_active(handle, swAe)) {
  770                         continue;
  771                 }
  772 
  773                 for (ii = swAe; ii < (swAe + AE_TG_NUM_CPM2X); ii++) {
  774                         ae_data0 = &obj_handle->ae_data[ii];
  775                         if (ae_data0->slice_num != 1) // not assigned
  776                                 continue;
  777 
  778                         for (jj = ii + 1; jj < (swAe + AE_TG_NUM_CPM2X); jj++) {
  779                                 ae_datax = &obj_handle->ae_data[jj];
  780                                 if (ae_datax->slice_num != 1) // not assigned
  781                                         continue;
  782                                 if (ae_data0->ae_slices[0]
  783                                         .encap_image->img_ptr !=
  784                                     ae_datax->ae_slices[0]
  785                                         .encap_image->img_ptr) {
  786                                         pr_err("Only 1 list is allowed in a ");
  787                                         pr_err("Tgroup for CPM2X;\n");
  788                                         pr_err("ME%d, %d is assigned", ii, jj);
  789                                         pr_err(" different list files\n");
  790                                         return EINVAL;
  791                                 }
  792                         }
  793                 }
  794         }
  795 
  796         return 0;
  797 }
  798 
  799 static int
  800 qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae)
  801 {
  802         int i;
  803         int ae = 0;
  804         unsigned long ae_mask = handle->hal_handle->ae_mask;
  805         unsigned long cfg_ae_mask = handle->cfg_ae_mask;
  806         int mflag = 0;
  807         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
  808 
  809         for_each_set_bit(ae, &ae_mask, max_ae)
  810         {
  811                 if (!test_bit(ae, &cfg_ae_mask))
  812                         continue;
  813 
  814                 for (i = 0; i < obj_handle->uimage_num; i++) {
  815                         unsigned long ae_assigned =
  816                             obj_handle->ae_uimage[i].img_ptr->ae_assigned;
  817                         if (!test_bit(ae, &ae_assigned))
  818                                 continue;
  819                         mflag = 1;
  820                         if (qat_uclo_init_ae_data(obj_handle, ae, i))
  821                                 return EINVAL;
  822                 }
  823         }
  824         if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
  825                 if (UcLo_checkTGroupList2X(handle)) {
  826                         return EINVAL;
  827                 }
  828         }
  829         if (!mflag) {
  830                 pr_err("QAT: uimage uses AE not set");
  831                 return EINVAL;
  832         }
  833         return 0;
  834 }
  835 
  836 static struct icp_qat_uof_strtable *
  837 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr,
  838                        char *tab_name,
  839                        struct icp_qat_uof_strtable *str_table)
  840 {
  841         struct icp_qat_uof_chunkhdr *chunk_hdr;
  842 
  843         chunk_hdr =
  844             qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)obj_hdr->file_buff,
  845                                 tab_name,
  846                                 NULL);
  847         if (chunk_hdr) {
  848                 int hdr_size;
  849 
  850                 memcpy(&str_table->table_len,
  851                        obj_hdr->file_buff + chunk_hdr->offset,
  852                        sizeof(str_table->table_len));
  853                 hdr_size = (char *)&str_table->strings - (char *)str_table;
  854                 str_table->strings = (uintptr_t)obj_hdr->file_buff +
  855                     chunk_hdr->offset + hdr_size;
  856                 return str_table;
  857         }
  858         return NULL;
  859 }
  860 
  861 static void
  862 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj,
  863                            struct icp_qat_uclo_init_mem_table *init_mem_tab)
  864 {
  865         struct icp_qat_uof_chunkhdr *chunk_hdr;
  866 
  867         chunk_hdr =
  868             qat_uclo_find_chunk(encap_uof_obj->obj_hdr, ICP_QAT_UOF_IMEM, NULL);
  869         if (chunk_hdr) {
  870                 memmove(&init_mem_tab->entry_num,
  871                         encap_uof_obj->beg_uof + chunk_hdr->offset,
  872                         sizeof(unsigned int));
  873                 init_mem_tab->init_mem =
  874                     (struct icp_qat_uof_initmem *)(encap_uof_obj->beg_uof +
  875                                                    chunk_hdr->offset +
  876                                                    sizeof(unsigned int));
  877         }
  878 }
  879 
  880 static unsigned int
  881 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
  882 {
  883         switch (pci_get_device(GET_DEV(handle->accel_dev))) {
  884         case ADF_DH895XCC_PCI_DEVICE_ID:
  885                 return ICP_QAT_AC_895XCC_DEV_TYPE;
  886         case ADF_C62X_PCI_DEVICE_ID:
  887                 return ICP_QAT_AC_C62X_DEV_TYPE;
  888         case ADF_C3XXX_PCI_DEVICE_ID:
  889                 return ICP_QAT_AC_C3XXX_DEV_TYPE;
  890         case ADF_200XX_PCI_DEVICE_ID:
  891                 return ICP_QAT_AC_200XX_DEV_TYPE;
  892         case ADF_C4XXX_PCI_DEVICE_ID:
  893                 return ICP_QAT_AC_C4XXX_DEV_TYPE;
  894         case ADF_4XXX_PCI_DEVICE_ID:
  895         case ADF_401XX_PCI_DEVICE_ID:
  896                 return ICP_QAT_AC_4XXX_A_DEV_TYPE;
  897         default:
  898                 pr_err("QAT: unsupported device 0x%x\n",
  899                        pci_get_device(GET_DEV(handle->accel_dev)));
  900                 return 0;
  901         }
  902 }
  903 
  904 static int
  905 qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle)
  906 {
  907         unsigned int maj_ver, prod_type = obj_handle->prod_type;
  908 
  909         if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) {
  910                 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n",
  911                        obj_handle->encap_uof_obj.obj_hdr->ac_dev_type,
  912                        prod_type);
  913                 return EINVAL;
  914         }
  915         maj_ver = obj_handle->prod_rev & 0xff;
  916         if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver ||
  917             obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) {
  918                 pr_err("QAT: UOF maj_ver 0x%x out of range\n", maj_ver);
  919                 return EINVAL;
  920         }
  921         return 0;
  922 }
  923 
  924 static int
  925 qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle,
  926                   unsigned char ae,
  927                   unsigned char ctx_mask,
  928                   enum icp_qat_uof_regtype reg_type,
  929                   unsigned short reg_addr,
  930                   unsigned int value)
  931 {
  932         switch (reg_type) {
  933         case ICP_GPA_ABS:
  934         case ICP_GPB_ABS:
  935                 ctx_mask = 0;
  936                 return qat_hal_init_gpr(
  937                     handle, ae, ctx_mask, reg_type, reg_addr, value);
  938         case ICP_GPA_REL:
  939         case ICP_GPB_REL:
  940                 return qat_hal_init_gpr(
  941                     handle, ae, ctx_mask, reg_type, reg_addr, value);
  942         case ICP_SR_ABS:
  943         case ICP_DR_ABS:
  944         case ICP_SR_RD_ABS:
  945         case ICP_DR_RD_ABS:
  946                 ctx_mask = 0;
  947                 return qat_hal_init_rd_xfer(
  948                     handle, ae, ctx_mask, reg_type, reg_addr, value);
  949         case ICP_SR_REL:
  950         case ICP_DR_REL:
  951         case ICP_SR_RD_REL:
  952         case ICP_DR_RD_REL:
  953                 return qat_hal_init_rd_xfer(
  954                     handle, ae, ctx_mask, reg_type, reg_addr, value);
  955         case ICP_SR_WR_ABS:
  956         case ICP_DR_WR_ABS:
  957                 ctx_mask = 0;
  958                 return qat_hal_init_wr_xfer(
  959                     handle, ae, ctx_mask, reg_type, reg_addr, value);
  960         case ICP_SR_WR_REL:
  961         case ICP_DR_WR_REL:
  962                 return qat_hal_init_wr_xfer(
  963                     handle, ae, ctx_mask, reg_type, reg_addr, value);
  964         case ICP_NEIGH_REL:
  965                 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value);
  966         default:
  967                 pr_err("QAT: UOF uses unsupported reg type 0x%x\n", reg_type);
  968                 return EFAULT;
  969         }
  970         return 0;
  971 }
  972 
  973 static int
  974 qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle,
  975                       unsigned int ae,
  976                       struct icp_qat_uclo_encapme *encap_ae)
  977 {
  978         unsigned int i;
  979         unsigned char ctx_mask;
  980         struct icp_qat_uof_init_regsym *init_regsym;
  981 
  982         if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) ==
  983             ICP_QAT_UCLO_MAX_CTX)
  984                 ctx_mask = 0xff;
  985         else
  986                 ctx_mask = 0x55;
  987 
  988         for (i = 0; i < encap_ae->init_regsym_num; i++) {
  989                 unsigned int exp_res;
  990 
  991                 init_regsym = &encap_ae->init_regsym[i];
  992                 exp_res = init_regsym->value;
  993                 switch (init_regsym->init_type) {
  994                 case ICP_QAT_UOF_INIT_REG:
  995                         qat_uclo_init_reg(handle,
  996                                           ae,
  997                                           ctx_mask,
  998                                           (enum icp_qat_uof_regtype)
  999                                               init_regsym->reg_type,
 1000                                           (unsigned short)init_regsym->reg_addr,
 1001                                           exp_res);
 1002                         break;
 1003                 case ICP_QAT_UOF_INIT_REG_CTX:
 1004                         /* check if ctx is appropriate for the ctxMode */
 1005                         if (!((1 << init_regsym->ctx) & ctx_mask)) {
 1006                                 pr_err("QAT: invalid ctx num = 0x%x\n",
 1007                                        init_regsym->ctx);
 1008                                 return EINVAL;
 1009                         }
 1010                         qat_uclo_init_reg(
 1011                             handle,
 1012                             ae,
 1013                             (unsigned char)(1 << init_regsym->ctx),
 1014                             (enum icp_qat_uof_regtype)init_regsym->reg_type,
 1015                             (unsigned short)init_regsym->reg_addr,
 1016                             exp_res);
 1017                         break;
 1018                 case ICP_QAT_UOF_INIT_EXPR:
 1019                         pr_err("QAT: INIT_EXPR feature not supported\n");
 1020                         return EINVAL;
 1021                 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP:
 1022                         pr_err("QAT: INIT_EXPR_ENDIAN_SWAP not supported\n");
 1023                         return EINVAL;
 1024                 default:
 1025                         break;
 1026                 }
 1027         }
 1028         return 0;
 1029 }
 1030 
 1031 static int
 1032 qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle)
 1033 {
 1034         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 1035         unsigned int s;
 1036         unsigned int ae = 0;
 1037         struct icp_qat_uclo_aedata *aed;
 1038         unsigned long ae_mask = handle->hal_handle->ae_mask;
 1039 
 1040         if (obj_handle->global_inited)
 1041                 return 0;
 1042         if (obj_handle->init_mem_tab.entry_num) {
 1043                 if (qat_uclo_init_memory(handle)) {
 1044                         pr_err("QAT: initialize memory failed\n");
 1045                         return EINVAL;
 1046                 }
 1047         }
 1048 
 1049         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
 1050         {
 1051                 aed = &obj_handle->ae_data[ae];
 1052                 for (s = 0; s < aed->slice_num; s++) {
 1053                         if (!aed->ae_slices[s].encap_image)
 1054                                 continue;
 1055                         if (qat_uclo_init_reg_sym(
 1056                                 handle, ae, aed->ae_slices[s].encap_image))
 1057                                 return EINVAL;
 1058                 }
 1059         }
 1060         obj_handle->global_inited = 1;
 1061         return 0;
 1062 }
 1063 
 1064 static int
 1065 qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle,
 1066                   struct icp_qat_uclo_objhandle *obj_handle,
 1067                   unsigned char ae,
 1068                   struct icp_qat_uof_image *uof_image)
 1069 {
 1070         unsigned char nn_mode;
 1071         char ae_mode = 0;
 1072 
 1073         ae_mode = (char)ICP_QAT_CTX_MODE(uof_image->ae_mode);
 1074         if (qat_hal_set_ae_ctx_mode(handle, ae, ae_mode)) {
 1075                 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n");
 1076                 return EFAULT;
 1077         }
 1078 
 1079         ae_mode = (char)ICP_QAT_SHARED_USTORE_MODE(uof_image->ae_mode);
 1080         qat_hal_set_ae_scs_mode(handle, ae, ae_mode);
 1081         if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
 1082                 nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
 1083 
 1084                 if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) {
 1085                         pr_err("QAT: qat_hal_set_ae_nn_mode error\n");
 1086                         return EFAULT;
 1087                 }
 1088         }
 1089         ae_mode = (char)ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode);
 1090         if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, ae_mode)) {
 1091                 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n");
 1092                 return EFAULT;
 1093         }
 1094         ae_mode = (char)ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode);
 1095         if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, ae_mode)) {
 1096                 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n");
 1097                 return EFAULT;
 1098         }
 1099         if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
 1100                 ae_mode = (char)ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode);
 1101                 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, ae_mode)) {
 1102                         pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n");
 1103                         return EFAULT;
 1104                 }
 1105                 ae_mode = (char)ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode);
 1106                 if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, ae_mode)) {
 1107                         pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n");
 1108                         return EFAULT;
 1109                 }
 1110                 ae_mode = (char)ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode);
 1111                 qat_hal_set_ae_tindex_mode(handle, ae, ae_mode);
 1112         }
 1113         return 0;
 1114 }
 1115 
 1116 static int
 1117 qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle)
 1118 {
 1119         int error;
 1120         unsigned char s;
 1121         unsigned char ae = 0;
 1122         struct icp_qat_uof_image *uof_image;
 1123         struct icp_qat_uclo_aedata *ae_data;
 1124         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 1125         unsigned long ae_mask = handle->hal_handle->ae_mask;
 1126 
 1127         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
 1128         {
 1129                 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
 1130 
 1131                 if (!test_bit(ae, &cfg_ae_mask))
 1132                         continue;
 1133 
 1134                 ae_data = &obj_handle->ae_data[ae];
 1135                 for (s = 0; s < min_t(unsigned int,
 1136                                       ae_data->slice_num,
 1137                                       ICP_QAT_UCLO_MAX_CTX);
 1138                      s++) {
 1139                         if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
 1140                                 continue;
 1141                         uof_image = ae_data->ae_slices[s].encap_image->img_ptr;
 1142                         error = qat_hal_set_modes(handle,
 1143                                                   obj_handle,
 1144                                                   ae,
 1145                                                   uof_image);
 1146                         if (error)
 1147                                 return error;
 1148                 }
 1149         }
 1150         return 0;
 1151 }
 1152 
 1153 static void
 1154 qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle)
 1155 {
 1156         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 1157         struct icp_qat_uclo_encapme *image;
 1158         int a;
 1159 
 1160         for (a = 0; a < obj_handle->uimage_num; a++) {
 1161                 image = &obj_handle->ae_uimage[a];
 1162                 image->uwords_num =
 1163                     image->page->beg_addr_p + image->page->micro_words_num;
 1164         }
 1165 }
 1166 
 1167 static int
 1168 qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
 1169 {
 1170         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 1171         unsigned int ae;
 1172 
 1173         obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff;
 1174         obj_handle->encap_uof_obj.obj_hdr =
 1175             (struct icp_qat_uof_objhdr *)obj_handle->obj_hdr->file_buff;
 1176         obj_handle->uword_in_bytes = 6;
 1177         obj_handle->prod_type = qat_uclo_get_dev_type(handle);
 1178         obj_handle->prod_rev =
 1179             PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id);
 1180         if (qat_uclo_check_uof_compat(obj_handle)) {
 1181                 pr_err("QAT: UOF incompatible\n");
 1182                 return EINVAL;
 1183         }
 1184         obj_handle->uword_buf = malloc(UWORD_CPYBUF_SIZE * sizeof(uint64_t),
 1185                                        M_QAT,
 1186                                        M_WAITOK | M_ZERO);
 1187         obj_handle->ustore_phy_size =
 1188             (obj_handle->prod_type == ICP_QAT_AC_C4XXX_DEV_TYPE) ? 0x2000 :
 1189                                                                    0x4000;
 1190         if (!obj_handle->obj_hdr->file_buff ||
 1191             !qat_uclo_map_str_table(obj_handle->obj_hdr,
 1192                                     ICP_QAT_UOF_STRT,
 1193                                     &obj_handle->str_table)) {
 1194                 pr_err("QAT: UOF doesn't have effective images\n");
 1195                 goto out_err;
 1196         }
 1197         obj_handle->uimage_num =
 1198             qat_uclo_map_uimage(obj_handle,
 1199                                 obj_handle->ae_uimage,
 1200                                 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX);
 1201         if (!obj_handle->uimage_num)
 1202                 goto out_err;
 1203         if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) {
 1204                 pr_err("QAT: Bad object\n");
 1205                 goto out_check_uof_aemask_err;
 1206         }
 1207         qat_uclo_init_uword_num(handle);
 1208         qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj,
 1209                                    &obj_handle->init_mem_tab);
 1210         if (qat_uclo_set_ae_mode(handle))
 1211                 goto out_check_uof_aemask_err;
 1212         return 0;
 1213 out_check_uof_aemask_err:
 1214         for (ae = 0; ae < obj_handle->uimage_num; ae++)
 1215                 free(obj_handle->ae_uimage[ae].page, M_QAT);
 1216 out_err:
 1217         free(obj_handle->uword_buf, M_QAT);
 1218         obj_handle->uword_buf = NULL;
 1219         return EFAULT;
 1220 }
 1221 
 1222 static int
 1223 qat_uclo_map_suof_file_hdr(const struct icp_qat_fw_loader_handle *handle,
 1224                            const struct icp_qat_suof_filehdr *suof_ptr,
 1225                            int suof_size)
 1226 {
 1227         unsigned int check_sum = 0;
 1228         unsigned int min_ver_offset = 0;
 1229         struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
 1230 
 1231         suof_handle->file_id = ICP_QAT_SUOF_FID;
 1232         suof_handle->suof_buf = (const char *)suof_ptr;
 1233         suof_handle->suof_size = suof_size;
 1234         min_ver_offset =
 1235             suof_size - offsetof(struct icp_qat_suof_filehdr, min_ver);
 1236         check_sum = qat_uclo_calc_str_checksum((const char *)&suof_ptr->min_ver,
 1237                                                min_ver_offset);
 1238         if (check_sum != suof_ptr->check_sum) {
 1239                 pr_err("QAT: incorrect SUOF checksum\n");
 1240                 return EINVAL;
 1241         }
 1242         suof_handle->check_sum = suof_ptr->check_sum;
 1243         suof_handle->min_ver = suof_ptr->min_ver;
 1244         suof_handle->maj_ver = suof_ptr->maj_ver;
 1245         suof_handle->fw_type = suof_ptr->fw_type;
 1246         return 0;
 1247 }
 1248 
 1249 static void
 1250 qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle,
 1251                   struct icp_qat_suof_img_hdr *suof_img_hdr,
 1252                   struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
 1253 {
 1254         struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
 1255         const struct icp_qat_simg_ae_mode *ae_mode;
 1256         struct icp_qat_suof_objhdr *suof_objhdr;
 1257         unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev));
 1258 
 1259         suof_img_hdr->simg_buf =
 1260             (suof_handle->suof_buf + suof_chunk_hdr->offset +
 1261              sizeof(*suof_objhdr));
 1262         suof_img_hdr->simg_len =
 1263             ((struct icp_qat_suof_objhdr *)(uintptr_t)(suof_handle->suof_buf +
 1264                                                        suof_chunk_hdr->offset))
 1265                 ->img_length;
 1266 
 1267         suof_img_hdr->css_header = suof_img_hdr->simg_buf;
 1268         suof_img_hdr->css_key =
 1269             (suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr));
 1270         suof_img_hdr->css_signature = suof_img_hdr->css_key +
 1271             ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) +
 1272             ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id);
 1273         suof_img_hdr->css_simg =
 1274             suof_img_hdr->css_signature + ICP_QAT_CSS_SIGNATURE_LEN(device_id);
 1275 
 1276         ae_mode = (const struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg);
 1277         suof_img_hdr->ae_mask = ae_mode->ae_mask;
 1278         suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name;
 1279         suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data;
 1280         suof_img_hdr->fw_type = ae_mode->fw_type;
 1281 }
 1282 
 1283 static void
 1284 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle,
 1285                           struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
 1286 {
 1287         char **sym_str = (char **)&suof_handle->sym_str;
 1288         unsigned int *sym_size = &suof_handle->sym_size;
 1289         struct icp_qat_suof_strtable *str_table_obj;
 1290 
 1291         *sym_size = *(unsigned int *)(uintptr_t)(suof_chunk_hdr->offset +
 1292                                                  suof_handle->suof_buf);
 1293         *sym_str =
 1294             (char *)(uintptr_t)(suof_handle->suof_buf + suof_chunk_hdr->offset +
 1295                                 sizeof(str_table_obj->tab_length));
 1296 }
 1297 
 1298 static int
 1299 qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle,
 1300                            struct icp_qat_suof_img_hdr *img_hdr)
 1301 {
 1302         const struct icp_qat_simg_ae_mode *img_ae_mode = NULL;
 1303         unsigned int prod_rev, maj_ver, prod_type;
 1304 
 1305         prod_type = qat_uclo_get_dev_type(handle);
 1306         img_ae_mode = (const struct icp_qat_simg_ae_mode *)img_hdr->css_simg;
 1307         prod_rev =
 1308             PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id);
 1309         if (img_ae_mode->dev_type != prod_type) {
 1310                 pr_err("QAT: incompatible product type %x\n",
 1311                        img_ae_mode->dev_type);
 1312                 return EINVAL;
 1313         }
 1314         maj_ver = prod_rev & 0xff;
 1315         if (maj_ver > img_ae_mode->devmax_ver ||
 1316             maj_ver < img_ae_mode->devmin_ver) {
 1317                 pr_err("QAT: incompatible device maj_ver 0x%x\n", maj_ver);
 1318                 return EINVAL;
 1319         }
 1320         return 0;
 1321 }
 1322 
 1323 static void
 1324 qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle)
 1325 {
 1326         struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
 1327 
 1328         free(sobj_handle->img_table.simg_hdr, M_QAT);
 1329         sobj_handle->img_table.simg_hdr = NULL;
 1330         free(handle->sobj_handle, M_QAT);
 1331         handle->sobj_handle = NULL;
 1332 }
 1333 
 1334 static void
 1335 qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr,
 1336                   unsigned int img_id,
 1337                   unsigned int num_simgs)
 1338 {
 1339         struct icp_qat_suof_img_hdr img_header;
 1340 
 1341         if ((img_id != num_simgs - 1) && img_id != ICP_QAT_UCLO_MAX_AE) {
 1342                 memcpy(&img_header,
 1343                        &suof_img_hdr[num_simgs - 1],
 1344                        sizeof(*suof_img_hdr));
 1345                 memcpy(&suof_img_hdr[num_simgs - 1],
 1346                        &suof_img_hdr[img_id],
 1347                        sizeof(*suof_img_hdr));
 1348                 memcpy(&suof_img_hdr[img_id],
 1349                        &img_header,
 1350                        sizeof(*suof_img_hdr));
 1351         }
 1352 }
 1353 
 1354 static int
 1355 qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle,
 1356                   const struct icp_qat_suof_filehdr *suof_ptr,
 1357                   int suof_size)
 1358 {
 1359         struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
 1360         struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL;
 1361         struct icp_qat_suof_img_hdr *suof_img_hdr = NULL;
 1362         int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE,
 1363             aeMax_img = ICP_QAT_UCLO_MAX_AE;
 1364         unsigned int i = 0;
 1365         struct icp_qat_suof_img_hdr img_header;
 1366 
 1367         if (!suof_ptr || suof_size == 0) {
 1368                 pr_err("QAT: input parameter SUOF pointer/size is NULL\n");
 1369                 return EINVAL;
 1370         }
 1371         if (qat_uclo_check_suof_format(suof_ptr))
 1372                 return EINVAL;
 1373         ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size);
 1374         if (ret)
 1375                 return ret;
 1376         suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)((uintptr_t)suof_ptr +
 1377                                                            sizeof(*suof_ptr));
 1378 
 1379         qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr);
 1380         suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1;
 1381 
 1382         if (suof_handle->img_table.num_simgs != 0) {
 1383                 suof_img_hdr = malloc(suof_handle->img_table.num_simgs *
 1384                                           sizeof(img_header),
 1385                                       M_QAT,
 1386                                       M_WAITOK | M_ZERO);
 1387                 suof_handle->img_table.simg_hdr = suof_img_hdr;
 1388         }
 1389 
 1390         for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
 1391                 qat_uclo_map_simg(handle,
 1392                                   &suof_img_hdr[i],
 1393                                   &suof_chunk_hdr[1 + i]);
 1394                 ret = qat_uclo_check_simg_compat(handle, &suof_img_hdr[i]);
 1395                 if (ret)
 1396                         return ret;
 1397                 suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask;
 1398                 if ((suof_img_hdr[i].ae_mask & 0x1) != 0)
 1399                         ae0_img = i;
 1400         }
 1401 
 1402         if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
 1403                 qat_uclo_tail_img(suof_img_hdr,
 1404                                   ae0_img,
 1405                                   suof_handle->img_table.num_simgs);
 1406         } else {
 1407                 if (suof_handle->img_table.num_simgs == 1)
 1408                         return 0;
 1409                 qat_uclo_tail_img(suof_img_hdr,
 1410                                   ae0_img,
 1411                                   suof_handle->img_table.num_simgs - 1);
 1412                 for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
 1413                         if ((suof_img_hdr[i].ae_mask &
 1414                              (0x1 << (handle->hal_handle->ae_max_num - 1))) !=
 1415                             0) {
 1416                                 aeMax_img = i;
 1417                                 break;
 1418                         }
 1419                 }
 1420                 qat_uclo_tail_img(suof_img_hdr,
 1421                                   aeMax_img,
 1422                                   suof_handle->img_table.num_simgs);
 1423         }
 1424         return 0;
 1425 }
 1426 
 1427 #define ADD_ADDR(high, low) ((((uint64_t)high) << 32) + (low))
 1428 #define BITS_IN_DWORD 32
 1429 
 1430 static int
 1431 qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle,
 1432                  struct icp_qat_fw_auth_desc *desc)
 1433 {
 1434         unsigned int fcu_sts, mem_cfg_err, retry = 0;
 1435         unsigned int fcu_ctl_csr, fcu_sts_csr;
 1436         unsigned int fcu_dram_hi_csr, fcu_dram_lo_csr;
 1437         u64 bus_addr;
 1438 
 1439         bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) -
 1440             sizeof(struct icp_qat_auth_chunk);
 1441         if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
 1442                 fcu_ctl_csr = FCU_CONTROL_C4XXX;
 1443                 fcu_sts_csr = FCU_STATUS_C4XXX;
 1444                 fcu_dram_hi_csr = FCU_DRAM_ADDR_HI_C4XXX;
 1445                 fcu_dram_lo_csr = FCU_DRAM_ADDR_LO_C4XXX;
 1446         } else {
 1447                 fcu_ctl_csr = FCU_CONTROL;
 1448                 fcu_sts_csr = FCU_STATUS;
 1449                 fcu_dram_hi_csr = FCU_DRAM_ADDR_HI;
 1450                 fcu_dram_lo_csr = FCU_DRAM_ADDR_LO;
 1451         }
 1452         SET_FCU_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD));
 1453         SET_FCU_CSR(handle, fcu_dram_lo_csr, bus_addr);
 1454         SET_FCU_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH);
 1455 
 1456         do {
 1457                 pause_ms("adfstop", FW_AUTH_WAIT_PERIOD);
 1458                 fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr);
 1459                 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL)
 1460                         goto auth_fail;
 1461                 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1))
 1462                         if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE)
 1463                                 return 0;
 1464         } while (retry++ < FW_AUTH_MAX_RETRY);
 1465 auth_fail:
 1466         pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n",
 1467                fcu_sts & FCU_AUTH_STS_MASK,
 1468                retry);
 1469         if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) {
 1470                 mem_cfg_err =
 1471                     (GET_FCU_CSR(handle, FCU_STATUS1_C4XXX) & MEM_CFG_ERR_BIT);
 1472                 if (mem_cfg_err)
 1473                         pr_err("QAT: MEM_CFG_ERR\n");
 1474         }
 1475         return EINVAL;
 1476 }
 1477 
 1478 static int
 1479 qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, int imgid)
 1480 {
 1481         struct icp_qat_suof_handle *sobj_handle;
 1482 
 1483         if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))))
 1484                 return 0;
 1485 
 1486         sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle;
 1487         if (handle->hal_handle->admin_ae_mask &
 1488             sobj_handle->img_table.simg_hdr[imgid].ae_mask)
 1489                 return 0;
 1490 
 1491         return 1;
 1492 }
 1493 
 1494 static int
 1495 qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle,
 1496                            struct icp_qat_fw_auth_desc *desc)
 1497 {
 1498         unsigned int i = 0;
 1499         unsigned int fcuSts = 0, fcuAeBroadcastMask = 0;
 1500         unsigned int retry = 0;
 1501         unsigned int fcuStsCsr = 0;
 1502         unsigned int fcuCtlCsr = 0;
 1503         unsigned int loadedAes = 0;
 1504         unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev));
 1505 
 1506         if (IS_QAT_GEN4(device_id)) {
 1507                 fcuCtlCsr = FCU_CONTROL_4XXX;
 1508                 fcuStsCsr = FCU_STATUS_4XXX;
 1509         } else {
 1510                 pr_err("Uclo_BroadcastLoadFW only applicable for CPM20\n");
 1511                 return EINVAL;
 1512         }
 1513 
 1514         for (i = 0; i < ICP_QAT_UCLO_MAX_AE; i++) {
 1515                 if (!test_bit(i, (unsigned long *)&handle->hal_handle->ae_mask))
 1516                         continue;
 1517 
 1518                 if (qat_hal_check_ae_active(handle, (unsigned char)i)) {
 1519                         pr_err(
 1520                             "Uclo_BroadcastLoadFW error (invalid AE status)\n");
 1521                         return EINVAL;
 1522                 }
 1523 
 1524                 if ((desc->ae_mask >> i) & 0x1) {
 1525                         fcuAeBroadcastMask |= 1 << i;
 1526                 }
 1527         }
 1528 
 1529         if (fcuAeBroadcastMask) {
 1530                 retry = 0;
 1531                 SET_FCU_CSR(handle,
 1532                             FCU_ME_BROADCAST_MASK_TYPE,
 1533                             fcuAeBroadcastMask);
 1534                 SET_FCU_CSR(handle, fcuCtlCsr, FCU_CTRL_CMD_LOAD);
 1535                 do {
 1536                         msleep(FW_AUTH_WAIT_PERIOD);
 1537                         fcuSts = GET_FCU_CSR(handle, fcuStsCsr);
 1538 
 1539                         if ((fcuSts & FCU_AUTH_STS_MASK) == FCU_STS_LOAD_FAIL) {
 1540                                 pr_err(
 1541                                     "Uclo_BroadcastLoadFW fail (fcu_status = 0x%x)\n",
 1542                                     fcuSts & FCU_AUTH_STS_MASK);
 1543                                 return EINVAL;
 1544                         } else if ((fcuSts & FCU_AUTH_STS_MASK) ==
 1545                                    FCU_STS_LOAD_DONE) {
 1546                                 if (IS_QAT_GEN4(device_id))
 1547                                         loadedAes =
 1548                                             GET_FCU_CSR(handle,
 1549                                                         FCU_AE_LOADED_4XXX);
 1550                                 else
 1551                                         loadedAes =
 1552                                             (fcuSts >> FCU_LOADED_AE_POS);
 1553 
 1554                                 if ((loadedAes & fcuAeBroadcastMask) ==
 1555                                     fcuAeBroadcastMask)
 1556                                         break;
 1557                         } else if ((fcuSts & FCU_AUTH_STS_MASK) ==
 1558                                    FCU_STS_VERI_DONE) {
 1559                                 SET_FCU_CSR(handle,
 1560                                             fcuCtlCsr,
 1561                                             FCU_CTRL_CMD_LOAD);
 1562                         }
 1563                 } while (retry++ < FW_BROADCAST_MAX_RETRY);
 1564                 if (retry > FW_BROADCAST_MAX_RETRY) {
 1565                         pr_err(
 1566                             "Uclo_BroadcastLoadFW fail(fcu_status = 0x%x),retry = %d\n",
 1567                             fcuSts & FCU_AUTH_STS_MASK,
 1568                             retry);
 1569                         return EINVAL;
 1570                 }
 1571         }
 1572         return 0;
 1573 }
 1574 
 1575 static int
 1576 qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
 1577                     struct icp_firml_dram_desc *dram_desc,
 1578                     unsigned int size)
 1579 {
 1580         int ret;
 1581 
 1582         ret = bus_dma_mem_create(&dram_desc->dram_mem,
 1583                                  handle->accel_dev->dma_tag,
 1584                                  1,
 1585                                  BUS_SPACE_MAXADDR,
 1586                                  size,
 1587                                  0);
 1588         if (ret != 0)
 1589                 return ret;
 1590         dram_desc->dram_base_addr_v = dram_desc->dram_mem.dma_vaddr;
 1591         dram_desc->dram_bus_addr = dram_desc->dram_mem.dma_baddr;
 1592         dram_desc->dram_size = size;
 1593         return 0;
 1594 }
 1595 
 1596 static void
 1597 qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle,
 1598                    struct icp_firml_dram_desc *dram_desc)
 1599 {
 1600         if (handle && dram_desc && dram_desc->dram_base_addr_v)
 1601                 bus_dma_mem_free(&dram_desc->dram_mem);
 1602 
 1603         if (dram_desc)
 1604                 explicit_bzero(dram_desc, sizeof(*dram_desc));
 1605 }
 1606 
 1607 static int
 1608 qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle,
 1609                      const char *image,
 1610                      unsigned int size,
 1611                      struct icp_firml_dram_desc *img_desc,
 1612                      struct icp_qat_fw_auth_desc **desc)
 1613 {
 1614         const struct icp_qat_css_hdr *css_hdr =
 1615             (const struct icp_qat_css_hdr *)image;
 1616         struct icp_qat_fw_auth_desc *auth_desc;
 1617         struct icp_qat_auth_chunk *auth_chunk;
 1618         u64 virt_addr, bus_addr, virt_base;
 1619         unsigned int length, simg_offset = sizeof(*auth_chunk);
 1620         unsigned int device_id = pci_get_device(GET_DEV(handle->accel_dev));
 1621 
 1622         if (size >
 1623             (ICP_QAT_AE_IMG_OFFSET(device_id) + ICP_QAT_CSS_MAX_IMAGE_LEN)) {
 1624                 pr_err("QAT: error, input image size overflow %d\n", size);
 1625                 return EINVAL;
 1626         }
 1627         length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ?
 1628             ICP_QAT_CSS_AE_SIMG_LEN(device_id) + simg_offset :
 1629             size + ICP_QAT_CSS_FWSK_PAD_LEN(device_id) + simg_offset;
 1630         if (qat_uclo_simg_alloc(handle, img_desc, length)) {
 1631                 pr_err("QAT: error, allocate continuous dram fail\n");
 1632                 return -ENOMEM;
 1633         }
 1634 
 1635         auth_chunk = img_desc->dram_base_addr_v;
 1636         auth_chunk->chunk_size = img_desc->dram_size;
 1637         auth_chunk->chunk_bus_addr = img_desc->dram_bus_addr;
 1638         virt_base = (uintptr_t)img_desc->dram_base_addr_v + simg_offset;
 1639         bus_addr = img_desc->dram_bus_addr + simg_offset;
 1640         auth_desc = img_desc->dram_base_addr_v;
 1641         auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
 1642         auth_desc->css_hdr_low = (unsigned int)bus_addr;
 1643         virt_addr = virt_base;
 1644 
 1645         memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr));
 1646         /* pub key */
 1647         bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) +
 1648             sizeof(*css_hdr);
 1649         virt_addr = virt_addr + sizeof(*css_hdr);
 1650 
 1651         auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
 1652         auth_desc->fwsk_pub_low = (unsigned int)bus_addr;
 1653 
 1654         memcpy((void *)(uintptr_t)virt_addr,
 1655                (const void *)(image + sizeof(*css_hdr)),
 1656                ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id));
 1657         /* padding */
 1658         explicit_bzero((void *)(uintptr_t)(
 1659                            virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)),
 1660                        ICP_QAT_CSS_FWSK_PAD_LEN(device_id));
 1661 
 1662         /* exponent */
 1663         memcpy((void *)(uintptr_t)(virt_addr +
 1664                                    ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) +
 1665                                    ICP_QAT_CSS_FWSK_PAD_LEN(device_id)),
 1666                (const void *)(image + sizeof(*css_hdr) +
 1667                               ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id)),
 1668                sizeof(unsigned int));
 1669 
 1670         /* signature */
 1671         bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) +
 1672             ICP_QAT_CSS_FWSK_PUB_LEN(device_id);
 1673         virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(device_id);
 1674         auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
 1675         auth_desc->signature_low = (unsigned int)bus_addr;
 1676 
 1677         memcpy((void *)(uintptr_t)virt_addr,
 1678                (const void *)(image + sizeof(*css_hdr) +
 1679                               ICP_QAT_CSS_FWSK_MODULUS_LEN(device_id) +
 1680                               ICP_QAT_CSS_FWSK_EXPONENT_LEN(device_id)),
 1681                ICP_QAT_CSS_SIGNATURE_LEN(device_id));
 1682 
 1683         bus_addr =
 1684             ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) +
 1685             ICP_QAT_CSS_SIGNATURE_LEN(device_id);
 1686         virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(device_id);
 1687 
 1688         auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
 1689         auth_desc->img_low = (unsigned int)bus_addr;
 1690         auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(device_id);
 1691         memcpy((void *)(uintptr_t)virt_addr,
 1692                (const void *)(image + ICP_QAT_AE_IMG_OFFSET(device_id)),
 1693                auth_desc->img_len);
 1694         virt_addr = virt_base;
 1695         /* AE firmware */
 1696         if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type ==
 1697             CSS_AE_FIRMWARE) {
 1698                 auth_desc->img_ae_mode_data_high = auth_desc->img_high;
 1699                 auth_desc->img_ae_mode_data_low = auth_desc->img_low;
 1700                 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high,
 1701                                     auth_desc->img_ae_mode_data_low) +
 1702                     sizeof(struct icp_qat_simg_ae_mode);
 1703 
 1704                 auth_desc->img_ae_init_data_high =
 1705                     (unsigned int)(bus_addr >> BITS_IN_DWORD);
 1706                 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr;
 1707                 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN;
 1708                 auth_desc->img_ae_insts_high =
 1709                     (unsigned int)(bus_addr >> BITS_IN_DWORD);
 1710                 auth_desc->img_ae_insts_low = (unsigned int)bus_addr;
 1711                 virt_addr += sizeof(struct icp_qat_css_hdr) +
 1712                     ICP_QAT_CSS_FWSK_PUB_LEN(device_id) +
 1713                     ICP_QAT_CSS_SIGNATURE_LEN(device_id);
 1714                 auth_desc->ae_mask =
 1715                     ((struct icp_qat_simg_ae_mode *)virt_addr)->ae_mask &
 1716                     handle->cfg_ae_mask;
 1717         } else {
 1718                 auth_desc->img_ae_insts_high = auth_desc->img_high;
 1719                 auth_desc->img_ae_insts_low = auth_desc->img_low;
 1720         }
 1721         *desc = auth_desc;
 1722         return 0;
 1723 }
 1724 
 1725 static int
 1726 qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle,
 1727                  struct icp_qat_fw_auth_desc *desc)
 1728 {
 1729         unsigned int i = 0;
 1730         unsigned int fcu_sts;
 1731         unsigned int fcu_sts_csr, fcu_ctl_csr;
 1732         unsigned int loaded_aes = FCU_LOADED_AE_POS;
 1733         unsigned long ae_mask = handle->hal_handle->ae_mask;
 1734 
 1735         if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
 1736                 fcu_ctl_csr = FCU_CONTROL_C4XXX;
 1737                 fcu_sts_csr = FCU_STATUS_C4XXX;
 1738 
 1739         } else {
 1740                 fcu_ctl_csr = FCU_CONTROL;
 1741                 fcu_sts_csr = FCU_STATUS;
 1742         }
 1743 
 1744         for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num)
 1745         {
 1746                 int retry = 0;
 1747 
 1748                 if (!((desc->ae_mask >> i) & 0x1))
 1749                         continue;
 1750                 if (qat_hal_check_ae_active(handle, i)) {
 1751                         pr_err("QAT: AE %d is active\n", i);
 1752                         return EINVAL;
 1753                 }
 1754                 SET_FCU_CSR(handle,
 1755                             fcu_ctl_csr,
 1756                             (FCU_CTRL_CMD_LOAD |
 1757                              (IS_QAT_GEN4(
 1758                                   pci_get_device(GET_DEV(handle->accel_dev))) ?
 1759                                   (1 << FCU_CTRL_BROADCAST_POS) :
 1760                                   0) |
 1761                              (i << FCU_CTRL_AE_POS)));
 1762 
 1763                 do {
 1764                         pause_ms("adfstop", FW_AUTH_WAIT_PERIOD);
 1765                         fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr);
 1766                         if ((fcu_sts & FCU_AUTH_STS_MASK) ==
 1767                             FCU_STS_LOAD_DONE) {
 1768                                 loaded_aes = IS_QAT_GEN3_OR_GEN4(pci_get_device(
 1769                                                  GET_DEV(handle->accel_dev))) ?
 1770                                     GET_FCU_CSR(handle, FCU_AE_LOADED_C4XXX) :
 1771                                     (fcu_sts >> FCU_LOADED_AE_POS);
 1772                                 if (loaded_aes & (1 << i))
 1773                                         break;
 1774                         }
 1775                 } while (retry++ < FW_AUTH_MAX_RETRY);
 1776                 if (retry > FW_AUTH_MAX_RETRY) {
 1777                         pr_err("QAT: firmware load failed timeout %x\n", retry);
 1778                         return EINVAL;
 1779                 }
 1780         }
 1781         return 0;
 1782 }
 1783 
 1784 static int
 1785 qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle,
 1786                       const void *addr_ptr,
 1787                       int mem_size)
 1788 {
 1789         struct icp_qat_suof_handle *suof_handle;
 1790 
 1791         suof_handle = malloc(sizeof(*suof_handle), M_QAT, M_WAITOK | M_ZERO);
 1792         handle->sobj_handle = suof_handle;
 1793         if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) {
 1794                 qat_uclo_del_suof(handle);
 1795                 pr_err("QAT: map SUOF failed\n");
 1796                 return EINVAL;
 1797         }
 1798         return 0;
 1799 }
 1800 
 1801 int
 1802 qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
 1803                    const void *addr_ptr,
 1804                    int mem_size)
 1805 {
 1806         struct icp_qat_fw_auth_desc *desc = NULL;
 1807         struct icp_firml_dram_desc img_desc;
 1808         int status = 0;
 1809 
 1810         if (handle->fw_auth) {
 1811                 status = qat_uclo_map_auth_fw(
 1812                     handle, addr_ptr, mem_size, &img_desc, &desc);
 1813                 if (!status)
 1814                         status = qat_uclo_auth_fw(handle, desc);
 1815 
 1816                 qat_uclo_simg_free(handle, &img_desc);
 1817         } else {
 1818                 if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) {
 1819                         device_printf(
 1820                             NULL, "QAT: PKE service is not allowed because ");
 1821                         device_printf(NULL, "MMP fw will not be loaded for ");
 1822                         device_printf(NULL,
 1823                                       "device 0x%x",
 1824                                       pci_get_device(
 1825                                           GET_DEV(handle->accel_dev)));
 1826                         return status;
 1827                 }
 1828                 if (pci_get_device(GET_DEV(handle->accel_dev)) ==
 1829                     ADF_C3XXX_PCI_DEVICE_ID) {
 1830                         pr_err("QAT: C3XXX doesn't support unsigned MMP\n");
 1831                         return EINVAL;
 1832                 }
 1833                 status = qat_uclo_wr_sram_by_words(handle,
 1834                                                    handle->hal_sram_offset,
 1835                                                    addr_ptr,
 1836                                                    mem_size);
 1837         }
 1838         return status;
 1839 }
 1840 
 1841 static int
 1842 qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle,
 1843                      const void *addr_ptr,
 1844                      int mem_size)
 1845 {
 1846         struct icp_qat_uof_filehdr *filehdr;
 1847         struct icp_qat_uclo_objhandle *objhdl;
 1848 
 1849         objhdl = malloc(sizeof(*objhdl), M_QAT, M_WAITOK | M_ZERO);
 1850         objhdl->obj_buf = malloc(mem_size, M_QAT, M_WAITOK);
 1851         bcopy(addr_ptr, objhdl->obj_buf, mem_size);
 1852         filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf;
 1853         if (qat_uclo_check_uof_format(filehdr))
 1854                 goto out_objhdr_err;
 1855         objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf,
 1856                                              filehdr,
 1857                                              ICP_QAT_UOF_OBJS);
 1858         if (!objhdl->obj_hdr) {
 1859                 pr_err("QAT: object file chunk is null\n");
 1860                 goto out_objhdr_err;
 1861         }
 1862         handle->obj_handle = objhdl;
 1863         if (qat_uclo_parse_uof_obj(handle))
 1864                 goto out_overlay_obj_err;
 1865         return 0;
 1866 
 1867 out_overlay_obj_err:
 1868         handle->obj_handle = NULL;
 1869         free(objhdl->obj_hdr, M_QAT);
 1870 out_objhdr_err:
 1871         free(objhdl->obj_buf, M_QAT);
 1872         free(objhdl, M_QAT);
 1873         return ENOMEM;
 1874 }
 1875 
 1876 static int
 1877 qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle,
 1878                           const struct icp_qat_mof_file_hdr *mof_ptr,
 1879                           u32 mof_size)
 1880 {
 1881         unsigned int checksum = 0;
 1882         unsigned int min_ver_offset = 0;
 1883         struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
 1884 
 1885         mobj_handle->file_id = ICP_QAT_MOF_FID;
 1886         mobj_handle->mof_buf = (const char *)mof_ptr;
 1887         mobj_handle->mof_size = mof_size;
 1888 
 1889         min_ver_offset =
 1890             mof_size - offsetof(struct icp_qat_mof_file_hdr, min_ver);
 1891         checksum = qat_uclo_calc_str_checksum((const char *)&mof_ptr->min_ver,
 1892                                               min_ver_offset);
 1893         if (checksum != mof_ptr->checksum) {
 1894                 pr_err("QAT: incorrect MOF checksum\n");
 1895                 return EINVAL;
 1896         }
 1897         mobj_handle->checksum = mof_ptr->checksum;
 1898         mobj_handle->min_ver = mof_ptr->min_ver;
 1899         mobj_handle->maj_ver = mof_ptr->maj_ver;
 1900         return 0;
 1901 }
 1902 
 1903 void
 1904 qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle)
 1905 {
 1906         struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
 1907 
 1908         free(mobj_handle->obj_table.obj_hdr, M_QAT);
 1909         mobj_handle->obj_table.obj_hdr = NULL;
 1910         free(handle->mobj_handle, M_QAT);
 1911         handle->mobj_handle = NULL;
 1912 }
 1913 
 1914 static int
 1915 qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle,
 1916                              const char *obj_name,
 1917                              const char **obj_ptr,
 1918                              unsigned int *obj_size)
 1919 {
 1920         unsigned int i;
 1921         struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr;
 1922 
 1923         for (i = 0; i < mobj_handle->obj_table.num_objs; i++) {
 1924                 if (!strncmp(obj_hdr[i].obj_name,
 1925                              obj_name,
 1926                              ICP_QAT_SUOF_OBJ_NAME_LEN)) {
 1927                         *obj_ptr = obj_hdr[i].obj_buf;
 1928                         *obj_size = obj_hdr[i].obj_size;
 1929                         break;
 1930                 }
 1931         }
 1932 
 1933         if (i >= mobj_handle->obj_table.num_objs) {
 1934                 pr_err("QAT: object %s is not found inside MOF\n", obj_name);
 1935                 return EFAULT;
 1936         }
 1937         return 0;
 1938 }
 1939 
 1940 static int
 1941 qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle,
 1942                           struct icp_qat_mof_objhdr *mobj_hdr,
 1943                           struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr)
 1944 {
 1945         if ((strncmp((char *)obj_chunkhdr->chunk_id,
 1946                      ICP_QAT_UOF_IMAG,
 1947                      ICP_QAT_MOF_OBJ_CHUNKID_LEN)) == 0) {
 1948                 mobj_hdr->obj_buf =
 1949                     (const char *)((unsigned long)obj_chunkhdr->offset +
 1950                                    mobj_handle->uobjs_hdr);
 1951         } else if ((strncmp((char *)(obj_chunkhdr->chunk_id),
 1952                             ICP_QAT_SUOF_IMAG,
 1953                             ICP_QAT_MOF_OBJ_CHUNKID_LEN)) == 0) {
 1954                 mobj_hdr->obj_buf =
 1955                     (const char *)((unsigned long)obj_chunkhdr->offset +
 1956                                    mobj_handle->sobjs_hdr);
 1957 
 1958         } else {
 1959                 pr_err("QAT: unsupported chunk id\n");
 1960                 return EINVAL;
 1961         }
 1962         mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size;
 1963         mobj_hdr->obj_name =
 1964             (char *)(obj_chunkhdr->name + mobj_handle->sym_str);
 1965         return 0;
 1966 }
 1967 
 1968 static int
 1969 qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle)
 1970 {
 1971         struct icp_qat_mof_objhdr *mof_obj_hdr;
 1972         const struct icp_qat_mof_obj_hdr *uobj_hdr;
 1973         const struct icp_qat_mof_obj_hdr *sobj_hdr;
 1974         struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr;
 1975         struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr;
 1976         unsigned int uobj_chunk_num = 0, sobj_chunk_num = 0;
 1977         unsigned int *valid_chunks = 0;
 1978         int ret, i;
 1979 
 1980         uobj_hdr = (const struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr;
 1981         sobj_hdr = (const struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr;
 1982         if (uobj_hdr)
 1983                 uobj_chunk_num = uobj_hdr->num_chunks;
 1984         if (sobj_hdr)
 1985                 sobj_chunk_num = sobj_hdr->num_chunks;
 1986 
 1987         mof_obj_hdr = (struct icp_qat_mof_objhdr *)
 1988             malloc((uobj_chunk_num + sobj_chunk_num) * sizeof(*mof_obj_hdr),
 1989                    M_QAT,
 1990                    M_WAITOK | M_ZERO);
 1991 
 1992         mobj_handle->obj_table.obj_hdr = mof_obj_hdr;
 1993         valid_chunks = &mobj_handle->obj_table.num_objs;
 1994         uobj_chunkhdr =
 1995             (struct icp_qat_mof_obj_chunkhdr *)((uintptr_t)uobj_hdr +
 1996                                                 sizeof(*uobj_hdr));
 1997         sobj_chunkhdr =
 1998             (struct icp_qat_mof_obj_chunkhdr *)((uintptr_t)sobj_hdr +
 1999                                                 sizeof(*sobj_hdr));
 2000 
 2001         /* map uof objects */
 2002         for (i = 0; i < uobj_chunk_num; i++) {
 2003                 ret = qat_uclo_map_obj_from_mof(mobj_handle,
 2004                                                 &mof_obj_hdr[*valid_chunks],
 2005                                                 &uobj_chunkhdr[i]);
 2006                 if (ret)
 2007                         return ret;
 2008                 (*valid_chunks)++;
 2009         }
 2010 
 2011         /* map suof objects */
 2012         for (i = 0; i < sobj_chunk_num; i++) {
 2013                 ret = qat_uclo_map_obj_from_mof(mobj_handle,
 2014                                                 &mof_obj_hdr[*valid_chunks],
 2015                                                 &sobj_chunkhdr[i]);
 2016                 if (ret)
 2017                         return ret;
 2018                 (*valid_chunks)++;
 2019         }
 2020 
 2021         if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunks) {
 2022                 pr_err("QAT: inconsistent UOF/SUOF chunk amount\n");
 2023                 return EINVAL;
 2024         }
 2025         return 0;
 2026 }
 2027 
 2028 static void
 2029 qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle,
 2030                          struct icp_qat_mof_chunkhdr *mof_chunkhdr)
 2031 {
 2032         char **sym_str = (char **)&mobj_handle->sym_str;
 2033         unsigned int *sym_size = &mobj_handle->sym_size;
 2034         struct icp_qat_mof_str_table *str_table_obj;
 2035 
 2036         *sym_size = *(unsigned int *)(uintptr_t)(mof_chunkhdr->offset +
 2037                                                  mobj_handle->mof_buf);
 2038         *sym_str =
 2039             (char *)(uintptr_t)(mobj_handle->mof_buf + mof_chunkhdr->offset +
 2040                                 sizeof(str_table_obj->tab_len));
 2041 }
 2042 
 2043 static void
 2044 qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle,
 2045                        struct icp_qat_mof_chunkhdr *mof_chunkhdr)
 2046 {
 2047         if (!strncmp(mof_chunkhdr->chunk_id,
 2048                      ICP_QAT_MOF_SYM_OBJS,
 2049                      ICP_QAT_MOF_OBJ_ID_LEN))
 2050                 qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr);
 2051         else if (!strncmp(mof_chunkhdr->chunk_id,
 2052                           ICP_QAT_UOF_OBJS,
 2053                           ICP_QAT_MOF_OBJ_ID_LEN))
 2054                 mobj_handle->uobjs_hdr =
 2055                     mobj_handle->mof_buf + (unsigned long)mof_chunkhdr->offset;
 2056         else if (!strncmp(mof_chunkhdr->chunk_id,
 2057                           ICP_QAT_SUOF_OBJS,
 2058                           ICP_QAT_MOF_OBJ_ID_LEN))
 2059                 mobj_handle->sobjs_hdr =
 2060                     mobj_handle->mof_buf + (unsigned long)mof_chunkhdr->offset;
 2061 }
 2062 
 2063 static int
 2064 qat_uclo_check_mof_format(const struct icp_qat_mof_file_hdr *mof_hdr)
 2065 {
 2066         int maj = mof_hdr->maj_ver & 0xff;
 2067         int min = mof_hdr->min_ver & 0xff;
 2068 
 2069         if (mof_hdr->file_id != ICP_QAT_MOF_FID) {
 2070                 pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id);
 2071                 return EINVAL;
 2072         }
 2073 
 2074         if (mof_hdr->num_chunks <= 0x1) {
 2075                 pr_err("QAT: MOF chunk amount is incorrect\n");
 2076                 return EINVAL;
 2077         }
 2078         if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) {
 2079                 pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n",
 2080                        maj,
 2081                        min);
 2082                 return EINVAL;
 2083         }
 2084         return 0;
 2085 }
 2086 
 2087 static int
 2088 qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle,
 2089                      const struct icp_qat_mof_file_hdr *mof_ptr,
 2090                      u32 mof_size,
 2091                      const char *obj_name,
 2092                      const char **obj_ptr,
 2093                      unsigned int *obj_size)
 2094 {
 2095         struct icp_qat_mof_handle *mobj_handle;
 2096         struct icp_qat_mof_chunkhdr *mof_chunkhdr;
 2097         unsigned short chunks_num;
 2098         int ret;
 2099         unsigned int i;
 2100 
 2101         if (mof_ptr->file_id == ICP_QAT_UOF_FID ||
 2102             mof_ptr->file_id == ICP_QAT_SUOF_FID) {
 2103                 if (obj_ptr)
 2104                         *obj_ptr = (const char *)mof_ptr;
 2105                 if (obj_size)
 2106                         *obj_size = (unsigned int)mof_size;
 2107                 return 0;
 2108         }
 2109         if (qat_uclo_check_mof_format(mof_ptr))
 2110                 return EINVAL;
 2111         mobj_handle = malloc(sizeof(*mobj_handle), M_QAT, M_WAITOK | M_ZERO);
 2112         handle->mobj_handle = mobj_handle;
 2113         ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size);
 2114         if (ret)
 2115                 return ret;
 2116         mof_chunkhdr = (struct icp_qat_mof_chunkhdr *)((uintptr_t)mof_ptr +
 2117                                                        sizeof(*mof_ptr));
 2118         chunks_num = mof_ptr->num_chunks;
 2119         /*Parse MOF file chunks*/
 2120         for (i = 0; i < chunks_num; i++)
 2121                 qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]);
 2122         /*All sym_objs uobjs and sobjs should be available*/
 2123         if (!mobj_handle->sym_str ||
 2124             (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr))
 2125                 return EINVAL;
 2126         ret = qat_uclo_map_objs_from_mof(mobj_handle);
 2127         if (ret)
 2128                 return ret;
 2129         /*Seek specified uof object in MOF*/
 2130         ret = qat_uclo_seek_obj_inside_mof(mobj_handle,
 2131                                            obj_name,
 2132                                            obj_ptr,
 2133                                            obj_size);
 2134         if (ret)
 2135                 return ret;
 2136         return 0;
 2137 }
 2138 
 2139 int
 2140 qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle,
 2141                  const void *addr_ptr,
 2142                  u32 mem_size,
 2143                  const char *obj_name)
 2144 {
 2145         const char *obj_addr;
 2146         u32 obj_size;
 2147         int ret;
 2148 
 2149         BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >
 2150                      (sizeof(handle->hal_handle->ae_mask) * 8));
 2151 
 2152         if (!handle || !addr_ptr || mem_size < 24)
 2153                 return EINVAL;
 2154 
 2155         if (obj_name) {
 2156                 ret = qat_uclo_map_mof_obj(
 2157                     handle, addr_ptr, mem_size, obj_name, &obj_addr, &obj_size);
 2158                 if (ret)
 2159                         return ret;
 2160         } else {
 2161                 obj_addr = addr_ptr;
 2162                 obj_size = mem_size;
 2163         }
 2164 
 2165         return (handle->fw_auth) ?
 2166             qat_uclo_map_suof_obj(handle, obj_addr, obj_size) :
 2167             qat_uclo_map_uof_obj(handle, obj_addr, obj_size);
 2168 }
 2169 
 2170 void
 2171 qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle)
 2172 {
 2173         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 2174         unsigned int a;
 2175         unsigned long ae_mask = handle->hal_handle->ae_mask;
 2176 
 2177         if (handle->mobj_handle)
 2178                 qat_uclo_del_mof(handle);
 2179         if (handle->sobj_handle)
 2180                 qat_uclo_del_suof(handle);
 2181         if (!obj_handle)
 2182                 return;
 2183 
 2184         free(obj_handle->uword_buf, M_QAT);
 2185         for (a = 0; a < obj_handle->uimage_num; a++)
 2186                 free(obj_handle->ae_uimage[a].page, M_QAT);
 2187 
 2188         for_each_set_bit(a, &ae_mask, handle->hal_handle->ae_max_num)
 2189         {
 2190                 qat_uclo_free_ae_data(&obj_handle->ae_data[a]);
 2191         }
 2192 
 2193         free(obj_handle->obj_hdr, M_QAT);
 2194         free(obj_handle->obj_buf, M_QAT);
 2195         free(obj_handle, M_QAT);
 2196         handle->obj_handle = NULL;
 2197 }
 2198 
 2199 static void
 2200 qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle,
 2201                      struct icp_qat_uclo_encap_page *encap_page,
 2202                      uint64_t *uword,
 2203                      unsigned int addr_p,
 2204                      unsigned int raddr,
 2205                      uint64_t fill)
 2206 {
 2207         uint64_t uwrd = 0;
 2208         unsigned int i, addr;
 2209 
 2210         if (!encap_page) {
 2211                 *uword = fill;
 2212                 return;
 2213         }
 2214         addr = (encap_page->page_region) ? raddr : addr_p;
 2215         for (i = 0; i < encap_page->uwblock_num; i++) {
 2216                 if (addr >= encap_page->uwblock[i].start_addr &&
 2217                     addr <= encap_page->uwblock[i].start_addr +
 2218                             encap_page->uwblock[i].words_num - 1) {
 2219                         addr -= encap_page->uwblock[i].start_addr;
 2220                         addr *= obj_handle->uword_in_bytes;
 2221                         memcpy(&uwrd,
 2222                                (void *)(((uintptr_t)encap_page->uwblock[i]
 2223                                              .micro_words) +
 2224                                         addr),
 2225                                obj_handle->uword_in_bytes);
 2226                         uwrd = uwrd & 0xbffffffffffull;
 2227                 }
 2228         }
 2229         *uword = uwrd;
 2230         if (*uword == INVLD_UWORD)
 2231                 *uword = fill;
 2232 }
 2233 
 2234 static void
 2235 qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle,
 2236                             struct icp_qat_uclo_encap_page *encap_page,
 2237                             unsigned int ae)
 2238 {
 2239         unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen;
 2240         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 2241         uint64_t fill_pat;
 2242 
 2243         /* load the page starting at appropriate ustore address */
 2244         /* get fill-pattern from an image -- they are all the same */
 2245         memcpy(&fill_pat,
 2246                obj_handle->ae_uimage[0].img_ptr->fill_pattern,
 2247                sizeof(uint64_t));
 2248         uw_physical_addr = encap_page->beg_addr_p;
 2249         uw_relative_addr = 0;
 2250         words_num = encap_page->micro_words_num;
 2251         while (words_num) {
 2252                 if (words_num < UWORD_CPYBUF_SIZE)
 2253                         cpylen = words_num;
 2254                 else
 2255                         cpylen = UWORD_CPYBUF_SIZE;
 2256 
 2257                 /* load the buffer */
 2258                 for (i = 0; i < cpylen; i++)
 2259                         qat_uclo_fill_uwords(obj_handle,
 2260                                              encap_page,
 2261                                              &obj_handle->uword_buf[i],
 2262                                              uw_physical_addr + i,
 2263                                              uw_relative_addr + i,
 2264                                              fill_pat);
 2265 
 2266                 if (obj_handle->ae_data[ae].shareable_ustore &&
 2267                     !IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev))))
 2268                         /* copy the buffer to ustore */
 2269                         qat_hal_wr_coalesce_uwords(handle,
 2270                                                    (unsigned char)ae,
 2271                                                    uw_physical_addr,
 2272                                                    cpylen,
 2273                                                    obj_handle->uword_buf);
 2274                 else
 2275                         /* copy the buffer to ustore */
 2276                         qat_hal_wr_uwords(handle,
 2277                                           (unsigned char)ae,
 2278                                           uw_physical_addr,
 2279                                           cpylen,
 2280                                           obj_handle->uword_buf);
 2281                 uw_physical_addr += cpylen;
 2282                 uw_relative_addr += cpylen;
 2283                 words_num -= cpylen;
 2284         }
 2285 }
 2286 
 2287 static void
 2288 qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle,
 2289                         struct icp_qat_uof_image *image)
 2290 {
 2291         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 2292         unsigned int ctx_mask, s;
 2293         struct icp_qat_uclo_page *page;
 2294         unsigned char ae = 0;
 2295         int ctx;
 2296         struct icp_qat_uclo_aedata *aed;
 2297         unsigned long ae_mask = handle->hal_handle->ae_mask;
 2298 
 2299         if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX)
 2300                 ctx_mask = 0xff;
 2301         else
 2302                 ctx_mask = 0x55;
 2303         /* load the default page and set assigned CTX PC
 2304          * to the entrypoint address
 2305          */
 2306         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num)
 2307         {
 2308                 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
 2309                 unsigned long ae_assigned = image->ae_assigned;
 2310 
 2311                 if (!test_bit(ae, &cfg_ae_mask))
 2312                         continue;
 2313 
 2314                 if (!test_bit(ae, &ae_assigned))
 2315                         continue;
 2316 
 2317                 aed = &obj_handle->ae_data[ae];
 2318                 /* find the slice to which this image is assigned */
 2319                 for (s = 0; s < aed->slice_num; s++) {
 2320                         if (image->ctx_assigned &
 2321                             aed->ae_slices[s].ctx_mask_assigned)
 2322                                 break;
 2323                 }
 2324                 if (s >= aed->slice_num)
 2325                         continue;
 2326                 page = aed->ae_slices[s].page;
 2327                 if (!page->encap_page->def_page)
 2328                         continue;
 2329                 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae);
 2330 
 2331                 page = aed->ae_slices[s].page;
 2332                 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++)
 2333                         aed->ae_slices[s].cur_page[ctx] =
 2334                             (ctx_mask & (1 << ctx)) ? page : NULL;
 2335                 qat_hal_set_live_ctx(handle,
 2336                                      (unsigned char)ae,
 2337                                      image->ctx_assigned);
 2338                 qat_hal_set_pc(handle,
 2339                                (unsigned char)ae,
 2340                                image->ctx_assigned,
 2341                                image->entry_address);
 2342         }
 2343 }
 2344 
 2345 static int
 2346 qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle)
 2347 {
 2348         unsigned int i;
 2349         struct icp_qat_fw_auth_desc *desc = NULL;
 2350         struct icp_firml_dram_desc img_desc;
 2351         struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
 2352         struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr;
 2353 
 2354         for (i = 0; i < sobj_handle->img_table.num_simgs; i++) {
 2355                 if (qat_uclo_map_auth_fw(handle,
 2356                                          (const char *)simg_hdr[i].simg_buf,
 2357                                          (unsigned int)(simg_hdr[i].simg_len),
 2358                                          &img_desc,
 2359                                          &desc))
 2360                         goto wr_err;
 2361                 if (qat_uclo_auth_fw(handle, desc))
 2362                         goto wr_err;
 2363                 if (qat_uclo_is_broadcast(handle, i)) {
 2364                         if (qat_uclo_broadcast_load_fw(handle, desc))
 2365                                 goto wr_err;
 2366                 } else {
 2367                         if (qat_uclo_load_fw(handle, desc))
 2368                                 goto wr_err;
 2369                 }
 2370                 qat_uclo_simg_free(handle, &img_desc);
 2371         }
 2372 
 2373         return 0;
 2374 wr_err:
 2375         qat_uclo_simg_free(handle, &img_desc);
 2376         return -EINVAL;
 2377 }
 2378 
 2379 static int
 2380 qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle)
 2381 {
 2382         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 2383         unsigned int i;
 2384 
 2385         if (qat_uclo_init_globals(handle))
 2386                 return EINVAL;
 2387         for (i = 0; i < obj_handle->uimage_num; i++) {
 2388                 if (!obj_handle->ae_uimage[i].img_ptr)
 2389                         return EINVAL;
 2390                 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i]))
 2391                         return EINVAL;
 2392                 qat_uclo_wr_uimage_page(handle,
 2393                                         obj_handle->ae_uimage[i].img_ptr);
 2394         }
 2395         return 0;
 2396 }
 2397 
 2398 int
 2399 qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle)
 2400 {
 2401         return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) :
 2402                                    qat_uclo_wr_uof_img(handle);
 2403 }
 2404 
 2405 int
 2406 qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle,
 2407                          unsigned int cfg_ae_mask)
 2408 {
 2409         if (!cfg_ae_mask)
 2410                 return EINVAL;
 2411 
 2412         handle->cfg_ae_mask = cfg_ae_mask;
 2413         return 0;
 2414 }
Cache object: b45fa845e631d000136c715eddd6c9d6
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/qat/qat_common/qat_uclo.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/qat/qat_common/qat_uclo.c
