ena_rss.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause
    3  *
    4  * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates.
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  *
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  *
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
   21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
   22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
   24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
   25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
   26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
   27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
   28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   29  *
   30  * $FreeBSD$
   31  *
   32  */
   33 
   34 #include <sys/cdefs.h>
   35 __FBSDID("$FreeBSD$");
   36 
   37 #include "opt_rss.h"
   38 
   39 #include "ena_rss.h"
   40 
   41 /*
   42  * This function should generate unique key for the whole driver.
   43  * If the key was already genereated in the previous call (for example
   44  * for another adapter), then it should be returned instead.
   45  */
   46 void
   47 ena_rss_key_fill(void *key, size_t size)
   48 {
   49         static bool key_generated;
   50         static uint8_t default_key[ENA_HASH_KEY_SIZE];
   51 
   52         KASSERT(size <= ENA_HASH_KEY_SIZE,
   53             ("Requested more bytes than ENA RSS key can hold"));
   54 
   55         if (!key_generated) {
   56                 arc4random_buf(default_key, ENA_HASH_KEY_SIZE);
   57                 key_generated = true;
   58         }
   59 
   60         memcpy(key, default_key, size);
   61 }
   62 
   63 /*
   64  * ENA HW expects the key to be in reverse-byte order.
   65  */
   66 static void
   67 ena_rss_reorder_hash_key(u8 *reordered_key, const u8 *key, size_t key_size)
   68 {
   69         int i;
   70 
   71         key = key + key_size - 1;
   72 
   73         for (i = 0; i < key_size; ++i)
   74                 *reordered_key++ = *key--;
   75 }
   76 
   77 int
   78 ena_rss_set_hash(struct ena_com_dev *ena_dev, const u8 *key)
   79 {
   80         enum ena_admin_hash_functions ena_func = ENA_ADMIN_TOEPLITZ;
   81         u8 hw_key[ENA_HASH_KEY_SIZE];
   82 
   83         ena_rss_reorder_hash_key(hw_key, key, ENA_HASH_KEY_SIZE);
   84 
   85         return (ena_com_fill_hash_function(ena_dev, ena_func, hw_key,
   86             ENA_HASH_KEY_SIZE, 0x0));
   87 }
   88 
   89 int
   90 ena_rss_get_hash_key(struct ena_com_dev *ena_dev, u8 *key)
   91 {
   92         u8 hw_key[ENA_HASH_KEY_SIZE];
   93         int rc;
   94 
   95         rc = ena_com_get_hash_key(ena_dev, hw_key);
   96         if (rc != 0)
   97                 return rc;
   98 
   99         ena_rss_reorder_hash_key(key, hw_key, ENA_HASH_KEY_SIZE);
  100 
  101         return (0);
  102 }
  103 
  104 static int
  105 ena_rss_init_default(struct ena_adapter *adapter)
  106 {
  107         struct ena_com_dev *ena_dev = adapter->ena_dev;
  108         device_t dev = adapter->pdev;
  109         int qid, rc, i;
  110 
  111         rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
  112         if (unlikely(rc != 0)) {
  113                 ena_log(dev, ERR, "Cannot init indirect table\n");
  114                 return (rc);
  115         }
  116 
  117         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
  118 #ifdef RSS
  119                 qid = rss_get_indirection_to_bucket(i) % adapter->num_io_queues;
  120 #else
  121                 qid = i % adapter->num_io_queues;
  122 #endif
  123                 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
  124                     ENA_IO_RXQ_IDX(qid));
  125                 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
  126                         ena_log(dev, ERR, "Cannot fill indirect table\n");
  127                         goto err_rss_destroy;
  128                 }
  129         }
  130 
  131 
  132 #ifdef RSS
  133         uint8_t rss_algo = rss_gethashalgo();
  134         if (rss_algo == RSS_HASH_TOEPLITZ) {
  135                 uint8_t hash_key[RSS_KEYSIZE];
  136 
  137                 rss_getkey(hash_key);
  138                 rc = ena_rss_set_hash(ena_dev, hash_key);
  139         } else
  140 #endif
  141                 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ,
  142                     NULL, ENA_HASH_KEY_SIZE, 0x0);
  143         if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
  144                 ena_log(dev, ERR, "Cannot fill hash function\n");
  145                 goto err_rss_destroy;
  146         }
  147 
  148         rc = ena_com_set_default_hash_ctrl(ena_dev);
  149         if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
  150                 ena_log(dev, ERR, "Cannot fill hash control\n");
  151                 goto err_rss_destroy;
  152         }
  153 
  154         rc = ena_rss_indir_init(adapter);
  155 
  156         return (rc == EOPNOTSUPP ? 0 : rc);
  157 
  158 err_rss_destroy:
  159         ena_com_rss_destroy(ena_dev);
  160         return (rc);
  161 }
  162 
  163 /* Configure the Rx forwarding */
  164 int
  165 ena_rss_configure(struct ena_adapter *adapter)
  166 {
  167         struct ena_com_dev *ena_dev = adapter->ena_dev;
  168         int rc;
  169 
  170         /* In case the RSS table was destroyed */
  171         if (!ena_dev->rss.tbl_log_size) {
  172                 rc = ena_rss_init_default(adapter);
  173                 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
  174                         ena_log(adapter->pdev, ERR,
  175                             "WARNING: RSS was not properly re-initialized,"
  176                             " it will affect bandwidth\n");
  177                         ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
  178                         return (rc);
  179                 }
  180         }
  181 
  182         /* Set indirect table */
  183         rc = ena_com_indirect_table_set(ena_dev);
  184         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
  185                 return (rc);
  186 
  187         /* Configure hash function (if supported) */
  188         rc = ena_com_set_hash_function(ena_dev);
  189         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
  190                 return (rc);
  191 
  192         /* Configure hash inputs (if supported) */
  193         rc = ena_com_set_hash_ctrl(ena_dev);
  194         if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
  195                 return (rc);
  196 
  197         return (0);
  198 }
  199 
  200 static void
  201 ena_rss_init_default_deferred(void *arg)
  202 {
  203         struct ena_adapter *adapter;
  204         devclass_t dc;
  205         int max;
  206         int rc;
  207 
  208         dc = devclass_find("ena");
  209         if (unlikely(dc == NULL)) {
  210                 ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__);
  211                 return;
  212         }
  213 
  214         max = devclass_get_maxunit(dc);
  215         while (max-- >= 0) {
  216                 adapter = devclass_get_softc(dc, max);
  217                 if (adapter != NULL) {
  218                         rc = ena_rss_init_default(adapter);
  219                         ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter);
  220                         if (unlikely(rc != 0)) {
  221                                 ena_log(adapter->pdev, WARN,
  222                                     "WARNING: RSS was not properly initialized,"
  223                                     " it will affect bandwidth\n");
  224                                 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE,
  225                                     adapter);
  226                         }
  227                 }
  228         }
  229 }
  230 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND,
  231     ena_rss_init_default_deferred, NULL);
  232 
  233 int
  234 ena_rss_indir_get(struct ena_adapter *adapter, uint32_t *table)
  235 {
  236         int rc, i;
  237 
  238         rc = ena_com_indirect_table_get(adapter->ena_dev, table);
  239         if (rc != 0) {
  240                 if (rc == EOPNOTSUPP)
  241                         device_printf(adapter->pdev,
  242                             "Reading from indirection table not supported\n");
  243                 else
  244                         device_printf(adapter->pdev,
  245                             "Unable to get indirection table\n");
  246                 return (rc);
  247         }
  248 
  249         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i)
  250                 table[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(table[i]);
  251 
  252         return (0);
  253 }
  254 
  255 int
  256 ena_rss_indir_set(struct ena_adapter *adapter, uint32_t *table)
  257 {
  258         int rc, i;
  259 
  260         for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; ++i) {
  261                 rc = ena_com_indirect_table_fill_entry(adapter->ena_dev, i,
  262                     ENA_IO_RXQ_IDX(table[i]));
  263                 if (rc != 0) {
  264                         device_printf(adapter->pdev,
  265                             "Cannot fill indirection table entry %d\n", i);
  266                         return (rc);
  267                 }
  268         }
  269 
  270         rc = ena_com_indirect_table_set(adapter->ena_dev);
  271         if (rc == EOPNOTSUPP)
  272                 device_printf(adapter->pdev,
  273                     "Writing to indirection table not supported\n");
  274         else if (rc != 0)
  275                 device_printf(adapter->pdev, "Cannot set indirection table\n");
  276 
  277         return (rc);
  278 }
  279 
  280 int
  281 ena_rss_indir_init(struct ena_adapter *adapter)
  282 {
  283         struct ena_indir *indir = adapter->rss_indir;
  284         int rc;
  285 
  286         if (indir == NULL) {
  287                 adapter->rss_indir = indir = malloc(sizeof(struct ena_indir),
  288                     M_DEVBUF, M_WAITOK | M_ZERO);
  289                 if (indir == NULL)
  290                         return (ENOMEM);
  291         }
  292 
  293         rc = ena_rss_indir_get(adapter, indir->table);
  294         if (rc != 0) {
  295                 free(adapter->rss_indir, M_DEVBUF);
  296                 adapter->rss_indir = NULL;
  297 
  298                 return (rc);
  299         }
  300 
  301         ena_rss_copy_indir_buf(indir->sysctl_buf, indir->table);
  302 
  303         return (0);
  304 }
Cache object: 9a3d789b5c593a5db79fcaae631ada22
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/ena/ena_rss.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/ena/ena_rss.c
