The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/net/rss_config.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2010-2011 Juniper Networks, Inc.
    3  * All rights reserved.
    4  *
    5  * This software was developed by Robert N. M. Watson under contract
    6  * to Juniper Networks, Inc.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 
   32 __FBSDID("$FreeBSD$");
   33 
   34 #include "opt_inet6.h"
   35 #include "opt_pcbgroup.h"
   36 
   37 #ifndef PCBGROUP
   38 #error "options RSS depends on options PCBGROUP"
   39 #endif
   40 
   41 #include <sys/param.h>
   42 #include <sys/mbuf.h>
   43 #include <sys/socket.h>
   44 #include <sys/priv.h>
   45 #include <sys/kernel.h>
   46 #include <sys/smp.h>
   47 #include <sys/sysctl.h>
   48 #include <sys/sbuf.h>
   49 
   50 #include <net/if.h>
   51 #include <net/if_var.h>
   52 #include <net/netisr.h>
   53 #include <net/rss_config.h>
   54 #include <net/toeplitz.h>
   55 
   56 /*-
   57  * Operating system parts of receiver-side scaling (RSS), which allows
   58  * network cards to direct flows to particular receive queues based on hashes
   59  * of header tuples.  This implementation aligns RSS buckets with connection
   60  * groups at the TCP/IP layer, so each bucket is associated with exactly one
   61  * group.  As a result, the group lookup structures (and lock) should have an
   62  * effective affinity with exactly one CPU.
   63  *
   64  * Network device drivers needing to configure RSS will query this framework
   65  * for parameters, such as the current RSS key, hashing policies, number of
   66  * bits, and indirection table mapping hashes to buckets and CPUs.  They may
   67  * provide their own supplementary information, such as queue<->CPU bindings.
   68  * It is the responsibility of the network device driver to inject packets
   69  * into the stack on as close to the right CPU as possible, if playing by RSS
   70  * rules.
   71  *
   72  * TODO:
   73  *
   74  * - Synchronization for rss_key and other future-configurable parameters.
   75  * - Event handler drivers can register to pick up RSS configuration changes.
   76  * - Should we allow rss_basecpu to be configured?
   77  * - Randomize key on boot.
   78  * - IPv6 support.
   79  * - Statistics on how often there's a misalignment between hardware
   80  *   placement and pcbgroup expectations.
   81  */
   82 
   83 SYSCTL_DECL(_net_inet);
   84 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
   85 
   86 /*
   87  * Toeplitz is the only required hash function in the RSS spec, so use it by
   88  * default.
   89  */
   90 static u_int    rss_hashalgo = RSS_HASH_TOEPLITZ;
   91 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
   92     "RSS hash algorithm");
   93 
   94 /*
   95  * Size of the indirection table; at most 128 entries per the RSS spec.  We
   96  * size it to at least 2 times the number of CPUs by default to allow useful
   97  * rebalancing.  If not set explicitly with a loader tunable, we tune based
   98  * on the number of CPUs present.
   99  *
  100  * XXXRW: buckets might be better to use for the tunable than bits.
  101  */
  102 static u_int    rss_bits;
  103 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
  104     "RSS bits");
  105 
  106 static u_int    rss_mask;
  107 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
  108     "RSS mask");
  109 
  110 static const u_int      rss_maxbits = RSS_MAXBITS;
  111 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
  112     __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
  113 
  114 /*
  115  * RSS's own count of the number of CPUs it could be using for processing.
  116  * Bounded to 64 by RSS constants.
  117  */
  118 static u_int    rss_ncpus;
  119 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
  120     "Number of CPUs available to RSS");
  121 
  122 #define RSS_MAXCPUS     (1 << (RSS_MAXBITS - 1))
  123 static const u_int      rss_maxcpus = RSS_MAXCPUS;
  124 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
  125     __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
  126 
  127 /*
  128  * Variable exists just for reporting rss_bits in a user-friendly way.
  129  */
  130 static u_int    rss_buckets;
  131 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
  132     "RSS buckets");
  133 
  134 /*
  135  * Base CPU number; devices will add this to all CPU numbers returned by the
  136  * RSS indirection table.  Currently unmodifable in FreeBSD.
  137  */
  138 static const u_int      rss_basecpu;
  139 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
  140     __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
  141 
  142 /*
  143  * Print verbose debugging messages.
  144  * 0 - disable
  145  * non-zero - enable
  146  */
  147 int     rss_debug = 0;
  148 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0,
  149     "RSS debug level");
  150 
  151 /*
  152  * RSS secret key, intended to prevent attacks on load-balancing.  Its
  153  * effectiveness may be limited by algorithm choice and available entropy
  154  * during the boot.
  155  *
  156  * XXXRW: And that we don't randomize it yet!
  157  *
  158  * This is the default Microsoft RSS specification key which is also
  159  * the Chelsio T5 firmware default key.
  160  */
  161 static uint8_t rss_key[RSS_KEYSIZE] = {
  162         0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
  163         0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
  164         0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
  165         0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
  166         0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
  167 };
  168 
  169 /*
  170  * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
  171  * Drivers may supplement this table with a separate CPU<->queue table when
  172  * programming devices.
  173  */
  174 struct rss_table_entry {
  175         uint8_t         rte_cpu;        /* CPU affinity of bucket. */
  176 };
  177 static struct rss_table_entry   rss_table[RSS_TABLE_MAXLEN];
  178 
  179 static void
  180 rss_init(__unused void *arg)
  181 {
  182         u_int i;
  183         u_int cpuid;
  184 
  185         /*
  186          * Validate tunables, coerce to sensible values.
  187          */
  188         switch (rss_hashalgo) {
  189         case RSS_HASH_TOEPLITZ:
  190         case RSS_HASH_NAIVE:
  191                 break;
  192 
  193         default:
  194                 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n",
  195                     rss_hashalgo, RSS_HASH_TOEPLITZ);
  196                 rss_hashalgo = RSS_HASH_TOEPLITZ;
  197         }
  198 
  199         /*
  200          * Count available CPUs.
  201          *
  202          * XXXRW: Note incorrect assumptions regarding contiguity of this set
  203          * elsewhere.
  204          */
  205         rss_ncpus = 0;
  206         for (i = 0; i <= mp_maxid; i++) {
  207                 if (CPU_ABSENT(i))
  208                         continue;
  209                 rss_ncpus++;
  210         }
  211         if (rss_ncpus > RSS_MAXCPUS)
  212                 rss_ncpus = RSS_MAXCPUS;
  213 
  214         /*
  215          * Tune RSS table entries to be no less than 2x the number of CPUs
  216          * -- unless we're running uniprocessor, in which case there's not
  217          * much point in having buckets to rearrange for load-balancing!
  218          */
  219         if (rss_ncpus > 1) {
  220                 if (rss_bits == 0)
  221                         rss_bits = fls(rss_ncpus - 1) + 1;
  222 
  223                 /*
  224                  * Microsoft limits RSS table entries to 128, so apply that
  225                  * limit to both auto-detected CPU counts and user-configured
  226                  * ones.
  227                  */
  228                 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
  229                         RSS_DEBUG("RSS bits %u not valid, coercing to %u\n",
  230                             rss_bits, RSS_MAXBITS);
  231                         rss_bits = RSS_MAXBITS;
  232                 }
  233 
  234                 /*
  235                  * Figure out how many buckets to use; warn if less than the
  236                  * number of configured CPUs, although this is not a fatal
  237                  * problem.
  238                  */
  239                 rss_buckets = (1 << rss_bits);
  240                 if (rss_buckets < rss_ncpus)
  241                         RSS_DEBUG("WARNING: rss_buckets (%u) less than "
  242                             "rss_ncpus (%u)\n", rss_buckets, rss_ncpus);
  243                 rss_mask = rss_buckets - 1;
  244         } else {
  245                 rss_bits = 0;
  246                 rss_buckets = 1;
  247                 rss_mask = 0;
  248         }
  249 
  250         /*
  251          * Set up initial CPU assignments: round-robin by default.
  252          */
  253         cpuid = CPU_FIRST();
  254         for (i = 0; i < rss_buckets; i++) {
  255                 rss_table[i].rte_cpu = cpuid;
  256                 cpuid = CPU_NEXT(cpuid);
  257         }
  258 
  259         /*
  260          * Randomize rrs_key.
  261          *
  262          * XXXRW: Not yet.  If nothing else, will require an rss_isbadkey()
  263          * loop to check for "bad" RSS keys.
  264          */
  265 }
  266 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
  267 
  268 static uint32_t
  269 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
  270     const uint8_t *data)
  271 {
  272         uint32_t v;
  273         u_int i;
  274 
  275         v = 0;
  276         for (i = 0; i < keylen; i++)
  277                 v += key[i];
  278         for (i = 0; i < datalen; i++)
  279                 v += data[i];
  280         return (v);
  281 }
  282 
  283 uint32_t
  284 rss_hash(u_int datalen, const uint8_t *data)
  285 {
  286  
  287         switch (rss_hashalgo) {
  288         case RSS_HASH_TOEPLITZ:
  289                 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
  290                     data));
  291 
  292         case RSS_HASH_NAIVE:
  293                 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
  294                     data));
  295 
  296         default:
  297                 panic("%s: unsupported/unknown hashalgo %d", __func__,
  298                     rss_hashalgo);
  299         }
  300 }
  301 
  302 /*
  303  * Query the number of RSS bits in use.
  304  */
  305 u_int
  306 rss_getbits(void)
  307 {
  308 
  309         return (rss_bits);
  310 }
  311 
  312 /*
  313  * Query the RSS bucket associated with an RSS hash.
  314  */
  315 u_int
  316 rss_getbucket(u_int hash)
  317 {
  318 
  319         return (hash & rss_mask);
  320 }
  321 
  322 /*
  323  * Query the RSS layer bucket associated with the given
  324  * entry in the RSS hash space.
  325  *
  326  * The RSS indirection table is 0 .. rss_buckets-1,
  327  * covering the low 'rss_bits' of the total 128 slot
  328  * RSS indirection table.  So just mask off rss_bits and
  329  * return that.
  330  *
  331  * NIC drivers can then iterate over the 128 slot RSS
  332  * indirection table and fetch which RSS bucket to
  333  * map it to.  This will typically be a CPU queue
  334  */
  335 u_int
  336 rss_get_indirection_to_bucket(u_int index)
  337 {
  338 
  339         return (index & rss_mask);
  340 }
  341 
  342 /*
  343  * Query the RSS CPU associated with an RSS bucket.
  344  */
  345 u_int
  346 rss_getcpu(u_int bucket)
  347 {
  348 
  349         return (rss_table[bucket].rte_cpu);
  350 }
  351 
  352 /*
  353  * netisr CPU affinity lookup given just the hash and hashtype.
  354  */
  355 u_int
  356 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
  357 {
  358 
  359         switch (hash_type) {
  360         case M_HASHTYPE_RSS_IPV4:
  361         case M_HASHTYPE_RSS_TCP_IPV4:
  362         case M_HASHTYPE_RSS_UDP_IPV4:
  363         case M_HASHTYPE_RSS_IPV6:
  364         case M_HASHTYPE_RSS_TCP_IPV6:
  365         case M_HASHTYPE_RSS_UDP_IPV6:
  366                 return (rss_getcpu(rss_getbucket(hash_val)));
  367         default:
  368                 return (NETISR_CPUID_NONE);
  369         }
  370 }
  371 
  372 /*
  373  * Query the RSS bucket associated with the given hash value and
  374  * type.
  375  */
  376 int
  377 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
  378 {
  379 
  380         switch (hash_type) {
  381         case M_HASHTYPE_RSS_IPV4:
  382         case M_HASHTYPE_RSS_TCP_IPV4:
  383         case M_HASHTYPE_RSS_UDP_IPV4:
  384         case M_HASHTYPE_RSS_IPV6:
  385         case M_HASHTYPE_RSS_TCP_IPV6:
  386         case M_HASHTYPE_RSS_UDP_IPV6:
  387                 *bucket_id = rss_getbucket(hash_val);
  388                 return (0);
  389         default:
  390                 return (-1);
  391         }
  392 }
  393 
  394 /*
  395  * netisr CPU affinity lookup routine for use by protocols.
  396  */
  397 struct mbuf *
  398 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
  399 {
  400 
  401         M_ASSERTPKTHDR(m);
  402         *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
  403         return (m);
  404 }
  405 
  406 int
  407 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
  408 {
  409 
  410         M_ASSERTPKTHDR(m);
  411 
  412         return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
  413             bucket_id));
  414 }
  415 
  416 /*
  417  * Query the RSS hash algorithm.
  418  */
  419 u_int
  420 rss_gethashalgo(void)
  421 {
  422 
  423         return (rss_hashalgo);
  424 }
  425 
  426 /*
  427  * Query the current RSS key; likely to be used by device drivers when
  428  * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
  429  *
  430  * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
  431  */
  432 void
  433 rss_getkey(uint8_t *key)
  434 {
  435 
  436         bcopy(rss_key, key, sizeof(rss_key));
  437 }
  438 
  439 /*
  440  * Query the number of buckets; this may be used by both network device
  441  * drivers, which will need to populate hardware shadows of the software
  442  * indirection table, and the network stack itself (such as when deciding how
  443  * many connection groups to allocate).
  444  */
  445 u_int
  446 rss_getnumbuckets(void)
  447 {
  448 
  449         return (rss_buckets);
  450 }
  451 
  452 /*
  453  * Query the number of CPUs in use by RSS; may be useful to device drivers
  454  * trying to figure out how to map a larger number of CPUs into a smaller
  455  * number of receive queues.
  456  */
  457 u_int
  458 rss_getnumcpus(void)
  459 {
  460 
  461         return (rss_ncpus);
  462 }
  463 
  464 /*
  465  * Return the supported RSS hash configuration.
  466  *
  467  * NICs should query this to determine what to configure in their redirection
  468  * matching table.
  469  */
  470 inline u_int
  471 rss_gethashconfig(void)
  472 {
  473 
  474         /* Return 4-tuple for TCP; 2-tuple for others */
  475         /*
  476          * UDP may fragment more often than TCP and thus we'll end up with
  477          * NICs returning 2-tuple fragments.
  478          * udp_init() and udplite_init() both currently initialise things
  479          * as 2-tuple.
  480          * So for now disable UDP 4-tuple hashing until all of the other
  481          * pieces are in place.
  482          */
  483         return (
  484             RSS_HASHTYPE_RSS_IPV4
  485         |    RSS_HASHTYPE_RSS_TCP_IPV4
  486         |    RSS_HASHTYPE_RSS_IPV6
  487         |    RSS_HASHTYPE_RSS_TCP_IPV6
  488         |    RSS_HASHTYPE_RSS_IPV6_EX
  489         |    RSS_HASHTYPE_RSS_TCP_IPV6_EX
  490 #if 0
  491         |    RSS_HASHTYPE_RSS_UDP_IPV4
  492         |    RSS_HASHTYPE_RSS_UDP_IPV4_EX
  493         |    RSS_HASHTYPE_RSS_UDP_IPV6
  494         |    RSS_HASHTYPE_RSS_UDP_IPV6_EX
  495 #endif
  496         );
  497 }
  498 
  499 /*
  500  * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
  501  * it appearing in debugging output unnecessarily.
  502  */
  503 static int
  504 sysctl_rss_key(SYSCTL_HANDLER_ARGS)
  505 {
  506         uint8_t temp_rss_key[RSS_KEYSIZE];
  507         int error;
  508 
  509         error = priv_check(req->td, PRIV_NETINET_HASHKEY);
  510         if (error)
  511                 return (error);
  512 
  513         bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
  514         error = sysctl_handle_opaque(oidp, temp_rss_key,
  515             sizeof(temp_rss_key), req);
  516         if (error)
  517                 return (error);
  518         if (req->newptr != NULL) {
  519                 /* XXXRW: Not yet. */
  520                 return (EINVAL);
  521         }
  522         return (0);
  523 }
  524 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
  525     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
  526     "", "RSS keying material");
  527 
  528 static int
  529 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
  530 {
  531         struct sbuf *sb;
  532         int error;
  533         int i;
  534 
  535         error = 0;
  536         error = sysctl_wire_old_buffer(req, 0);
  537         if (error != 0)
  538                 return (error);
  539         sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
  540         if (sb == NULL)
  541                 return (ENOMEM);
  542         for (i = 0; i < rss_buckets; i++) {
  543                 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
  544                     i,
  545                     rss_getcpu(i));
  546         }
  547         error = sbuf_finish(sb);
  548         sbuf_delete(sb);
  549 
  550         return (error);
  551 }
  552 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
  553     CTLTYPE_STRING | CTLFLAG_RD, NULL, 0,
  554     sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");

Cache object: 4bda42ac933f1a4b945d55bff49b3c34


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.