The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/netinet/in_rss.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2010-2011 Juniper Networks, Inc.
    3  * All rights reserved.
    4  *
    5  * This software was developed by Robert N. M. Watson under contract
    6  * to Juniper Networks, Inc.
    7  *
    8  * Redistribution and use in source and binary forms, with or without
    9  * modification, are permitted provided that the following conditions
   10  * are met:
   11  * 1. Redistributions of source code must retain the above copyright
   12  *    notice, this list of conditions and the following disclaimer.
   13  * 2. Redistributions in binary form must reproduce the above copyright
   14  *    notice, this list of conditions and the following disclaimer in the
   15  *    documentation and/or other materials provided with the distribution.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  */
   29 
   30 #include <sys/cdefs.h>
   31 
   32 __FBSDID("$FreeBSD$");
   33 
   34 #include "opt_inet6.h"
   35 
   36 #include <sys/param.h>
   37 #include <sys/mbuf.h>
   38 #include <sys/socket.h>
   39 #include <sys/priv.h>
   40 #include <sys/kernel.h>
   41 #include <sys/smp.h>
   42 #include <sys/sysctl.h>
   43 #include <sys/sbuf.h>
   44 
   45 #include <net/if.h>
   46 #include <net/if_var.h>
   47 #include <net/netisr.h>
   48 #include <net/rss_config.h>
   49 
   50 #include <netinet/in.h>
   51 #include <netinet/in_pcb.h>
   52 #include <netinet/in_rss.h>
   53 #include <netinet/in_var.h>
   54 
   55 /* for software rss hash support */
   56 #include <netinet/ip.h>
   57 #include <netinet/tcp.h>
   58 #include <netinet/udp.h>
   59 
   60 /*
   61  * Hash an IPv4 2-tuple.
   62  */
   63 uint32_t
   64 rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst)
   65 {
   66         uint8_t data[sizeof(src) + sizeof(dst)];
   67         u_int datalen;
   68 
   69         datalen = 0;
   70         bcopy(&src, &data[datalen], sizeof(src));
   71         datalen += sizeof(src);
   72         bcopy(&dst, &data[datalen], sizeof(dst));
   73         datalen += sizeof(dst);
   74         return (rss_hash(datalen, data));
   75 }
   76 
   77 /*
   78  * Hash an IPv4 4-tuple.
   79  */
   80 uint32_t
   81 rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst,
   82     u_short dstport)
   83 {
   84         uint8_t data[sizeof(src) + sizeof(dst) + sizeof(srcport) +
   85             sizeof(dstport)];
   86         u_int datalen;
   87 
   88         datalen = 0;
   89         bcopy(&src, &data[datalen], sizeof(src));
   90         datalen += sizeof(src);
   91         bcopy(&dst, &data[datalen], sizeof(dst));
   92         datalen += sizeof(dst);
   93         bcopy(&srcport, &data[datalen], sizeof(srcport));
   94         datalen += sizeof(srcport);
   95         bcopy(&dstport, &data[datalen], sizeof(dstport));
   96         datalen += sizeof(dstport);
   97         return (rss_hash(datalen, data));
   98 }
   99 
  100 /*
  101  * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
  102  * IPv4 source/destination address, UDP or TCP source/destination ports
  103  * and the protocol type.
  104  *
  105  * The protocol code may wish to do a software hash of the given
  106  * tuple.  This depends upon the currently configured RSS hash types.
  107  *
  108  * This assumes that the packet in question isn't a fragment.
  109  *
  110  * It also assumes the packet source/destination address
  111  * are in "incoming" packet order (ie, source is "far" address.)
  112  */
  113 int
  114 rss_proto_software_hash_v4(struct in_addr s, struct in_addr d,
  115     u_short sp, u_short dp, int proto,
  116     uint32_t *hashval, uint32_t *hashtype)
  117 {
  118         uint32_t hash;
  119 
  120         /*
  121          * Next, choose the hash type depending upon the protocol
  122          * identifier.
  123          */
  124         if ((proto == IPPROTO_TCP) &&
  125             (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
  126                 hash = rss_hash_ip4_4tuple(s, sp, d, dp);
  127                 *hashval = hash;
  128                 *hashtype = M_HASHTYPE_RSS_TCP_IPV4;
  129                 return (0);
  130         } else if ((proto == IPPROTO_UDP) &&
  131             (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
  132                 hash = rss_hash_ip4_4tuple(s, sp, d, dp);
  133                 *hashval = hash;
  134                 *hashtype = M_HASHTYPE_RSS_UDP_IPV4;
  135                 return (0);
  136         } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
  137                 /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
  138                 hash = rss_hash_ip4_2tuple(s, d);
  139                 *hashval = hash;
  140                 *hashtype = M_HASHTYPE_RSS_IPV4;
  141                 return (0);
  142         }
  143 
  144         /* No configured available hashtypes! */
  145         RSS_DEBUG("no available hashtypes!\n");
  146         return (-1);
  147 }
  148 
  149 /*
  150  * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
  151  * IPv4 source/destination address, UDP or TCP source/destination ports
  152  * and the protocol type.
  153  *
  154  * The protocol code may wish to do a software hash of the given
  155  * tuple.  This depends upon the currently configured RSS hash types.
  156  *
  157  * It assumes the packet source/destination address
  158  * are in "outgoing" packet order (ie, destination is "far" address.)
  159  */
  160 uint32_t
  161 xps_proto_software_hash_v4(struct in_addr s, struct in_addr d,
  162     u_short sp, u_short dp, int proto, uint32_t *hashtype)
  163 {
  164         uint32_t hash;
  165 
  166         /*
  167          * Next, choose the hash type depending upon the protocol
  168          * identifier.
  169          */
  170         if ((proto == IPPROTO_TCP) &&
  171             (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
  172                 hash = rss_hash_ip4_4tuple(d, dp, s, sp);
  173                 *hashtype = M_HASHTYPE_RSS_TCP_IPV4;
  174                 return (hash);
  175         } else if ((proto == IPPROTO_UDP) &&
  176             (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
  177                 hash = rss_hash_ip4_4tuple(d, dp, s, sp);
  178                 *hashtype = M_HASHTYPE_RSS_UDP_IPV4;
  179                 return (hash);
  180         } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
  181                 /* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
  182                 hash = rss_hash_ip4_2tuple(d, s);
  183                 *hashtype = M_HASHTYPE_RSS_IPV4;
  184                 return (hash);
  185         }
  186 
  187         *hashtype = M_HASHTYPE_NONE;
  188         return (0);
  189 }
  190 
  191 /*
  192  * Do a software calculation of the RSS for the given mbuf.
  193  *
  194  * This is typically used by the input path to recalculate the RSS after
  195  * some form of packet processing (eg de-capsulation, IP fragment reassembly.)
  196  *
  197  * dir is the packet direction - RSS_HASH_PKT_INGRESS for incoming and
  198  * RSS_HASH_PKT_EGRESS for outgoing.
  199  *
  200  * Returns 0 if a hash was done, -1 if no hash was done, +1 if
  201  * the mbuf already had a valid RSS flowid.
  202  *
  203  * This function doesn't modify the mbuf.  It's up to the caller to
  204  * assign flowid/flowtype as appropriate.
  205  */
  206 int
  207 rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
  208     uint32_t *hashtype)
  209 {
  210         const struct ip *ip;
  211         const struct tcphdr *th;
  212         const struct udphdr *uh;
  213         uint32_t flowtype;
  214         uint8_t proto;
  215         int iphlen;
  216         int is_frag = 0;
  217 
  218         /*
  219          * XXX For now this only handles hashing on incoming mbufs.
  220          */
  221         if (dir != RSS_HASH_PKT_INGRESS) {
  222                 RSS_DEBUG("called on EGRESS packet!\n");
  223                 return (-1);
  224         }
  225 
  226         /*
  227          * First, validate that the mbuf we have is long enough
  228          * to have an IPv4 header in it.
  229          */
  230         if (m->m_pkthdr.len < (sizeof(struct ip))) {
  231                 RSS_DEBUG("short mbuf pkthdr\n");
  232                 return (-1);
  233         }
  234         if (m->m_len < (sizeof(struct ip))) {
  235                 RSS_DEBUG("short mbuf len\n");
  236                 return (-1);
  237         }
  238 
  239         /* Ok, let's dereference that */
  240         ip = mtod(m, struct ip *);
  241         proto = ip->ip_p;
  242         iphlen = ip->ip_hl << 2;
  243 
  244         /*
  245          * If this is a fragment then it shouldn't be four-tuple
  246          * hashed just yet.  Once it's reassembled into a full
  247          * frame it should be re-hashed.
  248          */
  249         if (ip->ip_off & htons(IP_MF | IP_OFFMASK))
  250                 is_frag = 1;
  251 
  252         /*
  253          * If the mbuf flowid/flowtype matches the packet type,
  254          * and we don't support the 4-tuple version of the given protocol,
  255          * then signal to the owner that it can trust the flowid/flowtype
  256          * details.
  257          *
  258          * This is a little picky - eg, if TCPv4 / UDPv4 hashing
  259          * is supported but we got a TCP/UDP frame only 2-tuple hashed,
  260          * then we shouldn't just "trust" the 2-tuple hash.  We need
  261          * a 4-tuple hash.
  262          */
  263         flowtype = M_HASHTYPE_GET(m);
  264 
  265         if (flowtype != M_HASHTYPE_NONE) {
  266                 switch (proto) {
  267                 case IPPROTO_UDP:
  268                         if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
  269                             (flowtype == M_HASHTYPE_RSS_UDP_IPV4) &&
  270                             (is_frag == 0)) {
  271                                 return (1);
  272                         }
  273                         /*
  274                          * Only allow 2-tuple for UDP frames if we don't also
  275                          * support 4-tuple for UDP.
  276                          */
  277                         if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
  278                             ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) &&
  279                             flowtype == M_HASHTYPE_RSS_IPV4) {
  280                                 return (1);
  281                         }
  282                         break;
  283                 case IPPROTO_TCP:
  284                         if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
  285                             (flowtype == M_HASHTYPE_RSS_TCP_IPV4) &&
  286                             (is_frag == 0)) {
  287                                 return (1);
  288                         }
  289                         /*
  290                          * Only allow 2-tuple for TCP frames if we don't also
  291                          * support 2-tuple for TCP.
  292                          */
  293                         if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
  294                             ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) &&
  295                             flowtype == M_HASHTYPE_RSS_IPV4) {
  296                                 return (1);
  297                         }
  298                         break;
  299                 default:
  300                         if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
  301                             flowtype == M_HASHTYPE_RSS_IPV4) {
  302                                 return (1);
  303                         }
  304                         break;
  305                 }
  306         }
  307 
  308         /*
  309          * Decode enough information to make a hash decision.
  310          *
  311          * XXX TODO: does the hardware hash on 4-tuple if IP
  312          *    options are present?
  313          */
  314         if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
  315             (proto == IPPROTO_TCP) &&
  316             (is_frag == 0)) {
  317                 if (m->m_len < iphlen + sizeof(struct tcphdr)) {
  318                         RSS_DEBUG("short TCP frame?\n");
  319                         return (-1);
  320                 }
  321                 th = (const struct tcphdr *)((c_caddr_t)ip + iphlen);
  322                 return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
  323                     th->th_sport,
  324                     th->th_dport,
  325                     proto,
  326                     hashval,
  327                     hashtype);
  328         } else if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
  329             (proto == IPPROTO_UDP) &&
  330             (is_frag == 0)) {
  331                 uh = (const struct udphdr *)((c_caddr_t)ip + iphlen);
  332                 if (m->m_len < iphlen + sizeof(struct udphdr)) {
  333                         RSS_DEBUG("short UDP frame?\n");
  334                         return (-1);
  335                 }
  336                 return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
  337                     uh->uh_sport,
  338                     uh->uh_dport,
  339                     proto,
  340                     hashval,
  341                     hashtype);
  342         } else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
  343                 /* Default to 2-tuple hash */
  344                 return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
  345                     0,  /* source port */
  346                     0,  /* destination port */
  347                     0,  /* IPPROTO_IP */
  348                     hashval,
  349                     hashtype);
  350         } else {
  351                 RSS_DEBUG("no available hashtypes!\n");
  352                 return (-1);
  353         }
  354 }
  355 
  356 /*
  357  * Similar to rss_m2cpuid, but designed to be used by the IP NETISR
  358  * on incoming frames.
  359  *
  360  * If an existing RSS hash exists and it matches what the configured
  361  * hashing is, then use it.
  362  *
  363  * If there's an existing RSS hash but the desired hash is different,
  364  * or if there's no useful RSS hash, then calculate it via
  365  * the software path.
  366  *
  367  * XXX TODO: definitely want statistics here!
  368  */
  369 struct mbuf *
  370 rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source, u_int *cpuid)
  371 {
  372         uint32_t hash_val, hash_type;
  373         int ret;
  374 
  375         M_ASSERTPKTHDR(m);
  376 
  377         ret = rss_mbuf_software_hash_v4(m, RSS_HASH_PKT_INGRESS,
  378             &hash_val, &hash_type);
  379         if (ret > 0) {
  380                 /* mbuf has a valid hash already; don't need to modify it */
  381                 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
  382         } else if (ret == 0) {
  383                 /* hash was done; update */
  384                 m->m_pkthdr.flowid = hash_val;
  385                 M_HASHTYPE_SET(m, hash_type);
  386                 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
  387         } else { /* ret < 0 */
  388                 /* no hash was done */
  389                 *cpuid = NETISR_CPUID_NONE;
  390         }
  391         return (m);
  392 }

Cache object: 83d57fd87931e83b81ba5890cd8a9f4d


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.