The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/netinet/tcp_lro.h

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
    3  *
    4  * Copyright (c) 2006, Myricom Inc.
    5  * Copyright (c) 2008, Intel Corporation.
    6  * Copyright (c) 2016-2021 Mellanox Technologies.
    7  * All rights reserved.
    8  *
    9  * Redistribution and use in source and binary forms, with or without
   10  * modification, are permitted provided that the following conditions
   11  * are met:
   12  * 1. Redistributions of source code must retain the above copyright
   13  *    notice, this list of conditions and the following disclaimer.
   14  * 2. Redistributions in binary form must reproduce the above copyright
   15  *    notice, this list of conditions and the following disclaimer in the
   16  *    documentation and/or other materials provided with the distribution.
   17  *
   18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   28  * SUCH DAMAGE.
   29  *
   30  * $FreeBSD$
   31  */
   32 
   33 #ifndef _TCP_LRO_H_
   34 #define _TCP_LRO_H_
   35 
   36 #include <sys/time.h>
   37 #include <sys/param.h>
   38 
   39 #include <netinet/in.h>
   40 
   41 #ifndef TCP_LRO_ENTRIES
   42 /* Define default number of LRO entries per RX queue */
   43 #define TCP_LRO_ENTRIES 8
   44 #endif
   45 
   46 /*
   47  * Flags for ACK entry for compression
   48  * the bottom 12 bits has the th_x2|th_flags.
   49  * LRO itself adds only the TSTMP flags
   50  * to indicate if either of the types
   51  * of timestamps are filled and the
   52  * HAS_TSTMP option to indicate if the
   53  * TCP timestamp option is valid.
   54  *
   55  * The other 1 flag bits are for processing
   56  * by a stack.
   57  *
   58  */
   59 #define TSTMP_LRO               0x1000
   60 #define TSTMP_HDWR              0x2000
   61 #define HAS_TSTMP               0x4000
   62 /*
   63  * Default number of interrupts on the same cpu in a row
   64  * that will cause us to declare a "affinity cpu".
   65  */
   66 #define TCP_LRO_CPU_DECLARATION_THRESH 50
   67 
   68 struct inpcb;
   69 
   70 /* Precompute the LRO_RAW_ADDRESS_MAX value: */
   71 #define LRO_RAW_ADDRESS_MAX \
   72         howmany(12 + 2 * sizeof(struct in6_addr), sizeof(u_long))
   73 
   74 union lro_address {
   75         u_long raw[LRO_RAW_ADDRESS_MAX];
   76         struct {
   77                 uint8_t lro_type;       /* internal */
   78 #define LRO_TYPE_NONE     0
   79 #define LRO_TYPE_IPV4_TCP 1
   80 #define LRO_TYPE_IPV6_TCP 2
   81 #define LRO_TYPE_IPV4_UDP 3
   82 #define LRO_TYPE_IPV6_UDP 4
   83                 uint8_t lro_flags;
   84 #define LRO_FLAG_DECRYPTED 1
   85                 uint16_t vlan_id;       /* VLAN identifier */
   86                 uint16_t s_port;        /* source TCP/UDP port */
   87                 uint16_t d_port;        /* destination TCP/UDP port */
   88                 uint32_t vxlan_vni;     /* VXLAN virtual network identifier */
   89                 union {
   90                         struct in_addr v4;
   91                         struct in6_addr v6;
   92                 } s_addr;       /* source IPv4/IPv6 address */
   93                 union {
   94                         struct in_addr v4;
   95                         struct in6_addr v6;
   96                 } d_addr;       /* destination IPv4/IPv6 address */
   97         };
   98 };
   99 
  100 _Static_assert(sizeof(union lro_address) == sizeof(u_long) * LRO_RAW_ADDRESS_MAX,
  101     "The raw field in the lro_address union does not cover the whole structure.");
  102 
  103 /* Optimize address comparison by comparing one unsigned long at a time: */
  104 
  105 static inline bool
  106 lro_address_compare(const union lro_address *pa, const union lro_address *pb)
  107 {
  108         if (pa->lro_type == LRO_TYPE_NONE && pb->lro_type == LRO_TYPE_NONE) {
  109                 return (true);
  110         } else for (unsigned i = 0; i < LRO_RAW_ADDRESS_MAX; i++) {
  111                 if (pa->raw[i] != pb->raw[i])
  112                         return (false);
  113         }
  114         return (true);
  115 }
  116 
  117 struct lro_parser {
  118         union lro_address data;
  119         union {
  120                 uint8_t *l3;
  121                 struct ip *ip4;
  122                 struct ip6_hdr *ip6;
  123         };
  124         union {
  125                 uint8_t *l4;
  126                 struct tcphdr *tcp;
  127                 struct udphdr *udp;
  128         };
  129         uint16_t total_hdr_len;
  130 };
  131 
  132 /* This structure is zeroed frequently, try to keep it small. */
  133 struct lro_entry {
  134         LIST_ENTRY(lro_entry)   next;
  135         LIST_ENTRY(lro_entry)   hash_next;
  136         struct mbuf             *m_head;
  137         struct mbuf             *m_tail;
  138         struct mbuf             *m_last_mbuf;
  139         struct lro_parser       outer;
  140         struct lro_parser       inner;
  141         uint32_t                next_seq;       /* tcp_seq */
  142         uint32_t                ack_seq;        /* tcp_seq */
  143         uint32_t                tsval;
  144         uint32_t                tsecr;
  145         uint16_t                compressed;
  146         uint16_t                uncompressed;
  147         uint16_t                window;
  148         uint16_t                flags : 12,     /* 12 TCP header bits */
  149                                 timestamp : 1,
  150                                 needs_merge : 1,
  151                                 reserved : 2;   /* unused */
  152         struct bintime          alloc_time;     /* time when entry was allocated */
  153 };
  154 
  155 LIST_HEAD(lro_head, lro_entry);
  156 
  157 struct lro_mbuf_sort {
  158         uint64_t seq;
  159         struct mbuf *mb;
  160 };
  161 
  162 /* NB: This is part of driver structs. */
  163 struct lro_ctrl {
  164         struct ifnet    *ifp;
  165         struct lro_mbuf_sort *lro_mbuf_data;
  166         struct bintime  lro_last_queue_time;    /* last time data was queued */
  167         uint64_t        lro_queued;
  168         uint64_t        lro_flushed;
  169         uint64_t        lro_bad_csum;
  170         unsigned        lro_cnt;
  171         unsigned        lro_mbuf_count;
  172         unsigned        lro_mbuf_max;
  173         unsigned short  lro_ackcnt_lim;         /* max # of aggregated ACKs */
  174         unsigned short  lro_cpu;                /* Guess at the cpu we have affinity too */
  175         unsigned        lro_length_lim;         /* max len of aggregated data */
  176         u_long          lro_hashsz;
  177         uint32_t        lro_last_cpu;
  178         uint32_t        lro_cnt_of_same_cpu;
  179         struct lro_head *lro_hash;
  180         struct lro_head lro_active;
  181         struct lro_head lro_free;
  182         uint8_t         lro_cpu_is_set;         /* Flag to say its ok to set the CPU on the inp */
  183 };
  184 
  185 struct tcp_ackent {
  186         uint64_t timestamp;     /* hardware or sofware timestamp, valid if TSTMP_LRO or TSTMP_HDRW set */
  187         uint32_t seq;           /* th_seq value */
  188         uint32_t ack;           /* th_ack value */
  189         uint32_t ts_value;      /* If ts option value, valid if HAS_TSTMP is set */
  190         uint32_t ts_echo;       /* If ts option echo, valid if HAS_TSTMP is set */
  191         uint16_t win;           /* TCP window */
  192         uint16_t flags;         /* Flags to say if TS is present and type of timestamp and th_flags */
  193         uint8_t  codepoint;     /* IP level codepoint including ECN bits */
  194         uint8_t  ack_val_set;   /* Classification of ack used by the stack */
  195         uint8_t  pad[2];        /* To 32 byte boundary */
  196 };
  197 
  198 /* We use two M_PROTO on the mbuf */
  199 #define M_ACKCMP        M_PROTO4   /* Indicates LRO is sending in a  Ack-compression mbuf */
  200 #define M_LRO_EHDRSTRP  M_PROTO6   /* Indicates that LRO has stripped the etherenet header */
  201 
  202 #define TCP_LRO_LENGTH_MAX      (65535 - 255)   /* safe value with room for outer headers */
  203 #define TCP_LRO_ACKCNT_MAX      65535           /* unlimited */
  204 
  205 int tcp_lro_init(struct lro_ctrl *);
  206 int tcp_lro_init_args(struct lro_ctrl *, struct ifnet *, unsigned, unsigned);
  207 void tcp_lro_free(struct lro_ctrl *);
  208 void tcp_lro_flush_inactive(struct lro_ctrl *, const struct timeval *);
  209 void tcp_lro_flush(struct lro_ctrl *, struct lro_entry *);
  210 void tcp_lro_flush_all(struct lro_ctrl *);
  211 int tcp_lro_rx(struct lro_ctrl *, struct mbuf *, uint32_t);
  212 void tcp_lro_queue_mbuf(struct lro_ctrl *, struct mbuf *);
  213 void tcp_lro_reg_mbufq(void);
  214 void tcp_lro_dereg_mbufq(void);
  215 
  216 #define TCP_LRO_NO_ENTRIES      -2
  217 #define TCP_LRO_CANNOT          -1
  218 #define TCP_LRO_NOT_SUPPORTED   1
  219 
  220 #endif /* _TCP_LRO_H_ */

Cache object: c5824775dbe1aaa6fac741e8e6c17fe8


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.