cxgb_lro.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /**************************************************************************
    2 
    3 Copyright (c) 2007, Chelsio Inc.
    4 All rights reserved.
    5 
    6 Redistribution and use in source and binary forms, with or without
    7 modification, are permitted provided that the following conditions are met:
    8 
    9  1. Redistributions of source code must retain the above copyright notice,
   10     this list of conditions and the following disclaimer.
   11 
   12 2. Neither the name of the Chelsio Corporation nor the names of its
   13     contributors may be used to endorse or promote products derived from
   14     this software without specific prior written permission.
   15 
   16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   26 POSSIBILITY OF SUCH DAMAGE.
   27 
   28 ***************************************************************************/
   29 
   30 #include <sys/cdefs.h>
   31 __FBSDID("$FreeBSD$");
   32 
   33 
   34 #include <sys/param.h>
   35 #include <sys/systm.h>
   36 #include <sys/kernel.h>
   37 #include <sys/module.h>
   38 #include <sys/bus.h>
   39 #include <sys/conf.h>
   40 #include <machine/bus.h>
   41 #include <machine/resource.h>
   42 #include <sys/bus_dma.h>
   43 #include <sys/rman.h>
   44 #include <sys/queue.h>
   45 #include <sys/taskqueue.h>
   46 
   47 #include <netinet/in_systm.h>
   48 #include <netinet/in.h>
   49 #include <netinet/ip.h>
   50 #include <netinet/tcp.h>
   51 
   52 
   53 #ifdef CONFIG_DEFINED
   54 #include <cxgb_include.h>
   55 #else
   56 #include <dev/cxgb/cxgb_include.h>
   57 #endif
   58 
   59 #include <machine/in_cksum.h>
   60 
   61 
   62 #ifndef M_LRO
   63 #define M_LRO    0x0200
   64 #endif
   65 
   66 #ifdef DEBUG
   67 #define MBUF_HEADER_CHECK(m) do { \
   68         if ((m->m_len == 0) || (m->m_pkthdr.len == 0)   \
   69             || ((m->m_flags & M_PKTHDR) == 0))                          \
   70                 panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \
   71                     m->m_len, m->m_pkthdr.len, m->m_flags); \
   72         if ((m->m_flags & M_PKTHDR) == 0)                               \
   73                 panic("first mbuf is not packet header - flags=0x%x\n", \
   74                     m->m_flags);  \
   75         if ((m->m_len < ETHER_HDR_LEN) || (m->m_pkthdr.len < ETHER_HDR_LEN)) \
   76                 panic("packet too small len=%d pktlen=%d\n", \
   77                     m->m_len, m->m_pkthdr.len);\
   78 } while (0)
   79 #else
   80 #define MBUF_HEADER_CHECK(m)
   81 #endif    
   82 
   83 #define IPH_OFFSET (2 + sizeof (struct cpl_rx_pkt) + ETHER_HDR_LEN)
   84 #define LRO_SESSION_IDX_HINT_HASH(hash) (hash & (MAX_LRO_SES - 1))
   85 #define LRO_IDX_INC(idx) idx = (idx + 1) & (MAX_LRO_SES - 1)
   86 
   87 static __inline int
   88 lro_match(struct mbuf *m, struct ip *ih, struct tcphdr *th)
   89 {
   90         struct ip *sih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
   91         struct tcphdr *sth = (struct tcphdr *) (sih + 1);
   92 
   93         return (th->th_sport == sth->th_sport &&
   94             th->th_dport == sth->th_dport &&
   95             ih->ip_src.s_addr == sih->ip_src.s_addr &&
   96             ih->ip_dst.s_addr == sih->ip_dst.s_addr);
   97 }
   98 
   99 static __inline struct t3_lro_session *
  100 lro_lookup(struct lro_state *l, int idx, struct ip *ih, struct tcphdr *th)
  101 {
  102         struct t3_lro_session *s = NULL;
  103         int active = l->nactive;
  104 
  105         while (active) {
  106                 s = &l->sess[idx];
  107                 if (s->head) {
  108                         if (lro_match(s->head, ih, th)) 
  109                                 break;
  110                         active--;
  111                 }
  112                 LRO_IDX_INC(idx);
  113         }
  114 
  115         return (s);
  116 }
  117 
  118 static __inline int
  119 can_lro_packet(struct cpl_rx_pkt *cpl, unsigned int rss_hi)
  120 {
  121         struct ether_header *eh = (struct ether_header *)(cpl + 1);
  122         struct ip *ih = (struct ip *)(eh + 1);
  123 
  124         /* 
  125          * XXX VLAN support?
  126          */
  127         if (__predict_false(G_HASHTYPE(ntohl(rss_hi)) != RSS_HASH_4_TUPLE ||
  128                      (*((uint8_t *)cpl + 1) & 0x90) != 0x10 || 
  129                      cpl->csum != 0xffff || eh->ether_type != ntohs(ETHERTYPE_IP) ||
  130                      ih->ip_hl != (sizeof (*ih) >> 2))) {
  131                 return 0;
  132         }
  133 
  134         return 1;
  135 }
  136 
  137 static int
  138 can_lro_tcpsegment(struct tcphdr *th)
  139 {
  140         int olen = (th->th_off << 2) - sizeof (*th);
  141         u8 control_bits = *((u8 *)th + 13);
  142 
  143         if (__predict_false((control_bits & 0xB7) != 0x10))
  144                 goto no_lro;
  145 
  146         if (olen) {
  147                 uint32_t *ptr = (u32 *)(th + 1);
  148                 if (__predict_false(olen != TCPOLEN_TSTAMP_APPA || 
  149                              *ptr != ntohl((TCPOPT_NOP << 24) | 
  150                                            (TCPOPT_NOP << 16) | 
  151                                            (TCPOPT_TIMESTAMP << 8) | 
  152                                             TCPOLEN_TIMESTAMP)))
  153                         goto no_lro;
  154         }
  155 
  156         return 1;
  157 
  158  no_lro:
  159         return 0;       
  160 }
  161 
  162 static __inline void
  163 lro_new_session_init(struct t3_lro_session *s, struct mbuf *m)
  164 {
  165         struct ip *ih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
  166         struct tcphdr *th = (struct tcphdr *) (ih + 1);
  167         int ip_len = ntohs(ih->ip_len);
  168 
  169         DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
  170         
  171         s->head = m;
  172         
  173         MBUF_HEADER_CHECK(m);
  174         s->ip_len = ip_len;
  175         s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2);
  176 
  177 } 
  178 
  179 static void
  180 lro_flush_session(struct sge_qset *qs, struct t3_lro_session *s, struct mbuf *m)
  181 {
  182         struct lro_state *l = &qs->lro;
  183         struct mbuf *sm = s->head;
  184         struct ip *ih = (struct ip *)(mtod(sm, uint8_t *) + IPH_OFFSET);
  185 
  186         
  187         DPRINTF("%s(qs=%p, s=%p, ", __FUNCTION__,
  188             qs, s);
  189 
  190         if (m)
  191                 DPRINTF("m=%p)\n", m);
  192         else
  193                 DPRINTF("m=NULL)\n");
  194         
  195         ih->ip_len = htons(s->ip_len);
  196         ih->ip_sum = 0;
  197         ih->ip_sum = in_cksum_hdr(ih);
  198 
  199         MBUF_HEADER_CHECK(sm);
  200         
  201         sm->m_flags |= M_LRO;
  202         t3_rx_eth(qs->port->adapter, &qs->rspq, sm, 2);
  203         
  204         if (m) {
  205                 s->head = m;
  206                 lro_new_session_init(s, m);
  207         } else {
  208                 s->head = NULL;
  209                 l->nactive--;
  210         }
  211 
  212         qs->port_stats[SGE_PSTATS_LRO_FLUSHED]++;
  213 }
  214 
  215 static __inline struct t3_lro_session *
  216 lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash)
  217 {
  218         struct lro_state *l = &qs->lro;
  219         int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash); 
  220         struct t3_lro_session *s = &l->sess[idx];
  221 
  222         DPRINTF("%s(qs=%p,  m=%p, rss_hash=0x%x)\n", __FUNCTION__,
  223             qs, m, rss_hash);
  224         
  225         if (__predict_true(!s->head))
  226                 goto done;
  227 
  228         if (l->nactive > MAX_LRO_SES)
  229                 panic("MAX_LRO_PER_QSET exceeded");
  230         
  231         if (l->nactive == MAX_LRO_SES) {
  232                 lro_flush_session(qs, s, m);
  233                 qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++;
  234                 return s;
  235         }
  236 
  237         while (1) {
  238                 LRO_IDX_INC(idx);
  239                 s = &l->sess[idx];
  240                 if (!s->head)
  241                         break;
  242         }
  243 done:
  244         lro_new_session_init(s, m);
  245         l->nactive++;
  246 
  247         return s;
  248 }
  249 
  250 static __inline int
  251 lro_update_session(struct t3_lro_session *s, struct mbuf *m)
  252 {
  253         struct mbuf *sm = s->head;
  254         struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(sm, uint8_t *) + 2);
  255         struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + 2);
  256         struct ip *nih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
  257         struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1);
  258         uint32_t seq = ntohl(nth->th_seq);
  259         int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth);
  260         
  261         
  262         DPRINTF("%s(s=%p,  m=%p)\n", __FUNCTION__, s, m);       
  263         if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) {
  264                 return -1;
  265         }
  266         if (__predict_false(seq != s->seq)) {
  267                 DPRINTF("sequence mismatch\n");
  268                 return -1;
  269         }
  270 
  271         MBUF_HEADER_CHECK(sm);
  272         th = (struct tcphdr *)(mtod(sm, uint8_t *) + IPH_OFFSET + sizeof (struct ip));
  273 
  274         if (olen) {
  275                 uint32_t *ptr = (uint32_t *)(th + 1);
  276                 uint32_t *nptr = (uint32_t *)(nth + 1);
  277 
  278                 if (__predict_false(ntohl(*(ptr + 1)) > ntohl(*(nptr + 1)) || 
  279                              !*(nptr + 2))) {
  280                         return -1;
  281                 }
  282                 *(ptr + 1) = *(nptr + 1);
  283                 *(ptr + 2) = *(nptr + 2);
  284         }
  285         th->th_ack = nth->th_ack;
  286         th->th_win = nth->th_win;
  287 
  288         tcpiphlen = (nth->th_off << 2) + sizeof (*nih);
  289         plen = ntohs(nih->ip_len) - tcpiphlen;
  290         s->seq += plen;
  291         s->ip_len += plen;
  292         sm->m_pkthdr.len += plen;
  293 
  294         /*
  295          * XXX FIX ME
  296          *
  297          *
  298          */
  299 
  300 #if 0
  301         /* XXX this I *do not* understand */
  302         if (plen > skb_shinfo(s->skb)->gso_size)
  303                 skb_shinfo(s->skb)->gso_size = plen;
  304 #endif
  305 #if __FreeBSD_version > 700000  
  306         if (plen > sm->m_pkthdr.tso_segsz)
  307                 sm->m_pkthdr.tso_segsz = plen;
  308 #endif
  309         DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen));
  310         m_adj(m, IPH_OFFSET + tcpiphlen);
  311 #if 0 
  312         if (__predict_false(!skb_shinfo(s->skb)->frag_list))
  313                 skb_shinfo(s->skb)->frag_list = skb;
  314 
  315 #endif
  316 
  317 #if 0
  318         
  319         /* 
  320          * XXX we really need to be able to
  321          * support vectors of buffers in FreeBSD
  322          */
  323         int nr = skb_shinfo(s->skb)->nr_frags;
  324         skb_shinfo(s->skb)->frags[nr].page = frag->page;
  325         skb_shinfo(s->skb)->frags[nr].page_offset = 
  326             frag->page_offset + IPH_OFFSET + tcpiphlen;
  327         skb_shinfo(s->skb)->frags[nr].size = plen; 
  328         skb_shinfo(s->skb)->nr_frags = ++nr;
  329                 
  330 #endif
  331         return (0);
  332 }
  333 
  334 void
  335 t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
  336     int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro)
  337 {
  338         struct sge_qset *qs = rspq_to_qset(rq);
  339         struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad);
  340         struct ether_header *eh = (struct ether_header *)(cpl + 1);
  341         struct ip *ih;
  342         struct tcphdr *th; 
  343         struct t3_lro_session *s = NULL;
  344         
  345         if (lro == 0)
  346                 goto no_lro;
  347 
  348         if (!can_lro_packet(cpl, rss_csum))
  349                 goto no_lro;
  350         
  351         ih = (struct ip *)(eh + 1);
  352         th = (struct tcphdr *)(ih + 1);
  353         
  354         s = lro_lookup(&qs->lro,
  355             LRO_SESSION_IDX_HINT_HASH(rss_hash), ih, th);
  356         
  357         if (__predict_false(!can_lro_tcpsegment(th))) {
  358                 goto no_lro;
  359         } else if (__predict_false(!s)) {
  360                 s = lro_new_session(qs, m, rss_hash);
  361         } else {
  362                 if (lro_update_session(s, m)) {
  363                         lro_flush_session(qs, s, m);
  364                 }
  365 #ifdef notyet           
  366                 if (__predict_false(s->head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
  367                         lro_flush_session(qs, s, NULL);
  368                 }
  369 #endif          
  370         }
  371 
  372         qs->port_stats[SGE_PSTATS_LRO_QUEUED]++;
  373         return;
  374 no_lro:
  375         if (s)
  376                 lro_flush_session(qs, s, NULL);
  377         
  378         if (m->m_len == 0 || m->m_pkthdr.len == 0 || (m->m_flags & M_PKTHDR) == 0)
  379                 DPRINTF("rx_eth_lro mbuf len=%d pktlen=%d flags=0x%x\n",
  380                     m->m_len, m->m_pkthdr.len, m->m_flags);
  381 
  382         t3_rx_eth(adap, rq, m, ethpad);
  383 }
  384 
  385 void
  386 t3_lro_flush(adapter_t *adap, struct sge_qset *qs, struct lro_state *state)
  387 {
  388         unsigned int idx = state->active_idx;
  389 
  390         while (state->nactive) {
  391                 struct t3_lro_session *s = &state->sess[idx];
  392                 
  393                 if (s->head) 
  394                         lro_flush_session(qs, s, NULL);
  395                 LRO_IDX_INC(idx);
  396         }
  397 }
Cache object: ab50e5a1e8cb523ec800a8af596f25fd
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/dev/cxgb/cxgb_lro.c

FreeBSD/Linux Kernel Cross Reference
sys/dev/cxgb/cxgb_lro.c
