1 /**************************************************************************
2
3 Copyright (c) 2007, Chelsio Inc.
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Neither the name of the Chelsio Corporation nor the names of its
13 contributors may be used to endorse or promote products derived from
14 this software without specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 POSSIBILITY OF SUCH DAMAGE.
27
28 ***************************************************************************/
29
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD: releng/6.4/sys/dev/cxgb/cxgb_lro.c 171979 2007-08-25 23:55:21Z kmacy $");
32
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/module.h>
38 #include <sys/bus.h>
39 #include <sys/conf.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/bus_dma.h>
43 #include <sys/rman.h>
44 #include <sys/queue.h>
45 #include <sys/taskqueue.h>
46
47 #include <netinet/in_systm.h>
48 #include <netinet/in.h>
49 #include <netinet/ip.h>
50 #include <netinet/tcp.h>
51
52
53 #ifdef CONFIG_DEFINED
54 #include <cxgb_include.h>
55 #else
56 #include <dev/cxgb/cxgb_include.h>
57 #endif
58
59 #include <machine/in_cksum.h>
60
61
62 #ifndef M_LRO
63 #define M_LRO 0x0200
64 #endif
65
66 #ifdef DEBUG
67 #define MBUF_HEADER_CHECK(m) do { \
68 if ((m->m_len == 0) || (m->m_pkthdr.len == 0) \
69 || ((m->m_flags & M_PKTHDR) == 0)) \
70 panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \
71 m->m_len, m->m_pkthdr.len, m->m_flags); \
72 if ((m->m_flags & M_PKTHDR) == 0) \
73 panic("first mbuf is not packet header - flags=0x%x\n", \
74 m->m_flags); \
75 if ((m->m_len < ETHER_HDR_LEN) || (m->m_pkthdr.len < ETHER_HDR_LEN)) \
76 panic("packet too small len=%d pktlen=%d\n", \
77 m->m_len, m->m_pkthdr.len);\
78 } while (0)
79 #else
80 #define MBUF_HEADER_CHECK(m)
81 #endif
82
83 #define IPH_OFFSET (2 + sizeof (struct cpl_rx_pkt) + ETHER_HDR_LEN)
84 #define LRO_SESSION_IDX_HINT_HASH(hash) (hash & (MAX_LRO_SES - 1))
85 #define LRO_IDX_INC(idx) idx = (idx + 1) & (MAX_LRO_SES - 1)
86
87 static __inline int
88 lro_match(struct mbuf *m, struct ip *ih, struct tcphdr *th)
89 {
90 struct ip *sih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
91 struct tcphdr *sth = (struct tcphdr *) (sih + 1);
92
93 return (th->th_sport == sth->th_sport &&
94 th->th_dport == sth->th_dport &&
95 ih->ip_src.s_addr == sih->ip_src.s_addr &&
96 ih->ip_dst.s_addr == sih->ip_dst.s_addr);
97 }
98
99 static __inline struct t3_lro_session *
100 lro_lookup(struct lro_state *l, int idx, struct ip *ih, struct tcphdr *th)
101 {
102 struct t3_lro_session *s = NULL;
103 int active = l->nactive;
104
105 while (active) {
106 s = &l->sess[idx];
107 if (s->head) {
108 if (lro_match(s->head, ih, th))
109 break;
110 active--;
111 }
112 LRO_IDX_INC(idx);
113 }
114
115 return (s);
116 }
117
118 static __inline int
119 can_lro_packet(struct cpl_rx_pkt *cpl, unsigned int rss_hi)
120 {
121 struct ether_header *eh = (struct ether_header *)(cpl + 1);
122 struct ip *ih = (struct ip *)(eh + 1);
123
124 /*
125 * XXX VLAN support?
126 */
127 if (__predict_false(G_HASHTYPE(ntohl(rss_hi)) != RSS_HASH_4_TUPLE ||
128 (*((uint8_t *)cpl + 1) & 0x90) != 0x10 ||
129 cpl->csum != 0xffff || eh->ether_type != ntohs(ETHERTYPE_IP) ||
130 ih->ip_hl != (sizeof (*ih) >> 2))) {
131 return 0;
132 }
133
134 return 1;
135 }
136
137 static int
138 can_lro_tcpsegment(struct tcphdr *th)
139 {
140 int olen = (th->th_off << 2) - sizeof (*th);
141 u8 control_bits = *((u8 *)th + 13);
142
143 if (__predict_false((control_bits & 0xB7) != 0x10))
144 goto no_lro;
145
146 if (olen) {
147 uint32_t *ptr = (u32 *)(th + 1);
148 if (__predict_false(olen != TCPOLEN_TSTAMP_APPA ||
149 *ptr != ntohl((TCPOPT_NOP << 24) |
150 (TCPOPT_NOP << 16) |
151 (TCPOPT_TIMESTAMP << 8) |
152 TCPOLEN_TIMESTAMP)))
153 goto no_lro;
154 }
155
156 return 1;
157
158 no_lro:
159 return 0;
160 }
161
162 static __inline void
163 lro_new_session_init(struct t3_lro_session *s, struct mbuf *m)
164 {
165 struct ip *ih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
166 struct tcphdr *th = (struct tcphdr *) (ih + 1);
167 int ip_len = ntohs(ih->ip_len);
168
169 DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
170
171 s->head = m;
172
173 MBUF_HEADER_CHECK(m);
174 s->ip_len = ip_len;
175 s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2);
176
177 }
178
179 static void
180 lro_flush_session(struct sge_qset *qs, struct t3_lro_session *s, struct mbuf *m)
181 {
182 struct lro_state *l = &qs->lro;
183 struct mbuf *sm = s->head;
184 struct ip *ih = (struct ip *)(mtod(sm, uint8_t *) + IPH_OFFSET);
185
186
187 DPRINTF("%s(qs=%p, s=%p, ", __FUNCTION__,
188 qs, s);
189
190 if (m)
191 DPRINTF("m=%p)\n", m);
192 else
193 DPRINTF("m=NULL)\n");
194
195 ih->ip_len = htons(s->ip_len);
196 ih->ip_sum = 0;
197 ih->ip_sum = in_cksum_hdr(ih);
198
199 MBUF_HEADER_CHECK(sm);
200
201 sm->m_flags |= M_LRO;
202 t3_rx_eth(qs->port->adapter, &qs->rspq, sm, 2);
203
204 if (m) {
205 s->head = m;
206 lro_new_session_init(s, m);
207 } else {
208 s->head = NULL;
209 l->nactive--;
210 }
211
212 qs->port_stats[SGE_PSTATS_LRO_FLUSHED]++;
213 }
214
215 static __inline struct t3_lro_session *
216 lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash)
217 {
218 struct lro_state *l = &qs->lro;
219 int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash);
220 struct t3_lro_session *s = &l->sess[idx];
221
222 DPRINTF("%s(qs=%p, m=%p, rss_hash=0x%x)\n", __FUNCTION__,
223 qs, m, rss_hash);
224
225 if (__predict_true(!s->head))
226 goto done;
227
228 if (l->nactive > MAX_LRO_SES)
229 panic("MAX_LRO_PER_QSET exceeded");
230
231 if (l->nactive == MAX_LRO_SES) {
232 lro_flush_session(qs, s, m);
233 qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++;
234 return s;
235 }
236
237 while (1) {
238 LRO_IDX_INC(idx);
239 s = &l->sess[idx];
240 if (!s->head)
241 break;
242 }
243 done:
244 lro_new_session_init(s, m);
245 l->nactive++;
246
247 return s;
248 }
249
250 static __inline int
251 lro_update_session(struct t3_lro_session *s, struct mbuf *m)
252 {
253 struct mbuf *sm = s->head;
254 struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(sm, uint8_t *) + 2);
255 struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + 2);
256 struct ip *nih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
257 struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1);
258 uint32_t seq = ntohl(nth->th_seq);
259 int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth);
260
261
262 DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
263 if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) {
264 return -1;
265 }
266 if (__predict_false(seq != s->seq)) {
267 DPRINTF("sequence mismatch\n");
268 return -1;
269 }
270
271 MBUF_HEADER_CHECK(sm);
272 th = (struct tcphdr *)(mtod(sm, uint8_t *) + IPH_OFFSET + sizeof (struct ip));
273
274 if (olen) {
275 uint32_t *ptr = (uint32_t *)(th + 1);
276 uint32_t *nptr = (uint32_t *)(nth + 1);
277
278 if (__predict_false(ntohl(*(ptr + 1)) > ntohl(*(nptr + 1)) ||
279 !*(nptr + 2))) {
280 return -1;
281 }
282 *(ptr + 1) = *(nptr + 1);
283 *(ptr + 2) = *(nptr + 2);
284 }
285 th->th_ack = nth->th_ack;
286 th->th_win = nth->th_win;
287
288 tcpiphlen = (nth->th_off << 2) + sizeof (*nih);
289 plen = ntohs(nih->ip_len) - tcpiphlen;
290 s->seq += plen;
291 s->ip_len += plen;
292 sm->m_pkthdr.len += plen;
293
294 /*
295 * XXX FIX ME
296 *
297 *
298 */
299
300 #if 0
301 /* XXX this I *do not* understand */
302 if (plen > skb_shinfo(s->skb)->gso_size)
303 skb_shinfo(s->skb)->gso_size = plen;
304 #endif
305 #if __FreeBSD_version > 700000
306 if (plen > sm->m_pkthdr.tso_segsz)
307 sm->m_pkthdr.tso_segsz = plen;
308 #endif
309 DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen));
310 m_adj(m, IPH_OFFSET + tcpiphlen);
311 #if 0
312 if (__predict_false(!skb_shinfo(s->skb)->frag_list))
313 skb_shinfo(s->skb)->frag_list = skb;
314
315 #endif
316
317 #if 0
318
319 /*
320 * XXX we really need to be able to
321 * support vectors of buffers in FreeBSD
322 */
323 int nr = skb_shinfo(s->skb)->nr_frags;
324 skb_shinfo(s->skb)->frags[nr].page = frag->page;
325 skb_shinfo(s->skb)->frags[nr].page_offset =
326 frag->page_offset + IPH_OFFSET + tcpiphlen;
327 skb_shinfo(s->skb)->frags[nr].size = plen;
328 skb_shinfo(s->skb)->nr_frags = ++nr;
329
330 #endif
331 return (0);
332 }
333
334 void
335 t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
336 int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro)
337 {
338 struct sge_qset *qs = rspq_to_qset(rq);
339 struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad);
340 struct ether_header *eh = (struct ether_header *)(cpl + 1);
341 struct ip *ih;
342 struct tcphdr *th;
343 struct t3_lro_session *s = NULL;
344
345 if (lro == 0)
346 goto no_lro;
347
348 if (!can_lro_packet(cpl, rss_csum))
349 goto no_lro;
350
351 ih = (struct ip *)(eh + 1);
352 th = (struct tcphdr *)(ih + 1);
353
354 s = lro_lookup(&qs->lro,
355 LRO_SESSION_IDX_HINT_HASH(rss_hash), ih, th);
356
357 if (__predict_false(!can_lro_tcpsegment(th))) {
358 goto no_lro;
359 } else if (__predict_false(!s)) {
360 s = lro_new_session(qs, m, rss_hash);
361 } else {
362 if (lro_update_session(s, m)) {
363 lro_flush_session(qs, s, m);
364 }
365 #ifdef notyet
366 if (__predict_false(s->head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
367 lro_flush_session(qs, s, NULL);
368 }
369 #endif
370 }
371
372 qs->port_stats[SGE_PSTATS_LRO_QUEUED]++;
373 return;
374 no_lro:
375 if (s)
376 lro_flush_session(qs, s, NULL);
377
378 if (m->m_len == 0 || m->m_pkthdr.len == 0 || (m->m_flags & M_PKTHDR) == 0)
379 DPRINTF("rx_eth_lro mbuf len=%d pktlen=%d flags=0x%x\n",
380 m->m_len, m->m_pkthdr.len, m->m_flags);
381
382 t3_rx_eth(adap, rq, m, ethpad);
383 }
384
385 void
386 t3_lro_flush(adapter_t *adap, struct sge_qset *qs, struct lro_state *state)
387 {
388 unsigned int idx = state->active_idx;
389
390 while (state->nactive) {
391 struct t3_lro_session *s = &state->sess[idx];
392
393 if (s->head)
394 lro_flush_session(qs, s, NULL);
395 LRO_IDX_INC(idx);
396 }
397 }
Cache object: 18e8dfc7ef65d160f7d8ee9e7e642e29
|