1 /* $NetBSD: in_offload.c,v 1.14 2020/03/27 16:34:58 jdolecek Exp $ */
2
3 /*
4 * Copyright (c)2005, 2006 YAMAMOTO Takashi,
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: in_offload.c,v 1.14 2020/03/27 16:34:58 jdolecek Exp $");
31
32 #include <sys/param.h>
33 #include <sys/mbuf.h>
34
35 #include <net/if.h>
36
37 #include <netinet/in.h>
38 #include <netinet/in_systm.h>
39 #include <netinet/ip.h>
40 #include <netinet/ip_var.h>
41 #include <netinet/tcp.h>
42 #include <netinet/in_offload.h>
43
44 /*
45 * Handle M_CSUM_TSOv4 in software. Split the TCP payload in chunks of
46 * size MSS, and return mbuf chain consists of them.
47 */
48 struct mbuf *
49 tcp4_segment(struct mbuf *m, int off)
50 {
51 int mss;
52 int iphlen, thlen;
53 int hlen, len;
54 struct ip *ip;
55 struct tcphdr *th;
56 uint16_t ipid, phsum;
57 uint32_t tcpseq;
58 struct mbuf *hdr = NULL;
59 struct mbuf *m0 = NULL;
60 struct mbuf *prev = NULL;
61 struct mbuf *n, *t;
62 int nsegs;
63
64 KASSERT((m->m_flags & M_PKTHDR) != 0);
65 KASSERT((m->m_pkthdr.csum_flags & M_CSUM_TSOv4) != 0);
66
67 m->m_pkthdr.csum_flags = 0;
68
69 len = m->m_pkthdr.len;
70 KASSERT(len >= off + sizeof(*ip) + sizeof(*th));
71
72 hlen = off + sizeof(*ip);
73 if (m->m_len < hlen) {
74 m = m_pullup(m, hlen);
75 if (m == NULL)
76 goto quit;
77 }
78 ip = (void *)(mtod(m, char *) + off);
79 iphlen = ip->ip_hl * 4;
80 KASSERT(ip->ip_v == IPVERSION);
81 KASSERT(iphlen >= sizeof(*ip));
82 KASSERT(ip->ip_p == IPPROTO_TCP);
83 ipid = ntohs(ip->ip_id);
84
85 hlen = off + iphlen + sizeof(*th);
86 if (m->m_len < hlen) {
87 m = m_pullup(m, hlen);
88 if (m == NULL)
89 goto quit;
90 }
91 th = (void *)(mtod(m, char *) + off + iphlen);
92 tcpseq = ntohl(th->th_seq);
93 thlen = th->th_off * 4;
94 hlen = off + iphlen + thlen;
95
96 mss = m->m_pkthdr.segsz;
97 KASSERT(mss != 0);
98 KASSERT(len > hlen);
99
100 t = m_split(m, hlen, M_NOWAIT);
101 if (t == NULL)
102 goto quit;
103 hdr = m;
104 m = t;
105
106 len -= hlen;
107 KASSERT(len % mss == 0);
108
109 ip = (void *)(mtod(hdr, char *) + off);
110 ip->ip_len = htons(iphlen + thlen + mss);
111 phsum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr,
112 htons((uint16_t)(thlen + mss) + IPPROTO_TCP));
113
114 for (nsegs = len / mss; nsegs > 0; nsegs--) {
115 if (nsegs > 1) {
116 n = m_dup(hdr, 0, hlen, M_NOWAIT);
117 if (n == NULL)
118 goto quit;
119 } else
120 n = hdr;
121 KASSERT(n->m_len == hlen); /* XXX */
122
123 if (nsegs > 1) {
124 t = m_split(m, mss, M_NOWAIT);
125 if (t == NULL) {
126 m_freem(n);
127 goto quit;
128 }
129 } else
130 t = m;
131 m_cat(n, m);
132 m = t;
133
134 KASSERT(n->m_len >= hlen); /* XXX */
135
136 if (m0 == NULL)
137 m0 = n;
138
139 if (prev != NULL)
140 prev->m_nextpkt = n;
141
142 n->m_pkthdr.len = hlen + mss;
143 n->m_nextpkt = NULL; /* XXX */
144
145 ip = (void *)(mtod(n, char *) + off);
146 ip->ip_id = htons(ipid);
147 ip->ip_sum = 0;
148 ip->ip_sum = in4_cksum(n, 0, off, iphlen);
149
150 th = (void *)(mtod(n, char *) + off + iphlen);
151 th->th_seq = htonl(tcpseq);
152 th->th_sum = phsum;
153 th->th_sum = in4_cksum(n, 0, off + iphlen, thlen + mss);
154
155 tcpseq += mss;
156 ipid++;
157 prev = n;
158 }
159 return m0;
160
161 quit:
162 if (hdr != NULL)
163 m_freem(hdr);
164 if (m != NULL)
165 m_freem(m);
166 for (m = m0; m != NULL; m = n) {
167 n = m->m_nextpkt;
168 m_freem(m);
169 }
170
171 return NULL;
172 }
173
174 int
175 ip_tso_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa,
176 struct rtentry *rt)
177 {
178 struct mbuf *n;
179 int error = 0;
180
181 m = tcp4_segment(m, 0);
182 if (m == NULL)
183 return ENOMEM;
184 do {
185 n = m->m_nextpkt;
186 if (error == 0)
187 error = ip_if_output(ifp, m, sa, rt);
188 else
189 m_freem(m);
190 m = n;
191 } while (m != NULL);
192 return error;
193 }
194
195 /*
196 * Compute now in software the IP and TCP/UDP checksums. Cancel the
197 * hardware offloading.
198 */
199 void
200 in_undefer_cksum(struct mbuf *mh, size_t hdrlen, int csum_flags)
201 {
202 const size_t iphdrlen = M_CSUM_DATA_IPv4_IPHL(mh->m_pkthdr.csum_data);
203 uint16_t csum;
204 uint16_t ip_len;
205 uint16_t *csump;
206 struct mbuf *m = mh;
207
208 KASSERT(mh->m_flags & M_PKTHDR);
209 KASSERT(mh->m_pkthdr.len > hdrlen);
210 KASSERT((mh->m_pkthdr.csum_flags & csum_flags) == csum_flags);
211
212 /*
213 * Deal with prepended frame header as done by e.g. ether_output().
214 * If first mbuf in chain has just the header, use second mbuf
215 * for the actual checksum. in4_csum() expects the passed mbuf
216 * to have the whole (struct ip) area contiguous.
217 */
218 if (m->m_len <= hdrlen) {
219 hdrlen -= m->m_len;
220 m = m->m_next;
221 KASSERT(m != NULL);
222 }
223
224 if (__predict_true(hdrlen + sizeof(struct ip) <= m->m_len)) {
225 struct ip *ip = (struct ip *)(mtod(m, uint8_t *) + hdrlen);
226
227 ip_len = ip->ip_len;
228 csump = &ip->ip_sum;
229 } else {
230 const size_t ip_len_offset =
231 hdrlen + offsetof(struct ip, ip_len);
232
233 m_copydata(m, ip_len_offset, sizeof(ip_len), &ip_len);
234 csump = NULL;
235 }
236 ip_len = ntohs(ip_len);
237
238 if (csum_flags & M_CSUM_IPv4) {
239 csum = in4_cksum(m, 0, hdrlen, iphdrlen);
240 if (csump != NULL) {
241 *csump = csum;
242 } else {
243 const size_t offset = hdrlen +
244 offsetof(struct ip, ip_sum);
245
246 m_copyback(m, offset, sizeof(uint16_t), &csum);
247 }
248 }
249
250 if (csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) {
251 size_t l4offset = hdrlen + iphdrlen;
252
253 csum = in4_cksum(m, 0, l4offset, ip_len - iphdrlen);
254 if (csum == 0 && (csum_flags & M_CSUM_UDPv4) != 0)
255 csum = 0xffff;
256
257 l4offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
258
259 if (__predict_true(l4offset + sizeof(uint16_t) <= m->m_len)) {
260 *(uint16_t *)(mtod(m, char *) + l4offset) = csum;
261 } else {
262 m_copyback(m, l4offset, sizeof(csum), (void *)&csum);
263 }
264 }
265
266 mh->m_pkthdr.csum_flags ^= csum_flags;
267 }
268
269 /*
270 * Compute now in software the TCP/UDP checksum. Cancel the hardware
271 * offloading.
272 */
273 void
274 in_undefer_cksum_tcpudp(struct mbuf *m)
275 {
276 struct ip *ip;
277 uint16_t csum, offset;
278
279 KASSERT((m->m_flags & M_PKTHDR) != 0);
280 KASSERT((m->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_UDPv4)) != 0);
281 KASSERT((m->m_pkthdr.csum_flags & (M_CSUM_TCPv6|M_CSUM_UDPv6)) == 0);
282
283 ip = mtod(m, struct ip *);
284 offset = ip->ip_hl << 2;
285
286 csum = in4_cksum(m, 0, offset, ntohs(ip->ip_len) - offset);
287 if (csum == 0 && (m->m_pkthdr.csum_flags & M_CSUM_UDPv4) != 0)
288 csum = 0xffff;
289
290 offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
291
292 if ((offset + sizeof(uint16_t)) <= m->m_len) {
293 *(uint16_t *)(mtod(m, char *) + offset) = csum;
294 } else {
295 m_copyback(m, offset, sizeof(csum), (void *)&csum);
296 }
297 }
Cache object: 1804be29e80f32a4dc2f48c70e41ac37
|