The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/netinet/in_offload.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*      $NetBSD: in_offload.c,v 1.14 2020/03/27 16:34:58 jdolecek Exp $ */
    2 
    3 /*
    4  * Copyright (c)2005, 2006 YAMAMOTO Takashi,
    5  * All rights reserved.
    6  *
    7  * Redistribution and use in source and binary forms, with or without
    8  * modification, are permitted provided that the following conditions
    9  * are met:
   10  * 1. Redistributions of source code must retain the above copyright
   11  *    notice, this list of conditions and the following disclaimer.
   12  * 2. Redistributions in binary form must reproduce the above copyright
   13  *    notice, this list of conditions and the following disclaimer in the
   14  *    documentation and/or other materials provided with the distribution.
   15  *
   16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
   17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
   20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   26  * SUCH DAMAGE.
   27  */
   28 
   29 #include <sys/cdefs.h>
   30 __KERNEL_RCSID(0, "$NetBSD: in_offload.c,v 1.14 2020/03/27 16:34:58 jdolecek Exp $");
   31 
   32 #include <sys/param.h>
   33 #include <sys/mbuf.h>
   34 
   35 #include <net/if.h>
   36 
   37 #include <netinet/in.h>
   38 #include <netinet/in_systm.h>
   39 #include <netinet/ip.h>
   40 #include <netinet/ip_var.h>
   41 #include <netinet/tcp.h>
   42 #include <netinet/in_offload.h>
   43 
   44 /*
   45  * Handle M_CSUM_TSOv4 in software. Split the TCP payload in chunks of
   46  * size MSS, and return mbuf chain consists of them.
   47  */
   48 struct mbuf *
   49 tcp4_segment(struct mbuf *m, int off)
   50 {
   51         int mss;
   52         int iphlen, thlen;
   53         int hlen, len;
   54         struct ip *ip;
   55         struct tcphdr *th;
   56         uint16_t ipid, phsum;
   57         uint32_t tcpseq;
   58         struct mbuf *hdr = NULL;
   59         struct mbuf *m0 = NULL;
   60         struct mbuf *prev = NULL;
   61         struct mbuf *n, *t;
   62         int nsegs;
   63 
   64         KASSERT((m->m_flags & M_PKTHDR) != 0);
   65         KASSERT((m->m_pkthdr.csum_flags & M_CSUM_TSOv4) != 0);
   66 
   67         m->m_pkthdr.csum_flags = 0;
   68 
   69         len = m->m_pkthdr.len;
   70         KASSERT(len >= off + sizeof(*ip) + sizeof(*th));
   71 
   72         hlen = off + sizeof(*ip);
   73         if (m->m_len < hlen) {
   74                 m = m_pullup(m, hlen);
   75                 if (m == NULL)
   76                         goto quit;
   77         }
   78         ip = (void *)(mtod(m, char *) + off);
   79         iphlen = ip->ip_hl * 4;
   80         KASSERT(ip->ip_v == IPVERSION);
   81         KASSERT(iphlen >= sizeof(*ip));
   82         KASSERT(ip->ip_p == IPPROTO_TCP);
   83         ipid = ntohs(ip->ip_id);
   84 
   85         hlen = off + iphlen + sizeof(*th);
   86         if (m->m_len < hlen) {
   87                 m = m_pullup(m, hlen);
   88                 if (m == NULL)
   89                         goto quit;
   90         }
   91         th = (void *)(mtod(m, char *) + off + iphlen);
   92         tcpseq = ntohl(th->th_seq);
   93         thlen = th->th_off * 4;
   94         hlen = off + iphlen + thlen;
   95 
   96         mss = m->m_pkthdr.segsz;
   97         KASSERT(mss != 0);
   98         KASSERT(len > hlen);
   99 
  100         t = m_split(m, hlen, M_NOWAIT);
  101         if (t == NULL)
  102                 goto quit;
  103         hdr = m;
  104         m = t;
  105 
  106         len -= hlen;
  107         KASSERT(len % mss == 0);
  108 
  109         ip = (void *)(mtod(hdr, char *) + off);
  110         ip->ip_len = htons(iphlen + thlen + mss);
  111         phsum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr,
  112             htons((uint16_t)(thlen + mss) + IPPROTO_TCP));
  113 
  114         for (nsegs = len / mss; nsegs > 0; nsegs--) {
  115                 if (nsegs > 1) {
  116                         n = m_dup(hdr, 0, hlen, M_NOWAIT);
  117                         if (n == NULL)
  118                                 goto quit;
  119                 } else
  120                         n = hdr;
  121                 KASSERT(n->m_len == hlen); /* XXX */
  122 
  123                 if (nsegs > 1) {
  124                         t = m_split(m, mss, M_NOWAIT);
  125                         if (t == NULL) {
  126                                 m_freem(n);
  127                                 goto quit;
  128                         }
  129                 } else
  130                         t = m;
  131                 m_cat(n, m);
  132                 m = t;
  133 
  134                 KASSERT(n->m_len >= hlen); /* XXX */
  135 
  136                 if (m0 == NULL)
  137                         m0 = n;
  138 
  139                 if (prev != NULL)
  140                         prev->m_nextpkt = n;
  141 
  142                 n->m_pkthdr.len = hlen + mss;
  143                 n->m_nextpkt = NULL;    /* XXX */
  144 
  145                 ip = (void *)(mtod(n, char *) + off);
  146                 ip->ip_id = htons(ipid);
  147                 ip->ip_sum = 0;
  148                 ip->ip_sum = in4_cksum(n, 0, off, iphlen);
  149 
  150                 th = (void *)(mtod(n, char *) + off + iphlen);
  151                 th->th_seq = htonl(tcpseq);
  152                 th->th_sum = phsum;
  153                 th->th_sum = in4_cksum(n, 0, off + iphlen, thlen + mss);
  154 
  155                 tcpseq += mss;
  156                 ipid++;
  157                 prev = n;
  158         }
  159         return m0;
  160 
  161 quit:
  162         if (hdr != NULL)
  163                 m_freem(hdr);
  164         if (m != NULL)
  165                 m_freem(m);
  166         for (m = m0; m != NULL; m = n) {
  167                 n = m->m_nextpkt;
  168                 m_freem(m);
  169         }
  170 
  171         return NULL;
  172 }
  173 
  174 int
  175 ip_tso_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *sa,
  176     struct rtentry *rt)
  177 {
  178         struct mbuf *n;
  179         int error = 0;
  180 
  181         m = tcp4_segment(m, 0);
  182         if (m == NULL)
  183                 return ENOMEM;
  184         do {
  185                 n = m->m_nextpkt;
  186                 if (error == 0)
  187                         error = ip_if_output(ifp, m, sa, rt);
  188                 else
  189                         m_freem(m);
  190                 m = n;
  191         } while (m != NULL);
  192         return error;
  193 }
  194 
  195 /*
  196  * Compute now in software the IP and TCP/UDP checksums. Cancel the
  197  * hardware offloading.
  198  */
  199 void
  200 in_undefer_cksum(struct mbuf *mh, size_t hdrlen, int csum_flags)
  201 {
  202         const size_t iphdrlen = M_CSUM_DATA_IPv4_IPHL(mh->m_pkthdr.csum_data);
  203         uint16_t csum;
  204         uint16_t ip_len;
  205         uint16_t *csump;
  206         struct mbuf *m = mh;
  207 
  208         KASSERT(mh->m_flags & M_PKTHDR);
  209         KASSERT(mh->m_pkthdr.len > hdrlen);
  210         KASSERT((mh->m_pkthdr.csum_flags & csum_flags) == csum_flags);
  211 
  212         /*
  213          * Deal with prepended frame header as done by e.g. ether_output().
  214          * If first mbuf in chain has just the header, use second mbuf
  215          * for the actual checksum. in4_csum() expects the passed mbuf
  216          * to have the whole (struct ip) area contiguous.
  217          */
  218         if (m->m_len <= hdrlen) {
  219                 hdrlen -= m->m_len;
  220                 m = m->m_next;
  221                 KASSERT(m != NULL);
  222         }
  223 
  224         if (__predict_true(hdrlen + sizeof(struct ip) <= m->m_len)) {
  225                 struct ip *ip = (struct ip *)(mtod(m, uint8_t *) + hdrlen);
  226 
  227                 ip_len = ip->ip_len;
  228                 csump = &ip->ip_sum;
  229         } else {
  230                 const size_t ip_len_offset =
  231                     hdrlen + offsetof(struct ip, ip_len);
  232 
  233                 m_copydata(m, ip_len_offset, sizeof(ip_len), &ip_len);
  234                 csump = NULL;
  235         }
  236         ip_len = ntohs(ip_len);
  237 
  238         if (csum_flags & M_CSUM_IPv4) {
  239                 csum = in4_cksum(m, 0, hdrlen, iphdrlen);
  240                 if (csump != NULL) {
  241                         *csump = csum;
  242                 } else {
  243                         const size_t offset = hdrlen +
  244                             offsetof(struct ip, ip_sum);
  245 
  246                         m_copyback(m, offset, sizeof(uint16_t), &csum);
  247                 }
  248         }
  249 
  250         if (csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) {
  251                 size_t l4offset = hdrlen + iphdrlen;
  252 
  253                 csum = in4_cksum(m, 0, l4offset, ip_len - iphdrlen);
  254                 if (csum == 0 && (csum_flags & M_CSUM_UDPv4) != 0)
  255                         csum = 0xffff;
  256 
  257                 l4offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
  258 
  259                 if (__predict_true(l4offset + sizeof(uint16_t) <= m->m_len)) {
  260                         *(uint16_t *)(mtod(m, char *) + l4offset) = csum;
  261                 } else {
  262                         m_copyback(m, l4offset, sizeof(csum), (void *)&csum);
  263                 }
  264         }
  265 
  266         mh->m_pkthdr.csum_flags ^= csum_flags;
  267 }
  268 
  269 /*
  270  * Compute now in software the TCP/UDP checksum. Cancel the hardware
  271  * offloading.
  272  */
  273 void
  274 in_undefer_cksum_tcpudp(struct mbuf *m)
  275 {
  276         struct ip *ip;
  277         uint16_t csum, offset;
  278 
  279         KASSERT((m->m_flags & M_PKTHDR) != 0);
  280         KASSERT((m->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_UDPv4)) != 0);
  281         KASSERT((m->m_pkthdr.csum_flags & (M_CSUM_TCPv6|M_CSUM_UDPv6)) == 0);
  282 
  283         ip = mtod(m, struct ip *);
  284         offset = ip->ip_hl << 2;
  285 
  286         csum = in4_cksum(m, 0, offset, ntohs(ip->ip_len) - offset);
  287         if (csum == 0 && (m->m_pkthdr.csum_flags & M_CSUM_UDPv4) != 0)
  288                 csum = 0xffff;
  289 
  290         offset += M_CSUM_DATA_IPv4_OFFSET(m->m_pkthdr.csum_data);
  291 
  292         if ((offset + sizeof(uint16_t)) <= m->m_len) {
  293                 *(uint16_t *)(mtod(m, char *) + offset) = csum;
  294         } else {
  295                 m_copyback(m, offset, sizeof(csum), (void *)&csum);
  296         }
  297 }

Cache object: 1804be29e80f32a4dc2f48c70e41ac37


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.