in_cksum.c

Version: - FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
    1 /*-
    2  * Copyright (c) 1990 The Regents of the University of California.
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 3. All advertising materials mentioning features or use of this software
   14  *    must display the following acknowledgement:
   15  *      This product includes software developed by the University of
   16  *      California, Berkeley and its contributors.
   17  * 4. Neither the name of the University nor the names of its contributors
   18  *    may be used to endorse or promote products derived from this software
   19  *    without specific prior written permission.
   20  *
   21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   31  * SUCH DAMAGE.
   32  *
   33  *      from tahoe:     in_cksum.c      1.2     86/01/05
   34  *      from:           @(#)in_cksum.c  1.3 (Berkeley) 1/19/91
   35  * $FreeBSD: releng/5.1/sys/i386/i386/in_cksum.c 98648 2002-06-22 22:35:53Z jdp $
   36  */
   37 
   38 /*
   39  * MPsafe: alfred
   40  */
   41 #include <sys/param.h>
   42 #include <sys/systm.h>
   43 #include <sys/mbuf.h>
   44 
   45 #include <netinet/in.h>
   46 #include <netinet/in_systm.h>
   47 #include <netinet/ip.h>
   48 
   49 #include <machine/in_cksum.h>
   50 
   51 /*
   52  * Checksum routine for Internet Protocol family headers.
   53  *
   54  * This routine is very heavily used in the network
   55  * code and should be modified for each CPU to be as fast as possible.
   56  *
   57  * This implementation is 386 version.
   58  */
   59 
   60 #undef  ADDCARRY
   61 #define ADDCARRY(x)     if ((x) > 0xffff) (x) -= 0xffff
   62 #define REDUCE          {sum = (sum & 0xffff) + (sum >> 16); ADDCARRY(sum);}
   63 
   64 /*
   65  * These asm statements require __volatile because they pass information
   66  * via the condition codes.  GCC does not currently provide a way to specify
   67  * the condition codes as an input or output operand.
   68  *
   69  * The LOAD macro below is effectively a prefetch into cache.  GCC will
   70  * load the value into a register but will not use it.  Since modern CPUs
   71  * reorder operations, this will generally take place in parallel with
   72  * other calculations.
   73  */
   74 #define ADD(n)  __asm __volatile \
   75                 ("addl %1, %0" : "+r" (sum) : \
   76                 "g" (((const u_int32_t *)w)[n / 4]))
   77 #define ADDC(n) __asm __volatile \
   78                 ("adcl %1, %0" : "+r" (sum) : \
   79                 "g" (((const u_int32_t *)w)[n / 4]))
   80 #define LOAD(n) __asm __volatile \
   81                 ("" : : "r" (((const u_int32_t *)w)[n / 4]))
   82 #define MOP     __asm __volatile \
   83                 ("adcl         $0, %0" : "+r" (sum))
   84 
   85 u_short
   86 in_cksum_skip(m, len, skip)
   87         struct mbuf *m;
   88         int len;
   89         int skip;
   90 {
   91         register u_short *w;
   92         register unsigned sum = 0;
   93         register int mlen = 0;
   94         int byte_swapped = 0;
   95         union { char    c[2]; u_short   s; } su;
   96 
   97         len -= skip;
   98         for (; skip && m; m = m->m_next) {
   99                 if (m->m_len > skip) {
  100                         mlen = m->m_len - skip;
  101                         w = (u_short *)(mtod(m, u_char *) + skip);
  102                         goto skip_start;
  103                 } else {
  104                         skip -= m->m_len;
  105                 }
  106         }
  107 
  108         for (;m && len; m = m->m_next) {
  109                 if (m->m_len == 0)
  110                         continue;
  111                 w = mtod(m, u_short *);
  112                 if (mlen == -1) {
  113                         /*
  114                          * The first byte of this mbuf is the continuation
  115                          * of a word spanning between this mbuf and the
  116                          * last mbuf.
  117                          */
  118 
  119                         /* su.c[0] is already saved when scanning previous
  120                          * mbuf.  sum was REDUCEd when we found mlen == -1
  121                          */
  122                         su.c[1] = *(u_char *)w;
  123                         sum += su.s;
  124                         w = (u_short *)((char *)w + 1);
  125                         mlen = m->m_len - 1;
  126                         len--;
  127                 } else
  128                         mlen = m->m_len;
  129 skip_start:
  130                 if (len < mlen)
  131                         mlen = len;
  132                 len -= mlen;
  133                 /*
  134                  * Force to long boundary so we do longword aligned
  135                  * memory operations
  136                  */
  137                 if (3 & (int) w) {
  138                         REDUCE;
  139                         if ((1 & (int) w) && (mlen > 0)) {
  140                                 sum <<= 8;
  141                                 su.c[0] = *(char *)w;
  142                                 w = (u_short *)((char *)w + 1);
  143                                 mlen--;
  144                                 byte_swapped = 1;
  145                         }
  146                         if ((2 & (int) w) && (mlen >= 2)) {
  147                                 sum += *w++;
  148                                 mlen -= 2;
  149                         }
  150                 }
  151                 /*
  152                  * Advance to a 486 cache line boundary.
  153                  */
  154                 if (4 & (int) w && mlen >= 4) {
  155                         ADD(0);
  156                         MOP;
  157                         w += 2;
  158                         mlen -= 4;
  159                 }
  160                 if (8 & (int) w && mlen >= 8) {
  161                         ADD(0);
  162                         ADDC(4);
  163                         MOP;
  164                         w += 4;
  165                         mlen -= 8;
  166                 }
  167                 /*
  168                  * Do as much of the checksum as possible 32 bits at at time.
  169                  * In fact, this loop is unrolled to make overhead from
  170                  * branches &c small.
  171                  */
  172                 mlen -= 1;
  173                 while ((mlen -= 32) >= 0) {
  174                         /*
  175                          * Add with carry 16 words and fold in the last
  176                          * carry by adding a 0 with carry.
  177                          *
  178                          * The early ADD(16) and the LOAD(32) are to load
  179                          * the next 2 cache lines in advance on 486's.  The
  180                          * 486 has a penalty of 2 clock cycles for loading
  181                          * a cache line, plus whatever time the external
  182                          * memory takes to load the first word(s) addressed.
  183                          * These penalties are unavoidable.  Subsequent
  184                          * accesses to a cache line being loaded (and to
  185                          * other external memory?) are delayed until the
  186                          * whole load finishes.  These penalties are mostly
  187                          * avoided by not accessing external memory for
  188                          * 8 cycles after the ADD(16) and 12 cycles after
  189                          * the LOAD(32).  The loop terminates when mlen
  190                          * is initially 33 (not 32) to guaranteed that
  191                          * the LOAD(32) is within bounds.
  192                          */
  193                         ADD(16);
  194                         ADDC(0);
  195                         ADDC(4);
  196                         ADDC(8);
  197                         ADDC(12);
  198                         LOAD(32);
  199                         ADDC(20);
  200                         ADDC(24);
  201                         ADDC(28);
  202                         MOP;
  203                         w += 16;
  204                 }
  205                 mlen += 32 + 1;
  206                 if (mlen >= 32) {
  207                         ADD(16);
  208                         ADDC(0);
  209                         ADDC(4);
  210                         ADDC(8);
  211                         ADDC(12);
  212                         ADDC(20);
  213                         ADDC(24);
  214                         ADDC(28);
  215                         MOP;
  216                         w += 16;
  217                         mlen -= 32;
  218                 }
  219                 if (mlen >= 16) {
  220                         ADD(0);
  221                         ADDC(4);
  222                         ADDC(8);
  223                         ADDC(12);
  224                         MOP;
  225                         w += 8;
  226                         mlen -= 16;
  227                 }
  228                 if (mlen >= 8) {
  229                         ADD(0);
  230                         ADDC(4);
  231                         MOP;
  232                         w += 4;
  233                         mlen -= 8;
  234                 }
  235                 if (mlen == 0 && byte_swapped == 0)
  236                         continue;       /* worth 1% maybe ?? */
  237                 REDUCE;
  238                 while ((mlen -= 2) >= 0) {
  239                         sum += *w++;
  240                 }
  241                 if (byte_swapped) {
  242                         sum <<= 8;
  243                         byte_swapped = 0;
  244                         if (mlen == -1) {
  245                                 su.c[1] = *(char *)w;
  246                                 sum += su.s;
  247                                 mlen = 0;
  248                         } else
  249                                 mlen = -1;
  250                 } else if (mlen == -1)
  251                         /*
  252                          * This mbuf has odd number of bytes.
  253                          * There could be a word split betwen
  254                          * this mbuf and the next mbuf.
  255                          * Save the last byte (to prepend to next mbuf).
  256                          */
  257                         su.c[0] = *(char *)w;
  258         }
  259 
  260         if (len)
  261                 printf("%s: out of data by %d\n", __func__, len);
  262         if (mlen == -1) {
  263                 /* The last mbuf has odd # of bytes. Follow the
  264                    standard (the odd byte is shifted left by 8 bits) */
  265                 su.c[1] = 0;
  266                 sum += su.s;
  267         }
  268         REDUCE;
  269         return (~sum & 0xffff);
  270 }
Cache object: a0b439792cd95e5c3c8c15408262ea53
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.
FreeBSD/Linux Kernel Cross Reference sys/i386/i386/in_cksum.c

FreeBSD/Linux Kernel Cross Reference
sys/i386/i386/in_cksum.c
