The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/netinet/tcp_output.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
    3  *      The Regents of the University of California.  All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions
    7  * are met:
    8  * 1. Redistributions of source code must retain the above copyright
    9  *    notice, this list of conditions and the following disclaimer.
   10  * 2. Redistributions in binary form must reproduce the above copyright
   11  *    notice, this list of conditions and the following disclaimer in the
   12  *    documentation and/or other materials provided with the distribution.
   13  * 4. Neither the name of the University nor the names of its contributors
   14  *    may be used to endorse or promote products derived from this software
   15  *    without specific prior written permission.
   16  *
   17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   27  * SUCH DAMAGE.
   28  *
   29  *      @(#)tcp_output.c        8.4 (Berkeley) 5/24/95
   30  */
   31 
   32 #include <sys/cdefs.h>
   33 __FBSDID("$FreeBSD: releng/9.0/sys/netinet/tcp_output.c 225169 2011-08-25 09:20:13Z bz $");
   34 
   35 #include "opt_inet.h"
   36 #include "opt_inet6.h"
   37 #include "opt_ipsec.h"
   38 #include "opt_tcpdebug.h"
   39 
   40 #include <sys/param.h>
   41 #include <sys/systm.h>
   42 #include <sys/domain.h>
   43 #include <sys/hhook.h>
   44 #include <sys/kernel.h>
   45 #include <sys/lock.h>
   46 #include <sys/mbuf.h>
   47 #include <sys/mutex.h>
   48 #include <sys/protosw.h>
   49 #include <sys/socket.h>
   50 #include <sys/socketvar.h>
   51 #include <sys/sysctl.h>
   52 
   53 #include <net/if.h>
   54 #include <net/route.h>
   55 #include <net/vnet.h>
   56 
   57 #include <netinet/cc.h>
   58 #include <netinet/in.h>
   59 #include <netinet/in_systm.h>
   60 #include <netinet/ip.h>
   61 #include <netinet/in_pcb.h>
   62 #include <netinet/ip_var.h>
   63 #include <netinet/ip_options.h>
   64 #ifdef INET6
   65 #include <netinet6/in6_pcb.h>
   66 #include <netinet/ip6.h>
   67 #include <netinet6/ip6_var.h>
   68 #endif
   69 #define TCPOUTFLAGS
   70 #include <netinet/tcp_fsm.h>
   71 #include <netinet/tcp_seq.h>
   72 #include <netinet/tcp_timer.h>
   73 #include <netinet/tcp_var.h>
   74 #include <netinet/tcpip.h>
   75 #ifdef TCPDEBUG
   76 #include <netinet/tcp_debug.h>
   77 #endif
   78 
   79 #ifdef IPSEC
   80 #include <netipsec/ipsec.h>
   81 #endif /*IPSEC*/
   82 
   83 #include <machine/in_cksum.h>
   84 
   85 #include <security/mac/mac_framework.h>
   86 
   87 VNET_DEFINE(int, path_mtu_discovery) = 1;
   88 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, CTLFLAG_RW,
   89         &VNET_NAME(path_mtu_discovery), 1,
   90         "Enable Path MTU Discovery");
   91 
   92 VNET_DEFINE(int, ss_fltsz) = 1;
   93 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, slowstart_flightsize, CTLFLAG_RW,
   94         &VNET_NAME(ss_fltsz), 1,
   95         "Slow start flight size");
   96 
   97 VNET_DEFINE(int, ss_fltsz_local) = 4;
   98 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, local_slowstart_flightsize,
   99         CTLFLAG_RW, &VNET_NAME(ss_fltsz_local), 1,
  100         "Slow start flight size for local networks");
  101 
  102 VNET_DEFINE(int, tcp_do_tso) = 1;
  103 #define V_tcp_do_tso            VNET(tcp_do_tso)
  104 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_RW,
  105         &VNET_NAME(tcp_do_tso), 0,
  106         "Enable TCP Segmentation Offload");
  107 
  108 VNET_DEFINE(int, tcp_do_autosndbuf) = 1;
  109 #define V_tcp_do_autosndbuf     VNET(tcp_do_autosndbuf)
  110 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, sendbuf_auto, CTLFLAG_RW,
  111         &VNET_NAME(tcp_do_autosndbuf), 0,
  112         "Enable automatic send buffer sizing");
  113 
  114 VNET_DEFINE(int, tcp_autosndbuf_inc) = 8*1024;
  115 #define V_tcp_autosndbuf_inc    VNET(tcp_autosndbuf_inc)
  116 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, sendbuf_inc, CTLFLAG_RW,
  117         &VNET_NAME(tcp_autosndbuf_inc), 0,
  118         "Incrementor step size of automatic send buffer");
  119 
  120 VNET_DEFINE(int, tcp_autosndbuf_max) = 2*1024*1024;
  121 #define V_tcp_autosndbuf_max    VNET(tcp_autosndbuf_max)
  122 SYSCTL_VNET_INT(_net_inet_tcp, OID_AUTO, sendbuf_max, CTLFLAG_RW,
  123         &VNET_NAME(tcp_autosndbuf_max), 0,
  124         "Max size of automatic send buffer");
  125 
  126 static void inline      hhook_run_tcp_est_out(struct tcpcb *tp,
  127                             struct tcphdr *th, struct tcpopt *to,
  128                             long len, int tso);
  129 static void inline      cc_after_idle(struct tcpcb *tp);
  130 
  131 /*
  132  * Wrapper for the TCP established ouput helper hook.
  133  */
  134 static void inline
  135 hhook_run_tcp_est_out(struct tcpcb *tp, struct tcphdr *th,
  136     struct tcpopt *to, long len, int tso)
  137 {
  138         struct tcp_hhook_data hhook_data;
  139 
  140         if (V_tcp_hhh[HHOOK_TCP_EST_OUT]->hhh_nhooks > 0) {
  141                 hhook_data.tp = tp;
  142                 hhook_data.th = th;
  143                 hhook_data.to = to;
  144                 hhook_data.len = len;
  145                 hhook_data.tso = tso;
  146 
  147                 hhook_run_hooks(V_tcp_hhh[HHOOK_TCP_EST_OUT], &hhook_data,
  148                     tp->osd);
  149         }
  150 }
  151 
  152 /*
  153  * CC wrapper hook functions
  154  */
  155 static void inline
  156 cc_after_idle(struct tcpcb *tp)
  157 {
  158         INP_WLOCK_ASSERT(tp->t_inpcb);
  159 
  160         if (CC_ALGO(tp)->after_idle != NULL)
  161                 CC_ALGO(tp)->after_idle(tp->ccv);
  162 }
  163 
  164 /*
  165  * Tcp output routine: figure out what should be sent and send it.
  166  */
  167 int
  168 tcp_output(struct tcpcb *tp)
  169 {
  170         struct socket *so = tp->t_inpcb->inp_socket;
  171         long len, recwin, sendwin;
  172         int off, flags, error = 0;      /* Keep compiler happy */
  173         struct mbuf *m;
  174         struct ip *ip = NULL;
  175         struct ipovly *ipov = NULL;
  176         struct tcphdr *th;
  177         u_char opt[TCP_MAXOLEN];
  178         unsigned ipoptlen, optlen, hdrlen;
  179 #ifdef IPSEC
  180         unsigned ipsec_optlen = 0;
  181 #endif
  182         int idle, sendalot;
  183         int sack_rxmit, sack_bytes_rxmt;
  184         struct sackhole *p;
  185         int tso;
  186         struct tcpopt to;
  187 #if 0
  188         int maxburst = TCP_MAXBURST;
  189 #endif
  190 #ifdef INET6
  191         struct ip6_hdr *ip6 = NULL;
  192         int isipv6;
  193 
  194         isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0;
  195 #endif
  196 
  197         INP_WLOCK_ASSERT(tp->t_inpcb);
  198 
  199         /*
  200          * Determine length of data that should be transmitted,
  201          * and flags that will be used.
  202          * If there is some data or critical controls (SYN, RST)
  203          * to send, then transmit; otherwise, investigate further.
  204          */
  205         idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
  206         if (idle && ticks - tp->t_rcvtime >= tp->t_rxtcur)
  207                 cc_after_idle(tp);
  208         tp->t_flags &= ~TF_LASTIDLE;
  209         if (idle) {
  210                 if (tp->t_flags & TF_MORETOCOME) {
  211                         tp->t_flags |= TF_LASTIDLE;
  212                         idle = 0;
  213                 }
  214         }
  215 again:
  216         /*
  217          * If we've recently taken a timeout, snd_max will be greater than
  218          * snd_nxt.  There may be SACK information that allows us to avoid
  219          * resending already delivered data.  Adjust snd_nxt accordingly.
  220          */
  221         if ((tp->t_flags & TF_SACK_PERMIT) &&
  222             SEQ_LT(tp->snd_nxt, tp->snd_max))
  223                 tcp_sack_adjust(tp);
  224         sendalot = 0;
  225         tso = 0;
  226         off = tp->snd_nxt - tp->snd_una;
  227         sendwin = min(tp->snd_wnd, tp->snd_cwnd);
  228 
  229         flags = tcp_outflags[tp->t_state];
  230         /*
  231          * Send any SACK-generated retransmissions.  If we're explicitly trying
  232          * to send out new data (when sendalot is 1), bypass this function.
  233          * If we retransmit in fast recovery mode, decrement snd_cwnd, since
  234          * we're replacing a (future) new transmission with a retransmission
  235          * now, and we previously incremented snd_cwnd in tcp_input().
  236          */
  237         /*
  238          * Still in sack recovery , reset rxmit flag to zero.
  239          */
  240         sack_rxmit = 0;
  241         sack_bytes_rxmt = 0;
  242         len = 0;
  243         p = NULL;
  244         if ((tp->t_flags & TF_SACK_PERMIT) && IN_FASTRECOVERY(tp->t_flags) &&
  245             (p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
  246                 long cwin;
  247                 
  248                 cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
  249                 if (cwin < 0)
  250                         cwin = 0;
  251                 /* Do not retransmit SACK segments beyond snd_recover */
  252                 if (SEQ_GT(p->end, tp->snd_recover)) {
  253                         /*
  254                          * (At least) part of sack hole extends beyond
  255                          * snd_recover. Check to see if we can rexmit data
  256                          * for this hole.
  257                          */
  258                         if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
  259                                 /*
  260                                  * Can't rexmit any more data for this hole.
  261                                  * That data will be rexmitted in the next
  262                                  * sack recovery episode, when snd_recover
  263                                  * moves past p->rxmit.
  264                                  */
  265                                 p = NULL;
  266                                 goto after_sack_rexmit;
  267                         } else
  268                                 /* Can rexmit part of the current hole */
  269                                 len = ((long)ulmin(cwin,
  270                                                    tp->snd_recover - p->rxmit));
  271                 } else
  272                         len = ((long)ulmin(cwin, p->end - p->rxmit));
  273                 off = p->rxmit - tp->snd_una;
  274                 KASSERT(off >= 0,("%s: sack block to the left of una : %d",
  275                     __func__, off));
  276                 if (len > 0) {
  277                         sack_rxmit = 1;
  278                         sendalot = 1;
  279                         TCPSTAT_INC(tcps_sack_rexmits);
  280                         TCPSTAT_ADD(tcps_sack_rexmit_bytes,
  281                             min(len, tp->t_maxseg));
  282                 }
  283         }
  284 after_sack_rexmit:
  285         /*
  286          * Get standard flags, and add SYN or FIN if requested by 'hidden'
  287          * state flags.
  288          */
  289         if (tp->t_flags & TF_NEEDFIN)
  290                 flags |= TH_FIN;
  291         if (tp->t_flags & TF_NEEDSYN)
  292                 flags |= TH_SYN;
  293 
  294         SOCKBUF_LOCK(&so->so_snd);
  295         /*
  296          * If in persist timeout with window of 0, send 1 byte.
  297          * Otherwise, if window is small but nonzero
  298          * and timer expired, we will send what we can
  299          * and go to transmit state.
  300          */
  301         if (tp->t_flags & TF_FORCEDATA) {
  302                 if (sendwin == 0) {
  303                         /*
  304                          * If we still have some data to send, then
  305                          * clear the FIN bit.  Usually this would
  306                          * happen below when it realizes that we
  307                          * aren't sending all the data.  However,
  308                          * if we have exactly 1 byte of unsent data,
  309                          * then it won't clear the FIN bit below,
  310                          * and if we are in persist state, we wind
  311                          * up sending the packet without recording
  312                          * that we sent the FIN bit.
  313                          *
  314                          * We can't just blindly clear the FIN bit,
  315                          * because if we don't have any more data
  316                          * to send then the probe will be the FIN
  317                          * itself.
  318                          */
  319                         if (off < so->so_snd.sb_cc)
  320                                 flags &= ~TH_FIN;
  321                         sendwin = 1;
  322                 } else {
  323                         tcp_timer_activate(tp, TT_PERSIST, 0);
  324                         tp->t_rxtshift = 0;
  325                 }
  326         }
  327 
  328         /*
  329          * If snd_nxt == snd_max and we have transmitted a FIN, the
  330          * offset will be > 0 even if so_snd.sb_cc is 0, resulting in
  331          * a negative length.  This can also occur when TCP opens up
  332          * its congestion window while receiving additional duplicate
  333          * acks after fast-retransmit because TCP will reset snd_nxt
  334          * to snd_max after the fast-retransmit.
  335          *
  336          * In the normal retransmit-FIN-only case, however, snd_nxt will
  337          * be set to snd_una, the offset will be 0, and the length may
  338          * wind up 0.
  339          *
  340          * If sack_rxmit is true we are retransmitting from the scoreboard
  341          * in which case len is already set.
  342          */
  343         if (sack_rxmit == 0) {
  344                 if (sack_bytes_rxmt == 0)
  345                         len = ((long)ulmin(so->so_snd.sb_cc, sendwin) - off);
  346                 else {
  347                         long cwin;
  348 
  349                         /*
  350                          * We are inside of a SACK recovery episode and are
  351                          * sending new data, having retransmitted all the
  352                          * data possible in the scoreboard.
  353                          */
  354                         len = ((long)ulmin(so->so_snd.sb_cc, tp->snd_wnd) 
  355                                - off);
  356                         /*
  357                          * Don't remove this (len > 0) check !
  358                          * We explicitly check for len > 0 here (although it 
  359                          * isn't really necessary), to work around a gcc 
  360                          * optimization issue - to force gcc to compute
  361                          * len above. Without this check, the computation
  362                          * of len is bungled by the optimizer.
  363                          */
  364                         if (len > 0) {
  365                                 cwin = tp->snd_cwnd - 
  366                                         (tp->snd_nxt - tp->sack_newdata) -
  367                                         sack_bytes_rxmt;
  368                                 if (cwin < 0)
  369                                         cwin = 0;
  370                                 len = lmin(len, cwin);
  371                         }
  372                 }
  373         }
  374 
  375         /*
  376          * Lop off SYN bit if it has already been sent.  However, if this
  377          * is SYN-SENT state and if segment contains data and if we don't
  378          * know that foreign host supports TAO, suppress sending segment.
  379          */
  380         if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
  381                 if (tp->t_state != TCPS_SYN_RECEIVED)
  382                         flags &= ~TH_SYN;
  383                 off--, len++;
  384         }
  385 
  386         /*
  387          * Be careful not to send data and/or FIN on SYN segments.
  388          * This measure is needed to prevent interoperability problems
  389          * with not fully conformant TCP implementations.
  390          */
  391         if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
  392                 len = 0;
  393                 flags &= ~TH_FIN;
  394         }
  395 
  396         if (len < 0) {
  397                 /*
  398                  * If FIN has been sent but not acked,
  399                  * but we haven't been called to retransmit,
  400                  * len will be < 0.  Otherwise, window shrank
  401                  * after we sent into it.  If window shrank to 0,
  402                  * cancel pending retransmit, pull snd_nxt back
  403                  * to (closed) window, and set the persist timer
  404                  * if it isn't already going.  If the window didn't
  405                  * close completely, just wait for an ACK.
  406                  */
  407                 len = 0;
  408                 if (sendwin == 0) {
  409                         tcp_timer_activate(tp, TT_REXMT, 0);
  410                         tp->t_rxtshift = 0;
  411                         tp->snd_nxt = tp->snd_una;
  412                         if (!tcp_timer_active(tp, TT_PERSIST))
  413                                 tcp_setpersist(tp);
  414                 }
  415         }
  416 
  417         /* len will be >= 0 after this point. */
  418         KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
  419 
  420         /*
  421          * Automatic sizing of send socket buffer.  Often the send buffer
  422          * size is not optimally adjusted to the actual network conditions
  423          * at hand (delay bandwidth product).  Setting the buffer size too
  424          * small limits throughput on links with high bandwidth and high
  425          * delay (eg. trans-continental/oceanic links).  Setting the
  426          * buffer size too big consumes too much real kernel memory,
  427          * especially with many connections on busy servers.
  428          *
  429          * The criteria to step up the send buffer one notch are:
  430          *  1. receive window of remote host is larger than send buffer
  431          *     (with a fudge factor of 5/4th);
  432          *  2. send buffer is filled to 7/8th with data (so we actually
  433          *     have data to make use of it);
  434          *  3. send buffer fill has not hit maximal automatic size;
  435          *  4. our send window (slow start and cogestion controlled) is
  436          *     larger than sent but unacknowledged data in send buffer.
  437          *
  438          * The remote host receive window scaling factor may limit the
  439          * growing of the send buffer before it reaches its allowed
  440          * maximum.
  441          *
  442          * It scales directly with slow start or congestion window
  443          * and does at most one step per received ACK.  This fast
  444          * scaling has the drawback of growing the send buffer beyond
  445          * what is strictly necessary to make full use of a given
  446          * delay*bandwith product.  However testing has shown this not
  447          * to be much of an problem.  At worst we are trading wasting
  448          * of available bandwith (the non-use of it) for wasting some
  449          * socket buffer memory.
  450          *
  451          * TODO: Shrink send buffer during idle periods together
  452          * with congestion window.  Requires another timer.  Has to
  453          * wait for upcoming tcp timer rewrite.
  454          */
  455         if (V_tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) {
  456                 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
  457                     so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) &&
  458                     so->so_snd.sb_cc < V_tcp_autosndbuf_max &&
  459                     sendwin >= (so->so_snd.sb_cc - (tp->snd_nxt - tp->snd_una))) {
  460                         if (!sbreserve_locked(&so->so_snd,
  461                             min(so->so_snd.sb_hiwat + V_tcp_autosndbuf_inc,
  462                              V_tcp_autosndbuf_max), so, curthread))
  463                                 so->so_snd.sb_flags &= ~SB_AUTOSIZE;
  464                 }
  465         }
  466 
  467         /*
  468          * Decide if we can use TCP Segmentation Offloading (if supported by
  469          * hardware).
  470          *
  471          * TSO may only be used if we are in a pure bulk sending state.  The
  472          * presence of TCP-MD5, SACK retransmits, SACK advertizements and
  473          * IP options prevent using TSO.  With TSO the TCP header is the same
  474          * (except for the sequence number) for all generated packets.  This
  475          * makes it impossible to transmit any options which vary per generated
  476          * segment or packet.
  477          */
  478 #ifdef IPSEC
  479         /*
  480          * Pre-calculate here as we save another lookup into the darknesses
  481          * of IPsec that way and can actually decide if TSO is ok.
  482          */
  483         ipsec_optlen = ipsec_hdrsiz_tcp(tp);
  484 #endif
  485         if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > tp->t_maxseg &&
  486             ((tp->t_flags & TF_SIGNATURE) == 0) &&
  487             tp->rcv_numsacks == 0 && sack_rxmit == 0 &&
  488 #ifdef IPSEC
  489             ipsec_optlen == 0 &&
  490 #endif
  491             tp->t_inpcb->inp_options == NULL &&
  492             tp->t_inpcb->in6p_options == NULL)
  493                 tso = 1;
  494 
  495         if (sack_rxmit) {
  496                 if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc))
  497                         flags &= ~TH_FIN;
  498         } else {
  499                 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + so->so_snd.sb_cc))
  500                         flags &= ~TH_FIN;
  501         }
  502 
  503         recwin = sbspace(&so->so_rcv);
  504 
  505         /*
  506          * Sender silly window avoidance.   We transmit under the following
  507          * conditions when len is non-zero:
  508          *
  509          *      - We have a full segment (or more with TSO)
  510          *      - This is the last buffer in a write()/send() and we are
  511          *        either idle or running NODELAY
  512          *      - we've timed out (e.g. persist timer)
  513          *      - we have more then 1/2 the maximum send window's worth of
  514          *        data (receiver may be limited the window size)
  515          *      - we need to retransmit
  516          */
  517         if (len) {
  518                 if (len >= tp->t_maxseg)
  519                         goto send;
  520                 /*
  521                  * NOTE! on localhost connections an 'ack' from the remote
  522                  * end may occur synchronously with the output and cause
  523                  * us to flush a buffer queued with moretocome.  XXX
  524                  *
  525                  * note: the len + off check is almost certainly unnecessary.
  526                  */
  527                 if (!(tp->t_flags & TF_MORETOCOME) &&   /* normal case */
  528                     (idle || (tp->t_flags & TF_NODELAY)) &&
  529                     len + off >= so->so_snd.sb_cc &&
  530                     (tp->t_flags & TF_NOPUSH) == 0) {
  531                         goto send;
  532                 }
  533                 if (tp->t_flags & TF_FORCEDATA)         /* typ. timeout case */
  534                         goto send;
  535                 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
  536                         goto send;
  537                 if (SEQ_LT(tp->snd_nxt, tp->snd_max))   /* retransmit case */
  538                         goto send;
  539                 if (sack_rxmit)
  540                         goto send;
  541         }
  542 
  543         /*
  544          * Compare available window to amount of window
  545          * known to peer (as advertised window less
  546          * next expected input).  If the difference is at least two
  547          * max size segments, or at least 50% of the maximum possible
  548          * window, then want to send a window update to peer.
  549          * Skip this if the connection is in T/TCP half-open state.
  550          * Don't send pure window updates when the peer has closed
  551          * the connection and won't ever send more data.
  552          */
  553         if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
  554             !TCPS_HAVERCVDFIN(tp->t_state)) {
  555                 /*
  556                  * "adv" is the amount we can increase the window,
  557                  * taking into account that we are limited by
  558                  * TCP_MAXWIN << tp->rcv_scale.
  559                  */
  560                 long adv;
  561                 int oldwin;
  562 
  563                 adv = min(recwin, (long)TCP_MAXWIN << tp->rcv_scale);
  564                 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
  565                         oldwin = (tp->rcv_adv - tp->rcv_nxt);
  566                         adv -= oldwin;
  567                 } else
  568                         oldwin = 0;
  569 
  570                 /* 
  571                  * If the new window size ends up being the same as the old
  572                  * size when it is scaled, then don't force a window update.
  573                  */
  574                 if (oldwin >> tp->rcv_scale == (adv + oldwin) >> tp->rcv_scale)
  575                         goto dontupdate;
  576                 if (adv >= (long) (2 * tp->t_maxseg))
  577                         goto send;
  578                 if (2 * adv >= (long) so->so_rcv.sb_hiwat)
  579                         goto send;
  580         }
  581 dontupdate:
  582 
  583         /*
  584          * Send if we owe the peer an ACK, RST, SYN, or urgent data.  ACKNOW
  585          * is also a catch-all for the retransmit timer timeout case.
  586          */
  587         if (tp->t_flags & TF_ACKNOW)
  588                 goto send;
  589         if ((flags & TH_RST) ||
  590             ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0))
  591                 goto send;
  592         if (SEQ_GT(tp->snd_up, tp->snd_una))
  593                 goto send;
  594         /*
  595          * If our state indicates that FIN should be sent
  596          * and we have not yet done so, then we need to send.
  597          */
  598         if (flags & TH_FIN &&
  599             ((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una))
  600                 goto send;
  601         /*
  602          * In SACK, it is possible for tcp_output to fail to send a segment
  603          * after the retransmission timer has been turned off.  Make sure
  604          * that the retransmission timer is set.
  605          */
  606         if ((tp->t_flags & TF_SACK_PERMIT) &&
  607             SEQ_GT(tp->snd_max, tp->snd_una) &&
  608             !tcp_timer_active(tp, TT_REXMT) &&
  609             !tcp_timer_active(tp, TT_PERSIST)) {
  610                 tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur);
  611                 goto just_return;
  612         } 
  613         /*
  614          * TCP window updates are not reliable, rather a polling protocol
  615          * using ``persist'' packets is used to insure receipt of window
  616          * updates.  The three ``states'' for the output side are:
  617          *      idle                    not doing retransmits or persists
  618          *      persisting              to move a small or zero window
  619          *      (re)transmitting        and thereby not persisting
  620          *
  621          * tcp_timer_active(tp, TT_PERSIST)
  622          *      is true when we are in persist state.
  623          * (tp->t_flags & TF_FORCEDATA)
  624          *      is set when we are called to send a persist packet.
  625          * tcp_timer_active(tp, TT_REXMT)
  626          *      is set when we are retransmitting
  627          * The output side is idle when both timers are zero.
  628          *
  629          * If send window is too small, there is data to transmit, and no
  630          * retransmit or persist is pending, then go to persist state.
  631          * If nothing happens soon, send when timer expires:
  632          * if window is nonzero, transmit what we can,
  633          * otherwise force out a byte.
  634          */
  635         if (so->so_snd.sb_cc && !tcp_timer_active(tp, TT_REXMT) &&
  636             !tcp_timer_active(tp, TT_PERSIST)) {
  637                 tp->t_rxtshift = 0;
  638                 tcp_setpersist(tp);
  639         }
  640 
  641         /*
  642          * No reason to send a segment, just return.
  643          */
  644 just_return:
  645         SOCKBUF_UNLOCK(&so->so_snd);
  646         return (0);
  647 
  648 send:
  649         SOCKBUF_LOCK_ASSERT(&so->so_snd);
  650         /*
  651          * Before ESTABLISHED, force sending of initial options
  652          * unless TCP set not to do any options.
  653          * NOTE: we assume that the IP/TCP header plus TCP options
  654          * always fit in a single mbuf, leaving room for a maximum
  655          * link header, i.e.
  656          *      max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
  657          */
  658         optlen = 0;
  659 #ifdef INET6
  660         if (isipv6)
  661                 hdrlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
  662         else
  663 #endif
  664                 hdrlen = sizeof (struct tcpiphdr);
  665 
  666         /*
  667          * Compute options for segment.
  668          * We only have to care about SYN and established connection
  669          * segments.  Options for SYN-ACK segments are handled in TCP
  670          * syncache.
  671          */
  672         if ((tp->t_flags & TF_NOOPT) == 0) {
  673                 to.to_flags = 0;
  674                 /* Maximum segment size. */
  675                 if (flags & TH_SYN) {
  676                         tp->snd_nxt = tp->iss;
  677                         to.to_mss = tcp_mssopt(&tp->t_inpcb->inp_inc);
  678                         to.to_flags |= TOF_MSS;
  679                 }
  680                 /* Window scaling. */
  681                 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
  682                         to.to_wscale = tp->request_r_scale;
  683                         to.to_flags |= TOF_SCALE;
  684                 }
  685                 /* Timestamps. */
  686                 if ((tp->t_flags & TF_RCVD_TSTMP) ||
  687                     ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
  688                         to.to_tsval = ticks + tp->ts_offset;
  689                         to.to_tsecr = tp->ts_recent;
  690                         to.to_flags |= TOF_TS;
  691                         /* Set receive buffer autosizing timestamp. */
  692                         if (tp->rfbuf_ts == 0 &&
  693                             (so->so_rcv.sb_flags & SB_AUTOSIZE))
  694                                 tp->rfbuf_ts = ticks;
  695                 }
  696                 /* Selective ACK's. */
  697                 if (tp->t_flags & TF_SACK_PERMIT) {
  698                         if (flags & TH_SYN)
  699                                 to.to_flags |= TOF_SACKPERM;
  700                         else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
  701                             (tp->t_flags & TF_SACK_PERMIT) &&
  702                             tp->rcv_numsacks > 0) {
  703                                 to.to_flags |= TOF_SACK;
  704                                 to.to_nsacks = tp->rcv_numsacks;
  705                                 to.to_sacks = (u_char *)tp->sackblks;
  706                         }
  707                 }
  708 #ifdef TCP_SIGNATURE
  709                 /* TCP-MD5 (RFC2385). */
  710                 if (tp->t_flags & TF_SIGNATURE)
  711                         to.to_flags |= TOF_SIGNATURE;
  712 #endif /* TCP_SIGNATURE */
  713 
  714                 /* Processing the options. */
  715                 hdrlen += optlen = tcp_addoptions(&to, opt);
  716         }
  717 
  718 #ifdef INET6
  719         if (isipv6)
  720                 ipoptlen = ip6_optlen(tp->t_inpcb);
  721         else
  722 #endif
  723         if (tp->t_inpcb->inp_options)
  724                 ipoptlen = tp->t_inpcb->inp_options->m_len -
  725                                 offsetof(struct ipoption, ipopt_list);
  726         else
  727                 ipoptlen = 0;
  728 #ifdef IPSEC
  729         ipoptlen += ipsec_optlen;
  730 #endif
  731 
  732         /*
  733          * Adjust data length if insertion of options will
  734          * bump the packet length beyond the t_maxopd length.
  735          * Clear the FIN bit because we cut off the tail of
  736          * the segment.
  737          */
  738         if (len + optlen + ipoptlen > tp->t_maxopd) {
  739                 flags &= ~TH_FIN;
  740 
  741                 if (tso) {
  742                         KASSERT(ipoptlen == 0,
  743                             ("%s: TSO can't do IP options", __func__));
  744 
  745                         /*
  746                          * Limit a burst to IP_MAXPACKET minus IP,
  747                          * TCP and options length to keep ip->ip_len
  748                          * from overflowing.
  749                          */
  750                         if (len > IP_MAXPACKET - hdrlen) {
  751                                 len = IP_MAXPACKET - hdrlen;
  752                                 sendalot = 1;
  753                         }
  754 
  755                         /*
  756                          * Prevent the last segment from being
  757                          * fractional unless the send sockbuf can
  758                          * be emptied.
  759                          */
  760                         if (sendalot && off + len < so->so_snd.sb_cc) {
  761                                 len -= len % (tp->t_maxopd - optlen);
  762                                 sendalot = 1;
  763                         }
  764 
  765                         /*
  766                          * Send the FIN in a separate segment
  767                          * after the bulk sending is done.
  768                          * We don't trust the TSO implementations
  769                          * to clear the FIN flag on all but the
  770                          * last segment.
  771                          */
  772                         if (tp->t_flags & TF_NEEDFIN)
  773                                 sendalot = 1;
  774 
  775                 } else {
  776                         len = tp->t_maxopd - optlen - ipoptlen;
  777                         sendalot = 1;
  778                 }
  779         } else
  780                 tso = 0;
  781 
  782         KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET,
  783             ("%s: len > IP_MAXPACKET", __func__));
  784 
  785 /*#ifdef DIAGNOSTIC*/
  786 #ifdef INET6
  787         if (max_linkhdr + hdrlen > MCLBYTES)
  788 #else
  789         if (max_linkhdr + hdrlen > MHLEN)
  790 #endif
  791                 panic("tcphdr too big");
  792 /*#endif*/
  793 
  794         /*
  795          * This KASSERT is here to catch edge cases at a well defined place.
  796          * Before, those had triggered (random) panic conditions further down.
  797          */
  798         KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
  799 
  800         /*
  801          * Grab a header mbuf, attaching a copy of data to
  802          * be transmitted, and initialize the header from
  803          * the template for sends on this connection.
  804          */
  805         if (len) {
  806                 struct mbuf *mb;
  807                 u_int moff;
  808 
  809                 if ((tp->t_flags & TF_FORCEDATA) && len == 1)
  810                         TCPSTAT_INC(tcps_sndprobe);
  811                 else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
  812                         tp->t_sndrexmitpack++;
  813                         TCPSTAT_INC(tcps_sndrexmitpack);
  814                         TCPSTAT_ADD(tcps_sndrexmitbyte, len);
  815                 } else {
  816                         TCPSTAT_INC(tcps_sndpack);
  817                         TCPSTAT_ADD(tcps_sndbyte, len);
  818                 }
  819                 MGETHDR(m, M_DONTWAIT, MT_DATA);
  820                 if (m == NULL) {
  821                         SOCKBUF_UNLOCK(&so->so_snd);
  822                         error = ENOBUFS;
  823                         goto out;
  824                 }
  825 #ifdef INET6
  826                 if (MHLEN < hdrlen + max_linkhdr) {
  827                         MCLGET(m, M_DONTWAIT);
  828                         if ((m->m_flags & M_EXT) == 0) {
  829                                 SOCKBUF_UNLOCK(&so->so_snd);
  830                                 m_freem(m);
  831                                 error = ENOBUFS;
  832                                 goto out;
  833                         }
  834                 }
  835 #endif
  836                 m->m_data += max_linkhdr;
  837                 m->m_len = hdrlen;
  838 
  839                 /*
  840                  * Start the m_copy functions from the closest mbuf
  841                  * to the offset in the socket buffer chain.
  842                  */
  843                 mb = sbsndptr(&so->so_snd, off, len, &moff);
  844 
  845                 if (len <= MHLEN - hdrlen - max_linkhdr) {
  846                         m_copydata(mb, moff, (int)len,
  847                             mtod(m, caddr_t) + hdrlen);
  848                         m->m_len += len;
  849                 } else {
  850                         m->m_next = m_copy(mb, moff, (int)len);
  851                         if (m->m_next == NULL) {
  852                                 SOCKBUF_UNLOCK(&so->so_snd);
  853                                 (void) m_free(m);
  854                                 error = ENOBUFS;
  855                                 goto out;
  856                         }
  857                 }
  858 
  859                 /*
  860                  * If we're sending everything we've got, set PUSH.
  861                  * (This will keep happy those implementations which only
  862                  * give data to the user when a buffer fills or
  863                  * a PUSH comes in.)
  864                  */
  865                 if (off + len == so->so_snd.sb_cc)
  866                         flags |= TH_PUSH;
  867                 SOCKBUF_UNLOCK(&so->so_snd);
  868         } else {
  869                 SOCKBUF_UNLOCK(&so->so_snd);
  870                 if (tp->t_flags & TF_ACKNOW)
  871                         TCPSTAT_INC(tcps_sndacks);
  872                 else if (flags & (TH_SYN|TH_FIN|TH_RST))
  873                         TCPSTAT_INC(tcps_sndctrl);
  874                 else if (SEQ_GT(tp->snd_up, tp->snd_una))
  875                         TCPSTAT_INC(tcps_sndurg);
  876                 else
  877                         TCPSTAT_INC(tcps_sndwinup);
  878 
  879                 MGETHDR(m, M_DONTWAIT, MT_DATA);
  880                 if (m == NULL) {
  881                         error = ENOBUFS;
  882                         goto out;
  883                 }
  884 #ifdef INET6
  885                 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
  886                     MHLEN >= hdrlen) {
  887                         MH_ALIGN(m, hdrlen);
  888                 } else
  889 #endif
  890                 m->m_data += max_linkhdr;
  891                 m->m_len = hdrlen;
  892         }
  893         SOCKBUF_UNLOCK_ASSERT(&so->so_snd);
  894         m->m_pkthdr.rcvif = (struct ifnet *)0;
  895 #ifdef MAC
  896         mac_inpcb_create_mbuf(tp->t_inpcb, m);
  897 #endif
  898 #ifdef INET6
  899         if (isipv6) {
  900                 ip6 = mtod(m, struct ip6_hdr *);
  901                 th = (struct tcphdr *)(ip6 + 1);
  902                 tcpip_fillheaders(tp->t_inpcb, ip6, th);
  903         } else
  904 #endif /* INET6 */
  905         {
  906                 ip = mtod(m, struct ip *);
  907                 ipov = (struct ipovly *)ip;
  908                 th = (struct tcphdr *)(ip + 1);
  909                 tcpip_fillheaders(tp->t_inpcb, ip, th);
  910         }
  911 
  912         /*
  913          * Fill in fields, remembering maximum advertised
  914          * window for use in delaying messages about window sizes.
  915          * If resending a FIN, be sure not to use a new sequence number.
  916          */
  917         if (flags & TH_FIN && tp->t_flags & TF_SENTFIN &&
  918             tp->snd_nxt == tp->snd_max)
  919                 tp->snd_nxt--;
  920         /*
  921          * If we are starting a connection, send ECN setup
  922          * SYN packet. If we are on a retransmit, we may
  923          * resend those bits a number of times as per
  924          * RFC 3168.
  925          */
  926         if (tp->t_state == TCPS_SYN_SENT && V_tcp_do_ecn) {
  927                 if (tp->t_rxtshift >= 1) {
  928                         if (tp->t_rxtshift <= V_tcp_ecn_maxretries)
  929                                 flags |= TH_ECE|TH_CWR;
  930                 } else
  931                         flags |= TH_ECE|TH_CWR;
  932         }
  933         
  934         if (tp->t_state == TCPS_ESTABLISHED &&
  935             (tp->t_flags & TF_ECN_PERMIT)) {
  936                 /*
  937                  * If the peer has ECN, mark data packets with
  938                  * ECN capable transmission (ECT).
  939                  * Ignore pure ack packets, retransmissions and window probes.
  940                  */
  941                 if (len > 0 && SEQ_GEQ(tp->snd_nxt, tp->snd_max) &&
  942                     !((tp->t_flags & TF_FORCEDATA) && len == 1)) {
  943 #ifdef INET6
  944                         if (isipv6)
  945                                 ip6->ip6_flow |= htonl(IPTOS_ECN_ECT0 << 20);
  946                         else
  947 #endif
  948                                 ip->ip_tos |= IPTOS_ECN_ECT0;
  949                         TCPSTAT_INC(tcps_ecn_ect0);
  950                 }
  951                 
  952                 /*
  953                  * Reply with proper ECN notifications.
  954                  */
  955                 if (tp->t_flags & TF_ECN_SND_CWR) {
  956                         flags |= TH_CWR;
  957                         tp->t_flags &= ~TF_ECN_SND_CWR;
  958                 } 
  959                 if (tp->t_flags & TF_ECN_SND_ECE)
  960                         flags |= TH_ECE;
  961         }
  962         
  963         /*
  964          * If we are doing retransmissions, then snd_nxt will
  965          * not reflect the first unsent octet.  For ACK only
  966          * packets, we do not want the sequence number of the
  967          * retransmitted packet, we want the sequence number
  968          * of the next unsent octet.  So, if there is no data
  969          * (and no SYN or FIN), use snd_max instead of snd_nxt
  970          * when filling in ti_seq.  But if we are in persist
  971          * state, snd_max might reflect one byte beyond the
  972          * right edge of the window, so use snd_nxt in that
  973          * case, since we know we aren't doing a retransmission.
  974          * (retransmit and persist are mutually exclusive...)
  975          */
  976         if (sack_rxmit == 0) {
  977                 if (len || (flags & (TH_SYN|TH_FIN)) ||
  978                     tcp_timer_active(tp, TT_PERSIST))
  979                         th->th_seq = htonl(tp->snd_nxt);
  980                 else
  981                         th->th_seq = htonl(tp->snd_max);
  982         } else {
  983                 th->th_seq = htonl(p->rxmit);
  984                 p->rxmit += len;
  985                 tp->sackhint.sack_bytes_rexmit += len;
  986         }
  987         th->th_ack = htonl(tp->rcv_nxt);
  988         if (optlen) {
  989                 bcopy(opt, th + 1, optlen);
  990                 th->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
  991         }
  992         th->th_flags = flags;
  993         /*
  994          * Calculate receive window.  Don't shrink window,
  995          * but avoid silly window syndrome.
  996          */
  997         if (recwin < (long)(so->so_rcv.sb_hiwat / 4) &&
  998             recwin < (long)tp->t_maxseg)
  999                 recwin = 0;
 1000         if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
 1001             recwin < (long)(tp->rcv_adv - tp->rcv_nxt))
 1002                 recwin = (long)(tp->rcv_adv - tp->rcv_nxt);
 1003         if (recwin > (long)TCP_MAXWIN << tp->rcv_scale)
 1004                 recwin = (long)TCP_MAXWIN << tp->rcv_scale;
 1005 
 1006         /*
 1007          * According to RFC1323 the window field in a SYN (i.e., a <SYN>
 1008          * or <SYN,ACK>) segment itself is never scaled.  The <SYN,ACK>
 1009          * case is handled in syncache.
 1010          */
 1011         if (flags & TH_SYN)
 1012                 th->th_win = htons((u_short)
 1013                                 (min(sbspace(&so->so_rcv), TCP_MAXWIN)));
 1014         else
 1015                 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
 1016 
 1017         /*
 1018          * Adjust the RXWIN0SENT flag - indicate that we have advertised
 1019          * a 0 window.  This may cause the remote transmitter to stall.  This
 1020          * flag tells soreceive() to disable delayed acknowledgements when
 1021          * draining the buffer.  This can occur if the receiver is attempting
 1022          * to read more data than can be buffered prior to transmitting on
 1023          * the connection.
 1024          */
 1025         if (th->th_win == 0) {
 1026                 tp->t_sndzerowin++;
 1027                 tp->t_flags |= TF_RXWIN0SENT;
 1028         } else
 1029                 tp->t_flags &= ~TF_RXWIN0SENT;
 1030         if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
 1031                 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
 1032                 th->th_flags |= TH_URG;
 1033         } else
 1034                 /*
 1035                  * If no urgent pointer to send, then we pull
 1036                  * the urgent pointer to the left edge of the send window
 1037                  * so that it doesn't drift into the send window on sequence
 1038                  * number wraparound.
 1039                  */
 1040                 tp->snd_up = tp->snd_una;               /* drag it along */
 1041 
 1042 #ifdef TCP_SIGNATURE
 1043         if (tp->t_flags & TF_SIGNATURE) {
 1044                 int sigoff = to.to_signature - opt;
 1045                 tcp_signature_compute(m, 0, len, optlen,
 1046                     (u_char *)(th + 1) + sigoff, IPSEC_DIR_OUTBOUND);
 1047         }
 1048 #endif
 1049 
 1050         /*
 1051          * Put TCP length in extended header, and then
 1052          * checksum extended header and data.
 1053          */
 1054         m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
 1055 #ifdef INET6
 1056         if (isipv6)
 1057                 /*
 1058                  * ip6_plen is not need to be filled now, and will be filled
 1059                  * in ip6_output.
 1060                  */
 1061                 th->th_sum = in6_cksum(m, IPPROTO_TCP, sizeof(struct ip6_hdr),
 1062                                        sizeof(struct tcphdr) + optlen + len);
 1063         else
 1064 #endif /* INET6 */
 1065         {
 1066                 m->m_pkthdr.csum_flags = CSUM_TCP;
 1067                 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
 1068                 th->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
 1069                     htons(sizeof(struct tcphdr) + IPPROTO_TCP + len + optlen));
 1070 
 1071                 /* IP version must be set here for ipv4/ipv6 checking later */
 1072                 KASSERT(ip->ip_v == IPVERSION,
 1073                     ("%s: IP version incorrect: %d", __func__, ip->ip_v));
 1074         }
 1075 
 1076         /*
 1077          * Enable TSO and specify the size of the segments.
 1078          * The TCP pseudo header checksum is always provided.
 1079          * XXX: Fixme: This is currently not the case for IPv6.
 1080          */
 1081         if (tso) {
 1082                 KASSERT(len > tp->t_maxopd - optlen,
 1083                     ("%s: len <= tso_segsz", __func__));
 1084                 m->m_pkthdr.csum_flags |= CSUM_TSO;
 1085                 m->m_pkthdr.tso_segsz = tp->t_maxopd - optlen;
 1086         }
 1087 
 1088 #ifdef IPSEC
 1089         KASSERT(len + hdrlen + ipoptlen - ipsec_optlen == m_length(m, NULL),
 1090             ("%s: mbuf chain shorter than expected: %ld + %u + %u - %u != %u",
 1091             __func__, len, hdrlen, ipoptlen, ipsec_optlen, m_length(m, NULL)));
 1092 #else
 1093         KASSERT(len + hdrlen + ipoptlen == m_length(m, NULL),
 1094             ("%s: mbuf chain shorter than expected: %ld + %u + %u != %u",
 1095             __func__, len, hdrlen, ipoptlen, m_length(m, NULL)));
 1096 #endif
 1097 
 1098         /*
 1099          * In transmit state, time the transmission and arrange for
 1100          * the retransmit.  In persist state, just set snd_max.
 1101          */
 1102         if ((tp->t_flags & TF_FORCEDATA) == 0 || 
 1103             !tcp_timer_active(tp, TT_PERSIST)) {
 1104                 tcp_seq startseq = tp->snd_nxt;
 1105 
 1106                 /*
 1107                  * Advance snd_nxt over sequence space of this segment.
 1108                  */
 1109                 if (flags & (TH_SYN|TH_FIN)) {
 1110                         if (flags & TH_SYN)
 1111                                 tp->snd_nxt++;
 1112                         if (flags & TH_FIN) {
 1113                                 tp->snd_nxt++;
 1114                                 tp->t_flags |= TF_SENTFIN;
 1115                         }
 1116                 }
 1117                 if (sack_rxmit)
 1118                         goto timer;
 1119                 tp->snd_nxt += len;
 1120                 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
 1121                         tp->snd_max = tp->snd_nxt;
 1122                         /*
 1123                          * Time this transmission if not a retransmission and
 1124                          * not currently timing anything.
 1125                          */
 1126                         if (tp->t_rtttime == 0) {
 1127                                 tp->t_rtttime = ticks;
 1128                                 tp->t_rtseq = startseq;
 1129                                 TCPSTAT_INC(tcps_segstimed);
 1130                         }
 1131                 }
 1132 
 1133                 /*
 1134                  * Set retransmit timer if not currently set,
 1135                  * and not doing a pure ack or a keep-alive probe.
 1136                  * Initial value for retransmit timer is smoothed
 1137                  * round-trip time + 2 * round-trip time variance.
 1138                  * Initialize shift counter which is used for backoff
 1139                  * of retransmit time.
 1140                  */
 1141 timer:
 1142                 if (!tcp_timer_active(tp, TT_REXMT) &&
 1143                     ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
 1144                      (tp->snd_nxt != tp->snd_una))) {
 1145                         if (tcp_timer_active(tp, TT_PERSIST)) {
 1146                                 tcp_timer_activate(tp, TT_PERSIST, 0);
 1147                                 tp->t_rxtshift = 0;
 1148                         }
 1149                         tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur);
 1150                 }
 1151         } else {
 1152                 /*
 1153                  * Persist case, update snd_max but since we are in
 1154                  * persist mode (no window) we do not update snd_nxt.
 1155                  */
 1156                 int xlen = len;
 1157                 if (flags & TH_SYN)
 1158                         ++xlen;
 1159                 if (flags & TH_FIN) {
 1160                         ++xlen;
 1161                         tp->t_flags |= TF_SENTFIN;
 1162                 }
 1163                 if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max))
 1164                         tp->snd_max = tp->snd_nxt + len;
 1165         }
 1166 
 1167         /* Run HHOOK_TCP_ESTABLISHED_OUT helper hooks. */
 1168         hhook_run_tcp_est_out(tp, th, &to, len, tso);
 1169 
 1170 #ifdef TCPDEBUG
 1171         /*
 1172          * Trace.
 1173          */
 1174         if (so->so_options & SO_DEBUG) {
 1175                 u_short save = 0;
 1176 #ifdef INET6
 1177                 if (!isipv6)
 1178 #endif
 1179                 {
 1180                         save = ipov->ih_len;
 1181                         ipov->ih_len = htons(m->m_pkthdr.len /* - hdrlen + (th->th_off << 2) */);
 1182                 }
 1183                 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
 1184 #ifdef INET6
 1185                 if (!isipv6)
 1186 #endif
 1187                 ipov->ih_len = save;
 1188         }
 1189 #endif /* TCPDEBUG */
 1190 
 1191         /*
 1192          * Fill in IP length and desired time to live and
 1193          * send to IP level.  There should be a better way
 1194          * to handle ttl and tos; we could keep them in
 1195          * the template, but need a way to checksum without them.
 1196          */
 1197         /*
 1198          * m->m_pkthdr.len should have been set before cksum calcuration,
 1199          * because in6_cksum() need it.
 1200          */
 1201 #ifdef INET6
 1202         if (isipv6) {
 1203                 /*
 1204                  * we separately set hoplimit for every segment, since the
 1205                  * user might want to change the value via setsockopt.
 1206                  * Also, desired default hop limit might be changed via
 1207                  * Neighbor Discovery.
 1208                  */
 1209                 ip6->ip6_hlim = in6_selecthlim(tp->t_inpcb, NULL);
 1210 
 1211                 /* TODO: IPv6 IP6TOS_ECT bit on */
 1212                 error = ip6_output(m,
 1213                             tp->t_inpcb->in6p_outputopts, NULL,
 1214                             ((so->so_options & SO_DONTROUTE) ?
 1215                             IP_ROUTETOIF : 0), NULL, NULL, tp->t_inpcb);
 1216         }
 1217 #endif /* INET6 */
 1218 #if defined(INET) && defined(INET6)
 1219         else
 1220 #endif
 1221 #ifdef INET
 1222     {
 1223         ip->ip_len = m->m_pkthdr.len;
 1224 #ifdef INET6
 1225         if (tp->t_inpcb->inp_vflag & INP_IPV6PROTO)
 1226                 ip->ip_ttl = in6_selecthlim(tp->t_inpcb, NULL);
 1227 #endif /* INET6 */
 1228         /*
 1229          * If we do path MTU discovery, then we set DF on every packet.
 1230          * This might not be the best thing to do according to RFC3390
 1231          * Section 2. However the tcp hostcache migitates the problem
 1232          * so it affects only the first tcp connection with a host.
 1233          *
 1234          * NB: Don't set DF on small MTU/MSS to have a safe fallback.
 1235          */
 1236         if (V_path_mtu_discovery && tp->t_maxopd > V_tcp_minmss)
 1237                 ip->ip_off |= IP_DF;
 1238 
 1239         error = ip_output(m, tp->t_inpcb->inp_options, NULL,
 1240             ((so->so_options & SO_DONTROUTE) ? IP_ROUTETOIF : 0), 0,
 1241             tp->t_inpcb);
 1242     }
 1243 #endif /* INET */
 1244         if (error) {
 1245 
 1246                 /*
 1247                  * We know that the packet was lost, so back out the
 1248                  * sequence number advance, if any.
 1249                  *
 1250                  * If the error is EPERM the packet got blocked by the
 1251                  * local firewall.  Normally we should terminate the
 1252                  * connection but the blocking may have been spurious
 1253                  * due to a firewall reconfiguration cycle.  So we treat
 1254                  * it like a packet loss and let the retransmit timer and
 1255                  * timeouts do their work over time.
 1256                  * XXX: It is a POLA question whether calling tcp_drop right
 1257                  * away would be the really correct behavior instead.
 1258                  */
 1259                 if (((tp->t_flags & TF_FORCEDATA) == 0 ||
 1260                     !tcp_timer_active(tp, TT_PERSIST)) &&
 1261                     ((flags & TH_SYN) == 0) &&
 1262                     (error != EPERM)) {
 1263                         if (sack_rxmit) {
 1264                                 p->rxmit -= len;
 1265                                 tp->sackhint.sack_bytes_rexmit -= len;
 1266                                 KASSERT(tp->sackhint.sack_bytes_rexmit >= 0,
 1267                                     ("sackhint bytes rtx >= 0"));
 1268                         } else
 1269                                 tp->snd_nxt -= len;
 1270                 }
 1271 out:
 1272                 SOCKBUF_UNLOCK_ASSERT(&so->so_snd);     /* Check gotos. */
 1273                 switch (error) {
 1274                 case EPERM:
 1275                         tp->t_softerror = error;
 1276                         return (error);
 1277                 case ENOBUFS:
 1278                         if (!tcp_timer_active(tp, TT_REXMT) &&
 1279                             !tcp_timer_active(tp, TT_PERSIST))
 1280                                 tcp_timer_activate(tp, TT_REXMT, tp->t_rxtcur);
 1281                         tp->snd_cwnd = tp->t_maxseg;
 1282                         return (0);
 1283                 case EMSGSIZE:
 1284                         /*
 1285                          * For some reason the interface we used initially
 1286                          * to send segments changed to another or lowered
 1287                          * its MTU.
 1288                          *
 1289                          * tcp_mtudisc() will find out the new MTU and as
 1290                          * its last action, initiate retransmission, so it
 1291                          * is important to not do so here.
 1292                          *
 1293                          * If TSO was active we either got an interface
 1294                          * without TSO capabilits or TSO was turned off.
 1295                          * Disable it for this connection as too and
 1296                          * immediatly retry with MSS sized segments generated
 1297                          * by this function.
 1298                          */
 1299                         if (tso)
 1300                                 tp->t_flags &= ~TF_TSO;
 1301                         tcp_mtudisc(tp->t_inpcb, 0);
 1302                         return (0);
 1303                 case EHOSTDOWN:
 1304                 case EHOSTUNREACH:
 1305                 case ENETDOWN:
 1306                 case ENETUNREACH:
 1307                         if (TCPS_HAVERCVDSYN(tp->t_state)) {
 1308                                 tp->t_softerror = error;
 1309                                 return (0);
 1310                         }
 1311                         /* FALLTHROUGH */
 1312                 default:
 1313                         return (error);
 1314                 }
 1315         }
 1316         TCPSTAT_INC(tcps_sndtotal);
 1317 
 1318         /*
 1319          * Data sent (as far as we can tell).
 1320          * If this advertises a larger window than any other segment,
 1321          * then remember the size of the advertised window.
 1322          * Any pending ACK has now been sent.
 1323          */
 1324         if (recwin >= 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
 1325                 tp->rcv_adv = tp->rcv_nxt + recwin;
 1326         tp->last_ack_sent = tp->rcv_nxt;
 1327         tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
 1328         if (tcp_timer_active(tp, TT_DELACK))
 1329                 tcp_timer_activate(tp, TT_DELACK, 0);
 1330 #if 0
 1331         /*
 1332          * This completely breaks TCP if newreno is turned on.  What happens
 1333          * is that if delayed-acks are turned on on the receiver, this code
 1334          * on the transmitter effectively destroys the TCP window, forcing
 1335          * it to four packets (1.5Kx4 = 6K window).
 1336          */
 1337         if (sendalot && --maxburst)
 1338                 goto again;
 1339 #endif
 1340         if (sendalot)
 1341                 goto again;
 1342         return (0);
 1343 }
 1344 
 1345 void
 1346 tcp_setpersist(struct tcpcb *tp)
 1347 {
 1348         int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1;
 1349         int tt;
 1350 
 1351         tp->t_flags &= ~TF_PREVVALID;
 1352         if (tcp_timer_active(tp, TT_REXMT))
 1353                 panic("tcp_setpersist: retransmit pending");
 1354         /*
 1355          * Start/restart persistance timer.
 1356          */
 1357         TCPT_RANGESET(tt, t * tcp_backoff[tp->t_rxtshift],
 1358                       TCPTV_PERSMIN, TCPTV_PERSMAX);
 1359         tcp_timer_activate(tp, TT_PERSIST, tt);
 1360         if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
 1361                 tp->t_rxtshift++;
 1362 }
 1363 
 1364 /*
 1365  * Insert TCP options according to the supplied parameters to the place
 1366  * optp in a consistent way.  Can handle unaligned destinations.
 1367  *
 1368  * The order of the option processing is crucial for optimal packing and
 1369  * alignment for the scarce option space.
 1370  *
 1371  * The optimal order for a SYN/SYN-ACK segment is:
 1372  *   MSS (4) + NOP (1) + Window scale (3) + SACK permitted (2) +
 1373  *   Timestamp (10) + Signature (18) = 38 bytes out of a maximum of 40.
 1374  *
 1375  * The SACK options should be last.  SACK blocks consume 8*n+2 bytes.
 1376  * So a full size SACK blocks option is 34 bytes (with 4 SACK blocks).
 1377  * At minimum we need 10 bytes (to generate 1 SACK block).  If both
 1378  * TCP Timestamps (12 bytes) and TCP Signatures (18 bytes) are present,
 1379  * we only have 10 bytes for SACK options (40 - (12 + 18)).
 1380  */
 1381 int
 1382 tcp_addoptions(struct tcpopt *to, u_char *optp)
 1383 {
 1384         u_int mask, optlen = 0;
 1385 
 1386         for (mask = 1; mask < TOF_MAXOPT; mask <<= 1) {
 1387                 if ((to->to_flags & mask) != mask)
 1388                         continue;
 1389                 if (optlen == TCP_MAXOLEN)
 1390                         break;
 1391                 switch (to->to_flags & mask) {
 1392                 case TOF_MSS:
 1393                         while (optlen % 4) {
 1394                                 optlen += TCPOLEN_NOP;
 1395                                 *optp++ = TCPOPT_NOP;
 1396                         }
 1397                         if (TCP_MAXOLEN - optlen < TCPOLEN_MAXSEG)
 1398                                 continue;
 1399                         optlen += TCPOLEN_MAXSEG;
 1400                         *optp++ = TCPOPT_MAXSEG;
 1401                         *optp++ = TCPOLEN_MAXSEG;
 1402                         to->to_mss = htons(to->to_mss);
 1403                         bcopy((u_char *)&to->to_mss, optp, sizeof(to->to_mss));
 1404                         optp += sizeof(to->to_mss);
 1405                         break;
 1406                 case TOF_SCALE:
 1407                         while (!optlen || optlen % 2 != 1) {
 1408                                 optlen += TCPOLEN_NOP;
 1409                                 *optp++ = TCPOPT_NOP;
 1410                         }
 1411                         if (TCP_MAXOLEN - optlen < TCPOLEN_WINDOW)
 1412                                 continue;
 1413                         optlen += TCPOLEN_WINDOW;
 1414                         *optp++ = TCPOPT_WINDOW;
 1415                         *optp++ = TCPOLEN_WINDOW;
 1416                         *optp++ = to->to_wscale;
 1417                         break;
 1418                 case TOF_SACKPERM:
 1419                         while (optlen % 2) {
 1420                                 optlen += TCPOLEN_NOP;
 1421                                 *optp++ = TCPOPT_NOP;
 1422                         }
 1423                         if (TCP_MAXOLEN - optlen < TCPOLEN_SACK_PERMITTED)
 1424                                 continue;
 1425                         optlen += TCPOLEN_SACK_PERMITTED;
 1426                         *optp++ = TCPOPT_SACK_PERMITTED;
 1427                         *optp++ = TCPOLEN_SACK_PERMITTED;
 1428                         break;
 1429                 case TOF_TS:
 1430                         while (!optlen || optlen % 4 != 2) {
 1431                                 optlen += TCPOLEN_NOP;
 1432                                 *optp++ = TCPOPT_NOP;
 1433                         }
 1434                         if (TCP_MAXOLEN - optlen < TCPOLEN_TIMESTAMP)
 1435                                 continue;
 1436                         optlen += TCPOLEN_TIMESTAMP;
 1437                         *optp++ = TCPOPT_TIMESTAMP;
 1438                         *optp++ = TCPOLEN_TIMESTAMP;
 1439                         to->to_tsval = htonl(to->to_tsval);
 1440                         to->to_tsecr = htonl(to->to_tsecr);
 1441                         bcopy((u_char *)&to->to_tsval, optp, sizeof(to->to_tsval));
 1442                         optp += sizeof(to->to_tsval);
 1443                         bcopy((u_char *)&to->to_tsecr, optp, sizeof(to->to_tsecr));
 1444                         optp += sizeof(to->to_tsecr);
 1445                         break;
 1446                 case TOF_SIGNATURE:
 1447                         {
 1448                         int siglen = TCPOLEN_SIGNATURE - 2;
 1449 
 1450                         while (!optlen || optlen % 4 != 2) {
 1451                                 optlen += TCPOLEN_NOP;
 1452                                 *optp++ = TCPOPT_NOP;
 1453                         }
 1454                         if (TCP_MAXOLEN - optlen < TCPOLEN_SIGNATURE)
 1455                                 continue;
 1456                         optlen += TCPOLEN_SIGNATURE;
 1457                         *optp++ = TCPOPT_SIGNATURE;
 1458                         *optp++ = TCPOLEN_SIGNATURE;
 1459                         to->to_signature = optp;
 1460                         while (siglen--)
 1461                                  *optp++ = 0;
 1462                         break;
 1463                         }
 1464                 case TOF_SACK:
 1465                         {
 1466                         int sackblks = 0;
 1467                         struct sackblk *sack = (struct sackblk *)to->to_sacks;
 1468                         tcp_seq sack_seq;
 1469 
 1470                         while (!optlen || optlen % 4 != 2) {
 1471                                 optlen += TCPOLEN_NOP;
 1472                                 *optp++ = TCPOPT_NOP;
 1473                         }
 1474                         if (TCP_MAXOLEN - optlen < TCPOLEN_SACKHDR + TCPOLEN_SACK)
 1475                                 continue;
 1476                         optlen += TCPOLEN_SACKHDR;
 1477                         *optp++ = TCPOPT_SACK;
 1478                         sackblks = min(to->to_nsacks,
 1479                                         (TCP_MAXOLEN - optlen) / TCPOLEN_SACK);
 1480                         *optp++ = TCPOLEN_SACKHDR + sackblks * TCPOLEN_SACK;
 1481                         while (sackblks--) {
 1482                                 sack_seq = htonl(sack->start);
 1483                                 bcopy((u_char *)&sack_seq, optp, sizeof(sack_seq));
 1484                                 optp += sizeof(sack_seq);
 1485                                 sack_seq = htonl(sack->end);
 1486                                 bcopy((u_char *)&sack_seq, optp, sizeof(sack_seq));
 1487                                 optp += sizeof(sack_seq);
 1488                                 optlen += TCPOLEN_SACK;
 1489                                 sack++;
 1490                         }
 1491                         TCPSTAT_INC(tcps_sack_send_blocks);
 1492                         break;
 1493                         }
 1494                 default:
 1495                         panic("%s: unknown TCP option type", __func__);
 1496                         break;
 1497                 }
 1498         }
 1499 
 1500         /* Terminate and pad TCP options to a 4 byte boundary. */
 1501         if (optlen % 4) {
 1502                 optlen += TCPOLEN_EOL;
 1503                 *optp++ = TCPOPT_EOL;
 1504         }
 1505         /*
 1506          * According to RFC 793 (STD0007):
 1507          *   "The content of the header beyond the End-of-Option option
 1508          *    must be header padding (i.e., zero)."
 1509          *   and later: "The padding is composed of zeros."
 1510          */
 1511         while (optlen % 4) {
 1512                 optlen += TCPOLEN_PAD;
 1513                 *optp++ = TCPOPT_PAD;
 1514         }
 1515 
 1516         KASSERT(optlen <= TCP_MAXOLEN, ("%s: TCP options too long", __func__));
 1517         return (optlen);
 1518 }

Cache object: 01c04d9877b2f564f7d8a4932ad2f6df


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.