The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/netinet/tcp_hpts.h

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2016-2018 Netflix, Inc.
    3  *
    4  * Redistribution and use in source and binary forms, with or without
    5  * modification, are permitted provided that the following conditions
    6  * are met:
    7  * 1. Redistributions of source code must retain the above copyright
    8  *    notice, this list of conditions and the following disclaimer.
    9  * 2. Redistributions in binary form must reproduce the above copyright
   10  *    notice, this list of conditions and the following disclaimer in the
   11  *    documentation and/or other materials provided with the distribution.
   12  *
   13  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   23  * SUCH DAMAGE.
   24  *
   25  * $FreeBSD$
   26  */
   27 
   28 #ifndef __tcp_hpts_h__
   29 #define __tcp_hpts_h__
   30 
   31 /*
   32  * The hpts uses a 102400 wheel. The wheel
   33  * defines the time in 10 usec increments (102400 x 10).
   34  * This gives a range of 10usec - 1024ms to place
   35  * an entry within. If the user requests more than
   36  * 1.024 second, a remaineder is attached and the hpts
   37  * when seeing the remainder will re-insert the
   38  * inpcb forward in time from where it is until
   39  * the remainder is zero.
   40  */
   41 
   42 #define NUM_OF_HPTSI_SLOTS 102400
   43 
   44 TAILQ_HEAD(hptsh, inpcb);
   45 
   46 /* Number of useconds in a hpts tick */
   47 #define HPTS_TICKS_PER_USEC 10
   48 #define HPTS_MS_TO_SLOTS(x) ((x * 100) + 1)
   49 #define HPTS_USEC_TO_SLOTS(x) ((x+9) /10)
   50 #define HPTS_USEC_IN_SEC 1000000
   51 #define HPTS_MSEC_IN_SEC 1000
   52 #define HPTS_USEC_IN_MSEC 1000
   53 
   54 
   55 struct hpts_diag {
   56         uint32_t p_hpts_active;         /* bbr->flex7 x */
   57         uint32_t p_nxt_slot;            /* bbr->flex1 x */
   58         uint32_t p_cur_slot;            /* bbr->flex2 x */
   59         uint32_t p_prev_slot;           /* bbr->delivered */
   60         uint32_t p_runningtick;         /* bbr->inflight */
   61         uint32_t slot_req;              /* bbr->flex3 x */
   62         uint32_t inp_hptsslot;          /* bbr->flex4 x */
   63         uint32_t slot_remaining;        /* bbr->flex5 x */
   64         uint32_t have_slept;            /* bbr->epoch x */
   65         uint32_t hpts_sleep_time;       /* bbr->applimited x */
   66         uint32_t yet_to_sleep;          /* bbr->lt_epoch x */
   67         uint32_t need_new_to;           /* bbr->flex6 x  */
   68         uint32_t wheel_tick;            /* bbr->bw_inuse x */
   69         uint32_t maxticks;              /* bbr->delRate x */
   70         uint32_t wheel_cts;             /* bbr->rttProp x */
   71         int32_t co_ret;                 /* bbr->pkts_out x */
   72         uint32_t p_curtick;             /* upper bbr->cur_del_rate */
   73         uint32_t p_lasttick;            /* lower bbr->cur_del_rate */
   74         uint8_t p_on_min_sleep;         /* bbr->flex8 x */
   75 };
   76 
   77 /* Magic flags to tell whats cooking on the pacing wheel */
   78 #define PACE_TMR_DELACK 0x01    /* Delayed ack timer running */
   79 #define PACE_TMR_RACK   0x02    /* RACK timer running */
   80 #define PACE_TMR_TLP    0x04    /* TLP timer running */
   81 #define PACE_TMR_RXT    0x08    /* Retransmit timer running */
   82 #define PACE_TMR_PERSIT 0x10    /* Persists timer running */
   83 #define PACE_TMR_KEEP   0x20    /* Keep alive timer running */
   84 #define PACE_PKT_OUTPUT 0x40    /* Output Packets being paced */
   85 #define PACE_TMR_MASK   (PACE_TMR_KEEP|PACE_TMR_PERSIT|PACE_TMR_RXT|PACE_TMR_TLP|PACE_TMR_RACK|PACE_TMR_DELACK)
   86 
   87 #ifdef _KERNEL
   88 /* Each hpts has its own p_mtx which is used for locking */
   89 struct tcp_hpts_entry {
   90         /* Cache line 0x00 */
   91         struct mtx p_mtx;       /* Mutex for hpts */
   92         uint16_t p_hpts_active; /* Flag that says hpts is awake  */
   93         uint8_t p_hpts_wake_scheduled;  /* Have we scheduled a wakeup? */
   94         uint8_t p_wheel_complete; /* have we completed the wheel arc walk? */
   95         uint32_t p_curtick;     /* Tick in 10 us the hpts is going to */
   96         uint32_t p_runningtick; /* Current tick we are at if we are running */
   97         uint32_t p_prev_slot;   /* Previous slot we were on */
   98         uint32_t p_cur_slot;    /* Current slot in wheel hpts is draining */
   99         uint32_t p_nxt_slot;    /* The next slot outside the current range of
  100                                  * slots that the hpts is running on. */
  101         int32_t p_on_queue_cnt; /* Count on queue in this hpts */
  102         uint32_t p_lasttick;    /* Last tick before the current one */
  103         uint8_t p_direct_wake :1, /* boolean */
  104                 p_on_min_sleep:1, /* boolean */
  105                 p_avail:6; 
  106         uint8_t p_fill[3];        /* Fill to 32 bits */
  107         /* Cache line 0x40 */
  108         void *p_inp;
  109         struct hptsh p_input;   /* For the tcp-input runner */
  110         /* Hptsi wheel */
  111         struct hptsh *p_hptss;
  112         int32_t p_on_inqueue_cnt; /* Count on input queue in this hpts */
  113         uint32_t hit_no_enobuf;
  114         uint32_t p_dyn_adjust;
  115         uint32_t p_hpts_sleep_time;     /* Current sleep interval having a max
  116                                          * of 255ms */
  117         uint32_t overidden_sleep;       /* what was overrided by min-sleep for logging */
  118         uint32_t saved_lasttick;        /* for logging */
  119         uint32_t saved_curtick;         /* for logging */
  120         uint32_t saved_curslot;         /* for logging */
  121         uint32_t saved_prev_slot;       /* for logging */
  122         uint32_t p_delayed_by;  /* How much were we delayed by */
  123         /* Cache line 0x80 */
  124         struct sysctl_ctx_list hpts_ctx;
  125         struct sysctl_oid *hpts_root;
  126         struct intr_event *ie;
  127         void *ie_cookie;
  128         uint16_t p_num;         /* The hpts number one per cpu */
  129         uint16_t p_cpu;         /* The hpts CPU */
  130         /* There is extra space in here */
  131         /* Cache line 0x100 */
  132         struct callout co __aligned(CACHE_LINE_SIZE);
  133 }               __aligned(CACHE_LINE_SIZE);
  134 
  135 struct tcp_hptsi {
  136         struct proc *rp_proc;   /* Process structure for hpts */
  137         struct tcp_hpts_entry **rp_ent; /* Array of hptss */
  138         uint32_t rp_num_hptss;  /* Number of hpts threads */
  139 };
  140 
  141 #endif
  142 
  143 #define HPTS_REMOVE_INPUT  0x01
  144 #define HPTS_REMOVE_OUTPUT 0x02
  145 #define HPTS_REMOVE_ALL    (HPTS_REMOVE_INPUT | HPTS_REMOVE_OUTPUT)
  146 
  147 /*
  148  * When using the hpts, a TCP stack must make sure
  149  * that once a INP_DROPPED flag is applied to a INP
  150  * that it does not expect tcp_output() to ever be
  151  * called by the hpts. The hpts will *not* call
  152  * any output (or input) functions on a TCB that
  153  * is in the DROPPED state.
  154  *
  155  * This implies final ACK's and RST's that might
  156  * be sent when a TCB is still around must be
  157  * sent from a routine like tcp_respond().
  158  */
  159 #define DEFAULT_MIN_SLEEP 250   /* How many usec's is default for hpts sleep
  160                                  * this determines min granularity of the
  161                                  * hpts. If 0, granularity is 10useconds at
  162                                  * the cost of more CPU (context switching). */
  163 #ifdef _KERNEL
  164 #define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED)
  165 struct tcp_hpts_entry *tcp_hpts_lock(struct inpcb *inp);
  166 struct tcp_hpts_entry *tcp_input_lock(struct inpcb *inp);
  167 int __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line);
  168 #define tcp_queue_to_hpts_immediate(a)__tcp_queue_to_hpts_immediate(a, __LINE__)
  169 
  170 struct tcp_hpts_entry *tcp_cur_hpts(struct inpcb *inp);
  171 #define tcp_hpts_remove(a, b) __tcp_hpts_remove(a, b, __LINE__)
  172 void __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line);
  173 
  174 /*
  175  * To insert a TCB on the hpts you *must* be holding the
  176  * INP_WLOCK(). The hpts insert code will then acqurire
  177  * the hpts's lock and insert the TCB on the requested
  178  * slot possibly waking up the hpts if you are requesting
  179  * a time earlier than what the hpts is sleeping to (if
  180  * the hpts is sleeping). You may check the inp->inp_in_hpts
  181  * flag without the hpts lock. The hpts is the only one
  182  * that will clear this flag holding only the hpts lock. This
  183  * means that in your tcp_output() routine when you test for
  184  * it to be 1 (so you wont call output) it may be transitioning
  185  * to 0 (by the hpts). That will be fine since that will just
  186  * mean an extra call to tcp_output that most likely will find
  187  * the call you executed (when the mis-match occured) will have
  188  * put the TCB back on the hpts and it will return. If your
  189  * call did not add it back to the hpts then you will either
  190  * over-send or the cwnd will block you from sending more.
  191  *
  192  * Note you should also be holding the INP_WLOCK() when you
  193  * call the remove from the hpts as well. Thoug usually
  194  * you are either doing this from a timer, where you need
  195  * that INP_WLOCK() or from destroying your TCB where again
  196  * you should already have the INP_WLOCK().
  197  */
  198 uint32_t __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line);
  199 #define tcp_hpts_insert(a, b) __tcp_hpts_insert(a, b, __LINE__)
  200 
  201 uint32_t
  202 tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag);
  203 
  204 int
  205     __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line);
  206 #define tcp_queue_to_input_locked(a, b) __tcp_queue_to_input_locked(a, b, __LINE__);
  207 int
  208 __tcp_queue_to_input(struct inpcb *inp, int32_t line);
  209 #define tcp_queue_to_input(a) __tcp_queue_to_input(a, __LINE__)
  210 
  211 uint16_t tcp_hpts_delayedby(struct inpcb *inp);
  212 
  213 void __tcp_set_hpts(struct inpcb *inp, int32_t line);
  214 #define tcp_set_hpts(a) __tcp_set_hpts(a, __LINE__)
  215 
  216 void __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line);
  217 #define tcp_set_inp_to_drop(a, b) __tcp_set_inp_to_drop(a, b, __LINE__)
  218 
  219 extern int32_t tcp_min_hptsi_time;
  220 
  221 static __inline uint32_t
  222 tcp_tv_to_hptstick(struct timeval *sv)
  223 {
  224         return ((sv->tv_sec * 100000) + (sv->tv_usec / 10));
  225 }
  226 
  227 static __inline uint32_t
  228 tcp_gethptstick(struct timeval *sv)
  229 {
  230         struct timeval tv;
  231 
  232         if (sv == NULL)
  233                 sv = &tv;
  234         microuptime(sv);
  235         return (tcp_tv_to_hptstick(sv));
  236 }
  237 
  238 static __inline uint32_t
  239 tcp_tv_to_usectick(struct timeval *sv)
  240 {
  241         return ((uint32_t) ((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec));
  242 }
  243 
  244 static __inline uint32_t
  245 tcp_tv_to_mssectick(struct timeval *sv)
  246 {
  247         return ((uint32_t) ((sv->tv_sec * HPTS_MSEC_IN_SEC) + (sv->tv_usec/HPTS_USEC_IN_MSEC)));
  248 }
  249 
  250 static __inline void
  251 tcp_hpts_unlock(struct tcp_hpts_entry *hpts)
  252 {
  253         mtx_unlock(&hpts->p_mtx);
  254 }
  255 
  256 static __inline uint32_t
  257 tcp_get_usecs(struct timeval *tv)
  258 {
  259         struct timeval tvd;
  260 
  261         if (tv == NULL)
  262                 tv = &tvd;
  263         microuptime(tv);
  264         return (tcp_tv_to_usectick(tv));
  265 }
  266 
  267 #endif /* _KERNEL */
  268 #endif /* __tcp_hpts_h__ */

Cache object: eea08c2c47ef5e0c863a1cb8b1a24cd1


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.