1 /*-
2 * Copyright (c) 2016-2018 Netflix Inc.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 */
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD: releng/12.0/sys/netinet/tcp_hpts.c 339039 2018-10-01 10:46:00Z ae $");
28
29 #include "opt_inet.h"
30 #include "opt_inet6.h"
31 #include "opt_tcpdebug.h"
32 /**
33 * Some notes about usage.
34 *
35 * The tcp_hpts system is designed to provide a high precision timer
36 * system for tcp. Its main purpose is to provide a mechanism for
37 * pacing packets out onto the wire. It can be used in two ways
38 * by a given TCP stack (and those two methods can be used simultaneously).
39 *
40 * First, and probably the main thing its used by Rack and BBR for, it can
41 * be used to call tcp_output() of a transport stack at some time in the future.
42 * The normal way this is done is that tcp_output() of the stack schedules
43 * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The
44 * slot is the time from now that the stack wants to be called but it
45 * must be converted to tcp_hpts's notion of slot. This is done with
46 * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical
47 * call from the tcp_output() routine might look like:
48 *
49 * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550));
50 *
51 * The above would schedule tcp_ouput() to be called in 550 useconds.
52 * Note that if using this mechanism the stack will want to add near
53 * its top a check to prevent unwanted calls (from user land or the
54 * arrival of incoming ack's). So it would add something like:
55 *
56 * if (inp->inp_in_hpts)
57 * return;
58 *
59 * to prevent output processing until the time alotted has gone by.
60 * Of course this is a bare bones example and the stack will probably
61 * have more consideration then just the above.
62 *
63 * Now the tcp_hpts system will call tcp_output in one of two forms,
64 * it will first check to see if the stack as defined a
65 * tfb_tcp_output_wtime() function, if so that is the routine it
66 * will call, if that function is not defined then it will call the
67 * tfb_tcp_output() function. The only difference between these
68 * two calls is that the former passes the time in to the function
69 * so the function does not have to access the time (which tcp_hpts
70 * already has). What these functions do is of course totally up
71 * to the individual tcp stack.
72 *
73 * Now the second function (actually two functions I guess :D)
74 * the tcp_hpts system provides is the ability to either abort
75 * a connection (later) or process input on a connection.
76 * Why would you want to do this? To keep processor locality.
77 *
78 * So in order to use the input redirection function the
79 * stack changes its tcp_do_segment() routine to instead
80 * of process the data call the function:
81 *
82 * tcp_queue_pkt_to_input()
83 *
84 * You will note that the arguments to this function look
85 * a lot like tcp_do_segments's arguments. This function
86 * will assure that the tcp_hpts system will
87 * call the functions tfb_tcp_hpts_do_segment() from the
88 * correct CPU. Note that multiple calls can get pushed
89 * into the tcp_hpts system this will be indicated by
90 * the next to last argument to tfb_tcp_hpts_do_segment()
91 * (nxt_pkt). If nxt_pkt is a 1 then another packet is
92 * coming. If nxt_pkt is a 0 then this is the last call
93 * that the tcp_hpts system has available for the tcp stack.
94 *
95 * The other point of the input system is to be able to safely
96 * drop a tcp connection without worrying about the recursive
97 * locking that may be occuring on the INP_WLOCK. So if
98 * a stack wants to drop a connection it calls:
99 *
100 * tcp_set_inp_to_drop(tp, ETIMEDOUT)
101 *
102 * To schedule the tcp_hpts system to call
103 *
104 * tcp_drop(tp, drop_reason)
105 *
106 * at a future point. This is quite handy to prevent locking
107 * issues when dropping connections.
108 *
109 */
110
111 #include <sys/param.h>
112 #include <sys/bus.h>
113 #include <sys/interrupt.h>
114 #include <sys/module.h>
115 #include <sys/kernel.h>
116 #include <sys/hhook.h>
117 #include <sys/malloc.h>
118 #include <sys/mbuf.h>
119 #include <sys/proc.h> /* for proc0 declaration */
120 #include <sys/socket.h>
121 #include <sys/socketvar.h>
122 #include <sys/sysctl.h>
123 #include <sys/systm.h>
124 #include <sys/refcount.h>
125 #include <sys/sched.h>
126 #include <sys/queue.h>
127 #include <sys/smp.h>
128 #include <sys/counter.h>
129 #include <sys/time.h>
130 #include <sys/kthread.h>
131 #include <sys/kern_prefetch.h>
132
133 #include <vm/uma.h>
134
135 #include <net/route.h>
136 #include <net/vnet.h>
137
138 #define TCPSTATES /* for logging */
139
140 #include <netinet/in.h>
141 #include <netinet/in_kdtrace.h>
142 #include <netinet/in_pcb.h>
143 #include <netinet/ip.h>
144 #include <netinet/ip_icmp.h> /* required for icmp_var.h */
145 #include <netinet/icmp_var.h> /* for ICMP_BANDLIM */
146 #include <netinet/ip_var.h>
147 #include <netinet/ip6.h>
148 #include <netinet6/in6_pcb.h>
149 #include <netinet6/ip6_var.h>
150 #include <netinet/tcp.h>
151 #include <netinet/tcp_fsm.h>
152 #include <netinet/tcp_seq.h>
153 #include <netinet/tcp_timer.h>
154 #include <netinet/tcp_var.h>
155 #include <netinet/tcpip.h>
156 #include <netinet/cc/cc.h>
157 #include <netinet/tcp_hpts.h>
158
159 #ifdef tcpdebug
160 #include <netinet/tcp_debug.h>
161 #endif /* tcpdebug */
162 #ifdef tcp_offload
163 #include <netinet/tcp_offload.h>
164 #endif
165
166 #include "opt_rss.h"
167
168 MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts");
169 #ifdef RSS
170 #include <net/netisr.h>
171 #include <net/rss_config.h>
172 static int tcp_bind_threads = 1;
173 #else
174 static int tcp_bind_threads = 0;
175 #endif
176 TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads);
177
178 static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG;
179
180 TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size);
181
182 static struct tcp_hptsi tcp_pace;
183
184 static void tcp_wakehpts(struct tcp_hpts_entry *p);
185 static void tcp_wakeinput(struct tcp_hpts_entry *p);
186 static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv);
187 static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick);
188 static void tcp_hpts_thread(void *ctx);
189 static void tcp_init_hptsi(void *st);
190
191 int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP;
192 static int32_t tcp_hpts_callout_skip_swi = 0;
193
194 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW, 0, "TCP Hpts controls");
195
196 #define timersub(tvp, uvp, vvp) \
197 do { \
198 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \
199 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \
200 if ((vvp)->tv_usec < 0) { \
201 (vvp)->tv_sec--; \
202 (vvp)->tv_usec += 1000000; \
203 } \
204 } while (0)
205
206 static int32_t logging_on = 0;
207 static int32_t hpts_sleep_max = (NUM_OF_HPTSI_SLOTS - 2);
208 static int32_t tcp_hpts_precision = 120;
209
210 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW,
211 &tcp_hpts_precision, 120,
212 "Value for PRE() precision of callout");
213
214 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW,
215 &logging_on, 0,
216 "Turn on logging if compiled in");
217
218 counter_u64_t hpts_loops;
219
220 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, loops, CTLFLAG_RD,
221 &hpts_loops, "Number of times hpts had to loop to catch up");
222
223 counter_u64_t back_tosleep;
224
225 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, no_tcbsfound, CTLFLAG_RD,
226 &back_tosleep, "Number of times hpts found no tcbs");
227
228 static int32_t in_newts_every_tcb = 0;
229
230 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tsperpcb, CTLFLAG_RW,
231 &in_newts_every_tcb, 0,
232 "Do we have a new cts every tcb we process for input");
233 static int32_t in_ts_percision = 0;
234
235 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tspercision, CTLFLAG_RW,
236 &in_ts_percision, 0,
237 "Do we use percise timestamp for clients on input");
238 static int32_t out_newts_every_tcb = 0;
239
240 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tsperpcb, CTLFLAG_RW,
241 &out_newts_every_tcb, 0,
242 "Do we have a new cts every tcb we process for output");
243 static int32_t out_ts_percision = 0;
244
245 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tspercision, CTLFLAG_RW,
246 &out_ts_percision, 0,
247 "Do we use a percise timestamp for every output cts");
248
249 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLFLAG_RW,
250 &hpts_sleep_max, 0,
251 "The maximum time the hpts will sleep <1 - 254>");
252
253 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLFLAG_RW,
254 &tcp_min_hptsi_time, 0,
255 "The minimum time the hpts must sleep before processing more slots");
256
257 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, skip_swi, CTLFLAG_RW,
258 &tcp_hpts_callout_skip_swi, 0,
259 "Do we have the callout call directly to the hpts?");
260
261 static void
262 __tcp_hpts_log_it(struct tcp_hpts_entry *hpts, struct inpcb *inp, int event, uint32_t slot,
263 uint32_t ticknow, int32_t line)
264 {
265 struct hpts_log *pl;
266
267 HPTS_MTX_ASSERT(hpts);
268 if (hpts->p_log == NULL)
269 return;
270 pl = &hpts->p_log[hpts->p_log_at];
271 hpts->p_log_at++;
272 if (hpts->p_log_at >= hpts->p_logsize) {
273 hpts->p_log_at = 0;
274 hpts->p_log_wrapped = 1;
275 }
276 pl->inp = inp;
277 if (inp) {
278 pl->t_paceslot = inp->inp_hptsslot;
279 pl->t_hptsreq = inp->inp_hpts_request;
280 pl->p_onhpts = inp->inp_in_hpts;
281 pl->p_oninput = inp->inp_in_input;
282 } else {
283 pl->t_paceslot = 0;
284 pl->t_hptsreq = 0;
285 pl->p_onhpts = 0;
286 pl->p_oninput = 0;
287 }
288 pl->is_notempty = 1;
289 pl->event = event;
290 pl->line = line;
291 pl->cts = tcp_get_usecs(NULL);
292 pl->p_curtick = hpts->p_curtick;
293 pl->p_prevtick = hpts->p_prevtick;
294 pl->p_on_queue_cnt = hpts->p_on_queue_cnt;
295 pl->ticknow = ticknow;
296 pl->slot_req = slot;
297 pl->p_nxt_slot = hpts->p_nxt_slot;
298 pl->p_cur_slot = hpts->p_cur_slot;
299 pl->p_hpts_sleep_time = hpts->p_hpts_sleep_time;
300 pl->p_flags = (hpts->p_cpu & 0x7f);
301 pl->p_flags <<= 7;
302 pl->p_flags |= (hpts->p_num & 0x7f);
303 pl->p_flags <<= 2;
304 if (hpts->p_hpts_active) {
305 pl->p_flags |= HPTS_HPTS_ACTIVE;
306 }
307 }
308
309 #define tcp_hpts_log_it(a, b, c, d, e) __tcp_hpts_log_it(a, b, c, d, e, __LINE__)
310
311 static void
312 hpts_timeout_swi(void *arg)
313 {
314 struct tcp_hpts_entry *hpts;
315
316 hpts = (struct tcp_hpts_entry *)arg;
317 swi_sched(hpts->ie_cookie, 0);
318 }
319
320 static void
321 hpts_timeout_dir(void *arg)
322 {
323 tcp_hpts_thread(arg);
324 }
325
326 static inline void
327 hpts_sane_pace_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int clear)
328 {
329 #ifdef INVARIANTS
330 if (mtx_owned(&hpts->p_mtx) == 0) {
331 /* We don't own the mutex? */
332 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
333 }
334 if (hpts->p_cpu != inp->inp_hpts_cpu) {
335 /* It is not the right cpu/mutex? */
336 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
337 }
338 if (inp->inp_in_hpts == 0) {
339 /* We are not on the hpts? */
340 panic("%s: hpts:%p inp:%p not on the hpts?", __FUNCTION__, hpts, inp);
341 }
342 if (TAILQ_EMPTY(head) &&
343 (hpts->p_on_queue_cnt != 0)) {
344 /* We should not be empty with a queue count */
345 panic("%s hpts:%p hpts bucket empty but cnt:%d",
346 __FUNCTION__, hpts, hpts->p_on_queue_cnt);
347 }
348 #endif
349 TAILQ_REMOVE(head, inp, inp_hpts);
350 hpts->p_on_queue_cnt--;
351 if (hpts->p_on_queue_cnt < 0) {
352 /* Count should not go negative .. */
353 #ifdef INVARIANTS
354 panic("Hpts goes negative inp:%p hpts:%p",
355 inp, hpts);
356 #endif
357 hpts->p_on_queue_cnt = 0;
358 }
359 if (clear) {
360 inp->inp_hpts_request = 0;
361 inp->inp_in_hpts = 0;
362 }
363 }
364
365 static inline void
366 hpts_sane_pace_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int line, int noref)
367 {
368 #ifdef INVARIANTS
369 if (mtx_owned(&hpts->p_mtx) == 0) {
370 /* We don't own the mutex? */
371 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
372 }
373 if (hpts->p_cpu != inp->inp_hpts_cpu) {
374 /* It is not the right cpu/mutex? */
375 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
376 }
377 if ((noref == 0) && (inp->inp_in_hpts == 1)) {
378 /* We are already on the hpts? */
379 panic("%s: hpts:%p inp:%p already on the hpts?", __FUNCTION__, hpts, inp);
380 }
381 #endif
382 TAILQ_INSERT_TAIL(head, inp, inp_hpts);
383 inp->inp_in_hpts = 1;
384 hpts->p_on_queue_cnt++;
385 if (noref == 0) {
386 in_pcbref(inp);
387 }
388 }
389
390 static inline void
391 hpts_sane_input_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, int clear)
392 {
393 #ifdef INVARIANTS
394 if (mtx_owned(&hpts->p_mtx) == 0) {
395 /* We don't own the mutex? */
396 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
397 }
398 if (hpts->p_cpu != inp->inp_input_cpu) {
399 /* It is not the right cpu/mutex? */
400 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
401 }
402 if (inp->inp_in_input == 0) {
403 /* We are not on the input hpts? */
404 panic("%s: hpts:%p inp:%p not on the input hpts?", __FUNCTION__, hpts, inp);
405 }
406 #endif
407 TAILQ_REMOVE(&hpts->p_input, inp, inp_input);
408 hpts->p_on_inqueue_cnt--;
409 if (hpts->p_on_inqueue_cnt < 0) {
410 #ifdef INVARIANTS
411 panic("Hpts in goes negative inp:%p hpts:%p",
412 inp, hpts);
413 #endif
414 hpts->p_on_inqueue_cnt = 0;
415 }
416 #ifdef INVARIANTS
417 if (TAILQ_EMPTY(&hpts->p_input) &&
418 (hpts->p_on_inqueue_cnt != 0)) {
419 /* We should not be empty with a queue count */
420 panic("%s hpts:%p in_hpts input empty but cnt:%d",
421 __FUNCTION__, hpts, hpts->p_on_inqueue_cnt);
422 }
423 #endif
424 if (clear)
425 inp->inp_in_input = 0;
426 }
427
428 static inline void
429 hpts_sane_input_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, int line)
430 {
431 #ifdef INVARIANTS
432 if (mtx_owned(&hpts->p_mtx) == 0) {
433 /* We don't own the mutex? */
434 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
435 }
436 if (hpts->p_cpu != inp->inp_input_cpu) {
437 /* It is not the right cpu/mutex? */
438 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
439 }
440 if (inp->inp_in_input == 1) {
441 /* We are already on the input hpts? */
442 panic("%s: hpts:%p inp:%p already on the input hpts?", __FUNCTION__, hpts, inp);
443 }
444 #endif
445 TAILQ_INSERT_TAIL(&hpts->p_input, inp, inp_input);
446 inp->inp_in_input = 1;
447 hpts->p_on_inqueue_cnt++;
448 in_pcbref(inp);
449 }
450
451 static int
452 sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS)
453 {
454 struct tcp_hpts_entry *hpts;
455 size_t sz;
456 int32_t logging_was, i;
457 int32_t error = 0;
458
459 /*
460 * HACK: Turn off logging so no locks are required this really needs
461 * a memory barrier :)
462 */
463 logging_was = logging_on;
464 logging_on = 0;
465 if (!req->oldptr) {
466 /* How much? */
467 sz = 0;
468 for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
469 hpts = tcp_pace.rp_ent[i];
470 if (hpts->p_log == NULL)
471 continue;
472 sz += (sizeof(struct hpts_log) * hpts->p_logsize);
473 }
474 error = SYSCTL_OUT(req, 0, sz);
475 } else {
476 for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
477 hpts = tcp_pace.rp_ent[i];
478 if (hpts->p_log == NULL)
479 continue;
480 if (hpts->p_log_wrapped)
481 sz = (sizeof(struct hpts_log) * hpts->p_logsize);
482 else
483 sz = (sizeof(struct hpts_log) * hpts->p_log_at);
484 error = SYSCTL_OUT(req, hpts->p_log, sz);
485 }
486 }
487 logging_on = logging_was;
488 return error;
489 }
490
491 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
492 0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log");
493
494
495 static void
496 tcp_wakehpts(struct tcp_hpts_entry *hpts)
497 {
498 HPTS_MTX_ASSERT(hpts);
499 swi_sched(hpts->ie_cookie, 0);
500 if (hpts->p_hpts_active == 2) {
501 /* Rare sleeping on a ENOBUF */
502 wakeup_one(hpts);
503 }
504 }
505
506 static void
507 tcp_wakeinput(struct tcp_hpts_entry *hpts)
508 {
509 HPTS_MTX_ASSERT(hpts);
510 swi_sched(hpts->ie_cookie, 0);
511 if (hpts->p_hpts_active == 2) {
512 /* Rare sleeping on a ENOBUF */
513 wakeup_one(hpts);
514 }
515 }
516
517 struct tcp_hpts_entry *
518 tcp_cur_hpts(struct inpcb *inp)
519 {
520 int32_t hpts_num;
521 struct tcp_hpts_entry *hpts;
522
523 hpts_num = inp->inp_hpts_cpu;
524 hpts = tcp_pace.rp_ent[hpts_num];
525 return (hpts);
526 }
527
528 struct tcp_hpts_entry *
529 tcp_hpts_lock(struct inpcb *inp)
530 {
531 struct tcp_hpts_entry *hpts;
532 int32_t hpts_num;
533
534 again:
535 hpts_num = inp->inp_hpts_cpu;
536 hpts = tcp_pace.rp_ent[hpts_num];
537 #ifdef INVARIANTS
538 if (mtx_owned(&hpts->p_mtx)) {
539 panic("Hpts:%p owns mtx prior-to lock line:%d",
540 hpts, __LINE__);
541 }
542 #endif
543 mtx_lock(&hpts->p_mtx);
544 if (hpts_num != inp->inp_hpts_cpu) {
545 mtx_unlock(&hpts->p_mtx);
546 goto again;
547 }
548 return (hpts);
549 }
550
551 struct tcp_hpts_entry *
552 tcp_input_lock(struct inpcb *inp)
553 {
554 struct tcp_hpts_entry *hpts;
555 int32_t hpts_num;
556
557 again:
558 hpts_num = inp->inp_input_cpu;
559 hpts = tcp_pace.rp_ent[hpts_num];
560 #ifdef INVARIANTS
561 if (mtx_owned(&hpts->p_mtx)) {
562 panic("Hpts:%p owns mtx prior-to lock line:%d",
563 hpts, __LINE__);
564 }
565 #endif
566 mtx_lock(&hpts->p_mtx);
567 if (hpts_num != inp->inp_input_cpu) {
568 mtx_unlock(&hpts->p_mtx);
569 goto again;
570 }
571 return (hpts);
572 }
573
574 static void
575 tcp_remove_hpts_ref(struct inpcb *inp, struct tcp_hpts_entry *hpts, int line)
576 {
577 int32_t add_freed;
578
579 if (inp->inp_flags2 & INP_FREED) {
580 /*
581 * Need to play a special trick so that in_pcbrele_wlocked
582 * does not return 1 when it really should have returned 0.
583 */
584 add_freed = 1;
585 inp->inp_flags2 &= ~INP_FREED;
586 } else {
587 add_freed = 0;
588 }
589 #ifndef INP_REF_DEBUG
590 if (in_pcbrele_wlocked(inp)) {
591 /*
592 * This should not happen. We have the inpcb referred to by
593 * the main socket (why we are called) and the hpts. It
594 * should always return 0.
595 */
596 panic("inpcb:%p release ret 1",
597 inp);
598 }
599 #else
600 if (__in_pcbrele_wlocked(inp, line)) {
601 /*
602 * This should not happen. We have the inpcb referred to by
603 * the main socket (why we are called) and the hpts. It
604 * should always return 0.
605 */
606 panic("inpcb:%p release ret 1",
607 inp);
608 }
609 #endif
610 if (add_freed) {
611 inp->inp_flags2 |= INP_FREED;
612 }
613 }
614
615 static void
616 tcp_hpts_remove_locked_output(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
617 {
618 if (inp->inp_in_hpts) {
619 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], 1);
620 tcp_remove_hpts_ref(inp, hpts, line);
621 }
622 }
623
624 static void
625 tcp_hpts_remove_locked_input(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
626 {
627 HPTS_MTX_ASSERT(hpts);
628 if (inp->inp_in_input) {
629 hpts_sane_input_remove(hpts, inp, 1);
630 tcp_remove_hpts_ref(inp, hpts, line);
631 }
632 }
633
634 /*
635 * Called normally with the INP_LOCKED but it
636 * does not matter, the hpts lock is the key
637 * but the lock order allows us to hold the
638 * INP lock and then get the hpts lock.
639 *
640 * Valid values in the flags are
641 * HPTS_REMOVE_OUTPUT - remove from the output of the hpts.
642 * HPTS_REMOVE_INPUT - remove from the input of the hpts.
643 * Note that you can or both values together and get two
644 * actions.
645 */
646 void
647 __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line)
648 {
649 struct tcp_hpts_entry *hpts;
650
651 INP_WLOCK_ASSERT(inp);
652 if (flags & HPTS_REMOVE_OUTPUT) {
653 hpts = tcp_hpts_lock(inp);
654 tcp_hpts_remove_locked_output(hpts, inp, flags, line);
655 mtx_unlock(&hpts->p_mtx);
656 }
657 if (flags & HPTS_REMOVE_INPUT) {
658 hpts = tcp_input_lock(inp);
659 tcp_hpts_remove_locked_input(hpts, inp, flags, line);
660 mtx_unlock(&hpts->p_mtx);
661 }
662 }
663
664 static inline int
665 hpts_tick(struct tcp_hpts_entry *hpts, int32_t plus)
666 {
667 return ((hpts->p_prevtick + plus) % NUM_OF_HPTSI_SLOTS);
668 }
669
670 static int
671 tcp_queue_to_hpts_immediate_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line, int32_t noref)
672 {
673 int32_t need_wake = 0;
674 uint32_t ticknow = 0;
675
676 HPTS_MTX_ASSERT(hpts);
677 if (inp->inp_in_hpts == 0) {
678 /* Ok we need to set it on the hpts in the current slot */
679 if (hpts->p_hpts_active == 0) {
680 /* A sleeping hpts we want in next slot to run */
681 if (logging_on) {
682 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, 0,
683 hpts_tick(hpts, 1));
684 }
685 inp->inp_hptsslot = hpts_tick(hpts, 1);
686 inp->inp_hpts_request = 0;
687 if (logging_on) {
688 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEP_BEFORE, 1, ticknow);
689 }
690 need_wake = 1;
691 } else if ((void *)inp == hpts->p_inp) {
692 /*
693 * We can't allow you to go into the same slot we
694 * are in. We must put you out.
695 */
696 inp->inp_hptsslot = hpts->p_nxt_slot;
697 } else
698 inp->inp_hptsslot = hpts->p_cur_slot;
699 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
700 inp->inp_hpts_request = 0;
701 if (logging_on) {
702 tcp_hpts_log_it(hpts, inp, HPTSLOG_IMMEDIATE, 0, 0);
703 }
704 if (need_wake) {
705 /*
706 * Activate the hpts if it is sleeping and its
707 * timeout is not 1.
708 */
709 if (logging_on) {
710 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_HPTS, 0, ticknow);
711 }
712 hpts->p_direct_wake = 1;
713 tcp_wakehpts(hpts);
714 }
715 }
716 return (need_wake);
717 }
718
719 int
720 __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line)
721 {
722 int32_t ret;
723 struct tcp_hpts_entry *hpts;
724
725 INP_WLOCK_ASSERT(inp);
726 hpts = tcp_hpts_lock(inp);
727 ret = tcp_queue_to_hpts_immediate_locked(inp, hpts, line, 0);
728 mtx_unlock(&hpts->p_mtx);
729 return (ret);
730 }
731
732 static void
733 tcp_hpts_insert_locked(struct tcp_hpts_entry *hpts, struct inpcb *inp, uint32_t slot, uint32_t cts, int32_t line,
734 struct hpts_diag *diag, int32_t noref)
735 {
736 int32_t need_new_to = 0;
737 int32_t need_wakeup = 0;
738 uint32_t largest_slot;
739 uint32_t ticknow = 0;
740 uint32_t slot_calc;
741
742 HPTS_MTX_ASSERT(hpts);
743 if (diag) {
744 memset(diag, 0, sizeof(struct hpts_diag));
745 diag->p_hpts_active = hpts->p_hpts_active;
746 diag->p_nxt_slot = hpts->p_nxt_slot;
747 diag->p_cur_slot = hpts->p_cur_slot;
748 diag->slot_req = slot;
749 }
750 if ((inp->inp_in_hpts == 0) || noref) {
751 inp->inp_hpts_request = slot;
752 if (slot == 0) {
753 /* Immediate */
754 tcp_queue_to_hpts_immediate_locked(inp, hpts, line, noref);
755 return;
756 }
757 if (hpts->p_hpts_active) {
758 /*
759 * Its slot - 1 since nxt_slot is the next tick that
760 * will go off since the hpts is awake
761 */
762 if (logging_on) {
763 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_NORMAL, slot, 0);
764 }
765 /*
766 * We want to make sure that we don't place a inp in
767 * the range of p_cur_slot <-> p_nxt_slot. If we
768 * take from p_nxt_slot to the end, plus p_cur_slot
769 * and then take away 2, we will know how many is
770 * the max slots we can use.
771 */
772 if (hpts->p_nxt_slot > hpts->p_cur_slot) {
773 /*
774 * Non-wrap case nxt_slot <-> cur_slot we
775 * don't want to land in. So the diff gives
776 * us what is taken away from the number of
777 * slots.
778 */
779 largest_slot = NUM_OF_HPTSI_SLOTS - (hpts->p_nxt_slot - hpts->p_cur_slot);
780 } else if (hpts->p_nxt_slot == hpts->p_cur_slot) {
781 largest_slot = NUM_OF_HPTSI_SLOTS - 2;
782 } else {
783 /*
784 * Wrap case so the diff gives us the number
785 * of slots that we can land in.
786 */
787 largest_slot = hpts->p_cur_slot - hpts->p_nxt_slot;
788 }
789 /*
790 * We take away two so we never have a problem (20
791 * usec's) out of 1024000 usecs
792 */
793 largest_slot -= 2;
794 if (inp->inp_hpts_request > largest_slot) {
795 /*
796 * Restrict max jump of slots and remember
797 * leftover
798 */
799 slot = largest_slot;
800 inp->inp_hpts_request -= largest_slot;
801 } else {
802 /* This one will run when we hit it */
803 inp->inp_hpts_request = 0;
804 }
805 if (hpts->p_nxt_slot == hpts->p_cur_slot)
806 slot_calc = (hpts->p_nxt_slot + slot) % NUM_OF_HPTSI_SLOTS;
807 else
808 slot_calc = (hpts->p_nxt_slot + slot - 1) % NUM_OF_HPTSI_SLOTS;
809 if (slot_calc == hpts->p_cur_slot) {
810 #ifdef INVARIANTS
811 /* TSNH */
812 panic("Hpts:%p impossible slot calculation slot_calc:%u slot:%u largest:%u\n",
813 hpts, slot_calc, slot, largest_slot);
814 #endif
815 if (slot_calc)
816 slot_calc--;
817 else
818 slot_calc = NUM_OF_HPTSI_SLOTS - 1;
819 }
820 inp->inp_hptsslot = slot_calc;
821 if (diag) {
822 diag->inp_hptsslot = inp->inp_hptsslot;
823 }
824 } else {
825 /*
826 * The hpts is sleeping, we need to figure out where
827 * it will wake up at and if we need to reschedule
828 * its time-out.
829 */
830 uint32_t have_slept, yet_to_sleep;
831 uint32_t slot_now;
832 struct timeval tv;
833
834 ticknow = tcp_gethptstick(&tv);
835 slot_now = ticknow % NUM_OF_HPTSI_SLOTS;
836 /*
837 * The user wants to be inserted at (slot_now +
838 * slot) % NUM_OF_HPTSI_SLOTS, so lets set that up.
839 */
840 largest_slot = NUM_OF_HPTSI_SLOTS - 2;
841 if (inp->inp_hpts_request > largest_slot) {
842 /* Adjust the residual in inp_hpts_request */
843 slot = largest_slot;
844 inp->inp_hpts_request -= largest_slot;
845 } else {
846 /* No residual it all fits */
847 inp->inp_hpts_request = 0;
848 }
849 inp->inp_hptsslot = (slot_now + slot) % NUM_OF_HPTSI_SLOTS;
850 if (diag) {
851 diag->slot_now = slot_now;
852 diag->inp_hptsslot = inp->inp_hptsslot;
853 diag->p_on_min_sleep = hpts->p_on_min_sleep;
854 }
855 if (logging_on) {
856 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, slot, ticknow);
857 }
858 /* Now do we need to restart the hpts's timer? */
859 if (TSTMP_GT(ticknow, hpts->p_curtick))
860 have_slept = ticknow - hpts->p_curtick;
861 else
862 have_slept = 0;
863 if (have_slept < hpts->p_hpts_sleep_time) {
864 /* This should be what happens */
865 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept;
866 } else {
867 /* We are over-due */
868 yet_to_sleep = 0;
869 need_wakeup = 1;
870 }
871 if (diag) {
872 diag->have_slept = have_slept;
873 diag->yet_to_sleep = yet_to_sleep;
874 diag->hpts_sleep_time = hpts->p_hpts_sleep_time;
875 }
876 if ((hpts->p_on_min_sleep == 0) && (yet_to_sleep > slot)) {
877 /*
878 * We need to reschedule the hptss time-out.
879 */
880 hpts->p_hpts_sleep_time = slot;
881 need_new_to = slot * HPTS_TICKS_PER_USEC;
882 }
883 }
884 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
885 if (logging_on) {
886 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERTED, slot, ticknow);
887 }
888 /*
889 * Now how far is the hpts sleeping to? if active is 1, its
890 * up and ticking we do nothing, otherwise we may need to
891 * reschedule its callout if need_new_to is set from above.
892 */
893 if (need_wakeup) {
894 if (logging_on) {
895 tcp_hpts_log_it(hpts, inp, HPTSLOG_RESCHEDULE, 1, 0);
896 }
897 hpts->p_direct_wake = 1;
898 tcp_wakehpts(hpts);
899 if (diag) {
900 diag->need_new_to = 0;
901 diag->co_ret = 0xffff0000;
902 }
903 } else if (need_new_to) {
904 int32_t co_ret;
905 struct timeval tv;
906 sbintime_t sb;
907
908 tv.tv_sec = 0;
909 tv.tv_usec = 0;
910 while (need_new_to > HPTS_USEC_IN_SEC) {
911 tv.tv_sec++;
912 need_new_to -= HPTS_USEC_IN_SEC;
913 }
914 tv.tv_usec = need_new_to;
915 sb = tvtosbt(tv);
916 if (tcp_hpts_callout_skip_swi == 0) {
917 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
918 hpts_timeout_swi, hpts, hpts->p_cpu,
919 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
920 } else {
921 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
922 hpts_timeout_dir, hpts,
923 hpts->p_cpu,
924 C_PREL(tcp_hpts_precision));
925 }
926 if (diag) {
927 diag->need_new_to = need_new_to;
928 diag->co_ret = co_ret;
929 }
930 }
931 } else {
932 #ifdef INVARIANTS
933 panic("Hpts:%p tp:%p already on hpts and add?", hpts, inp);
934 #endif
935 }
936 }
937
938 uint32_t
939 tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag){
940 struct tcp_hpts_entry *hpts;
941 uint32_t slot_on, cts;
942 struct timeval tv;
943
944 /*
945 * We now return the next-slot the hpts will be on, beyond its
946 * current run (if up) or where it was when it stopped if it is
947 * sleeping.
948 */
949 INP_WLOCK_ASSERT(inp);
950 hpts = tcp_hpts_lock(inp);
951 if (in_ts_percision)
952 microuptime(&tv);
953 else
954 getmicrouptime(&tv);
955 cts = tcp_tv_to_usectick(&tv);
956 tcp_hpts_insert_locked(hpts, inp, slot, cts, line, diag, 0);
957 slot_on = hpts->p_nxt_slot;
958 mtx_unlock(&hpts->p_mtx);
959 return (slot_on);
960 }
961
962 uint32_t
963 __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line){
964 return (tcp_hpts_insert_diag(inp, slot, line, NULL));
965 }
966
967 int
968 __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line)
969 {
970 int32_t retval = 0;
971
972 HPTS_MTX_ASSERT(hpts);
973 if (inp->inp_in_input == 0) {
974 /* Ok we need to set it on the hpts in the current slot */
975 hpts_sane_input_insert(hpts, inp, line);
976 retval = 1;
977 if (hpts->p_hpts_active == 0) {
978 /*
979 * Activate the hpts if it is sleeping.
980 */
981 if (logging_on) {
982 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_INPUT, 0, 0);
983 }
984 retval = 2;
985 hpts->p_direct_wake = 1;
986 tcp_wakeinput(hpts);
987 }
988 } else if (hpts->p_hpts_active == 0) {
989 retval = 4;
990 hpts->p_direct_wake = 1;
991 tcp_wakeinput(hpts);
992 }
993 return (retval);
994 }
995
996 void
997 tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
998 int32_t tlen, int32_t drop_hdrlen, uint8_t iptos)
999 {
1000 /* Setup packet for input first */
1001 INP_WLOCK_ASSERT(tp->t_inpcb);
1002 m->m_pkthdr.pace_thoff = (uint16_t) ((caddr_t)th - mtod(m, caddr_t));
1003 m->m_pkthdr.pace_tlen = (uint16_t) tlen;
1004 m->m_pkthdr.pace_drphdrlen = drop_hdrlen;
1005 m->m_pkthdr.pace_tos = iptos;
1006 m->m_pkthdr.pace_lock = (curthread->td_epochnest != 0);
1007 if (tp->t_in_pkt == NULL) {
1008 tp->t_in_pkt = m;
1009 tp->t_tail_pkt = m;
1010 } else {
1011 tp->t_tail_pkt->m_nextpkt = m;
1012 tp->t_tail_pkt = m;
1013 }
1014 }
1015
1016
1017 int32_t
1018 __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
1019 int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, int32_t line){
1020 struct tcp_hpts_entry *hpts;
1021 int32_t ret;
1022
1023 tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos);
1024 hpts = tcp_input_lock(tp->t_inpcb);
1025 ret = __tcp_queue_to_input_locked(tp->t_inpcb, hpts, line);
1026 mtx_unlock(&hpts->p_mtx);
1027 return (ret);
1028 }
1029
1030 void
1031 __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line)
1032 {
1033 struct tcp_hpts_entry *hpts;
1034 struct tcpcb *tp;
1035
1036 tp = intotcpcb(inp);
1037 hpts = tcp_input_lock(tp->t_inpcb);
1038 if (inp->inp_in_input == 0) {
1039 /* Ok we need to set it on the hpts in the current slot */
1040 hpts_sane_input_insert(hpts, inp, line);
1041 if (hpts->p_hpts_active == 0) {
1042 /*
1043 * Activate the hpts if it is sleeping.
1044 */
1045 hpts->p_direct_wake = 1;
1046 tcp_wakeinput(hpts);
1047 }
1048 } else if (hpts->p_hpts_active == 0) {
1049 hpts->p_direct_wake = 1;
1050 tcp_wakeinput(hpts);
1051 }
1052 inp->inp_hpts_drop_reas = reason;
1053 mtx_unlock(&hpts->p_mtx);
1054 }
1055
1056 static uint16_t
1057 hpts_random_cpu(struct inpcb *inp){
1058 /*
1059 * No flow type set distribute the load randomly.
1060 */
1061 uint16_t cpuid;
1062 uint32_t ran;
1063
1064 /*
1065 * If one has been set use it i.e. we want both in and out on the
1066 * same hpts.
1067 */
1068 if (inp->inp_input_cpu_set) {
1069 return (inp->inp_input_cpu);
1070 } else if (inp->inp_hpts_cpu_set) {
1071 return (inp->inp_hpts_cpu);
1072 }
1073 /* Nothing set use a random number */
1074 ran = arc4random();
1075 cpuid = (ran & 0xffff) % mp_ncpus;
1076 return (cpuid);
1077 }
1078
1079 static uint16_t
1080 hpts_cpuid(struct inpcb *inp){
1081 u_int cpuid;
1082
1083
1084 /*
1085 * If one has been set use it i.e. we want both in and out on the
1086 * same hpts.
1087 */
1088 if (inp->inp_input_cpu_set) {
1089 return (inp->inp_input_cpu);
1090 } else if (inp->inp_hpts_cpu_set) {
1091 return (inp->inp_hpts_cpu);
1092 }
1093 /* If one is set the other must be the same */
1094 #ifdef RSS
1095 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
1096 if (cpuid == NETISR_CPUID_NONE)
1097 return (hpts_random_cpu(inp));
1098 else
1099 return (cpuid);
1100 #else
1101 /*
1102 * We don't have a flowid -> cpuid mapping, so cheat and just map
1103 * unknown cpuids to curcpu. Not the best, but apparently better
1104 * than defaulting to swi 0.
1105 */
1106 if (inp->inp_flowtype != M_HASHTYPE_NONE) {
1107 cpuid = inp->inp_flowid % mp_ncpus;
1108 return (cpuid);
1109 }
1110 cpuid = hpts_random_cpu(inp);
1111 return (cpuid);
1112 #endif
1113 }
1114
1115 /*
1116 * Do NOT try to optimize the processing of inp's
1117 * by first pulling off all the inp's into a temporary
1118 * list (e.g. TAILQ_CONCAT). If you do that the subtle
1119 * interactions of switching CPU's will kill because of
1120 * problems in the linked list manipulation. Basically
1121 * you would switch cpu's with the hpts mutex locked
1122 * but then while you were processing one of the inp's
1123 * some other one that you switch will get a new
1124 * packet on the different CPU. It will insert it
1125 * on the new hptss input list. Creating a temporary
1126 * link in the inp will not fix it either, since
1127 * the other hpts will be doing the same thing and
1128 * you will both end up using the temporary link.
1129 *
1130 * You will die in an ASSERT for tailq corruption if you
1131 * run INVARIANTS or you will die horribly without
1132 * INVARIANTS in some unknown way with a corrupt linked
1133 * list.
1134 */
1135 static void
1136 tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv)
1137 {
1138 struct mbuf *m, *n;
1139 struct tcpcb *tp;
1140 struct inpcb *inp;
1141 uint16_t drop_reason;
1142 int16_t set_cpu;
1143 uint32_t did_prefetch = 0;
1144 int32_t ti_locked = TI_UNLOCKED;
1145 struct epoch_tracker et;
1146
1147 HPTS_MTX_ASSERT(hpts);
1148 while ((inp = TAILQ_FIRST(&hpts->p_input)) != NULL) {
1149 HPTS_MTX_ASSERT(hpts);
1150 hpts_sane_input_remove(hpts, inp, 0);
1151 if (inp->inp_input_cpu_set == 0) {
1152 set_cpu = 1;
1153 } else {
1154 set_cpu = 0;
1155 }
1156 hpts->p_inp = inp;
1157 drop_reason = inp->inp_hpts_drop_reas;
1158 inp->inp_in_input = 0;
1159 mtx_unlock(&hpts->p_mtx);
1160 CURVNET_SET(inp->inp_vnet);
1161 if (drop_reason) {
1162 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1163 ti_locked = TI_RLOCKED;
1164 } else {
1165 ti_locked = TI_UNLOCKED;
1166 }
1167 INP_WLOCK(inp);
1168 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1169 (inp->inp_flags2 & INP_FREED)) {
1170 out:
1171 hpts->p_inp = NULL;
1172 if (ti_locked == TI_RLOCKED) {
1173 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1174 }
1175 if (in_pcbrele_wlocked(inp) == 0) {
1176 INP_WUNLOCK(inp);
1177 }
1178 ti_locked = TI_UNLOCKED;
1179 CURVNET_RESTORE();
1180 mtx_lock(&hpts->p_mtx);
1181 continue;
1182 }
1183 tp = intotcpcb(inp);
1184 if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1185 goto out;
1186 }
1187 if (drop_reason) {
1188 /* This tcb is being destroyed for drop_reason */
1189 m = tp->t_in_pkt;
1190 if (m)
1191 n = m->m_nextpkt;
1192 else
1193 n = NULL;
1194 tp->t_in_pkt = NULL;
1195 while (m) {
1196 m_freem(m);
1197 m = n;
1198 if (m)
1199 n = m->m_nextpkt;
1200 }
1201 tp = tcp_drop(tp, drop_reason);
1202 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1203 if (tp == NULL) {
1204 INP_WLOCK(inp);
1205 }
1206 if (in_pcbrele_wlocked(inp) == 0)
1207 INP_WUNLOCK(inp);
1208 CURVNET_RESTORE();
1209 mtx_lock(&hpts->p_mtx);
1210 continue;
1211 }
1212 if (set_cpu) {
1213 /*
1214 * Setup so the next time we will move to the right
1215 * CPU. This should be a rare event. It will
1216 * sometimes happens when we are the client side
1217 * (usually not the server). Somehow tcp_output()
1218 * gets called before the tcp_do_segment() sets the
1219 * intial state. This means the r_cpu and r_hpts_cpu
1220 * is 0. We get on the hpts, and then tcp_input()
1221 * gets called setting up the r_cpu to the correct
1222 * value. The hpts goes off and sees the mis-match.
1223 * We simply correct it here and the CPU will switch
1224 * to the new hpts nextime the tcb gets added to the
1225 * the hpts (not this time) :-)
1226 */
1227 tcp_set_hpts(inp);
1228 }
1229 m = tp->t_in_pkt;
1230 n = NULL;
1231 if (m != NULL &&
1232 (m->m_pkthdr.pace_lock == TI_RLOCKED ||
1233 tp->t_state != TCPS_ESTABLISHED)) {
1234 ti_locked = TI_RLOCKED;
1235 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1236 m = tp->t_in_pkt;
1237 }
1238 if (in_newts_every_tcb) {
1239 if (in_ts_percision)
1240 microuptime(tv);
1241 else
1242 getmicrouptime(tv);
1243 }
1244 if (tp->t_fb_ptr != NULL) {
1245 kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1246 did_prefetch = 1;
1247 }
1248 /* Any input work to do, if so do it first */
1249 if ((m != NULL) && (m == tp->t_in_pkt)) {
1250 struct tcphdr *th;
1251 int32_t tlen, drop_hdrlen, nxt_pkt;
1252 uint8_t iptos;
1253
1254 n = m->m_nextpkt;
1255 tp->t_in_pkt = tp->t_tail_pkt = NULL;
1256 while (m) {
1257 th = (struct tcphdr *)(mtod(m, caddr_t)+m->m_pkthdr.pace_thoff);
1258 tlen = m->m_pkthdr.pace_tlen;
1259 drop_hdrlen = m->m_pkthdr.pace_drphdrlen;
1260 iptos = m->m_pkthdr.pace_tos;
1261 m->m_nextpkt = NULL;
1262 if (n)
1263 nxt_pkt = 1;
1264 else
1265 nxt_pkt = 0;
1266 inp->inp_input_calls = 1;
1267 if (tp->t_fb->tfb_tcp_hpts_do_segment) {
1268 /* Use the hpts specific do_segment */
1269 (*tp->t_fb->tfb_tcp_hpts_do_segment) (m, th, inp->inp_socket,
1270 tp, drop_hdrlen,
1271 tlen, iptos, nxt_pkt, tv);
1272 } else {
1273 /* Use the default do_segment */
1274 (*tp->t_fb->tfb_tcp_do_segment) (m, th, inp->inp_socket,
1275 tp, drop_hdrlen,
1276 tlen, iptos);
1277 }
1278 if (ti_locked == TI_RLOCKED)
1279 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1280 /*
1281 * Do segment returns unlocked we need the
1282 * lock again but we also need some kasserts
1283 * here.
1284 */
1285 INP_INFO_WUNLOCK_ASSERT(&V_tcbinfo);
1286 INP_UNLOCK_ASSERT(inp);
1287 m = n;
1288 if (m)
1289 n = m->m_nextpkt;
1290 if (m != NULL &&
1291 m->m_pkthdr.pace_lock == TI_RLOCKED) {
1292 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1293 ti_locked = TI_RLOCKED;
1294 } else
1295 ti_locked = TI_UNLOCKED;
1296 INP_WLOCK(inp);
1297 /*
1298 * Since we have an opening here we must
1299 * re-check if the tcb went away while we
1300 * were getting the lock(s).
1301 */
1302 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1303 (inp->inp_flags2 & INP_FREED)) {
1304 while (m) {
1305 m_freem(m);
1306 m = n;
1307 if (m)
1308 n = m->m_nextpkt;
1309 }
1310 goto out;
1311 }
1312 /*
1313 * Now that we hold the INP lock, check if
1314 * we need to upgrade our lock.
1315 */
1316 if (ti_locked == TI_UNLOCKED &&
1317 (tp->t_state != TCPS_ESTABLISHED)) {
1318 ti_locked = TI_RLOCKED;
1319 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1320 }
1321 } /** end while(m) */
1322 } /** end if ((m != NULL) && (m == tp->t_in_pkt)) */
1323 if (in_pcbrele_wlocked(inp) == 0)
1324 INP_WUNLOCK(inp);
1325 if (ti_locked == TI_RLOCKED)
1326 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1327 INP_INFO_WUNLOCK_ASSERT(&V_tcbinfo);
1328 INP_UNLOCK_ASSERT(inp);
1329 ti_locked = TI_UNLOCKED;
1330 mtx_lock(&hpts->p_mtx);
1331 hpts->p_inp = NULL;
1332 CURVNET_RESTORE();
1333 }
1334 }
1335
1336 static int
1337 tcp_hpts_est_run(struct tcp_hpts_entry *hpts)
1338 {
1339 int32_t ticks_to_run;
1340
1341 if (hpts->p_prevtick && (SEQ_GT(hpts->p_curtick, hpts->p_prevtick))) {
1342 ticks_to_run = hpts->p_curtick - hpts->p_prevtick;
1343 if (ticks_to_run >= (NUM_OF_HPTSI_SLOTS - 1)) {
1344 ticks_to_run = NUM_OF_HPTSI_SLOTS - 2;
1345 }
1346 } else {
1347 if (hpts->p_prevtick == hpts->p_curtick) {
1348 /* This happens when we get woken up right away */
1349 return (-1);
1350 }
1351 ticks_to_run = 1;
1352 }
1353 /* Set in where we will be when we catch up */
1354 hpts->p_nxt_slot = (hpts->p_cur_slot + ticks_to_run) % NUM_OF_HPTSI_SLOTS;
1355 if (hpts->p_nxt_slot == hpts->p_cur_slot) {
1356 panic("Impossible math -- hpts:%p p_nxt_slot:%d p_cur_slot:%d ticks_to_run:%d",
1357 hpts, hpts->p_nxt_slot, hpts->p_cur_slot, ticks_to_run);
1358 }
1359 return (ticks_to_run);
1360 }
1361
1362 static void
1363 tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick)
1364 {
1365 struct tcpcb *tp;
1366 struct inpcb *inp = NULL, *ninp;
1367 struct timeval tv;
1368 int32_t ticks_to_run, i, error, tick_now, interum_tick;
1369 int32_t paced_cnt = 0;
1370 int32_t did_prefetch = 0;
1371 int32_t prefetch_ninp = 0;
1372 int32_t prefetch_tp = 0;
1373 uint32_t cts;
1374 int16_t set_cpu;
1375
1376 HPTS_MTX_ASSERT(hpts);
1377 hpts->p_curtick = tcp_tv_to_hptstick(ctick);
1378 cts = tcp_tv_to_usectick(ctick);
1379 memcpy(&tv, ctick, sizeof(struct timeval));
1380 hpts->p_cur_slot = hpts_tick(hpts, 1);
1381
1382 /* Figure out if we had missed ticks */
1383 again:
1384 HPTS_MTX_ASSERT(hpts);
1385 ticks_to_run = tcp_hpts_est_run(hpts);
1386 if (!TAILQ_EMPTY(&hpts->p_input)) {
1387 tcp_input_data(hpts, &tv);
1388 }
1389 #ifdef INVARIANTS
1390 if (TAILQ_EMPTY(&hpts->p_input) &&
1391 (hpts->p_on_inqueue_cnt != 0)) {
1392 panic("tp:%p in_hpts input empty but cnt:%d",
1393 hpts, hpts->p_on_inqueue_cnt);
1394 }
1395 #endif
1396 HPTS_MTX_ASSERT(hpts);
1397 /* Reset the ticks to run and time if we need too */
1398 interum_tick = tcp_gethptstick(&tv);
1399 if (interum_tick != hpts->p_curtick) {
1400 /* Save off the new time we execute to */
1401 *ctick = tv;
1402 hpts->p_curtick = interum_tick;
1403 cts = tcp_tv_to_usectick(&tv);
1404 hpts->p_cur_slot = hpts_tick(hpts, 1);
1405 ticks_to_run = tcp_hpts_est_run(hpts);
1406 }
1407 if (ticks_to_run == -1) {
1408 goto no_run;
1409 }
1410 if (logging_on) {
1411 tcp_hpts_log_it(hpts, inp, HPTSLOG_SETTORUN, ticks_to_run, 0);
1412 }
1413 if (hpts->p_on_queue_cnt == 0) {
1414 goto no_one;
1415 }
1416 HPTS_MTX_ASSERT(hpts);
1417 for (i = 0; i < ticks_to_run; i++) {
1418 /*
1419 * Calculate our delay, if there are no extra ticks there
1420 * was not any
1421 */
1422 hpts->p_delayed_by = (ticks_to_run - (i + 1)) * HPTS_TICKS_PER_USEC;
1423 HPTS_MTX_ASSERT(hpts);
1424 while ((inp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1425 /* For debugging */
1426 if (logging_on) {
1427 tcp_hpts_log_it(hpts, inp, HPTSLOG_HPTSI, ticks_to_run, i);
1428 }
1429 hpts->p_inp = inp;
1430 paced_cnt++;
1431 if (hpts->p_cur_slot != inp->inp_hptsslot) {
1432 panic("Hpts:%p inp:%p slot mis-aligned %u vs %u",
1433 hpts, inp, hpts->p_cur_slot, inp->inp_hptsslot);
1434 }
1435 /* Now pull it */
1436 if (inp->inp_hpts_cpu_set == 0) {
1437 set_cpu = 1;
1438 } else {
1439 set_cpu = 0;
1440 }
1441 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[hpts->p_cur_slot], 0);
1442 if ((ninp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1443 /* We prefetch the next inp if possible */
1444 kern_prefetch(ninp, &prefetch_ninp);
1445 prefetch_ninp = 1;
1446 }
1447 if (inp->inp_hpts_request) {
1448 /*
1449 * This guy is deferred out further in time
1450 * then our wheel had on it. Push him back
1451 * on the wheel.
1452 */
1453 int32_t remaining_slots;
1454
1455 remaining_slots = ticks_to_run - (i + 1);
1456 if (inp->inp_hpts_request > remaining_slots) {
1457 /*
1458 * Keep INVARIANTS happy by clearing
1459 * the flag
1460 */
1461 tcp_hpts_insert_locked(hpts, inp, inp->inp_hpts_request, cts, __LINE__, NULL, 1);
1462 hpts->p_inp = NULL;
1463 continue;
1464 }
1465 inp->inp_hpts_request = 0;
1466 }
1467 /*
1468 * We clear the hpts flag here after dealing with
1469 * remaining slots. This way anyone looking with the
1470 * TCB lock will see its on the hpts until just
1471 * before we unlock.
1472 */
1473 inp->inp_in_hpts = 0;
1474 mtx_unlock(&hpts->p_mtx);
1475 INP_WLOCK(inp);
1476 if (in_pcbrele_wlocked(inp)) {
1477 mtx_lock(&hpts->p_mtx);
1478 if (logging_on)
1479 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 1);
1480 hpts->p_inp = NULL;
1481 continue;
1482 }
1483 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
1484 out_now:
1485 #ifdef INVARIANTS
1486 if (mtx_owned(&hpts->p_mtx)) {
1487 panic("Hpts:%p owns mtx prior-to lock line:%d",
1488 hpts, __LINE__);
1489 }
1490 #endif
1491 INP_WUNLOCK(inp);
1492 mtx_lock(&hpts->p_mtx);
1493 if (logging_on)
1494 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 3);
1495 hpts->p_inp = NULL;
1496 continue;
1497 }
1498 tp = intotcpcb(inp);
1499 if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1500 goto out_now;
1501 }
1502 if (set_cpu) {
1503 /*
1504 * Setup so the next time we will move to
1505 * the right CPU. This should be a rare
1506 * event. It will sometimes happens when we
1507 * are the client side (usually not the
1508 * server). Somehow tcp_output() gets called
1509 * before the tcp_do_segment() sets the
1510 * intial state. This means the r_cpu and
1511 * r_hpts_cpu is 0. We get on the hpts, and
1512 * then tcp_input() gets called setting up
1513 * the r_cpu to the correct value. The hpts
1514 * goes off and sees the mis-match. We
1515 * simply correct it here and the CPU will
1516 * switch to the new hpts nextime the tcb
1517 * gets added to the the hpts (not this one)
1518 * :-)
1519 */
1520 tcp_set_hpts(inp);
1521 }
1522 if (out_newts_every_tcb) {
1523 struct timeval sv;
1524
1525 if (out_ts_percision)
1526 microuptime(&sv);
1527 else
1528 getmicrouptime(&sv);
1529 cts = tcp_tv_to_usectick(&sv);
1530 }
1531 CURVNET_SET(inp->inp_vnet);
1532 /*
1533 * There is a hole here, we get the refcnt on the
1534 * inp so it will still be preserved but to make
1535 * sure we can get the INP we need to hold the p_mtx
1536 * above while we pull out the tp/inp, as long as
1537 * fini gets the lock first we are assured of having
1538 * a sane INP we can lock and test.
1539 */
1540 #ifdef INVARIANTS
1541 if (mtx_owned(&hpts->p_mtx)) {
1542 panic("Hpts:%p owns mtx before tcp-output:%d",
1543 hpts, __LINE__);
1544 }
1545 #endif
1546 if (tp->t_fb_ptr != NULL) {
1547 kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1548 did_prefetch = 1;
1549 }
1550 inp->inp_hpts_calls = 1;
1551 if (tp->t_fb->tfb_tcp_output_wtime != NULL) {
1552 error = (*tp->t_fb->tfb_tcp_output_wtime) (tp, &tv);
1553 } else {
1554 error = tp->t_fb->tfb_tcp_output(tp);
1555 }
1556 if (ninp && ninp->inp_ppcb) {
1557 /*
1558 * If we have a nxt inp, see if we can
1559 * prefetch its ppcb. Note this may seem
1560 * "risky" since we have no locks (other
1561 * than the previous inp) and there no
1562 * assurance that ninp was not pulled while
1563 * we were processing inp and freed. If this
1564 * occured it could mean that either:
1565 *
1566 * a) Its NULL (which is fine we won't go
1567 * here) <or> b) Its valid (which is cool we
1568 * will prefetch it) <or> c) The inp got
1569 * freed back to the slab which was
1570 * reallocated. Then the piece of memory was
1571 * re-used and something else (not an
1572 * address) is in inp_ppcb. If that occurs
1573 * we don't crash, but take a TLB shootdown
1574 * performance hit (same as if it was NULL
1575 * and we tried to pre-fetch it).
1576 *
1577 * Considering that the likelyhood of <c> is
1578 * quite rare we will take a risk on doing
1579 * this. If performance drops after testing
1580 * we can always take this out. NB: the
1581 * kern_prefetch on amd64 actually has
1582 * protection against a bad address now via
1583 * the DMAP_() tests. This will prevent the
1584 * TLB hit, and instead if <c> occurs just
1585 * cause us to load cache with a useless
1586 * address (to us).
1587 */
1588 kern_prefetch(ninp->inp_ppcb, &prefetch_tp);
1589 prefetch_tp = 1;
1590 }
1591 INP_WUNLOCK(inp);
1592 INP_UNLOCK_ASSERT(inp);
1593 CURVNET_RESTORE();
1594 #ifdef INVARIANTS
1595 if (mtx_owned(&hpts->p_mtx)) {
1596 panic("Hpts:%p owns mtx prior-to lock line:%d",
1597 hpts, __LINE__);
1598 }
1599 #endif
1600 mtx_lock(&hpts->p_mtx);
1601 if (logging_on)
1602 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 4);
1603 hpts->p_inp = NULL;
1604 }
1605 HPTS_MTX_ASSERT(hpts);
1606 hpts->p_inp = NULL;
1607 hpts->p_cur_slot++;
1608 if (hpts->p_cur_slot >= NUM_OF_HPTSI_SLOTS) {
1609 hpts->p_cur_slot = 0;
1610 }
1611 }
1612 no_one:
1613 HPTS_MTX_ASSERT(hpts);
1614 hpts->p_prevtick = hpts->p_curtick;
1615 hpts->p_delayed_by = 0;
1616 /*
1617 * Check to see if we took an excess amount of time and need to run
1618 * more ticks (if we did not hit eno-bufs).
1619 */
1620 /* Re-run any input that may be there */
1621 (void)tcp_gethptstick(&tv);
1622 if (!TAILQ_EMPTY(&hpts->p_input)) {
1623 tcp_input_data(hpts, &tv);
1624 }
1625 #ifdef INVARIANTS
1626 if (TAILQ_EMPTY(&hpts->p_input) &&
1627 (hpts->p_on_inqueue_cnt != 0)) {
1628 panic("tp:%p in_hpts input empty but cnt:%d",
1629 hpts, hpts->p_on_inqueue_cnt);
1630 }
1631 #endif
1632 tick_now = tcp_gethptstick(&tv);
1633 if (SEQ_GT(tick_now, hpts->p_prevtick)) {
1634 struct timeval res;
1635
1636 /* Did we really spend a full tick or more in here? */
1637 timersub(&tv, ctick, &res);
1638 if (res.tv_sec || (res.tv_usec >= HPTS_TICKS_PER_USEC)) {
1639 counter_u64_add(hpts_loops, 1);
1640 if (logging_on) {
1641 tcp_hpts_log_it(hpts, inp, HPTSLOG_TOLONG, (uint32_t) res.tv_usec, tick_now);
1642 }
1643 *ctick = res;
1644 hpts->p_curtick = tick_now;
1645 goto again;
1646 }
1647 }
1648 no_run:
1649 {
1650 uint32_t t = 0, i, fnd = 0;
1651
1652 if (hpts->p_on_queue_cnt) {
1653
1654
1655 /*
1656 * Find next slot that is occupied and use that to
1657 * be the sleep time.
1658 */
1659 for (i = 1, t = hpts->p_nxt_slot; i < NUM_OF_HPTSI_SLOTS; i++) {
1660 if (TAILQ_EMPTY(&hpts->p_hptss[t]) == 0) {
1661 fnd = 1;
1662 break;
1663 }
1664 t = (t + 1) % NUM_OF_HPTSI_SLOTS;
1665 }
1666 if (fnd) {
1667 hpts->p_hpts_sleep_time = i;
1668 } else {
1669 counter_u64_add(back_tosleep, 1);
1670 #ifdef INVARIANTS
1671 panic("Hpts:%p cnt:%d but non found", hpts, hpts->p_on_queue_cnt);
1672 #endif
1673 hpts->p_on_queue_cnt = 0;
1674 goto non_found;
1675 }
1676 t++;
1677 } else {
1678 /* No one on the wheel sleep for all but 2 slots */
1679 non_found:
1680 if (hpts_sleep_max == 0)
1681 hpts_sleep_max = 1;
1682 hpts->p_hpts_sleep_time = min((NUM_OF_HPTSI_SLOTS - 2), hpts_sleep_max);
1683 t = 0;
1684 }
1685 if (logging_on) {
1686 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEPSET, t, (hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC));
1687 }
1688 }
1689 }
1690
1691 void
1692 __tcp_set_hpts(struct inpcb *inp, int32_t line)
1693 {
1694 struct tcp_hpts_entry *hpts;
1695
1696 INP_WLOCK_ASSERT(inp);
1697 hpts = tcp_hpts_lock(inp);
1698 if ((inp->inp_in_hpts == 0) &&
1699 (inp->inp_hpts_cpu_set == 0)) {
1700 inp->inp_hpts_cpu = hpts_cpuid(inp);
1701 inp->inp_hpts_cpu_set = 1;
1702 }
1703 mtx_unlock(&hpts->p_mtx);
1704 hpts = tcp_input_lock(inp);
1705 if ((inp->inp_input_cpu_set == 0) &&
1706 (inp->inp_in_input == 0)) {
1707 inp->inp_input_cpu = hpts_cpuid(inp);
1708 inp->inp_input_cpu_set = 1;
1709 }
1710 mtx_unlock(&hpts->p_mtx);
1711 }
1712
1713 uint16_t
1714 tcp_hpts_delayedby(struct inpcb *inp){
1715 return (tcp_pace.rp_ent[inp->inp_hpts_cpu]->p_delayed_by);
1716 }
1717
1718 static void
1719 tcp_hpts_thread(void *ctx)
1720 {
1721 struct tcp_hpts_entry *hpts;
1722 struct timeval tv;
1723 sbintime_t sb;
1724
1725 hpts = (struct tcp_hpts_entry *)ctx;
1726 mtx_lock(&hpts->p_mtx);
1727 if (hpts->p_direct_wake) {
1728 /* Signaled by input */
1729 if (logging_on)
1730 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 1, 1);
1731 callout_stop(&hpts->co);
1732 } else {
1733 /* Timed out */
1734 if (callout_pending(&hpts->co) ||
1735 !callout_active(&hpts->co)) {
1736 if (logging_on)
1737 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 2, 2);
1738 mtx_unlock(&hpts->p_mtx);
1739 return;
1740 }
1741 callout_deactivate(&hpts->co);
1742 if (logging_on)
1743 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 3, 3);
1744 }
1745 hpts->p_hpts_active = 1;
1746 (void)tcp_gethptstick(&tv);
1747 tcp_hptsi(hpts, &tv);
1748 HPTS_MTX_ASSERT(hpts);
1749 tv.tv_sec = 0;
1750 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1751 if (tcp_min_hptsi_time && (tv.tv_usec < tcp_min_hptsi_time)) {
1752 tv.tv_usec = tcp_min_hptsi_time;
1753 hpts->p_on_min_sleep = 1;
1754 } else {
1755 /* Clear the min sleep flag */
1756 hpts->p_on_min_sleep = 0;
1757 }
1758 hpts->p_hpts_active = 0;
1759 sb = tvtosbt(tv);
1760 if (tcp_hpts_callout_skip_swi == 0) {
1761 callout_reset_sbt_on(&hpts->co, sb, 0,
1762 hpts_timeout_swi, hpts, hpts->p_cpu,
1763 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1764 } else {
1765 callout_reset_sbt_on(&hpts->co, sb, 0,
1766 hpts_timeout_dir, hpts,
1767 hpts->p_cpu,
1768 C_PREL(tcp_hpts_precision));
1769 }
1770 hpts->p_direct_wake = 0;
1771 mtx_unlock(&hpts->p_mtx);
1772 }
1773
1774 #undef timersub
1775
1776 static void
1777 tcp_init_hptsi(void *st)
1778 {
1779 int32_t i, j, error, bound = 0, created = 0;
1780 size_t sz, asz;
1781 struct timeval tv;
1782 sbintime_t sb;
1783 struct tcp_hpts_entry *hpts;
1784 char unit[16];
1785 uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
1786
1787 tcp_pace.rp_proc = NULL;
1788 tcp_pace.rp_num_hptss = ncpus;
1789 hpts_loops = counter_u64_alloc(M_WAITOK);
1790 back_tosleep = counter_u64_alloc(M_WAITOK);
1791
1792 sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *));
1793 tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1794 asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS;
1795 for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
1796 tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry),
1797 M_TCPHPTS, M_WAITOK | M_ZERO);
1798 tcp_pace.rp_ent[i]->p_hptss = malloc(asz,
1799 M_TCPHPTS, M_WAITOK);
1800 hpts = tcp_pace.rp_ent[i];
1801 /*
1802 * Init all the hpts structures that are not specifically
1803 * zero'd by the allocations. Also lets attach them to the
1804 * appropriate sysctl block as well.
1805 */
1806 mtx_init(&hpts->p_mtx, "tcp_hpts_lck",
1807 "hpts", MTX_DEF | MTX_DUPOK);
1808 TAILQ_INIT(&hpts->p_input);
1809 for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) {
1810 TAILQ_INIT(&hpts->p_hptss[j]);
1811 }
1812 sysctl_ctx_init(&hpts->hpts_ctx);
1813 sprintf(unit, "%d", i);
1814 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx,
1815 SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts),
1816 OID_AUTO,
1817 unit,
1818 CTLFLAG_RW, 0,
1819 "");
1820 SYSCTL_ADD_INT(&hpts->hpts_ctx,
1821 SYSCTL_CHILDREN(hpts->hpts_root),
1822 OID_AUTO, "in_qcnt", CTLFLAG_RD,
1823 &hpts->p_on_inqueue_cnt, 0,
1824 "Count TCB's awaiting input processing");
1825 SYSCTL_ADD_INT(&hpts->hpts_ctx,
1826 SYSCTL_CHILDREN(hpts->hpts_root),
1827 OID_AUTO, "out_qcnt", CTLFLAG_RD,
1828 &hpts->p_on_queue_cnt, 0,
1829 "Count TCB's awaiting output processing");
1830 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1831 SYSCTL_CHILDREN(hpts->hpts_root),
1832 OID_AUTO, "active", CTLFLAG_RD,
1833 &hpts->p_hpts_active, 0,
1834 "Is the hpts active");
1835 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1836 SYSCTL_CHILDREN(hpts->hpts_root),
1837 OID_AUTO, "curslot", CTLFLAG_RD,
1838 &hpts->p_cur_slot, 0,
1839 "What the current slot is if active");
1840 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1841 SYSCTL_CHILDREN(hpts->hpts_root),
1842 OID_AUTO, "curtick", CTLFLAG_RD,
1843 &hpts->p_curtick, 0,
1844 "What the current tick on if active");
1845 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1846 SYSCTL_CHILDREN(hpts->hpts_root),
1847 OID_AUTO, "logsize", CTLFLAG_RD,
1848 &hpts->p_logsize, 0,
1849 "Hpts logging buffer size");
1850 hpts->p_hpts_sleep_time = NUM_OF_HPTSI_SLOTS - 2;
1851 hpts->p_num = i;
1852 hpts->p_prevtick = hpts->p_curtick = tcp_gethptstick(&tv);
1853 hpts->p_prevtick -= 1;
1854 hpts->p_prevtick %= NUM_OF_HPTSI_SLOTS;
1855 hpts->p_cpu = 0xffff;
1856 hpts->p_nxt_slot = 1;
1857 hpts->p_logsize = tcp_hpts_logging_size;
1858 if (hpts->p_logsize) {
1859 sz = (sizeof(struct hpts_log) * hpts->p_logsize);
1860 hpts->p_log = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1861 }
1862 callout_init(&hpts->co, 1);
1863 }
1864 /*
1865 * Now lets start ithreads to handle the hptss.
1866 */
1867 CPU_FOREACH(i) {
1868 hpts = tcp_pace.rp_ent[i];
1869 hpts->p_cpu = i;
1870 error = swi_add(&hpts->ie, "hpts",
1871 tcp_hpts_thread, (void *)hpts,
1872 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie);
1873 if (error) {
1874 panic("Can't add hpts:%p i:%d err:%d",
1875 hpts, i, error);
1876 }
1877 created++;
1878 if (tcp_bind_threads) {
1879 if (intr_event_bind(hpts->ie, i) == 0)
1880 bound++;
1881 }
1882 tv.tv_sec = 0;
1883 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1884 sb = tvtosbt(tv);
1885 if (tcp_hpts_callout_skip_swi == 0) {
1886 callout_reset_sbt_on(&hpts->co, sb, 0,
1887 hpts_timeout_swi, hpts, hpts->p_cpu,
1888 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1889 } else {
1890 callout_reset_sbt_on(&hpts->co, sb, 0,
1891 hpts_timeout_dir, hpts,
1892 hpts->p_cpu,
1893 C_PREL(tcp_hpts_precision));
1894 }
1895 }
1896 printf("TCP Hpts created %d swi interrupt thread and bound %d\n",
1897 created, bound);
1898 return;
1899 }
1900
1901 SYSINIT(tcphptsi, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, tcp_init_hptsi, NULL);
1902 MODULE_VERSION(tcphpts, 1);
Cache object: ccb9e46596918371c91c8648bb4752b6
|