1 /*-
2 * Copyright (c) 2015-2017 Patrick Kelsey
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 /*
28 * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include
29 * this code, add the following line to your kernel config:
30 *
31 * options TCP_RFC7413
32 *
33 *
34 * The generated TFO cookies are the 64-bit output of
35 * SipHash24(key=<16-byte-key>, msg=<client-ip>). Multiple concurrent valid
36 * keys are supported so that time-based rolling cookie invalidation
37 * policies can be implemented in the system. The default number of
38 * concurrent keys is 2. This can be adjusted in the kernel config as
39 * follows:
40 *
41 * options TCP_RFC7413_MAX_KEYS=<num-keys>
42 *
43 *
44 * In addition to the facilities defined in RFC7413, this implementation
45 * supports a pre-shared key (PSK) mode of operation in which the TFO server
46 * requires the client to be in posession of a shared secret in order for
47 * the client to be able to successfully open TFO connections with the
48 * server. This is useful, for example, in environments where TFO servers
49 * are exposed to both internal and external clients and only wish to allow
50 * TFO connections from internal clients.
51 *
52 * In the PSK mode of operation, the server generates and sends TFO cookies
53 * to requesting clients as usual. However, when validating cookies
54 * received in TFO SYNs from clients, the server requires the
55 * client-supplied cookie to equal SipHash24(key=<16-byte-psk>,
56 * msg=<cookie-sent-to-client>).
57 *
58 * Multiple concurrent valid pre-shared keys are supported so that
59 * time-based rolling PSK invalidation policies can be implemented in the
60 * system. The default number of concurrent pre-shared keys is 2. This can
61 * be adjusted in the kernel config as follows:
62 *
63 * options TCP_RFC7413_MAX_PSKS=<num-psks>
64 *
65 *
66 * The following TFO-specific sysctls are defined:
67 *
68 * net.inet.tcp.fastopen.acceptany (RW, default 0)
69 * When non-zero, all client-supplied TFO cookies will be considered to
70 * be valid.
71 *
72 * net.inet.tcp.fastopen.autokey (RW, default 120)
73 * When this and net.inet.tcp.fastopen.server_enable are non-zero, a new
74 * key will be automatically generated after this many seconds.
75 *
76 * net.inet.tcp.fastopen.ccache_bucket_limit
77 * (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT)
78 * The maximum number of entries in a client cookie cache bucket.
79 *
80 * net.inet.tcp.fastopen.ccache_buckets
81 * (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT)
82 * The number of client cookie cache buckets.
83 *
84 * net.inet.tcp.fastopen.ccache_list (RO)
85 * Print the client cookie cache.
86 *
87 * net.inet.tcp.fastopen.client_enable (RW, default 0)
88 * When zero, no new active (i.e., client) TFO connections can be
89 * created. On the transition from enabled to disabled, the client
90 * cookie cache is cleared and disabled. The transition from enabled to
91 * disabled does not affect any active TFO connections in progress; it
92 * only prevents new ones from being made.
93 *
94 * net.inet.tcp.fastopen.keylen (RD)
95 * The key length in bytes.
96 *
97 * net.inet.tcp.fastopen.maxkeys (RD)
98 * The maximum number of keys supported.
99 *
100 * net.inet.tcp.fastopen.maxpsks (RD)
101 * The maximum number of pre-shared keys supported.
102 *
103 * net.inet.tcp.fastopen.numkeys (RD)
104 * The current number of keys installed.
105 *
106 * net.inet.tcp.fastopen.numpsks (RD)
107 * The current number of pre-shared keys installed.
108 *
109 * net.inet.tcp.fastopen.path_disable_time
110 * (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT)
111 * When a failure occurs while trying to create a new active (i.e.,
112 * client) TFO connection, new active connections on the same path, as
113 * determined by the tuple {client_ip, server_ip, server_port}, will be
114 * forced to be non-TFO for this many seconds. Note that the path
115 * disable mechanism relies on state stored in client cookie cache
116 * entries, so it is possible for the disable time for a given path to
117 * be reduced if the corresponding client cookie cache entry is reused
118 * due to resource pressure before the disable period has elapsed.
119 *
120 * net.inet.tcp.fastopen.psk_enable (RW, default 0)
121 * When non-zero, pre-shared key (PSK) mode is enabled for all TFO
122 * servers. On the transition from enabled to disabled, all installed
123 * pre-shared keys are removed.
124 *
125 * net.inet.tcp.fastopen.server_enable (RW, default 0)
126 * When zero, no new passive (i.e., server) TFO connections can be
127 * created. On the transition from enabled to disabled, all installed
128 * keys and pre-shared keys are removed. On the transition from
129 * disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and
130 * there are no keys installed, a new key will be generated immediately.
131 * The transition from enabled to disabled does not affect any passive
132 * TFO connections in progress; it only prevents new ones from being
133 * made.
134 *
135 * net.inet.tcp.fastopen.setkey (WR)
136 * Install a new key by writing net.inet.tcp.fastopen.keylen bytes to
137 * this sysctl.
138 *
139 * net.inet.tcp.fastopen.setpsk (WR)
140 * Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen
141 * bytes to this sysctl.
142 *
143 * In order for TFO connections to be created via a listen socket, that
144 * socket must have the TCP_FASTOPEN socket option set on it. This option
145 * can be set on the socket either before or after the listen() is invoked.
146 * Clearing this option on a listen socket after it has been set has no
147 * effect on existing TFO connections or TFO connections in progress; it
148 * only prevents new TFO connections from being made.
149 *
150 * For passively-created sockets, the TCP_FASTOPEN socket option can be
151 * queried to determine whether the connection was established using TFO.
152 * Note that connections that are established via a TFO SYN, but that fall
153 * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option
154 * set.
155 *
156 * Per the RFC, this implementation limits the number of TFO connections
157 * that can be in the SYN_RECEIVED state on a per listen-socket basis.
158 * Whenever this limit is exceeded, requests for new TFO connections are
159 * serviced as non-TFO requests. Without such a limit, given a valid TFO
160 * cookie, an attacker could keep the listen queue in an overflow condition
161 * using a TFO SYN flood. This implementation sets the limit at half the
162 * configured listen backlog.
163 *
164 */
165
166 #include <sys/cdefs.h>
167 __FBSDID("$FreeBSD$");
168
169 #include "opt_inet.h"
170
171 #include <sys/param.h>
172 #include <sys/jail.h>
173 #include <sys/kernel.h>
174 #include <sys/hash.h>
175 #include <sys/limits.h>
176 #include <sys/lock.h>
177 #include <sys/proc.h>
178 #include <sys/rmlock.h>
179 #include <sys/sbuf.h>
180 #include <sys/socket.h>
181 #include <sys/socketvar.h>
182 #include <sys/sysctl.h>
183 #include <sys/systm.h>
184
185 #include <crypto/siphash/siphash.h>
186
187 #include <net/vnet.h>
188
189 #include <netinet/in.h>
190 #include <netinet/in_pcb.h>
191 #include <netinet/tcp_var.h>
192 #include <netinet/tcp_fastopen.h>
193
194 #define TCP_FASTOPEN_KEY_LEN SIPHASH_KEY_LENGTH
195
196 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN
197 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN
198 #endif
199
200 /*
201 * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold
202 * the PSK until the connect occurs.
203 */
204 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN
205 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN
206 #endif
207
208 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT 16
209 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT 2048 /* must be power of 2 */
210
211 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT 900 /* seconds */
212
213 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1)
214 #define TCP_FASTOPEN_MAX_KEYS 2
215 #else
216 #define TCP_FASTOPEN_MAX_KEYS TCP_RFC7413_MAX_KEYS
217 #endif
218
219 #if TCP_FASTOPEN_MAX_KEYS > 10
220 #undef TCP_FASTOPEN_MAX_KEYS
221 #define TCP_FASTOPEN_MAX_KEYS 10
222 #endif
223
224 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1)
225 #define TCP_FASTOPEN_MAX_PSKS 2
226 #else
227 #define TCP_FASTOPEN_MAX_PSKS TCP_RFC7413_MAX_PSKS
228 #endif
229
230 #if TCP_FASTOPEN_MAX_PSKS > 10
231 #undef TCP_FASTOPEN_MAX_PSKS
232 #define TCP_FASTOPEN_MAX_PSKS 10
233 #endif
234
235 struct tcp_fastopen_keylist {
236 unsigned int newest;
237 unsigned int newest_psk;
238 uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN];
239 uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN];
240 };
241
242 struct tcp_fastopen_callout {
243 struct callout c;
244 struct vnet *v;
245 };
246
247 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup(
248 struct in_conninfo *, struct tcp_fastopen_ccache_bucket **);
249 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create(
250 struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t,
251 uint8_t *);
252 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *,
253 unsigned int);
254 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *,
255 struct tcp_fastopen_ccache_bucket *);
256
257 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
258 "TCP Fast Open");
259
260 VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany) = 0;
261 #define V_tcp_fastopen_acceptany VNET(tcp_fastopen_acceptany)
262 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany,
263 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0,
264 "Accept any non-empty cookie");
265
266 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_autokey) = 120;
267 #define V_tcp_fastopen_autokey VNET(tcp_fastopen_autokey)
268 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS);
269 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey,
270 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
271 NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU",
272 "Number of seconds between auto-generation of a new key; zero disables");
273
274 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS);
275 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit,
276 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT,
277 NULL, 0, &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU",
278 "Max entries per bucket in client cookie cache");
279
280 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_ccache_buckets) =
281 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
282 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets)
283 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets,
284 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0,
285 "Client cookie cache number of buckets (power of 2)");
286
287 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 1;
288 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS);
289 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable,
290 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
291 NULL, 0, &sysctl_net_inet_tcp_fastopen_client_enable, "IU",
292 "Enable/disable TCP Fast Open client functionality");
293
294 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen,
295 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN,
296 "Key length in bytes");
297
298 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys,
299 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS,
300 "Maximum number of keys supported");
301
302 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks,
303 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS,
304 "Maximum number of pre-shared keys supported");
305
306 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numkeys) = 0;
307 #define V_tcp_fastopen_numkeys VNET(tcp_fastopen_numkeys)
308 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys,
309 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0,
310 "Number of keys installed");
311
312 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numpsks) = 0;
313 #define V_tcp_fastopen_numpsks VNET(tcp_fastopen_numpsks)
314 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks,
315 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0,
316 "Number of pre-shared keys installed");
317
318 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_path_disable_time) =
319 TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT;
320 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time)
321 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time,
322 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0,
323 "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path");
324
325 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_psk_enable) = 0;
326 #define V_tcp_fastopen_psk_enable VNET(tcp_fastopen_psk_enable)
327 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS);
328 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable,
329 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
330 NULL, 0, &sysctl_net_inet_tcp_fastopen_psk_enable, "IU",
331 "Enable/disable TCP Fast Open server pre-shared key mode");
332
333 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0;
334 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS);
335 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable,
336 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
337 NULL, 0, &sysctl_net_inet_tcp_fastopen_server_enable, "IU",
338 "Enable/disable TCP Fast Open server functionality");
339
340 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS);
341 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey,
342 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
343 NULL, 0, &sysctl_net_inet_tcp_fastopen_setkey, "",
344 "Install a new key");
345
346 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS);
347 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk,
348 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
349 NULL, 0, &sysctl_net_inet_tcp_fastopen_setpsk, "",
350 "Install a new pre-shared key");
351
352 static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS);
353 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_list,
354 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
355 NULL, 0, sysctl_net_inet_tcp_fastopen_ccache_list, "A",
356 "List of all client cookie cache entries");
357
358 VNET_DEFINE_STATIC(struct rmlock, tcp_fastopen_keylock);
359 #define V_tcp_fastopen_keylock VNET(tcp_fastopen_keylock)
360
361 #define TCP_FASTOPEN_KEYS_RLOCK(t) rm_rlock(&V_tcp_fastopen_keylock, (t))
362 #define TCP_FASTOPEN_KEYS_RUNLOCK(t) rm_runlock(&V_tcp_fastopen_keylock, (t))
363 #define TCP_FASTOPEN_KEYS_WLOCK() rm_wlock(&V_tcp_fastopen_keylock)
364 #define TCP_FASTOPEN_KEYS_WUNLOCK() rm_wunlock(&V_tcp_fastopen_keylock)
365
366 VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys);
367 #define V_tcp_fastopen_keys VNET(tcp_fastopen_keys)
368
369 VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx);
370 #define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx)
371
372 VNET_DEFINE_STATIC(uma_zone_t, counter_zone);
373 #define V_counter_zone VNET(counter_zone)
374
375 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets");
376
377 VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache);
378 #define V_tcp_fastopen_ccache VNET(tcp_fastopen_ccache)
379
380 #define CCB_LOCK(ccb) mtx_lock(&(ccb)->ccb_mtx)
381 #define CCB_UNLOCK(ccb) mtx_unlock(&(ccb)->ccb_mtx)
382 #define CCB_LOCK_ASSERT(ccb) mtx_assert(&(ccb)->ccb_mtx, MA_OWNED)
383
384 void
385 tcp_fastopen_init(void)
386 {
387 unsigned int i;
388
389 V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
390 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
391 rm_init(&V_tcp_fastopen_keylock, "tfo_keylock");
392 callout_init_rm(&V_tcp_fastopen_autokey_ctx.c,
393 &V_tcp_fastopen_keylock, 0);
394 V_tcp_fastopen_autokey_ctx.v = curvnet;
395 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
396 V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1;
397
398 /* May already be non-zero if kernel tunable was set */
399 if (V_tcp_fastopen_ccache.bucket_limit == 0)
400 V_tcp_fastopen_ccache.bucket_limit =
401 TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT;
402
403 /* May already be non-zero if kernel tunable was set */
404 if ((V_tcp_fastopen_ccache_buckets == 0) ||
405 !powerof2(V_tcp_fastopen_ccache_buckets))
406 V_tcp_fastopen_ccache.buckets =
407 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
408 else
409 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets;
410
411 V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1;
412 V_tcp_fastopen_ccache.secret = arc4random();
413
414 V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets *
415 sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE,
416 M_WAITOK | M_ZERO);
417
418 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
419 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries);
420 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket",
421 NULL, MTX_DEF);
422 if (V_tcp_fastopen_client_enable) {
423 /* enable bucket */
424 V_tcp_fastopen_ccache.base[i].ccb_num_entries = 0;
425 } else {
426 /* disable bucket */
427 V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1;
428 }
429 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache;
430 }
431
432 /*
433 * Note that while the total number of entries in the cookie cache
434 * is limited by the table management logic to
435 * V_tcp_fastopen_ccache.buckets *
436 * V_tcp_fastopen_ccache.bucket_limit, the total number of items in
437 * this zone can exceed that amount by the number of CPUs in the
438 * system times the maximum number of unallocated items that can be
439 * present in each UMA per-CPU cache for this zone.
440 */
441 V_tcp_fastopen_ccache.zone = uma_zcreate("tfo_ccache_entries",
442 sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
443 UMA_ALIGN_CACHE, 0);
444 }
445
446 void
447 tcp_fastopen_destroy(void)
448 {
449 struct tcp_fastopen_ccache_bucket *ccb;
450 unsigned int i;
451
452 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
453 ccb = &V_tcp_fastopen_ccache.base[i];
454 tcp_fastopen_ccache_bucket_trim(ccb, 0);
455 mtx_destroy(&ccb->ccb_mtx);
456 }
457
458 KASSERT(uma_zone_get_cur(V_tcp_fastopen_ccache.zone) == 0,
459 ("%s: TFO ccache zone allocation count not 0", __func__));
460 uma_zdestroy(V_tcp_fastopen_ccache.zone);
461 free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE);
462
463 callout_drain(&V_tcp_fastopen_autokey_ctx.c);
464 rm_destroy(&V_tcp_fastopen_keylock);
465 uma_zdestroy(V_counter_zone);
466 }
467
468 unsigned int *
469 tcp_fastopen_alloc_counter(void)
470 {
471 unsigned int *counter;
472 counter = uma_zalloc(V_counter_zone, M_NOWAIT);
473 if (counter)
474 *counter = 1;
475 return (counter);
476 }
477
478 void
479 tcp_fastopen_decrement_counter(unsigned int *counter)
480 {
481 if (*counter == 1)
482 uma_zfree(V_counter_zone, counter);
483 else
484 atomic_subtract_int(counter, 1);
485 }
486
487 static void
488 tcp_fastopen_addkey_locked(uint8_t *key)
489 {
490
491 V_tcp_fastopen_keys.newest++;
492 if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS)
493 V_tcp_fastopen_keys.newest = 0;
494 memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key,
495 TCP_FASTOPEN_KEY_LEN);
496 if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS)
497 V_tcp_fastopen_numkeys++;
498 }
499
500 static void
501 tcp_fastopen_addpsk_locked(uint8_t *psk)
502 {
503
504 V_tcp_fastopen_keys.newest_psk++;
505 if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS)
506 V_tcp_fastopen_keys.newest_psk = 0;
507 memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk,
508 TCP_FASTOPEN_KEY_LEN);
509 if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS)
510 V_tcp_fastopen_numpsks++;
511 }
512
513 static void
514 tcp_fastopen_autokey_locked(void)
515 {
516 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
517
518 arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0);
519 tcp_fastopen_addkey_locked(newkey);
520 }
521
522 static void
523 tcp_fastopen_autokey_callout(void *arg)
524 {
525 struct tcp_fastopen_callout *ctx = arg;
526
527 CURVNET_SET(ctx->v);
528 tcp_fastopen_autokey_locked();
529 callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz,
530 tcp_fastopen_autokey_callout, ctx);
531 CURVNET_RESTORE();
532 }
533
534 static uint64_t
535 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc)
536 {
537 SIPHASH_CTX ctx;
538 uint64_t siphash;
539
540 SipHash24_Init(&ctx);
541 SipHash_SetKey(&ctx, key);
542 switch (inc->inc_flags & INC_ISIPV6) {
543 #ifdef INET
544 case 0:
545 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr));
546 break;
547 #endif
548 #ifdef INET6
549 case INC_ISIPV6:
550 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr));
551 break;
552 #endif
553 }
554 SipHash_Final((u_int8_t *)&siphash, &ctx);
555
556 return (siphash);
557 }
558
559 static uint64_t
560 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len)
561 {
562 SIPHASH_CTX ctx;
563 uint64_t psk_cookie;
564
565 SipHash24_Init(&ctx);
566 SipHash_SetKey(&ctx, psk);
567 SipHash_Update(&ctx, cookie, cookie_len);
568 SipHash_Final((u_int8_t *)&psk_cookie, &ctx);
569
570 return (psk_cookie);
571 }
572
573 static int
574 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie)
575 {
576 unsigned int i, psk_index;
577 uint64_t psk_cookie;
578
579 if (V_tcp_fastopen_psk_enable) {
580 psk_index = V_tcp_fastopen_keys.newest_psk;
581 for (i = 0; i < V_tcp_fastopen_numpsks; i++) {
582 psk_cookie =
583 tcp_fastopen_make_psk_cookie(
584 V_tcp_fastopen_keys.psk[psk_index],
585 (uint8_t *)cur_cookie,
586 TCP_FASTOPEN_COOKIE_LEN);
587
588 if (memcmp(wire_cookie, &psk_cookie,
589 TCP_FASTOPEN_COOKIE_LEN) == 0)
590 return (1);
591
592 if (psk_index == 0)
593 psk_index = TCP_FASTOPEN_MAX_PSKS - 1;
594 else
595 psk_index--;
596 }
597 } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0)
598 return (1);
599
600 return (0);
601 }
602
603 /*
604 * Return values:
605 * -1 the cookie is invalid and no valid cookie is available
606 * 0 the cookie is invalid and the latest cookie has been returned
607 * 1 the cookie is valid and the latest cookie has been returned
608 */
609 int
610 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
611 unsigned int len, uint64_t *latest_cookie)
612 {
613 struct rm_priotracker tracker;
614 unsigned int i, key_index;
615 int rv;
616 uint64_t cur_cookie;
617
618 if (V_tcp_fastopen_acceptany) {
619 *latest_cookie = 0;
620 return (1);
621 }
622
623 TCP_FASTOPEN_KEYS_RLOCK(&tracker);
624 if (len != TCP_FASTOPEN_COOKIE_LEN) {
625 if (V_tcp_fastopen_numkeys > 0) {
626 *latest_cookie =
627 tcp_fastopen_make_cookie(
628 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
629 inc);
630 rv = 0;
631 } else
632 rv = -1;
633 goto out;
634 }
635
636 /*
637 * Check against each available key, from newest to oldest.
638 */
639 key_index = V_tcp_fastopen_keys.newest;
640 for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
641 cur_cookie =
642 tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
643 inc);
644 if (i == 0)
645 *latest_cookie = cur_cookie;
646 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie);
647 if (rv)
648 goto out;
649 if (key_index == 0)
650 key_index = TCP_FASTOPEN_MAX_KEYS - 1;
651 else
652 key_index--;
653 }
654 rv = 0;
655
656 out:
657 TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
658 return (rv);
659 }
660
661 static int
662 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
663 {
664 int error;
665 unsigned int new;
666
667 new = V_tcp_fastopen_autokey;
668 error = sysctl_handle_int(oidp, &new, 0, req);
669 if (error == 0 && req->newptr) {
670 if (new > (INT_MAX / hz))
671 return (EINVAL);
672
673 TCP_FASTOPEN_KEYS_WLOCK();
674 if (V_tcp_fastopen_server_enable) {
675 if (V_tcp_fastopen_autokey && !new)
676 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
677 else if (new)
678 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
679 new * hz, tcp_fastopen_autokey_callout,
680 &V_tcp_fastopen_autokey_ctx);
681 }
682 V_tcp_fastopen_autokey = new;
683 TCP_FASTOPEN_KEYS_WUNLOCK();
684 }
685
686 return (error);
687 }
688
689 static int
690 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)
691 {
692 int error;
693 unsigned int new;
694
695 new = V_tcp_fastopen_psk_enable;
696 error = sysctl_handle_int(oidp, &new, 0, req);
697 if (error == 0 && req->newptr) {
698 if (V_tcp_fastopen_psk_enable && !new) {
699 /* enabled -> disabled */
700 TCP_FASTOPEN_KEYS_WLOCK();
701 V_tcp_fastopen_numpsks = 0;
702 V_tcp_fastopen_keys.newest_psk =
703 TCP_FASTOPEN_MAX_PSKS - 1;
704 V_tcp_fastopen_psk_enable = 0;
705 TCP_FASTOPEN_KEYS_WUNLOCK();
706 } else if (!V_tcp_fastopen_psk_enable && new) {
707 /* disabled -> enabled */
708 TCP_FASTOPEN_KEYS_WLOCK();
709 V_tcp_fastopen_psk_enable = 1;
710 TCP_FASTOPEN_KEYS_WUNLOCK();
711 }
712 }
713 return (error);
714 }
715
716 static int
717 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)
718 {
719 int error;
720 unsigned int new;
721
722 new = V_tcp_fastopen_server_enable;
723 error = sysctl_handle_int(oidp, &new, 0, req);
724 if (error == 0 && req->newptr) {
725 if (V_tcp_fastopen_server_enable && !new) {
726 /* enabled -> disabled */
727 TCP_FASTOPEN_KEYS_WLOCK();
728 V_tcp_fastopen_numkeys = 0;
729 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
730 if (V_tcp_fastopen_autokey)
731 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
732 V_tcp_fastopen_numpsks = 0;
733 V_tcp_fastopen_keys.newest_psk =
734 TCP_FASTOPEN_MAX_PSKS - 1;
735 V_tcp_fastopen_server_enable = 0;
736 TCP_FASTOPEN_KEYS_WUNLOCK();
737 } else if (!V_tcp_fastopen_server_enable && new) {
738 /* disabled -> enabled */
739 TCP_FASTOPEN_KEYS_WLOCK();
740 if (V_tcp_fastopen_autokey &&
741 (V_tcp_fastopen_numkeys == 0)) {
742 tcp_fastopen_autokey_locked();
743 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
744 V_tcp_fastopen_autokey * hz,
745 tcp_fastopen_autokey_callout,
746 &V_tcp_fastopen_autokey_ctx);
747 }
748 V_tcp_fastopen_server_enable = 1;
749 TCP_FASTOPEN_KEYS_WUNLOCK();
750 }
751 }
752 return (error);
753 }
754
755 static int
756 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)
757 {
758 int error;
759 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
760
761 if (req->oldptr != NULL || req->oldlen != 0)
762 return (EINVAL);
763 if (req->newptr == NULL)
764 return (EPERM);
765 if (req->newlen != sizeof(newkey))
766 return (EINVAL);
767 error = SYSCTL_IN(req, newkey, sizeof(newkey));
768 if (error)
769 return (error);
770
771 TCP_FASTOPEN_KEYS_WLOCK();
772 tcp_fastopen_addkey_locked(newkey);
773 TCP_FASTOPEN_KEYS_WUNLOCK();
774
775 return (0);
776 }
777
778 static int
779 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)
780 {
781 int error;
782 uint8_t newpsk[TCP_FASTOPEN_KEY_LEN];
783
784 if (req->oldptr != NULL || req->oldlen != 0)
785 return (EINVAL);
786 if (req->newptr == NULL)
787 return (EPERM);
788 if (req->newlen != sizeof(newpsk))
789 return (EINVAL);
790 error = SYSCTL_IN(req, newpsk, sizeof(newpsk));
791 if (error)
792 return (error);
793
794 TCP_FASTOPEN_KEYS_WLOCK();
795 tcp_fastopen_addpsk_locked(newpsk);
796 TCP_FASTOPEN_KEYS_WUNLOCK();
797
798 return (0);
799 }
800
801 static int
802 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)
803 {
804 struct tcp_fastopen_ccache_bucket *ccb;
805 int error;
806 unsigned int new;
807 unsigned int i;
808
809 new = V_tcp_fastopen_ccache.bucket_limit;
810 error = sysctl_handle_int(oidp, &new, 0, req);
811 if (error == 0 && req->newptr) {
812 if ((new == 0) || (new > INT_MAX))
813 error = EINVAL;
814 else {
815 if (new < V_tcp_fastopen_ccache.bucket_limit) {
816 for (i = 0; i < V_tcp_fastopen_ccache.buckets;
817 i++) {
818 ccb = &V_tcp_fastopen_ccache.base[i];
819 tcp_fastopen_ccache_bucket_trim(ccb, new);
820 }
821 }
822 V_tcp_fastopen_ccache.bucket_limit = new;
823 }
824 }
825 return (error);
826 }
827
828 static int
829 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)
830 {
831 struct tcp_fastopen_ccache_bucket *ccb;
832 int error;
833 unsigned int new, i;
834
835 new = V_tcp_fastopen_client_enable;
836 error = sysctl_handle_int(oidp, &new, 0, req);
837 if (error == 0 && req->newptr) {
838 if (V_tcp_fastopen_client_enable && !new) {
839 /* enabled -> disabled */
840 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
841 ccb = &V_tcp_fastopen_ccache.base[i];
842 KASSERT(ccb->ccb_num_entries > -1,
843 ("%s: ccb->ccb_num_entries %d is negative",
844 __func__, ccb->ccb_num_entries));
845 tcp_fastopen_ccache_bucket_trim(ccb, 0);
846 }
847 V_tcp_fastopen_client_enable = 0;
848 } else if (!V_tcp_fastopen_client_enable && new) {
849 /* disabled -> enabled */
850 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
851 ccb = &V_tcp_fastopen_ccache.base[i];
852 CCB_LOCK(ccb);
853 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
854 ("%s: ccb->ccb_entries not empty", __func__));
855 KASSERT(ccb->ccb_num_entries == -1,
856 ("%s: ccb->ccb_num_entries %d not -1", __func__,
857 ccb->ccb_num_entries));
858 ccb->ccb_num_entries = 0; /* enable bucket */
859 CCB_UNLOCK(ccb);
860 }
861 V_tcp_fastopen_client_enable = 1;
862 }
863 }
864 return (error);
865 }
866
867 void
868 tcp_fastopen_connect(struct tcpcb *tp)
869 {
870 struct inpcb *inp = tptoinpcb(tp);
871 struct tcp_fastopen_ccache_bucket *ccb;
872 struct tcp_fastopen_ccache_entry *cce;
873 sbintime_t now;
874 uint16_t server_mss;
875 uint64_t psk_cookie;
876
877 psk_cookie = 0;
878 cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb);
879 if (cce) {
880 if (cce->disable_time == 0) {
881 if ((cce->cookie_len > 0) &&
882 (tp->t_tfo_client_cookie_len ==
883 TCP_FASTOPEN_PSK_LEN)) {
884 psk_cookie =
885 tcp_fastopen_make_psk_cookie(
886 tp->t_tfo_cookie.client,
887 cce->cookie, cce->cookie_len);
888 } else {
889 tp->t_tfo_client_cookie_len = cce->cookie_len;
890 memcpy(tp->t_tfo_cookie.client, cce->cookie,
891 cce->cookie_len);
892 }
893 server_mss = cce->server_mss;
894 CCB_UNLOCK(ccb);
895 if (tp->t_tfo_client_cookie_len ==
896 TCP_FASTOPEN_PSK_LEN && psk_cookie) {
897 tp->t_tfo_client_cookie_len =
898 TCP_FASTOPEN_COOKIE_LEN;
899 memcpy(tp->t_tfo_cookie.client, &psk_cookie,
900 TCP_FASTOPEN_COOKIE_LEN);
901 }
902 tcp_mss(tp, server_mss ? server_mss : -1);
903 tp->snd_wnd = tp->t_maxseg;
904 } else {
905 /*
906 * The path is disabled. Check the time and
907 * possibly re-enable.
908 */
909 now = getsbinuptime();
910 if (now - cce->disable_time >
911 ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) {
912 /*
913 * Re-enable path. Force a TFO cookie
914 * request. Forget the old MSS as it may be
915 * bogus now, and we will rediscover it in
916 * the SYN|ACK.
917 */
918 cce->disable_time = 0;
919 cce->server_mss = 0;
920 cce->cookie_len = 0;
921 /*
922 * tp->t_tfo... cookie details are already
923 * zero from the tcpcb init.
924 */
925 } else {
926 /*
927 * Path is disabled, so disable TFO on this
928 * connection.
929 */
930 tp->t_flags &= ~TF_FASTOPEN;
931 }
932 CCB_UNLOCK(ccb);
933 tcp_mss(tp, -1);
934 /*
935 * snd_wnd is irrelevant since we are either forcing
936 * a TFO cookie request or disabling TFO - either
937 * way, no data with the SYN.
938 */
939 }
940 } else {
941 /*
942 * A new entry for this path will be created when a SYN|ACK
943 * comes back, or the attempt otherwise fails.
944 */
945 CCB_UNLOCK(ccb);
946 tcp_mss(tp, -1);
947 /*
948 * snd_wnd is irrelevant since we are forcing a TFO cookie
949 * request.
950 */
951 }
952 }
953
954 void
955 tcp_fastopen_disable_path(struct tcpcb *tp)
956 {
957 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc;
958 struct tcp_fastopen_ccache_bucket *ccb;
959 struct tcp_fastopen_ccache_entry *cce;
960
961 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
962 if (cce) {
963 cce->server_mss = 0;
964 cce->cookie_len = 0;
965 /*
966 * Preserve the existing disable time if it is already
967 * disabled.
968 */
969 if (cce->disable_time == 0)
970 cce->disable_time = getsbinuptime();
971 } else /* use invalid cookie len to create disabled entry */
972 tcp_fastopen_ccache_create(ccb, inc, 0,
973 TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL);
974
975 CCB_UNLOCK(ccb);
976 tp->t_flags &= ~TF_FASTOPEN;
977 }
978
979 void
980 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss,
981 uint8_t cookie_len, uint8_t *cookie)
982 {
983 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc;
984 struct tcp_fastopen_ccache_bucket *ccb;
985 struct tcp_fastopen_ccache_entry *cce;
986
987 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
988 if (cce) {
989 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
990 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
991 ((cookie_len & 0x1) == 0)) {
992 cce->server_mss = mss;
993 cce->cookie_len = cookie_len;
994 memcpy(cce->cookie, cookie, cookie_len);
995 cce->disable_time = 0;
996 } else {
997 /* invalid cookie length, disable entry */
998 cce->server_mss = 0;
999 cce->cookie_len = 0;
1000 /*
1001 * Preserve the existing disable time if it is
1002 * already disabled.
1003 */
1004 if (cce->disable_time == 0)
1005 cce->disable_time = getsbinuptime();
1006 }
1007 } else
1008 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie);
1009
1010 CCB_UNLOCK(ccb);
1011 }
1012
1013 static struct tcp_fastopen_ccache_entry *
1014 tcp_fastopen_ccache_lookup(struct in_conninfo *inc,
1015 struct tcp_fastopen_ccache_bucket **ccbp)
1016 {
1017 struct tcp_fastopen_ccache_bucket *ccb;
1018 struct tcp_fastopen_ccache_entry *cce;
1019 uint32_t last_word;
1020 uint32_t hash;
1021
1022 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4,
1023 V_tcp_fastopen_ccache.secret);
1024 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4,
1025 hash);
1026 last_word = inc->inc_fport;
1027 hash = jenkins_hash32(&last_word, 1, hash);
1028 ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask];
1029 *ccbp = ccb;
1030 CCB_LOCK(ccb);
1031
1032 /*
1033 * Always returns with locked bucket.
1034 */
1035 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link)
1036 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) &&
1037 (cce->server_port == inc->inc_ie.ie_fport) &&
1038 (((cce->af == AF_INET) &&
1039 (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) &&
1040 (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) ||
1041 ((cce->af == AF_INET6) &&
1042 IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) &&
1043 IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr))))
1044 break;
1045
1046 return (cce);
1047 }
1048
1049 static struct tcp_fastopen_ccache_entry *
1050 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb,
1051 struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
1052 {
1053 struct tcp_fastopen_ccache_entry *cce;
1054
1055 /*
1056 * 1. Create a new entry, or
1057 * 2. Reclaim an existing entry, or
1058 * 3. Fail
1059 */
1060
1061 CCB_LOCK_ASSERT(ccb);
1062
1063 cce = NULL;
1064 if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit)
1065 cce = uma_zalloc(V_tcp_fastopen_ccache.zone, M_NOWAIT);
1066
1067 if (cce == NULL) {
1068 /*
1069 * At bucket limit, or out of memory - reclaim last
1070 * entry in bucket.
1071 */
1072 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries);
1073 if (cce == NULL) {
1074 /* XXX count this event */
1075 return (NULL);
1076 }
1077
1078 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1079 } else
1080 ccb->ccb_num_entries++;
1081
1082 TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link);
1083 cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET;
1084 if (cce->af == AF_INET) {
1085 cce->cce_client_ip.v4 = inc->inc_laddr;
1086 cce->cce_server_ip.v4 = inc->inc_faddr;
1087 } else {
1088 cce->cce_client_ip.v6 = inc->inc6_laddr;
1089 cce->cce_server_ip.v6 = inc->inc6_faddr;
1090 }
1091 cce->server_port = inc->inc_fport;
1092 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1093 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1094 ((cookie_len & 0x1) == 0)) {
1095 cce->server_mss = mss;
1096 cce->cookie_len = cookie_len;
1097 memcpy(cce->cookie, cookie, cookie_len);
1098 cce->disable_time = 0;
1099 } else {
1100 /* invalid cookie length, disable cce */
1101 cce->server_mss = 0;
1102 cce->cookie_len = 0;
1103 cce->disable_time = getsbinuptime();
1104 }
1105
1106 return (cce);
1107 }
1108
1109 static void
1110 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb,
1111 unsigned int limit)
1112 {
1113 struct tcp_fastopen_ccache_entry *cce, *cce_tmp;
1114 unsigned int entries;
1115
1116 CCB_LOCK(ccb);
1117 entries = 0;
1118 TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) {
1119 entries++;
1120 if (entries > limit)
1121 tcp_fastopen_ccache_entry_drop(cce, ccb);
1122 }
1123 KASSERT(ccb->ccb_num_entries <= (int)limit,
1124 ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__,
1125 ccb->ccb_num_entries, limit));
1126 if (limit == 0) {
1127 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
1128 ("%s: ccb->ccb_entries not empty", __func__));
1129 ccb->ccb_num_entries = -1; /* disable bucket */
1130 }
1131 CCB_UNLOCK(ccb);
1132 }
1133
1134 static void
1135 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce,
1136 struct tcp_fastopen_ccache_bucket *ccb)
1137 {
1138
1139 CCB_LOCK_ASSERT(ccb);
1140
1141 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1142 ccb->ccb_num_entries--;
1143 uma_zfree(V_tcp_fastopen_ccache.zone, cce);
1144 }
1145
1146 static int
1147 sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)
1148 {
1149 struct sbuf sb;
1150 struct tcp_fastopen_ccache_bucket *ccb;
1151 struct tcp_fastopen_ccache_entry *cce;
1152 sbintime_t now, duration, limit;
1153 const int linesize = 128;
1154 int i, error, num_entries;
1155 unsigned int j;
1156 #ifdef INET6
1157 char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN];
1158 #else
1159 char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN];
1160 #endif
1161
1162 if (jailed_without_vnet(curthread->td_ucred) != 0)
1163 return (EPERM);
1164
1165 /* Only allow root to read the client cookie cache */
1166 if (curthread->td_ucred->cr_uid != 0)
1167 return (EPERM);
1168
1169 num_entries = 0;
1170 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1171 ccb = &V_tcp_fastopen_ccache.base[i];
1172 CCB_LOCK(ccb);
1173 if (ccb->ccb_num_entries > 0)
1174 num_entries += ccb->ccb_num_entries;
1175 CCB_UNLOCK(ccb);
1176 }
1177 sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL);
1178
1179 sbuf_printf(&sb,
1180 "\nLocal IP address Remote IP address Port MSS"
1181 " Disabled Cookie\n");
1182
1183 now = getsbinuptime();
1184 limit = (sbintime_t)V_tcp_fastopen_path_disable_time << 32;
1185 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1186 ccb = &V_tcp_fastopen_ccache.base[i];
1187 CCB_LOCK(ccb);
1188 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) {
1189 if (cce->disable_time != 0) {
1190 duration = now - cce->disable_time;
1191 if (limit >= duration)
1192 duration = limit - duration;
1193 else
1194 duration = 0;
1195 } else
1196 duration = 0;
1197 sbuf_printf(&sb,
1198 "%-20s %-20s %5u %5u ",
1199 inet_ntop(cce->af, &cce->cce_client_ip,
1200 clt_buf, sizeof(clt_buf)),
1201 inet_ntop(cce->af, &cce->cce_server_ip,
1202 srv_buf, sizeof(srv_buf)),
1203 ntohs(cce->server_port),
1204 cce->server_mss);
1205 if (duration > 0)
1206 sbuf_printf(&sb, "%7ds ", sbintime_getsec(duration));
1207 else
1208 sbuf_printf(&sb, "%8s ", "No");
1209 for (j = 0; j < cce->cookie_len; j++)
1210 sbuf_printf(&sb, "%02x", cce->cookie[j]);
1211 sbuf_putc(&sb, '\n');
1212 }
1213 CCB_UNLOCK(ccb);
1214 }
1215 error = sbuf_finish(&sb);
1216 if (error == 0)
1217 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
1218 sbuf_delete(&sb);
1219 return (error);
1220 }
Cache object: 73160e5e00375c73441b7c0ece9ae30e
|