1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1988, 1990, 1993
5 * The Regents of the University of California.
6 * Copyright (c) 2004 The FreeBSD Foundation
7 * Copyright (c) 2004-2008 Robert N. M. Watson
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94
35 */
36
37 /*
38 * Comments on the socket life cycle:
39 *
40 * soalloc() sets of socket layer state for a socket, called only by
41 * socreate() and sonewconn(). Socket layer private.
42 *
43 * sodealloc() tears down socket layer state for a socket, called only by
44 * sofree() and sonewconn(). Socket layer private.
45 *
46 * pru_attach() associates protocol layer state with an allocated socket;
47 * called only once, may fail, aborting socket allocation. This is called
48 * from socreate() and sonewconn(). Socket layer private.
49 *
50 * pru_detach() disassociates protocol layer state from an attached socket,
51 * and will be called exactly once for sockets in which pru_attach() has
52 * been successfully called. If pru_attach() returned an error,
53 * pru_detach() will not be called. Socket layer private.
54 *
55 * pru_abort() and pru_close() notify the protocol layer that the last
56 * consumer of a socket is starting to tear down the socket, and that the
57 * protocol should terminate the connection. Historically, pru_abort() also
58 * detached protocol state from the socket state, but this is no longer the
59 * case.
60 *
61 * socreate() creates a socket and attaches protocol state. This is a public
62 * interface that may be used by socket layer consumers to create new
63 * sockets.
64 *
65 * sonewconn() creates a socket and attaches protocol state. This is a
66 * public interface that may be used by protocols to create new sockets when
67 * a new connection is received and will be available for accept() on a
68 * listen socket.
69 *
70 * soclose() destroys a socket after possibly waiting for it to disconnect.
71 * This is a public interface that socket consumers should use to close and
72 * release a socket when done with it.
73 *
74 * soabort() destroys a socket without waiting for it to disconnect (used
75 * only for incoming connections that are already partially or fully
76 * connected). This is used internally by the socket layer when clearing
77 * listen socket queues (due to overflow or close on the listen socket), but
78 * is also a public interface protocols may use to abort connections in
79 * their incomplete listen queues should they no longer be required. Sockets
80 * placed in completed connection listen queues should not be aborted for
81 * reasons described in the comment above the soclose() implementation. This
82 * is not a general purpose close routine, and except in the specific
83 * circumstances described here, should not be used.
84 *
85 * sofree() will free a socket and its protocol state if all references on
86 * the socket have been released, and is the public interface to attempt to
87 * free a socket when a reference is removed. This is a socket layer private
88 * interface.
89 *
90 * NOTE: In addition to socreate() and soclose(), which provide a single
91 * socket reference to the consumer to be managed as required, there are two
92 * calls to explicitly manage socket references, soref(), and sorele().
93 * Currently, these are generally required only when transitioning a socket
94 * from a listen queue to a file descriptor, in order to prevent garbage
95 * collection of the socket at an untimely moment. For a number of reasons,
96 * these interfaces are not preferred, and should be avoided.
97 *
98 * NOTE: With regard to VNETs the general rule is that callers do not set
99 * curvnet. Exceptions to this rule include soabort(), sodisconnect(),
100 * sofree() (and with that sorele(), sotryfree()), as well as sonewconn()
101 * and sorflush(), which are usually called from a pre-set VNET context.
102 * sopoll() currently does not need a VNET context to be set.
103 */
104
105 #include <sys/cdefs.h>
106 __FBSDID("$FreeBSD$");
107
108 #include "opt_inet.h"
109 #include "opt_inet6.h"
110 #include "opt_kern_tls.h"
111 #include "opt_sctp.h"
112
113 #include <sys/param.h>
114 #include <sys/systm.h>
115 #include <sys/capsicum.h>
116 #include <sys/fcntl.h>
117 #include <sys/limits.h>
118 #include <sys/lock.h>
119 #include <sys/mac.h>
120 #include <sys/malloc.h>
121 #include <sys/mbuf.h>
122 #include <sys/mutex.h>
123 #include <sys/domain.h>
124 #include <sys/file.h> /* for struct knote */
125 #include <sys/hhook.h>
126 #include <sys/kernel.h>
127 #include <sys/khelp.h>
128 #include <sys/ktls.h>
129 #include <sys/event.h>
130 #include <sys/eventhandler.h>
131 #include <sys/poll.h>
132 #include <sys/proc.h>
133 #include <sys/protosw.h>
134 #include <sys/sbuf.h>
135 #include <sys/socket.h>
136 #include <sys/socketvar.h>
137 #include <sys/resourcevar.h>
138 #include <net/route.h>
139 #include <sys/signalvar.h>
140 #include <sys/stat.h>
141 #include <sys/sx.h>
142 #include <sys/sysctl.h>
143 #include <sys/taskqueue.h>
144 #include <sys/uio.h>
145 #include <sys/un.h>
146 #include <sys/unpcb.h>
147 #include <sys/jail.h>
148 #include <sys/syslog.h>
149 #include <netinet/in.h>
150 #include <netinet/in_pcb.h>
151 #include <netinet/tcp.h>
152
153 #include <net/vnet.h>
154
155 #include <security/mac/mac_framework.h>
156
157 #include <vm/uma.h>
158
159 #ifdef COMPAT_FREEBSD32
160 #include <sys/mount.h>
161 #include <sys/sysent.h>
162 #include <compat/freebsd32/freebsd32.h>
163 #endif
164
165 static int soreceive_rcvoob(struct socket *so, struct uio *uio,
166 int flags);
167 static void so_rdknl_lock(void *);
168 static void so_rdknl_unlock(void *);
169 static void so_rdknl_assert_lock(void *, int);
170 static void so_wrknl_lock(void *);
171 static void so_wrknl_unlock(void *);
172 static void so_wrknl_assert_lock(void *, int);
173
174 static void filt_sordetach(struct knote *kn);
175 static int filt_soread(struct knote *kn, long hint);
176 static void filt_sowdetach(struct knote *kn);
177 static int filt_sowrite(struct knote *kn, long hint);
178 static int filt_soempty(struct knote *kn, long hint);
179 static int inline hhook_run_socket(struct socket *so, void *hctx, int32_t h_id);
180 fo_kqfilter_t soo_kqfilter;
181
182 static struct filterops soread_filtops = {
183 .f_isfd = 1,
184 .f_detach = filt_sordetach,
185 .f_event = filt_soread,
186 };
187 static struct filterops sowrite_filtops = {
188 .f_isfd = 1,
189 .f_detach = filt_sowdetach,
190 .f_event = filt_sowrite,
191 };
192 static struct filterops soempty_filtops = {
193 .f_isfd = 1,
194 .f_detach = filt_sowdetach,
195 .f_event = filt_soempty,
196 };
197
198 so_gen_t so_gencnt; /* generation count for sockets */
199
200 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
201 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block");
202
203 #define VNET_SO_ASSERT(so) \
204 VNET_ASSERT(curvnet != NULL, \
205 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));
206
207 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]);
208 #define V_socket_hhh VNET(socket_hhh)
209
210 /*
211 * Limit on the number of connections in the listen queue waiting
212 * for accept(2).
213 * NB: The original sysctl somaxconn is still available but hidden
214 * to prevent confusion about the actual purpose of this number.
215 */
216 static u_int somaxconn = SOMAXCONN;
217
218 static int
219 sysctl_somaxconn(SYSCTL_HANDLER_ARGS)
220 {
221 int error;
222 int val;
223
224 val = somaxconn;
225 error = sysctl_handle_int(oidp, &val, 0, req);
226 if (error || !req->newptr )
227 return (error);
228
229 /*
230 * The purpose of the UINT_MAX / 3 limit, is so that the formula
231 * 3 * so_qlimit / 2
232 * below, will not overflow.
233 */
234
235 if (val < 1 || val > UINT_MAX / 3)
236 return (EINVAL);
237
238 somaxconn = val;
239 return (0);
240 }
241 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue,
242 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, sizeof(int),
243 sysctl_somaxconn, "I",
244 "Maximum listen socket pending connection accept queue size");
245 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn,
246 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 0,
247 sizeof(int), sysctl_somaxconn, "I",
248 "Maximum listen socket pending connection accept queue size (compat)");
249
250 static int numopensockets;
251 SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
252 &numopensockets, 0, "Number of open sockets");
253
254 /*
255 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket
256 * so_gencnt field.
257 */
258 static struct mtx so_global_mtx;
259 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF);
260
261 /*
262 * General IPC sysctl name space, used by sockets and a variety of other IPC
263 * types.
264 */
265 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
266 "IPC");
267
268 /*
269 * Initialize the socket subsystem and set up the socket
270 * memory allocator.
271 */
272 static uma_zone_t socket_zone;
273 int maxsockets;
274
275 static void
276 socket_zone_change(void *tag)
277 {
278
279 maxsockets = uma_zone_set_max(socket_zone, maxsockets);
280 }
281
282 static void
283 socket_hhook_register(int subtype)
284 {
285
286 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype,
287 &V_socket_hhh[subtype],
288 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
289 printf("%s: WARNING: unable to register hook\n", __func__);
290 }
291
292 static void
293 socket_hhook_deregister(int subtype)
294 {
295
296 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0)
297 printf("%s: WARNING: unable to deregister hook\n", __func__);
298 }
299
300 static void
301 socket_init(void *tag)
302 {
303
304 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL,
305 NULL, NULL, UMA_ALIGN_PTR, 0);
306 maxsockets = uma_zone_set_max(socket_zone, maxsockets);
307 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached");
308 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL,
309 EVENTHANDLER_PRI_FIRST);
310 }
311 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL);
312
313 static void
314 socket_vnet_init(const void *unused __unused)
315 {
316 int i;
317
318 /* We expect a contiguous range */
319 for (i = 0; i <= HHOOK_SOCKET_LAST; i++)
320 socket_hhook_register(i);
321 }
322 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY,
323 socket_vnet_init, NULL);
324
325 static void
326 socket_vnet_uninit(const void *unused __unused)
327 {
328 int i;
329
330 for (i = 0; i <= HHOOK_SOCKET_LAST; i++)
331 socket_hhook_deregister(i);
332 }
333 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY,
334 socket_vnet_uninit, NULL);
335
336 /*
337 * Initialise maxsockets. This SYSINIT must be run after
338 * tunable_mbinit().
339 */
340 static void
341 init_maxsockets(void *ignored)
342 {
343
344 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets);
345 maxsockets = imax(maxsockets, maxfiles);
346 }
347 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL);
348
349 /*
350 * Sysctl to get and set the maximum global sockets limit. Notify protocols
351 * of the change so that they can update their dependent limits as required.
352 */
353 static int
354 sysctl_maxsockets(SYSCTL_HANDLER_ARGS)
355 {
356 int error, newmaxsockets;
357
358 newmaxsockets = maxsockets;
359 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req);
360 if (error == 0 && req->newptr && newmaxsockets != maxsockets) {
361 if (newmaxsockets > maxsockets &&
362 newmaxsockets <= maxfiles) {
363 maxsockets = newmaxsockets;
364 EVENTHANDLER_INVOKE(maxsockets_change);
365 } else
366 error = EINVAL;
367 }
368 return (error);
369 }
370 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets,
371 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, &maxsockets, 0,
372 sysctl_maxsockets, "IU",
373 "Maximum number of sockets available");
374
375 /*
376 * Socket operation routines. These routines are called by the routines in
377 * sys_socket.c or from a system process, and implement the semantics of
378 * socket operations by switching out to the protocol specific routines.
379 */
380
381 /*
382 * Get a socket structure from our zone, and initialize it. Note that it
383 * would probably be better to allocate socket and PCB at the same time, but
384 * I'm not convinced that all the protocols can be easily modified to do
385 * this.
386 *
387 * soalloc() returns a socket with a ref count of 0.
388 */
389 static struct socket *
390 soalloc(struct vnet *vnet)
391 {
392 struct socket *so;
393
394 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO);
395 if (so == NULL)
396 return (NULL);
397 #ifdef MAC
398 if (mac_socket_init(so, M_NOWAIT) != 0) {
399 uma_zfree(socket_zone, so);
400 return (NULL);
401 }
402 #endif
403 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) {
404 uma_zfree(socket_zone, so);
405 return (NULL);
406 }
407
408 /*
409 * The socket locking protocol allows to lock 2 sockets at a time,
410 * however, the first one must be a listening socket. WITNESS lacks
411 * a feature to change class of an existing lock, so we use DUPOK.
412 */
413 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK);
414 mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF);
415 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF);
416 so->so_rcv.sb_sel = &so->so_rdsel;
417 so->so_snd.sb_sel = &so->so_wrsel;
418 sx_init(&so->so_snd_sx, "so_snd_sx");
419 sx_init(&so->so_rcv_sx, "so_rcv_sx");
420 TAILQ_INIT(&so->so_snd.sb_aiojobq);
421 TAILQ_INIT(&so->so_rcv.sb_aiojobq);
422 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so);
423 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so);
424 #ifdef VIMAGE
425 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p",
426 __func__, __LINE__, so));
427 so->so_vnet = vnet;
428 #endif
429 /* We shouldn't need the so_global_mtx */
430 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) {
431 /* Do we need more comprehensive error returns? */
432 uma_zfree(socket_zone, so);
433 return (NULL);
434 }
435 mtx_lock(&so_global_mtx);
436 so->so_gencnt = ++so_gencnt;
437 ++numopensockets;
438 #ifdef VIMAGE
439 vnet->vnet_sockcnt++;
440 #endif
441 mtx_unlock(&so_global_mtx);
442
443 return (so);
444 }
445
446 /*
447 * Free the storage associated with a socket at the socket layer, tear down
448 * locks, labels, etc. All protocol state is assumed already to have been
449 * torn down (and possibly never set up) by the caller.
450 */
451 void
452 sodealloc(struct socket *so)
453 {
454
455 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count));
456 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL"));
457
458 mtx_lock(&so_global_mtx);
459 so->so_gencnt = ++so_gencnt;
460 --numopensockets; /* Could be below, but faster here. */
461 #ifdef VIMAGE
462 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p",
463 __func__, __LINE__, so));
464 so->so_vnet->vnet_sockcnt--;
465 #endif
466 mtx_unlock(&so_global_mtx);
467 #ifdef MAC
468 mac_socket_destroy(so);
469 #endif
470 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE);
471
472 khelp_destroy_osd(&so->osd);
473 if (SOLISTENING(so)) {
474 if (so->sol_accept_filter != NULL)
475 accept_filt_setopt(so, NULL);
476 } else {
477 if (so->so_rcv.sb_hiwat)
478 (void)chgsbsize(so->so_cred->cr_uidinfo,
479 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY);
480 if (so->so_snd.sb_hiwat)
481 (void)chgsbsize(so->so_cred->cr_uidinfo,
482 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY);
483 sx_destroy(&so->so_snd_sx);
484 sx_destroy(&so->so_rcv_sx);
485 mtx_destroy(&so->so_snd_mtx);
486 mtx_destroy(&so->so_rcv_mtx);
487 }
488 crfree(so->so_cred);
489 mtx_destroy(&so->so_lock);
490 uma_zfree(socket_zone, so);
491 }
492
493 /*
494 * socreate returns a socket with a ref count of 1 and a file descriptor
495 * reference. The socket should be closed with soclose().
496 */
497 int
498 socreate(int dom, struct socket **aso, int type, int proto,
499 struct ucred *cred, struct thread *td)
500 {
501 struct protosw *prp;
502 struct socket *so;
503 int error;
504
505 /*
506 * XXX: divert(4) historically abused PF_INET. Keep this compatibility
507 * shim until all applications have been updated.
508 */
509 if (__predict_false(dom == PF_INET && type == SOCK_RAW &&
510 proto == IPPROTO_DIVERT)) {
511 dom = PF_DIVERT;
512 printf("%s uses obsolete way to create divert(4) socket\n",
513 td->td_proc->p_comm);
514 }
515
516 prp = pffindproto(dom, type, proto);
517 if (prp == NULL) {
518 /* No support for domain. */
519 if (pffinddomain(dom) == NULL)
520 return (EAFNOSUPPORT);
521 /* No support for socket type. */
522 if (proto == 0 && type != 0)
523 return (EPROTOTYPE);
524 return (EPROTONOSUPPORT);
525 }
526
527 MPASS(prp->pr_attach);
528
529 if (IN_CAPABILITY_MODE(td) && (prp->pr_flags & PR_CAPATTACH) == 0)
530 return (ECAPMODE);
531
532 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0)
533 return (EPROTONOSUPPORT);
534
535 so = soalloc(CRED_TO_VNET(cred));
536 if (so == NULL)
537 return (ENOBUFS);
538
539 so->so_type = type;
540 so->so_cred = crhold(cred);
541 if ((prp->pr_domain->dom_family == PF_INET) ||
542 (prp->pr_domain->dom_family == PF_INET6) ||
543 (prp->pr_domain->dom_family == PF_ROUTE))
544 so->so_fibnum = td->td_proc->p_fibnum;
545 else
546 so->so_fibnum = 0;
547 so->so_proto = prp;
548 #ifdef MAC
549 mac_socket_create(cred, so);
550 #endif
551 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
552 so_rdknl_assert_lock);
553 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
554 so_wrknl_assert_lock);
555 if ((prp->pr_flags & PR_SOCKBUF) == 0) {
556 so->so_snd.sb_mtx = &so->so_snd_mtx;
557 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
558 }
559 /*
560 * Auto-sizing of socket buffers is managed by the protocols and
561 * the appropriate flags must be set in the pru_attach function.
562 */
563 CURVNET_SET(so->so_vnet);
564 error = prp->pr_attach(so, proto, td);
565 CURVNET_RESTORE();
566 if (error) {
567 sodealloc(so);
568 return (error);
569 }
570 soref(so);
571 *aso = so;
572 return (0);
573 }
574
575 #ifdef REGRESSION
576 static int regression_sonewconn_earlytest = 1;
577 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW,
578 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test");
579 #endif
580
581 static struct timeval overinterval = { 60, 0 };
582 SYSCTL_TIMEVAL_SEC(_kern_ipc, OID_AUTO, sooverinterval, CTLFLAG_RW,
583 &overinterval,
584 "Delay in seconds between warnings for listen socket overflows");
585
586 /*
587 * When an attempt at a new connection is noted on a socket which supports
588 * accept(2), the protocol has two options:
589 * 1) Call legacy sonewconn() function, which would call protocol attach
590 * method, same as used for socket(2).
591 * 2) Call solisten_clone(), do attach that is specific to a cloned connection,
592 * and then call solisten_enqueue().
593 *
594 * Note: the ref count on the socket is 0 on return.
595 */
596 struct socket *
597 solisten_clone(struct socket *head)
598 {
599 struct sbuf descrsb;
600 struct socket *so;
601 int len, overcount;
602 u_int qlen;
603 const char localprefix[] = "local:";
604 char descrbuf[SUNPATHLEN + sizeof(localprefix)];
605 #if defined(INET6)
606 char addrbuf[INET6_ADDRSTRLEN];
607 #elif defined(INET)
608 char addrbuf[INET_ADDRSTRLEN];
609 #endif
610 bool dolog, over;
611
612 SOLISTEN_LOCK(head);
613 over = (head->sol_qlen > 3 * head->sol_qlimit / 2);
614 #ifdef REGRESSION
615 if (regression_sonewconn_earlytest && over) {
616 #else
617 if (over) {
618 #endif
619 head->sol_overcount++;
620 dolog = !!ratecheck(&head->sol_lastover, &overinterval);
621
622 /*
623 * If we're going to log, copy the overflow count and queue
624 * length from the listen socket before dropping the lock.
625 * Also, reset the overflow count.
626 */
627 if (dolog) {
628 overcount = head->sol_overcount;
629 head->sol_overcount = 0;
630 qlen = head->sol_qlen;
631 }
632 SOLISTEN_UNLOCK(head);
633
634 if (dolog) {
635 /*
636 * Try to print something descriptive about the
637 * socket for the error message.
638 */
639 sbuf_new(&descrsb, descrbuf, sizeof(descrbuf),
640 SBUF_FIXEDLEN);
641 switch (head->so_proto->pr_domain->dom_family) {
642 #if defined(INET) || defined(INET6)
643 #ifdef INET
644 case AF_INET:
645 #endif
646 #ifdef INET6
647 case AF_INET6:
648 if (head->so_proto->pr_domain->dom_family ==
649 AF_INET6 ||
650 (sotoinpcb(head)->inp_inc.inc_flags &
651 INC_ISIPV6)) {
652 ip6_sprintf(addrbuf,
653 &sotoinpcb(head)->inp_inc.inc6_laddr);
654 sbuf_printf(&descrsb, "[%s]", addrbuf);
655 } else
656 #endif
657 {
658 #ifdef INET
659 inet_ntoa_r(
660 sotoinpcb(head)->inp_inc.inc_laddr,
661 addrbuf);
662 sbuf_cat(&descrsb, addrbuf);
663 #endif
664 }
665 sbuf_printf(&descrsb, ":%hu (proto %u)",
666 ntohs(sotoinpcb(head)->inp_inc.inc_lport),
667 head->so_proto->pr_protocol);
668 break;
669 #endif /* INET || INET6 */
670 case AF_UNIX:
671 sbuf_cat(&descrsb, localprefix);
672 if (sotounpcb(head)->unp_addr != NULL)
673 len =
674 sotounpcb(head)->unp_addr->sun_len -
675 offsetof(struct sockaddr_un,
676 sun_path);
677 else
678 len = 0;
679 if (len > 0)
680 sbuf_bcat(&descrsb,
681 sotounpcb(head)->unp_addr->sun_path,
682 len);
683 else
684 sbuf_cat(&descrsb, "(unknown)");
685 break;
686 }
687
688 /*
689 * If we can't print something more specific, at least
690 * print the domain name.
691 */
692 if (sbuf_finish(&descrsb) != 0 ||
693 sbuf_len(&descrsb) <= 0) {
694 sbuf_clear(&descrsb);
695 sbuf_cat(&descrsb,
696 head->so_proto->pr_domain->dom_name ?:
697 "unknown");
698 sbuf_finish(&descrsb);
699 }
700 KASSERT(sbuf_len(&descrsb) > 0,
701 ("%s: sbuf creation failed", __func__));
702 /*
703 * Preserve the historic listen queue overflow log
704 * message, that starts with "sonewconn:". It has
705 * been known to sysadmins for years and also test
706 * sys/kern/sonewconn_overflow checks for it.
707 */
708 if (head->so_cred == 0) {
709 log(LOG_DEBUG, "sonewconn: pcb %p (%s): "
710 "Listen queue overflow: %i already in "
711 "queue awaiting acceptance (%d "
712 "occurrences)\n", head->so_pcb,
713 sbuf_data(&descrsb),
714 qlen, overcount);
715 } else {
716 log(LOG_DEBUG, "sonewconn: pcb %p (%s): "
717 "Listen queue overflow: "
718 "%i already in queue awaiting acceptance "
719 "(%d occurrences), euid %d, rgid %d, jail %s\n",
720 head->so_pcb, sbuf_data(&descrsb), qlen,
721 overcount, head->so_cred->cr_uid,
722 head->so_cred->cr_rgid,
723 head->so_cred->cr_prison ?
724 head->so_cred->cr_prison->pr_name :
725 "not_jailed");
726 }
727 sbuf_delete(&descrsb);
728
729 overcount = 0;
730 }
731
732 return (NULL);
733 }
734 SOLISTEN_UNLOCK(head);
735 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL",
736 __func__, head));
737 so = soalloc(head->so_vnet);
738 if (so == NULL) {
739 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: "
740 "limit reached or out of memory\n",
741 __func__, head->so_pcb);
742 return (NULL);
743 }
744 so->so_listen = head;
745 so->so_type = head->so_type;
746 so->so_options = head->so_options & ~SO_ACCEPTCONN;
747 so->so_linger = head->so_linger;
748 so->so_state = head->so_state;
749 so->so_fibnum = head->so_fibnum;
750 so->so_proto = head->so_proto;
751 so->so_cred = crhold(head->so_cred);
752 #ifdef MAC
753 mac_socket_newconn(head, so);
754 #endif
755 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
756 so_rdknl_assert_lock);
757 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
758 so_wrknl_assert_lock);
759 VNET_SO_ASSERT(head);
760 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) {
761 sodealloc(so);
762 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n",
763 __func__, head->so_pcb);
764 return (NULL);
765 }
766 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat;
767 so->so_snd.sb_lowat = head->sol_sbsnd_lowat;
768 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo;
769 so->so_snd.sb_timeo = head->sol_sbsnd_timeo;
770 so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE;
771 so->so_snd.sb_flags = head->sol_sbsnd_flags & SB_AUTOSIZE;
772 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) {
773 so->so_snd.sb_mtx = &so->so_snd_mtx;
774 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
775 }
776
777 return (so);
778 }
779
780 /* Connstatus may be 0, or SS_ISCONFIRMING, or SS_ISCONNECTED. */
781 struct socket *
782 sonewconn(struct socket *head, int connstatus)
783 {
784 struct socket *so;
785
786 if ((so = solisten_clone(head)) == NULL)
787 return (NULL);
788
789 if (so->so_proto->pr_attach(so, 0, NULL) != 0) {
790 sodealloc(so);
791 log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n",
792 __func__, head->so_pcb);
793 return (NULL);
794 }
795
796 (void)solisten_enqueue(so, connstatus);
797
798 return (so);
799 }
800
801 /*
802 * Enqueue socket cloned by solisten_clone() to the listen queue of the
803 * listener it has been cloned from.
804 *
805 * Return 'true' if socket landed on complete queue, otherwise 'false'.
806 */
807 bool
808 solisten_enqueue(struct socket *so, int connstatus)
809 {
810 struct socket *head = so->so_listen;
811
812 MPASS(refcount_load(&so->so_count) == 0);
813 refcount_init(&so->so_count, 1);
814
815 SOLISTEN_LOCK(head);
816 if (head->sol_accept_filter != NULL)
817 connstatus = 0;
818 so->so_state |= connstatus;
819 soref(head); /* A socket on (in)complete queue refs head. */
820 if (connstatus) {
821 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
822 so->so_qstate = SQ_COMP;
823 head->sol_qlen++;
824 solisten_wakeup(head); /* unlocks */
825 return (true);
826 } else {
827 /*
828 * Keep removing sockets from the head until there's room for
829 * us to insert on the tail. In pre-locking revisions, this
830 * was a simple if(), but as we could be racing with other
831 * threads and soabort() requires dropping locks, we must
832 * loop waiting for the condition to be true.
833 */
834 while (head->sol_incqlen > head->sol_qlimit) {
835 struct socket *sp;
836
837 sp = TAILQ_FIRST(&head->sol_incomp);
838 TAILQ_REMOVE(&head->sol_incomp, sp, so_list);
839 head->sol_incqlen--;
840 SOCK_LOCK(sp);
841 sp->so_qstate = SQ_NONE;
842 sp->so_listen = NULL;
843 SOCK_UNLOCK(sp);
844 sorele_locked(head); /* does SOLISTEN_UNLOCK, head stays */
845 soabort(sp);
846 SOLISTEN_LOCK(head);
847 }
848 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list);
849 so->so_qstate = SQ_INCOMP;
850 head->sol_incqlen++;
851 SOLISTEN_UNLOCK(head);
852 return (false);
853 }
854 }
855
856 #if defined(SCTP) || defined(SCTP_SUPPORT)
857 /*
858 * Socket part of sctp_peeloff(). Detach a new socket from an
859 * association. The new socket is returned with a reference.
860 *
861 * XXXGL: reduce copy-paste with solisten_clone().
862 */
863 struct socket *
864 sopeeloff(struct socket *head)
865 {
866 struct socket *so;
867
868 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p",
869 __func__, __LINE__, head));
870 so = soalloc(head->so_vnet);
871 if (so == NULL) {
872 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: "
873 "limit reached or out of memory\n",
874 __func__, head->so_pcb);
875 return (NULL);
876 }
877 so->so_type = head->so_type;
878 so->so_options = head->so_options;
879 so->so_linger = head->so_linger;
880 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED;
881 so->so_fibnum = head->so_fibnum;
882 so->so_proto = head->so_proto;
883 so->so_cred = crhold(head->so_cred);
884 #ifdef MAC
885 mac_socket_newconn(head, so);
886 #endif
887 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
888 so_rdknl_assert_lock);
889 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
890 so_wrknl_assert_lock);
891 VNET_SO_ASSERT(head);
892 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
893 sodealloc(so);
894 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n",
895 __func__, head->so_pcb);
896 return (NULL);
897 }
898 if ((*so->so_proto->pr_attach)(so, 0, NULL)) {
899 sodealloc(so);
900 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n",
901 __func__, head->so_pcb);
902 return (NULL);
903 }
904 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat;
905 so->so_snd.sb_lowat = head->so_snd.sb_lowat;
906 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo;
907 so->so_snd.sb_timeo = head->so_snd.sb_timeo;
908 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE;
909 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE;
910
911 soref(so);
912
913 return (so);
914 }
915 #endif /* SCTP */
916
917 int
918 sobind(struct socket *so, struct sockaddr *nam, struct thread *td)
919 {
920 int error;
921
922 CURVNET_SET(so->so_vnet);
923 error = so->so_proto->pr_bind(so, nam, td);
924 CURVNET_RESTORE();
925 return (error);
926 }
927
928 int
929 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td)
930 {
931 int error;
932
933 CURVNET_SET(so->so_vnet);
934 error = so->so_proto->pr_bindat(fd, so, nam, td);
935 CURVNET_RESTORE();
936 return (error);
937 }
938
939 /*
940 * solisten() transitions a socket from a non-listening state to a listening
941 * state, but can also be used to update the listen queue depth on an
942 * existing listen socket. The protocol will call back into the sockets
943 * layer using solisten_proto_check() and solisten_proto() to check and set
944 * socket-layer listen state. Call backs are used so that the protocol can
945 * acquire both protocol and socket layer locks in whatever order is required
946 * by the protocol.
947 *
948 * Protocol implementors are advised to hold the socket lock across the
949 * socket-layer test and set to avoid races at the socket layer.
950 */
951 int
952 solisten(struct socket *so, int backlog, struct thread *td)
953 {
954 int error;
955
956 CURVNET_SET(so->so_vnet);
957 error = so->so_proto->pr_listen(so, backlog, td);
958 CURVNET_RESTORE();
959 return (error);
960 }
961
962 /*
963 * Prepare for a call to solisten_proto(). Acquire all socket buffer locks in
964 * order to interlock with socket I/O.
965 */
966 int
967 solisten_proto_check(struct socket *so)
968 {
969 SOCK_LOCK_ASSERT(so);
970
971 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
972 SS_ISDISCONNECTING)) != 0)
973 return (EINVAL);
974
975 /*
976 * Sleeping is not permitted here, so simply fail if userspace is
977 * attempting to transmit or receive on the socket. This kind of
978 * transient failure is not ideal, but it should occur only if userspace
979 * is misusing the socket interfaces.
980 */
981 if (!sx_try_xlock(&so->so_snd_sx))
982 return (EAGAIN);
983 if (!sx_try_xlock(&so->so_rcv_sx)) {
984 sx_xunlock(&so->so_snd_sx);
985 return (EAGAIN);
986 }
987 mtx_lock(&so->so_snd_mtx);
988 mtx_lock(&so->so_rcv_mtx);
989
990 /* Interlock with soo_aio_queue(). */
991 if (!SOLISTENING(so) &&
992 ((so->so_snd.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0 ||
993 (so->so_rcv.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0)) {
994 solisten_proto_abort(so);
995 return (EINVAL);
996 }
997 return (0);
998 }
999
1000 /*
1001 * Undo the setup done by solisten_proto_check().
1002 */
1003 void
1004 solisten_proto_abort(struct socket *so)
1005 {
1006 mtx_unlock(&so->so_snd_mtx);
1007 mtx_unlock(&so->so_rcv_mtx);
1008 sx_xunlock(&so->so_snd_sx);
1009 sx_xunlock(&so->so_rcv_sx);
1010 }
1011
1012 void
1013 solisten_proto(struct socket *so, int backlog)
1014 {
1015 int sbrcv_lowat, sbsnd_lowat;
1016 u_int sbrcv_hiwat, sbsnd_hiwat;
1017 short sbrcv_flags, sbsnd_flags;
1018 sbintime_t sbrcv_timeo, sbsnd_timeo;
1019
1020 SOCK_LOCK_ASSERT(so);
1021 KASSERT((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
1022 SS_ISDISCONNECTING)) == 0,
1023 ("%s: bad socket state %p", __func__, so));
1024
1025 if (SOLISTENING(so))
1026 goto listening;
1027
1028 /*
1029 * Change this socket to listening state.
1030 */
1031 sbrcv_lowat = so->so_rcv.sb_lowat;
1032 sbsnd_lowat = so->so_snd.sb_lowat;
1033 sbrcv_hiwat = so->so_rcv.sb_hiwat;
1034 sbsnd_hiwat = so->so_snd.sb_hiwat;
1035 sbrcv_flags = so->so_rcv.sb_flags;
1036 sbsnd_flags = so->so_snd.sb_flags;
1037 sbrcv_timeo = so->so_rcv.sb_timeo;
1038 sbsnd_timeo = so->so_snd.sb_timeo;
1039
1040 sbdestroy(so, SO_SND);
1041 sbdestroy(so, SO_RCV);
1042
1043 #ifdef INVARIANTS
1044 bzero(&so->so_rcv,
1045 sizeof(struct socket) - offsetof(struct socket, so_rcv));
1046 #endif
1047
1048 so->sol_sbrcv_lowat = sbrcv_lowat;
1049 so->sol_sbsnd_lowat = sbsnd_lowat;
1050 so->sol_sbrcv_hiwat = sbrcv_hiwat;
1051 so->sol_sbsnd_hiwat = sbsnd_hiwat;
1052 so->sol_sbrcv_flags = sbrcv_flags;
1053 so->sol_sbsnd_flags = sbsnd_flags;
1054 so->sol_sbrcv_timeo = sbrcv_timeo;
1055 so->sol_sbsnd_timeo = sbsnd_timeo;
1056
1057 so->sol_qlen = so->sol_incqlen = 0;
1058 TAILQ_INIT(&so->sol_incomp);
1059 TAILQ_INIT(&so->sol_comp);
1060
1061 so->sol_accept_filter = NULL;
1062 so->sol_accept_filter_arg = NULL;
1063 so->sol_accept_filter_str = NULL;
1064
1065 so->sol_upcall = NULL;
1066 so->sol_upcallarg = NULL;
1067
1068 so->so_options |= SO_ACCEPTCONN;
1069
1070 listening:
1071 if (backlog < 0 || backlog > somaxconn)
1072 backlog = somaxconn;
1073 so->sol_qlimit = backlog;
1074
1075 mtx_unlock(&so->so_snd_mtx);
1076 mtx_unlock(&so->so_rcv_mtx);
1077 sx_xunlock(&so->so_snd_sx);
1078 sx_xunlock(&so->so_rcv_sx);
1079 }
1080
1081 /*
1082 * Wakeup listeners/subsystems once we have a complete connection.
1083 * Enters with lock, returns unlocked.
1084 */
1085 void
1086 solisten_wakeup(struct socket *sol)
1087 {
1088
1089 if (sol->sol_upcall != NULL)
1090 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT);
1091 else {
1092 selwakeuppri(&sol->so_rdsel, PSOCK);
1093 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0);
1094 }
1095 SOLISTEN_UNLOCK(sol);
1096 wakeup_one(&sol->sol_comp);
1097 if ((sol->so_state & SS_ASYNC) && sol->so_sigio != NULL)
1098 pgsigio(&sol->so_sigio, SIGIO, 0);
1099 }
1100
1101 /*
1102 * Return single connection off a listening socket queue. Main consumer of
1103 * the function is kern_accept4(). Some modules, that do their own accept
1104 * management also use the function. The socket reference held by the
1105 * listen queue is handed to the caller.
1106 *
1107 * Listening socket must be locked on entry and is returned unlocked on
1108 * return.
1109 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT.
1110 */
1111 int
1112 solisten_dequeue(struct socket *head, struct socket **ret, int flags)
1113 {
1114 struct socket *so;
1115 int error;
1116
1117 SOLISTEN_LOCK_ASSERT(head);
1118
1119 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) &&
1120 head->so_error == 0) {
1121 error = msleep(&head->sol_comp, SOCK_MTX(head), PSOCK | PCATCH,
1122 "accept", 0);
1123 if (error != 0) {
1124 SOLISTEN_UNLOCK(head);
1125 return (error);
1126 }
1127 }
1128 if (head->so_error) {
1129 error = head->so_error;
1130 head->so_error = 0;
1131 } else if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp))
1132 error = EWOULDBLOCK;
1133 else
1134 error = 0;
1135 if (error) {
1136 SOLISTEN_UNLOCK(head);
1137 return (error);
1138 }
1139 so = TAILQ_FIRST(&head->sol_comp);
1140 SOCK_LOCK(so);
1141 KASSERT(so->so_qstate == SQ_COMP,
1142 ("%s: so %p not SQ_COMP", __func__, so));
1143 head->sol_qlen--;
1144 so->so_qstate = SQ_NONE;
1145 so->so_listen = NULL;
1146 TAILQ_REMOVE(&head->sol_comp, so, so_list);
1147 if (flags & ACCEPT4_INHERIT)
1148 so->so_state |= (head->so_state & SS_NBIO);
1149 else
1150 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0;
1151 SOCK_UNLOCK(so);
1152 sorele_locked(head);
1153
1154 *ret = so;
1155 return (0);
1156 }
1157
1158 /*
1159 * Free socket upon release of the very last reference.
1160 */
1161 static void
1162 sofree(struct socket *so)
1163 {
1164 struct protosw *pr = so->so_proto;
1165
1166 SOCK_LOCK_ASSERT(so);
1167 KASSERT(refcount_load(&so->so_count) == 0,
1168 ("%s: so %p has references", __func__, so));
1169 KASSERT(SOLISTENING(so) || so->so_qstate == SQ_NONE,
1170 ("%s: so %p is on listen queue", __func__, so));
1171
1172 SOCK_UNLOCK(so);
1173
1174 if (so->so_dtor != NULL)
1175 so->so_dtor(so);
1176
1177 VNET_SO_ASSERT(so);
1178 if ((pr->pr_flags & PR_RIGHTS) && !SOLISTENING(so)) {
1179 MPASS(pr->pr_domain->dom_dispose != NULL);
1180 (*pr->pr_domain->dom_dispose)(so);
1181 }
1182 if (pr->pr_detach != NULL)
1183 pr->pr_detach(so);
1184
1185 /*
1186 * From this point on, we assume that no other references to this
1187 * socket exist anywhere else in the stack. Therefore, no locks need
1188 * to be acquired or held.
1189 */
1190 if (!(pr->pr_flags & PR_SOCKBUF) && !SOLISTENING(so)) {
1191 sbdestroy(so, SO_SND);
1192 sbdestroy(so, SO_RCV);
1193 }
1194 seldrain(&so->so_rdsel);
1195 seldrain(&so->so_wrsel);
1196 knlist_destroy(&so->so_rdsel.si_note);
1197 knlist_destroy(&so->so_wrsel.si_note);
1198 sodealloc(so);
1199 }
1200
1201 /*
1202 * Release a reference on a socket while holding the socket lock.
1203 * Unlocks the socket lock before returning.
1204 */
1205 void
1206 sorele_locked(struct socket *so)
1207 {
1208 SOCK_LOCK_ASSERT(so);
1209 if (refcount_release(&so->so_count))
1210 sofree(so);
1211 else
1212 SOCK_UNLOCK(so);
1213 }
1214
1215 /*
1216 * Close a socket on last file table reference removal. Initiate disconnect
1217 * if connected. Free socket when disconnect complete.
1218 *
1219 * This function will sorele() the socket. Note that soclose() may be called
1220 * prior to the ref count reaching zero. The actual socket structure will
1221 * not be freed until the ref count reaches zero.
1222 */
1223 int
1224 soclose(struct socket *so)
1225 {
1226 struct accept_queue lqueue;
1227 int error = 0;
1228 bool listening, last __diagused;
1229
1230 CURVNET_SET(so->so_vnet);
1231 funsetown(&so->so_sigio);
1232 if (so->so_state & SS_ISCONNECTED) {
1233 if ((so->so_state & SS_ISDISCONNECTING) == 0) {
1234 error = sodisconnect(so);
1235 if (error) {
1236 if (error == ENOTCONN)
1237 error = 0;
1238 goto drop;
1239 }
1240 }
1241
1242 if ((so->so_options & SO_LINGER) != 0 && so->so_linger != 0) {
1243 if ((so->so_state & SS_ISDISCONNECTING) &&
1244 (so->so_state & SS_NBIO))
1245 goto drop;
1246 while (so->so_state & SS_ISCONNECTED) {
1247 error = tsleep(&so->so_timeo,
1248 PSOCK | PCATCH, "soclos",
1249 so->so_linger * hz);
1250 if (error)
1251 break;
1252 }
1253 }
1254 }
1255
1256 drop:
1257 if (so->so_proto->pr_close != NULL)
1258 so->so_proto->pr_close(so);
1259
1260 SOCK_LOCK(so);
1261 if ((listening = SOLISTENING(so))) {
1262 struct socket *sp;
1263
1264 TAILQ_INIT(&lqueue);
1265 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list);
1266 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list);
1267
1268 so->sol_qlen = so->sol_incqlen = 0;
1269
1270 TAILQ_FOREACH(sp, &lqueue, so_list) {
1271 SOCK_LOCK(sp);
1272 sp->so_qstate = SQ_NONE;
1273 sp->so_listen = NULL;
1274 SOCK_UNLOCK(sp);
1275 last = refcount_release(&so->so_count);
1276 KASSERT(!last, ("%s: released last reference for %p",
1277 __func__, so));
1278 }
1279 }
1280 sorele_locked(so);
1281 if (listening) {
1282 struct socket *sp, *tsp;
1283
1284 TAILQ_FOREACH_SAFE(sp, &lqueue, so_list, tsp)
1285 soabort(sp);
1286 }
1287 CURVNET_RESTORE();
1288 return (error);
1289 }
1290
1291 /*
1292 * soabort() is used to abruptly tear down a connection, such as when a
1293 * resource limit is reached (listen queue depth exceeded), or if a listen
1294 * socket is closed while there are sockets waiting to be accepted.
1295 *
1296 * This interface is tricky, because it is called on an unreferenced socket,
1297 * and must be called only by a thread that has actually removed the socket
1298 * from the listen queue it was on. Likely this thread holds the last
1299 * reference on the socket and soabort() will proceed with sofree(). But
1300 * it might be not the last, as the sockets on the listen queues are seen
1301 * from the protocol side.
1302 *
1303 * This interface will call into the protocol code, so must not be called
1304 * with any socket locks held. Protocols do call it while holding their own
1305 * recursible protocol mutexes, but this is something that should be subject
1306 * to review in the future.
1307 *
1308 * Usually socket should have a single reference left, but this is not a
1309 * requirement. In the past, when we have had named references for file
1310 * descriptor and protocol, we asserted that none of them are being held.
1311 */
1312 void
1313 soabort(struct socket *so)
1314 {
1315
1316 VNET_SO_ASSERT(so);
1317
1318 if (so->so_proto->pr_abort != NULL)
1319 so->so_proto->pr_abort(so);
1320 SOCK_LOCK(so);
1321 sorele_locked(so);
1322 }
1323
1324 int
1325 soaccept(struct socket *so, struct sockaddr **nam)
1326 {
1327 int error;
1328
1329 CURVNET_SET(so->so_vnet);
1330 error = so->so_proto->pr_accept(so, nam);
1331 CURVNET_RESTORE();
1332 return (error);
1333 }
1334
1335 int
1336 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td)
1337 {
1338
1339 return (soconnectat(AT_FDCWD, so, nam, td));
1340 }
1341
1342 int
1343 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td)
1344 {
1345 int error;
1346
1347 CURVNET_SET(so->so_vnet);
1348 /*
1349 * If protocol is connection-based, can only connect once.
1350 * Otherwise, if connected, try to disconnect first. This allows
1351 * user to disconnect by connecting to, e.g., a null address.
1352 */
1353 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
1354 ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
1355 (error = sodisconnect(so)))) {
1356 error = EISCONN;
1357 } else {
1358 /*
1359 * Prevent accumulated error from previous connection from
1360 * biting us.
1361 */
1362 so->so_error = 0;
1363 if (fd == AT_FDCWD) {
1364 error = so->so_proto->pr_connect(so, nam, td);
1365 } else {
1366 error = so->so_proto->pr_connectat(fd, so, nam, td);
1367 }
1368 }
1369 CURVNET_RESTORE();
1370
1371 return (error);
1372 }
1373
1374 int
1375 soconnect2(struct socket *so1, struct socket *so2)
1376 {
1377 int error;
1378
1379 CURVNET_SET(so1->so_vnet);
1380 error = so1->so_proto->pr_connect2(so1, so2);
1381 CURVNET_RESTORE();
1382 return (error);
1383 }
1384
1385 int
1386 sodisconnect(struct socket *so)
1387 {
1388 int error;
1389
1390 if ((so->so_state & SS_ISCONNECTED) == 0)
1391 return (ENOTCONN);
1392 if (so->so_state & SS_ISDISCONNECTING)
1393 return (EALREADY);
1394 VNET_SO_ASSERT(so);
1395 error = so->so_proto->pr_disconnect(so);
1396 return (error);
1397 }
1398
1399 int
1400 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio,
1401 struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
1402 {
1403 long space;
1404 ssize_t resid;
1405 int clen = 0, error, dontroute;
1406
1407 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM"));
1408 KASSERT(so->so_proto->pr_flags & PR_ATOMIC,
1409 ("sosend_dgram: !PR_ATOMIC"));
1410
1411 if (uio != NULL)
1412 resid = uio->uio_resid;
1413 else
1414 resid = top->m_pkthdr.len;
1415 /*
1416 * In theory resid should be unsigned. However, space must be
1417 * signed, as it might be less than 0 if we over-committed, and we
1418 * must use a signed comparison of space and resid. On the other
1419 * hand, a negative resid causes us to loop sending 0-length
1420 * segments to the protocol.
1421 */
1422 if (resid < 0) {
1423 error = EINVAL;
1424 goto out;
1425 }
1426
1427 dontroute =
1428 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0;
1429 if (td != NULL)
1430 td->td_ru.ru_msgsnd++;
1431 if (control != NULL)
1432 clen = control->m_len;
1433
1434 SOCKBUF_LOCK(&so->so_snd);
1435 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
1436 SOCKBUF_UNLOCK(&so->so_snd);
1437 error = EPIPE;
1438 goto out;
1439 }
1440 if (so->so_error) {
1441 error = so->so_error;
1442 so->so_error = 0;
1443 SOCKBUF_UNLOCK(&so->so_snd);
1444 goto out;
1445 }
1446 if ((so->so_state & SS_ISCONNECTED) == 0) {
1447 /*
1448 * `sendto' and `sendmsg' is allowed on a connection-based
1449 * socket if it supports implied connect. Return ENOTCONN if
1450 * not connected and no address is supplied.
1451 */
1452 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
1453 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
1454 if ((so->so_state & SS_ISCONFIRMING) == 0 &&
1455 !(resid == 0 && clen != 0)) {
1456 SOCKBUF_UNLOCK(&so->so_snd);
1457 error = ENOTCONN;
1458 goto out;
1459 }
1460 } else if (addr == NULL) {
1461 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
1462 error = ENOTCONN;
1463 else
1464 error = EDESTADDRREQ;
1465 SOCKBUF_UNLOCK(&so->so_snd);
1466 goto out;
1467 }
1468 }
1469
1470 /*
1471 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a
1472 * problem and need fixing.
1473 */
1474 space = sbspace(&so->so_snd);
1475 if (flags & MSG_OOB)
1476 space += 1024;
1477 space -= clen;
1478 SOCKBUF_UNLOCK(&so->so_snd);
1479 if (resid > space) {
1480 error = EMSGSIZE;
1481 goto out;
1482 }
1483 if (uio == NULL) {
1484 resid = 0;
1485 if (flags & MSG_EOR)
1486 top->m_flags |= M_EOR;
1487 } else {
1488 /*
1489 * Copy the data from userland into a mbuf chain.
1490 * If no data is to be copied in, a single empty mbuf
1491 * is returned.
1492 */
1493 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr,
1494 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0)));
1495 if (top == NULL) {
1496 error = EFAULT; /* only possible error */
1497 goto out;
1498 }
1499 space -= resid - uio->uio_resid;
1500 resid = uio->uio_resid;
1501 }
1502 KASSERT(resid == 0, ("sosend_dgram: resid != 0"));
1503 /*
1504 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock
1505 * than with.
1506 */
1507 if (dontroute) {
1508 SOCK_LOCK(so);
1509 so->so_options |= SO_DONTROUTE;
1510 SOCK_UNLOCK(so);
1511 }
1512 /*
1513 * XXX all the SBS_CANTSENDMORE checks previously done could be out
1514 * of date. We could have received a reset packet in an interrupt or
1515 * maybe we slept while doing page faults in uiomove() etc. We could
1516 * probably recheck again inside the locking protection here, but
1517 * there are probably other places that this also happens. We must
1518 * rethink this.
1519 */
1520 VNET_SO_ASSERT(so);
1521 error = so->so_proto->pr_send(so, (flags & MSG_OOB) ? PRUS_OOB :
1522 /*
1523 * If the user set MSG_EOF, the protocol understands this flag and
1524 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND.
1525 */
1526 ((flags & MSG_EOF) &&
1527 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
1528 (resid <= 0)) ?
1529 PRUS_EOF :
1530 /* If there is more to send set PRUS_MORETOCOME */
1531 (flags & MSG_MORETOCOME) ||
1532 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
1533 top, addr, control, td);
1534 if (dontroute) {
1535 SOCK_LOCK(so);
1536 so->so_options &= ~SO_DONTROUTE;
1537 SOCK_UNLOCK(so);
1538 }
1539 clen = 0;
1540 control = NULL;
1541 top = NULL;
1542 out:
1543 if (top != NULL)
1544 m_freem(top);
1545 if (control != NULL)
1546 m_freem(control);
1547 return (error);
1548 }
1549
1550 /*
1551 * Send on a socket. If send must go all at once and message is larger than
1552 * send buffering, then hard error. Lock against other senders. If must go
1553 * all at once and not enough room now, then inform user that this would
1554 * block and do nothing. Otherwise, if nonblocking, send as much as
1555 * possible. The data to be sent is described by "uio" if nonzero, otherwise
1556 * by the mbuf chain "top" (which must be null if uio is not). Data provided
1557 * in mbuf chain must be small enough to send all at once.
1558 *
1559 * Returns nonzero on error, timeout or signal; callers must check for short
1560 * counts if EINTR/ERESTART are returned. Data and control buffers are freed
1561 * on return.
1562 */
1563 int
1564 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
1565 struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
1566 {
1567 long space;
1568 ssize_t resid;
1569 int clen = 0, error, dontroute;
1570 int atomic = sosendallatonce(so) || top;
1571 int pr_send_flag;
1572 #ifdef KERN_TLS
1573 struct ktls_session *tls;
1574 int tls_enq_cnt, tls_send_flag;
1575 uint8_t tls_rtype;
1576
1577 tls = NULL;
1578 tls_rtype = TLS_RLTYPE_APP;
1579 #endif
1580 if (uio != NULL)
1581 resid = uio->uio_resid;
1582 else if ((top->m_flags & M_PKTHDR) != 0)
1583 resid = top->m_pkthdr.len;
1584 else
1585 resid = m_length(top, NULL);
1586 /*
1587 * In theory resid should be unsigned. However, space must be
1588 * signed, as it might be less than 0 if we over-committed, and we
1589 * must use a signed comparison of space and resid. On the other
1590 * hand, a negative resid causes us to loop sending 0-length
1591 * segments to the protocol.
1592 *
1593 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM
1594 * type sockets since that's an error.
1595 */
1596 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) {
1597 error = EINVAL;
1598 goto out;
1599 }
1600
1601 dontroute =
1602 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
1603 (so->so_proto->pr_flags & PR_ATOMIC);
1604 if (td != NULL)
1605 td->td_ru.ru_msgsnd++;
1606 if (control != NULL)
1607 clen = control->m_len;
1608
1609 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags));
1610 if (error)
1611 goto out;
1612
1613 #ifdef KERN_TLS
1614 tls_send_flag = 0;
1615 tls = ktls_hold(so->so_snd.sb_tls_info);
1616 if (tls != NULL) {
1617 if (tls->mode == TCP_TLS_MODE_SW)
1618 tls_send_flag = PRUS_NOTREADY;
1619
1620 if (control != NULL) {
1621 struct cmsghdr *cm = mtod(control, struct cmsghdr *);
1622
1623 if (clen >= sizeof(*cm) &&
1624 cm->cmsg_type == TLS_SET_RECORD_TYPE) {
1625 tls_rtype = *((uint8_t *)CMSG_DATA(cm));
1626 clen = 0;
1627 m_freem(control);
1628 control = NULL;
1629 atomic = 1;
1630 }
1631 }
1632
1633 if (resid == 0 && !ktls_permit_empty_frames(tls)) {
1634 error = EINVAL;
1635 goto release;
1636 }
1637 }
1638 #endif
1639
1640 restart:
1641 do {
1642 SOCKBUF_LOCK(&so->so_snd);
1643 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
1644 SOCKBUF_UNLOCK(&so->so_snd);
1645 error = EPIPE;
1646 goto release;
1647 }
1648 if (so->so_error) {
1649 error = so->so_error;
1650 so->so_error = 0;
1651 SOCKBUF_UNLOCK(&so->so_snd);
1652 goto release;
1653 }
1654 if ((so->so_state & SS_ISCONNECTED) == 0) {
1655 /*
1656 * `sendto' and `sendmsg' is allowed on a connection-
1657 * based socket if it supports implied connect.
1658 * Return ENOTCONN if not connected and no address is
1659 * supplied.
1660 */
1661 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
1662 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
1663 if ((so->so_state & SS_ISCONFIRMING) == 0 &&
1664 !(resid == 0 && clen != 0)) {
1665 SOCKBUF_UNLOCK(&so->so_snd);
1666 error = ENOTCONN;
1667 goto release;
1668 }
1669 } else if (addr == NULL) {
1670 SOCKBUF_UNLOCK(&so->so_snd);
1671 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
1672 error = ENOTCONN;
1673 else
1674 error = EDESTADDRREQ;
1675 goto release;
1676 }
1677 }
1678 space = sbspace(&so->so_snd);
1679 if (flags & MSG_OOB)
1680 space += 1024;
1681 if ((atomic && resid > so->so_snd.sb_hiwat) ||
1682 clen > so->so_snd.sb_hiwat) {
1683 SOCKBUF_UNLOCK(&so->so_snd);
1684 error = EMSGSIZE;
1685 goto release;
1686 }
1687 if (space < resid + clen &&
1688 (atomic || space < so->so_snd.sb_lowat || space < clen)) {
1689 if ((so->so_state & SS_NBIO) ||
1690 (flags & (MSG_NBIO | MSG_DONTWAIT)) != 0) {
1691 SOCKBUF_UNLOCK(&so->so_snd);
1692 error = EWOULDBLOCK;
1693 goto release;
1694 }
1695 error = sbwait(so, SO_SND);
1696 SOCKBUF_UNLOCK(&so->so_snd);
1697 if (error)
1698 goto release;
1699 goto restart;
1700 }
1701 SOCKBUF_UNLOCK(&so->so_snd);
1702 space -= clen;
1703 do {
1704 if (uio == NULL) {
1705 resid = 0;
1706 if (flags & MSG_EOR)
1707 top->m_flags |= M_EOR;
1708 #ifdef KERN_TLS
1709 if (tls != NULL) {
1710 ktls_frame(top, tls, &tls_enq_cnt,
1711 tls_rtype);
1712 tls_rtype = TLS_RLTYPE_APP;
1713 }
1714 #endif
1715 } else {
1716 /*
1717 * Copy the data from userland into a mbuf
1718 * chain. If resid is 0, which can happen
1719 * only if we have control to send, then
1720 * a single empty mbuf is returned. This
1721 * is a workaround to prevent protocol send
1722 * methods to panic.
1723 */
1724 #ifdef KERN_TLS
1725 if (tls != NULL) {
1726 top = m_uiotombuf(uio, M_WAITOK, space,
1727 tls->params.max_frame_len,
1728 M_EXTPG |
1729 ((flags & MSG_EOR) ? M_EOR : 0));
1730 if (top != NULL) {
1731 ktls_frame(top, tls,
1732 &tls_enq_cnt, tls_rtype);
1733 }
1734 tls_rtype = TLS_RLTYPE_APP;
1735 } else
1736 #endif
1737 top = m_uiotombuf(uio, M_WAITOK, space,
1738 (atomic ? max_hdr : 0),
1739 (atomic ? M_PKTHDR : 0) |
1740 ((flags & MSG_EOR) ? M_EOR : 0));
1741 if (top == NULL) {
1742 error = EFAULT; /* only possible error */
1743 goto release;
1744 }
1745 space -= resid - uio->uio_resid;
1746 resid = uio->uio_resid;
1747 }
1748 if (dontroute) {
1749 SOCK_LOCK(so);
1750 so->so_options |= SO_DONTROUTE;
1751 SOCK_UNLOCK(so);
1752 }
1753 /*
1754 * XXX all the SBS_CANTSENDMORE checks previously
1755 * done could be out of date. We could have received
1756 * a reset packet in an interrupt or maybe we slept
1757 * while doing page faults in uiomove() etc. We
1758 * could probably recheck again inside the locking
1759 * protection here, but there are probably other
1760 * places that this also happens. We must rethink
1761 * this.
1762 */
1763 VNET_SO_ASSERT(so);
1764
1765 pr_send_flag = (flags & MSG_OOB) ? PRUS_OOB :
1766 /*
1767 * If the user set MSG_EOF, the protocol understands
1768 * this flag and nothing left to send then use
1769 * PRU_SEND_EOF instead of PRU_SEND.
1770 */
1771 ((flags & MSG_EOF) &&
1772 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
1773 (resid <= 0)) ?
1774 PRUS_EOF :
1775 /* If there is more to send set PRUS_MORETOCOME. */
1776 (flags & MSG_MORETOCOME) ||
1777 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0;
1778
1779 #ifdef KERN_TLS
1780 pr_send_flag |= tls_send_flag;
1781 #endif
1782
1783 error = so->so_proto->pr_send(so, pr_send_flag, top,
1784 addr, control, td);
1785
1786 if (dontroute) {
1787 SOCK_LOCK(so);
1788 so->so_options &= ~SO_DONTROUTE;
1789 SOCK_UNLOCK(so);
1790 }
1791
1792 #ifdef KERN_TLS
1793 if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
1794 if (error != 0) {
1795 m_freem(top);
1796 top = NULL;
1797 } else {
1798 soref(so);
1799 ktls_enqueue(top, so, tls_enq_cnt);
1800 }
1801 }
1802 #endif
1803 clen = 0;
1804 control = NULL;
1805 top = NULL;
1806 if (error)
1807 goto release;
1808 } while (resid && space > 0);
1809 } while (resid);
1810
1811 release:
1812 SOCK_IO_SEND_UNLOCK(so);
1813 out:
1814 #ifdef KERN_TLS
1815 if (tls != NULL)
1816 ktls_free(tls);
1817 #endif
1818 if (top != NULL)
1819 m_freem(top);
1820 if (control != NULL)
1821 m_freem(control);
1822 return (error);
1823 }
1824
1825 /*
1826 * Send to a socket from a kernel thread.
1827 *
1828 * XXXGL: in almost all cases uio is NULL and the mbuf is supplied.
1829 * Exception is nfs/bootp_subr.c. It is arguable that the VNET context needs
1830 * to be set at all. This function should just boil down to a static inline
1831 * calling the protocol method.
1832 */
1833 int
1834 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
1835 struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
1836 {
1837 int error;
1838
1839 CURVNET_SET(so->so_vnet);
1840 error = so->so_proto->pr_sosend(so, addr, uio,
1841 top, control, flags, td);
1842 CURVNET_RESTORE();
1843 return (error);
1844 }
1845
1846 /*
1847 * send(2), write(2) or aio_write(2) on a socket.
1848 */
1849 int
1850 sousrsend(struct socket *so, struct sockaddr *addr, struct uio *uio,
1851 struct mbuf *control, int flags, struct proc *userproc)
1852 {
1853 struct thread *td;
1854 ssize_t len;
1855 int error;
1856
1857 td = uio->uio_td;
1858 len = uio->uio_resid;
1859 CURVNET_SET(so->so_vnet);
1860 error = so->so_proto->pr_sosend(so, addr, uio, NULL, control, flags,
1861 td);
1862 CURVNET_RESTORE();
1863 if (error != 0) {
1864 if (uio->uio_resid != len &&
1865 (so->so_proto->pr_flags & PR_ATOMIC) == 0 &&
1866 (error == ERESTART || error == EINTR ||
1867 error == EWOULDBLOCK))
1868 error = 0;
1869 /* Generation of SIGPIPE can be controlled per socket. */
1870 if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0 &&
1871 (flags & MSG_NOSIGNAL) == 0) {
1872 if (userproc != NULL) {
1873 /* aio(4) job */
1874 PROC_LOCK(userproc);
1875 kern_psignal(userproc, SIGPIPE);
1876 PROC_UNLOCK(userproc);
1877 } else {
1878 PROC_LOCK(td->td_proc);
1879 tdsignal(td, SIGPIPE);
1880 PROC_UNLOCK(td->td_proc);
1881 }
1882 }
1883 }
1884 return (error);
1885 }
1886
1887 /*
1888 * The part of soreceive() that implements reading non-inline out-of-band
1889 * data from a socket. For more complete comments, see soreceive(), from
1890 * which this code originated.
1891 *
1892 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is
1893 * unable to return an mbuf chain to the caller.
1894 */
1895 static int
1896 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags)
1897 {
1898 struct protosw *pr = so->so_proto;
1899 struct mbuf *m;
1900 int error;
1901
1902 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0"));
1903 VNET_SO_ASSERT(so);
1904
1905 m = m_get(M_WAITOK, MT_DATA);
1906 error = pr->pr_rcvoob(so, m, flags & MSG_PEEK);
1907 if (error)
1908 goto bad;
1909 do {
1910 error = uiomove(mtod(m, void *),
1911 (int) min(uio->uio_resid, m->m_len), uio);
1912 m = m_free(m);
1913 } while (uio->uio_resid && error == 0 && m);
1914 bad:
1915 if (m != NULL)
1916 m_freem(m);
1917 return (error);
1918 }
1919
1920 /*
1921 * Following replacement or removal of the first mbuf on the first mbuf chain
1922 * of a socket buffer, push necessary state changes back into the socket
1923 * buffer so that other consumers see the values consistently. 'nextrecord'
1924 * is the callers locally stored value of the original value of
1925 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes.
1926 * NOTE: 'nextrecord' may be NULL.
1927 */
1928 static __inline void
1929 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord)
1930 {
1931
1932 SOCKBUF_LOCK_ASSERT(sb);
1933 /*
1934 * First, update for the new value of nextrecord. If necessary, make
1935 * it the first record.
1936 */
1937 if (sb->sb_mb != NULL)
1938 sb->sb_mb->m_nextpkt = nextrecord;
1939 else
1940 sb->sb_mb = nextrecord;
1941
1942 /*
1943 * Now update any dependent socket buffer fields to reflect the new
1944 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the
1945 * addition of a second clause that takes care of the case where
1946 * sb_mb has been updated, but remains the last record.
1947 */
1948 if (sb->sb_mb == NULL) {
1949 sb->sb_mbtail = NULL;
1950 sb->sb_lastrecord = NULL;
1951 } else if (sb->sb_mb->m_nextpkt == NULL)
1952 sb->sb_lastrecord = sb->sb_mb;
1953 }
1954
1955 /*
1956 * Implement receive operations on a socket. We depend on the way that
1957 * records are added to the sockbuf by sbappend. In particular, each record
1958 * (mbufs linked through m_next) must begin with an address if the protocol
1959 * so specifies, followed by an optional mbuf or mbufs containing ancillary
1960 * data, and then zero or more mbufs of data. In order to allow parallelism
1961 * between network receive and copying to user space, as well as avoid
1962 * sleeping with a mutex held, we release the socket buffer mutex during the
1963 * user space copy. Although the sockbuf is locked, new data may still be
1964 * appended, and thus we must maintain consistency of the sockbuf during that
1965 * time.
1966 *
1967 * The caller may receive the data as a single mbuf chain by supplying an
1968 * mbuf **mp0 for use in returning the chain. The uio is then used only for
1969 * the count in uio_resid.
1970 */
1971 int
1972 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio,
1973 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
1974 {
1975 struct mbuf *m, **mp;
1976 int flags, error, offset;
1977 ssize_t len;
1978 struct protosw *pr = so->so_proto;
1979 struct mbuf *nextrecord;
1980 int moff, type = 0;
1981 ssize_t orig_resid = uio->uio_resid;
1982 bool report_real_len = false;
1983
1984 mp = mp0;
1985 if (psa != NULL)
1986 *psa = NULL;
1987 if (controlp != NULL)
1988 *controlp = NULL;
1989 if (flagsp != NULL) {
1990 report_real_len = *flagsp & MSG_TRUNC;
1991 *flagsp &= ~MSG_TRUNC;
1992 flags = *flagsp &~ MSG_EOR;
1993 } else
1994 flags = 0;
1995 if (flags & MSG_OOB)
1996 return (soreceive_rcvoob(so, uio, flags));
1997 if (mp != NULL)
1998 *mp = NULL;
1999 if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING)
2000 && uio->uio_resid) {
2001 VNET_SO_ASSERT(so);
2002 pr->pr_rcvd(so, 0);
2003 }
2004
2005 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
2006 if (error)
2007 return (error);
2008
2009 restart:
2010 SOCKBUF_LOCK(&so->so_rcv);
2011 m = so->so_rcv.sb_mb;
2012 /*
2013 * If we have less data than requested, block awaiting more (subject
2014 * to any timeout) if:
2015 * 1. the current count is less than the low water mark, or
2016 * 2. MSG_DONTWAIT is not set
2017 */
2018 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 &&
2019 sbavail(&so->so_rcv) < uio->uio_resid) &&
2020 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat &&
2021 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
2022 KASSERT(m != NULL || !sbavail(&so->so_rcv),
2023 ("receive: m == %p sbavail == %u",
2024 m, sbavail(&so->so_rcv)));
2025 if (so->so_error || so->so_rerror) {
2026 if (m != NULL)
2027 goto dontblock;
2028 if (so->so_error)
2029 error = so->so_error;
2030 else
2031 error = so->so_rerror;
2032 if ((flags & MSG_PEEK) == 0) {
2033 if (so->so_error)
2034 so->so_error = 0;
2035 else
2036 so->so_rerror = 0;
2037 }
2038 SOCKBUF_UNLOCK(&so->so_rcv);
2039 goto release;
2040 }
2041 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2042 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
2043 if (m != NULL)
2044 goto dontblock;
2045 #ifdef KERN_TLS
2046 else if (so->so_rcv.sb_tlsdcc == 0 &&
2047 so->so_rcv.sb_tlscc == 0) {
2048 #else
2049 else {
2050 #endif
2051 SOCKBUF_UNLOCK(&so->so_rcv);
2052 goto release;
2053 }
2054 }
2055 for (; m != NULL; m = m->m_next)
2056 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
2057 m = so->so_rcv.sb_mb;
2058 goto dontblock;
2059 }
2060 if ((so->so_state & (SS_ISCONNECTING | SS_ISCONNECTED |
2061 SS_ISDISCONNECTING | SS_ISDISCONNECTED)) == 0 &&
2062 (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) {
2063 SOCKBUF_UNLOCK(&so->so_rcv);
2064 error = ENOTCONN;
2065 goto release;
2066 }
2067 if (uio->uio_resid == 0 && !report_real_len) {
2068 SOCKBUF_UNLOCK(&so->so_rcv);
2069 goto release;
2070 }
2071 if ((so->so_state & SS_NBIO) ||
2072 (flags & (MSG_DONTWAIT|MSG_NBIO))) {
2073 SOCKBUF_UNLOCK(&so->so_rcv);
2074 error = EWOULDBLOCK;
2075 goto release;
2076 }
2077 SBLASTRECORDCHK(&so->so_rcv);
2078 SBLASTMBUFCHK(&so->so_rcv);
2079 error = sbwait(so, SO_RCV);
2080 SOCKBUF_UNLOCK(&so->so_rcv);
2081 if (error)
2082 goto release;
2083 goto restart;
2084 }
2085 dontblock:
2086 /*
2087 * From this point onward, we maintain 'nextrecord' as a cache of the
2088 * pointer to the next record in the socket buffer. We must keep the
2089 * various socket buffer pointers and local stack versions of the
2090 * pointers in sync, pushing out modifications before dropping the
2091 * socket buffer mutex, and re-reading them when picking it up.
2092 *
2093 * Otherwise, we will race with the network stack appending new data
2094 * or records onto the socket buffer by using inconsistent/stale
2095 * versions of the field, possibly resulting in socket buffer
2096 * corruption.
2097 *
2098 * By holding the high-level sblock(), we prevent simultaneous
2099 * readers from pulling off the front of the socket buffer.
2100 */
2101 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2102 if (uio->uio_td)
2103 uio->uio_td->td_ru.ru_msgrcv++;
2104 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb"));
2105 SBLASTRECORDCHK(&so->so_rcv);
2106 SBLASTMBUFCHK(&so->so_rcv);
2107 nextrecord = m->m_nextpkt;
2108 if (pr->pr_flags & PR_ADDR) {
2109 KASSERT(m->m_type == MT_SONAME,
2110 ("m->m_type == %d", m->m_type));
2111 orig_resid = 0;
2112 if (psa != NULL)
2113 *psa = sodupsockaddr(mtod(m, struct sockaddr *),
2114 M_NOWAIT);
2115 if (flags & MSG_PEEK) {
2116 m = m->m_next;
2117 } else {
2118 sbfree(&so->so_rcv, m);
2119 so->so_rcv.sb_mb = m_free(m);
2120 m = so->so_rcv.sb_mb;
2121 sockbuf_pushsync(&so->so_rcv, nextrecord);
2122 }
2123 }
2124
2125 /*
2126 * Process one or more MT_CONTROL mbufs present before any data mbufs
2127 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we
2128 * just copy the data; if !MSG_PEEK, we call into the protocol to
2129 * perform externalization (or freeing if controlp == NULL).
2130 */
2131 if (m != NULL && m->m_type == MT_CONTROL) {
2132 struct mbuf *cm = NULL, *cmn;
2133 struct mbuf **cme = &cm;
2134 #ifdef KERN_TLS
2135 struct cmsghdr *cmsg;
2136 struct tls_get_record tgr;
2137
2138 /*
2139 * For MSG_TLSAPPDATA, check for an alert record.
2140 * If found, return ENXIO without removing
2141 * it from the receive queue. This allows a subsequent
2142 * call without MSG_TLSAPPDATA to receive it.
2143 * Note that, for TLS, there should only be a single
2144 * control mbuf with the TLS_GET_RECORD message in it.
2145 */
2146 if (flags & MSG_TLSAPPDATA) {
2147 cmsg = mtod(m, struct cmsghdr *);
2148 if (cmsg->cmsg_type == TLS_GET_RECORD &&
2149 cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) {
2150 memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr));
2151 if (__predict_false(tgr.tls_type ==
2152 TLS_RLTYPE_ALERT)) {
2153 SOCKBUF_UNLOCK(&so->so_rcv);
2154 error = ENXIO;
2155 goto release;
2156 }
2157 }
2158 }
2159 #endif
2160
2161 do {
2162 if (flags & MSG_PEEK) {
2163 if (controlp != NULL) {
2164 *controlp = m_copym(m, 0, m->m_len,
2165 M_NOWAIT);
2166 controlp = &(*controlp)->m_next;
2167 }
2168 m = m->m_next;
2169 } else {
2170 sbfree(&so->so_rcv, m);
2171 so->so_rcv.sb_mb = m->m_next;
2172 m->m_next = NULL;
2173 *cme = m;
2174 cme = &(*cme)->m_next;
2175 m = so->so_rcv.sb_mb;
2176 }
2177 } while (m != NULL && m->m_type == MT_CONTROL);
2178 if ((flags & MSG_PEEK) == 0)
2179 sockbuf_pushsync(&so->so_rcv, nextrecord);
2180 while (cm != NULL) {
2181 cmn = cm->m_next;
2182 cm->m_next = NULL;
2183 if (pr->pr_domain->dom_externalize != NULL) {
2184 SOCKBUF_UNLOCK(&so->so_rcv);
2185 VNET_SO_ASSERT(so);
2186 error = (*pr->pr_domain->dom_externalize)
2187 (cm, controlp, flags);
2188 SOCKBUF_LOCK(&so->so_rcv);
2189 } else if (controlp != NULL)
2190 *controlp = cm;
2191 else
2192 m_freem(cm);
2193 if (controlp != NULL) {
2194 while (*controlp != NULL)
2195 controlp = &(*controlp)->m_next;
2196 }
2197 cm = cmn;
2198 }
2199 if (m != NULL)
2200 nextrecord = so->so_rcv.sb_mb->m_nextpkt;
2201 else
2202 nextrecord = so->so_rcv.sb_mb;
2203 orig_resid = 0;
2204 }
2205 if (m != NULL) {
2206 if ((flags & MSG_PEEK) == 0) {
2207 KASSERT(m->m_nextpkt == nextrecord,
2208 ("soreceive: post-control, nextrecord !sync"));
2209 if (nextrecord == NULL) {
2210 KASSERT(so->so_rcv.sb_mb == m,
2211 ("soreceive: post-control, sb_mb!=m"));
2212 KASSERT(so->so_rcv.sb_lastrecord == m,
2213 ("soreceive: post-control, lastrecord!=m"));
2214 }
2215 }
2216 type = m->m_type;
2217 if (type == MT_OOBDATA)
2218 flags |= MSG_OOB;
2219 } else {
2220 if ((flags & MSG_PEEK) == 0) {
2221 KASSERT(so->so_rcv.sb_mb == nextrecord,
2222 ("soreceive: sb_mb != nextrecord"));
2223 if (so->so_rcv.sb_mb == NULL) {
2224 KASSERT(so->so_rcv.sb_lastrecord == NULL,
2225 ("soreceive: sb_lastercord != NULL"));
2226 }
2227 }
2228 }
2229 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2230 SBLASTRECORDCHK(&so->so_rcv);
2231 SBLASTMBUFCHK(&so->so_rcv);
2232
2233 /*
2234 * Now continue to read any data mbufs off of the head of the socket
2235 * buffer until the read request is satisfied. Note that 'type' is
2236 * used to store the type of any mbuf reads that have happened so far
2237 * such that soreceive() can stop reading if the type changes, which
2238 * causes soreceive() to return only one of regular data and inline
2239 * out-of-band data in a single socket receive operation.
2240 */
2241 moff = 0;
2242 offset = 0;
2243 while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0
2244 && error == 0) {
2245 /*
2246 * If the type of mbuf has changed since the last mbuf
2247 * examined ('type'), end the receive operation.
2248 */
2249 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2250 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) {
2251 if (type != m->m_type)
2252 break;
2253 } else if (type == MT_OOBDATA)
2254 break;
2255 else
2256 KASSERT(m->m_type == MT_DATA,
2257 ("m->m_type == %d", m->m_type));
2258 so->so_rcv.sb_state &= ~SBS_RCVATMARK;
2259 len = uio->uio_resid;
2260 if (so->so_oobmark && len > so->so_oobmark - offset)
2261 len = so->so_oobmark - offset;
2262 if (len > m->m_len - moff)
2263 len = m->m_len - moff;
2264 /*
2265 * If mp is set, just pass back the mbufs. Otherwise copy
2266 * them out via the uio, then free. Sockbuf must be
2267 * consistent here (points to current mbuf, it points to next
2268 * record) when we drop priority; we must note any additions
2269 * to the sockbuf when we block interrupts again.
2270 */
2271 if (mp == NULL) {
2272 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2273 SBLASTRECORDCHK(&so->so_rcv);
2274 SBLASTMBUFCHK(&so->so_rcv);
2275 SOCKBUF_UNLOCK(&so->so_rcv);
2276 if ((m->m_flags & M_EXTPG) != 0)
2277 error = m_unmapped_uiomove(m, moff, uio,
2278 (int)len);
2279 else
2280 error = uiomove(mtod(m, char *) + moff,
2281 (int)len, uio);
2282 SOCKBUF_LOCK(&so->so_rcv);
2283 if (error) {
2284 /*
2285 * The MT_SONAME mbuf has already been removed
2286 * from the record, so it is necessary to
2287 * remove the data mbufs, if any, to preserve
2288 * the invariant in the case of PR_ADDR that
2289 * requires MT_SONAME mbufs at the head of
2290 * each record.
2291 */
2292 if (pr->pr_flags & PR_ATOMIC &&
2293 ((flags & MSG_PEEK) == 0))
2294 (void)sbdroprecord_locked(&so->so_rcv);
2295 SOCKBUF_UNLOCK(&so->so_rcv);
2296 goto release;
2297 }
2298 } else
2299 uio->uio_resid -= len;
2300 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2301 if (len == m->m_len - moff) {
2302 if (m->m_flags & M_EOR)
2303 flags |= MSG_EOR;
2304 if (flags & MSG_PEEK) {
2305 m = m->m_next;
2306 moff = 0;
2307 } else {
2308 nextrecord = m->m_nextpkt;
2309 sbfree(&so->so_rcv, m);
2310 if (mp != NULL) {
2311 m->m_nextpkt = NULL;
2312 *mp = m;
2313 mp = &m->m_next;
2314 so->so_rcv.sb_mb = m = m->m_next;
2315 *mp = NULL;
2316 } else {
2317 so->so_rcv.sb_mb = m_free(m);
2318 m = so->so_rcv.sb_mb;
2319 }
2320 sockbuf_pushsync(&so->so_rcv, nextrecord);
2321 SBLASTRECORDCHK(&so->so_rcv);
2322 SBLASTMBUFCHK(&so->so_rcv);
2323 }
2324 } else {
2325 if (flags & MSG_PEEK)
2326 moff += len;
2327 else {
2328 if (mp != NULL) {
2329 if (flags & MSG_DONTWAIT) {
2330 *mp = m_copym(m, 0, len,
2331 M_NOWAIT);
2332 if (*mp == NULL) {
2333 /*
2334 * m_copym() couldn't
2335 * allocate an mbuf.
2336 * Adjust uio_resid back
2337 * (it was adjusted
2338 * down by len bytes,
2339 * which we didn't end
2340 * up "copying" over).
2341 */
2342 uio->uio_resid += len;
2343 break;
2344 }
2345 } else {
2346 SOCKBUF_UNLOCK(&so->so_rcv);
2347 *mp = m_copym(m, 0, len,
2348 M_WAITOK);
2349 SOCKBUF_LOCK(&so->so_rcv);
2350 }
2351 }
2352 sbcut_locked(&so->so_rcv, len);
2353 }
2354 }
2355 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2356 if (so->so_oobmark) {
2357 if ((flags & MSG_PEEK) == 0) {
2358 so->so_oobmark -= len;
2359 if (so->so_oobmark == 0) {
2360 so->so_rcv.sb_state |= SBS_RCVATMARK;
2361 break;
2362 }
2363 } else {
2364 offset += len;
2365 if (offset == so->so_oobmark)
2366 break;
2367 }
2368 }
2369 if (flags & MSG_EOR)
2370 break;
2371 /*
2372 * If the MSG_WAITALL flag is set (for non-atomic socket), we
2373 * must not quit until "uio->uio_resid == 0" or an error
2374 * termination. If a signal/timeout occurs, return with a
2375 * short count but without error. Keep sockbuf locked
2376 * against other readers.
2377 */
2378 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
2379 !sosendallatonce(so) && nextrecord == NULL) {
2380 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2381 if (so->so_error || so->so_rerror ||
2382 so->so_rcv.sb_state & SBS_CANTRCVMORE)
2383 break;
2384 /*
2385 * Notify the protocol that some data has been
2386 * drained before blocking.
2387 */
2388 if (pr->pr_flags & PR_WANTRCVD) {
2389 SOCKBUF_UNLOCK(&so->so_rcv);
2390 VNET_SO_ASSERT(so);
2391 pr->pr_rcvd(so, flags);
2392 SOCKBUF_LOCK(&so->so_rcv);
2393 }
2394 SBLASTRECORDCHK(&so->so_rcv);
2395 SBLASTMBUFCHK(&so->so_rcv);
2396 /*
2397 * We could receive some data while was notifying
2398 * the protocol. Skip blocking in this case.
2399 */
2400 if (so->so_rcv.sb_mb == NULL) {
2401 error = sbwait(so, SO_RCV);
2402 if (error) {
2403 SOCKBUF_UNLOCK(&so->so_rcv);
2404 goto release;
2405 }
2406 }
2407 m = so->so_rcv.sb_mb;
2408 if (m != NULL)
2409 nextrecord = m->m_nextpkt;
2410 }
2411 }
2412
2413 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2414 if (m != NULL && pr->pr_flags & PR_ATOMIC) {
2415 if (report_real_len)
2416 uio->uio_resid -= m_length(m, NULL) - moff;
2417 flags |= MSG_TRUNC;
2418 if ((flags & MSG_PEEK) == 0)
2419 (void) sbdroprecord_locked(&so->so_rcv);
2420 }
2421 if ((flags & MSG_PEEK) == 0) {
2422 if (m == NULL) {
2423 /*
2424 * First part is an inline SB_EMPTY_FIXUP(). Second
2425 * part makes sure sb_lastrecord is up-to-date if
2426 * there is still data in the socket buffer.
2427 */
2428 so->so_rcv.sb_mb = nextrecord;
2429 if (so->so_rcv.sb_mb == NULL) {
2430 so->so_rcv.sb_mbtail = NULL;
2431 so->so_rcv.sb_lastrecord = NULL;
2432 } else if (nextrecord->m_nextpkt == NULL)
2433 so->so_rcv.sb_lastrecord = nextrecord;
2434 }
2435 SBLASTRECORDCHK(&so->so_rcv);
2436 SBLASTMBUFCHK(&so->so_rcv);
2437 /*
2438 * If soreceive() is being done from the socket callback,
2439 * then don't need to generate ACK to peer to update window,
2440 * since ACK will be generated on return to TCP.
2441 */
2442 if (!(flags & MSG_SOCALLBCK) &&
2443 (pr->pr_flags & PR_WANTRCVD)) {
2444 SOCKBUF_UNLOCK(&so->so_rcv);
2445 VNET_SO_ASSERT(so);
2446 pr->pr_rcvd(so, flags);
2447 SOCKBUF_LOCK(&so->so_rcv);
2448 }
2449 }
2450 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2451 if (orig_resid == uio->uio_resid && orig_resid &&
2452 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
2453 SOCKBUF_UNLOCK(&so->so_rcv);
2454 goto restart;
2455 }
2456 SOCKBUF_UNLOCK(&so->so_rcv);
2457
2458 if (flagsp != NULL)
2459 *flagsp |= flags;
2460 release:
2461 SOCK_IO_RECV_UNLOCK(so);
2462 return (error);
2463 }
2464
2465 /*
2466 * Optimized version of soreceive() for stream (TCP) sockets.
2467 */
2468 int
2469 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio,
2470 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2471 {
2472 int len = 0, error = 0, flags, oresid;
2473 struct sockbuf *sb;
2474 struct mbuf *m, *n = NULL;
2475
2476 /* We only do stream sockets. */
2477 if (so->so_type != SOCK_STREAM)
2478 return (EINVAL);
2479 if (psa != NULL)
2480 *psa = NULL;
2481 if (flagsp != NULL)
2482 flags = *flagsp &~ MSG_EOR;
2483 else
2484 flags = 0;
2485 if (controlp != NULL)
2486 *controlp = NULL;
2487 if (flags & MSG_OOB)
2488 return (soreceive_rcvoob(so, uio, flags));
2489 if (mp0 != NULL)
2490 *mp0 = NULL;
2491
2492 sb = &so->so_rcv;
2493
2494 #ifdef KERN_TLS
2495 /*
2496 * KTLS store TLS records as records with a control message to
2497 * describe the framing.
2498 *
2499 * We check once here before acquiring locks to optimize the
2500 * common case.
2501 */
2502 if (sb->sb_tls_info != NULL)
2503 return (soreceive_generic(so, psa, uio, mp0, controlp,
2504 flagsp));
2505 #endif
2506
2507 /* Prevent other readers from entering the socket. */
2508 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
2509 if (error)
2510 return (error);
2511 SOCKBUF_LOCK(sb);
2512
2513 #ifdef KERN_TLS
2514 if (sb->sb_tls_info != NULL) {
2515 SOCKBUF_UNLOCK(sb);
2516 SOCK_IO_RECV_UNLOCK(so);
2517 return (soreceive_generic(so, psa, uio, mp0, controlp,
2518 flagsp));
2519 }
2520 #endif
2521
2522 /* Easy one, no space to copyout anything. */
2523 if (uio->uio_resid == 0) {
2524 error = EINVAL;
2525 goto out;
2526 }
2527 oresid = uio->uio_resid;
2528
2529 /* We will never ever get anything unless we are or were connected. */
2530 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
2531 error = ENOTCONN;
2532 goto out;
2533 }
2534
2535 restart:
2536 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2537
2538 /* Abort if socket has reported problems. */
2539 if (so->so_error) {
2540 if (sbavail(sb) > 0)
2541 goto deliver;
2542 if (oresid > uio->uio_resid)
2543 goto out;
2544 error = so->so_error;
2545 if (!(flags & MSG_PEEK))
2546 so->so_error = 0;
2547 goto out;
2548 }
2549
2550 /* Door is closed. Deliver what is left, if any. */
2551 if (sb->sb_state & SBS_CANTRCVMORE) {
2552 if (sbavail(sb) > 0)
2553 goto deliver;
2554 else
2555 goto out;
2556 }
2557
2558 /* Socket buffer is empty and we shall not block. */
2559 if (sbavail(sb) == 0 &&
2560 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) {
2561 error = EAGAIN;
2562 goto out;
2563 }
2564
2565 /* Socket buffer got some data that we shall deliver now. */
2566 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) &&
2567 ((so->so_state & SS_NBIO) ||
2568 (flags & (MSG_DONTWAIT|MSG_NBIO)) ||
2569 sbavail(sb) >= sb->sb_lowat ||
2570 sbavail(sb) >= uio->uio_resid ||
2571 sbavail(sb) >= sb->sb_hiwat) ) {
2572 goto deliver;
2573 }
2574
2575 /* On MSG_WAITALL we must wait until all data or error arrives. */
2576 if ((flags & MSG_WAITALL) &&
2577 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat))
2578 goto deliver;
2579
2580 /*
2581 * Wait and block until (more) data comes in.
2582 * NB: Drops the sockbuf lock during wait.
2583 */
2584 error = sbwait(so, SO_RCV);
2585 if (error)
2586 goto out;
2587 goto restart;
2588
2589 deliver:
2590 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2591 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__));
2592 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__));
2593
2594 /* Statistics. */
2595 if (uio->uio_td)
2596 uio->uio_td->td_ru.ru_msgrcv++;
2597
2598 /* Fill uio until full or current end of socket buffer is reached. */
2599 len = min(uio->uio_resid, sbavail(sb));
2600 if (mp0 != NULL) {
2601 /* Dequeue as many mbufs as possible. */
2602 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) {
2603 if (*mp0 == NULL)
2604 *mp0 = sb->sb_mb;
2605 else
2606 m_cat(*mp0, sb->sb_mb);
2607 for (m = sb->sb_mb;
2608 m != NULL && m->m_len <= len;
2609 m = m->m_next) {
2610 KASSERT(!(m->m_flags & M_NOTAVAIL),
2611 ("%s: m %p not available", __func__, m));
2612 len -= m->m_len;
2613 uio->uio_resid -= m->m_len;
2614 sbfree(sb, m);
2615 n = m;
2616 }
2617 n->m_next = NULL;
2618 sb->sb_mb = m;
2619 sb->sb_lastrecord = sb->sb_mb;
2620 if (sb->sb_mb == NULL)
2621 SB_EMPTY_FIXUP(sb);
2622 }
2623 /* Copy the remainder. */
2624 if (len > 0) {
2625 KASSERT(sb->sb_mb != NULL,
2626 ("%s: len > 0 && sb->sb_mb empty", __func__));
2627
2628 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT);
2629 if (m == NULL)
2630 len = 0; /* Don't flush data from sockbuf. */
2631 else
2632 uio->uio_resid -= len;
2633 if (*mp0 != NULL)
2634 m_cat(*mp0, m);
2635 else
2636 *mp0 = m;
2637 if (*mp0 == NULL) {
2638 error = ENOBUFS;
2639 goto out;
2640 }
2641 }
2642 } else {
2643 /* NB: Must unlock socket buffer as uiomove may sleep. */
2644 SOCKBUF_UNLOCK(sb);
2645 error = m_mbuftouio(uio, sb->sb_mb, len);
2646 SOCKBUF_LOCK(sb);
2647 if (error)
2648 goto out;
2649 }
2650 SBLASTRECORDCHK(sb);
2651 SBLASTMBUFCHK(sb);
2652
2653 /*
2654 * Remove the delivered data from the socket buffer unless we
2655 * were only peeking.
2656 */
2657 if (!(flags & MSG_PEEK)) {
2658 if (len > 0)
2659 sbdrop_locked(sb, len);
2660
2661 /* Notify protocol that we drained some data. */
2662 if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
2663 (((flags & MSG_WAITALL) && uio->uio_resid > 0) ||
2664 !(flags & MSG_SOCALLBCK))) {
2665 SOCKBUF_UNLOCK(sb);
2666 VNET_SO_ASSERT(so);
2667 so->so_proto->pr_rcvd(so, flags);
2668 SOCKBUF_LOCK(sb);
2669 }
2670 }
2671
2672 /*
2673 * For MSG_WAITALL we may have to loop again and wait for
2674 * more data to come in.
2675 */
2676 if ((flags & MSG_WAITALL) && uio->uio_resid > 0)
2677 goto restart;
2678 out:
2679 SBLASTRECORDCHK(sb);
2680 SBLASTMBUFCHK(sb);
2681 SOCKBUF_UNLOCK(sb);
2682 SOCK_IO_RECV_UNLOCK(so);
2683 return (error);
2684 }
2685
2686 /*
2687 * Optimized version of soreceive() for simple datagram cases from userspace.
2688 * Unlike in the stream case, we're able to drop a datagram if copyout()
2689 * fails, and because we handle datagrams atomically, we don't need to use a
2690 * sleep lock to prevent I/O interlacing.
2691 */
2692 int
2693 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio,
2694 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2695 {
2696 struct mbuf *m, *m2;
2697 int flags, error;
2698 ssize_t len;
2699 struct protosw *pr = so->so_proto;
2700 struct mbuf *nextrecord;
2701
2702 if (psa != NULL)
2703 *psa = NULL;
2704 if (controlp != NULL)
2705 *controlp = NULL;
2706 if (flagsp != NULL)
2707 flags = *flagsp &~ MSG_EOR;
2708 else
2709 flags = 0;
2710
2711 /*
2712 * For any complicated cases, fall back to the full
2713 * soreceive_generic().
2714 */
2715 if (mp0 != NULL || (flags & (MSG_PEEK | MSG_OOB | MSG_TRUNC)))
2716 return (soreceive_generic(so, psa, uio, mp0, controlp,
2717 flagsp));
2718
2719 /*
2720 * Enforce restrictions on use.
2721 */
2722 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0,
2723 ("soreceive_dgram: wantrcvd"));
2724 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic"));
2725 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0,
2726 ("soreceive_dgram: SBS_RCVATMARK"));
2727 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0,
2728 ("soreceive_dgram: P_CONNREQUIRED"));
2729
2730 /*
2731 * Loop blocking while waiting for a datagram.
2732 */
2733 SOCKBUF_LOCK(&so->so_rcv);
2734 while ((m = so->so_rcv.sb_mb) == NULL) {
2735 KASSERT(sbavail(&so->so_rcv) == 0,
2736 ("soreceive_dgram: sb_mb NULL but sbavail %u",
2737 sbavail(&so->so_rcv)));
2738 if (so->so_error) {
2739 error = so->so_error;
2740 so->so_error = 0;
2741 SOCKBUF_UNLOCK(&so->so_rcv);
2742 return (error);
2743 }
2744 if (so->so_rcv.sb_state & SBS_CANTRCVMORE ||
2745 uio->uio_resid == 0) {
2746 SOCKBUF_UNLOCK(&so->so_rcv);
2747 return (0);
2748 }
2749 if ((so->so_state & SS_NBIO) ||
2750 (flags & (MSG_DONTWAIT|MSG_NBIO))) {
2751 SOCKBUF_UNLOCK(&so->so_rcv);
2752 return (EWOULDBLOCK);
2753 }
2754 SBLASTRECORDCHK(&so->so_rcv);
2755 SBLASTMBUFCHK(&so->so_rcv);
2756 error = sbwait(so, SO_RCV);
2757 if (error) {
2758 SOCKBUF_UNLOCK(&so->so_rcv);
2759 return (error);
2760 }
2761 }
2762 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2763
2764 if (uio->uio_td)
2765 uio->uio_td->td_ru.ru_msgrcv++;
2766 SBLASTRECORDCHK(&so->so_rcv);
2767 SBLASTMBUFCHK(&so->so_rcv);
2768 nextrecord = m->m_nextpkt;
2769 if (nextrecord == NULL) {
2770 KASSERT(so->so_rcv.sb_lastrecord == m,
2771 ("soreceive_dgram: lastrecord != m"));
2772 }
2773
2774 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord,
2775 ("soreceive_dgram: m_nextpkt != nextrecord"));
2776
2777 /*
2778 * Pull 'm' and its chain off the front of the packet queue.
2779 */
2780 so->so_rcv.sb_mb = NULL;
2781 sockbuf_pushsync(&so->so_rcv, nextrecord);
2782
2783 /*
2784 * Walk 'm's chain and free that many bytes from the socket buffer.
2785 */
2786 for (m2 = m; m2 != NULL; m2 = m2->m_next)
2787 sbfree(&so->so_rcv, m2);
2788
2789 /*
2790 * Do a few last checks before we let go of the lock.
2791 */
2792 SBLASTRECORDCHK(&so->so_rcv);
2793 SBLASTMBUFCHK(&so->so_rcv);
2794 SOCKBUF_UNLOCK(&so->so_rcv);
2795
2796 if (pr->pr_flags & PR_ADDR) {
2797 KASSERT(m->m_type == MT_SONAME,
2798 ("m->m_type == %d", m->m_type));
2799 if (psa != NULL)
2800 *psa = sodupsockaddr(mtod(m, struct sockaddr *),
2801 M_NOWAIT);
2802 m = m_free(m);
2803 }
2804 if (m == NULL) {
2805 /* XXXRW: Can this happen? */
2806 return (0);
2807 }
2808
2809 /*
2810 * Packet to copyout() is now in 'm' and it is disconnected from the
2811 * queue.
2812 *
2813 * Process one or more MT_CONTROL mbufs present before any data mbufs
2814 * in the first mbuf chain on the socket buffer. We call into the
2815 * protocol to perform externalization (or freeing if controlp ==
2816 * NULL). In some cases there can be only MT_CONTROL mbufs without
2817 * MT_DATA mbufs.
2818 */
2819 if (m->m_type == MT_CONTROL) {
2820 struct mbuf *cm = NULL, *cmn;
2821 struct mbuf **cme = &cm;
2822
2823 do {
2824 m2 = m->m_next;
2825 m->m_next = NULL;
2826 *cme = m;
2827 cme = &(*cme)->m_next;
2828 m = m2;
2829 } while (m != NULL && m->m_type == MT_CONTROL);
2830 while (cm != NULL) {
2831 cmn = cm->m_next;
2832 cm->m_next = NULL;
2833 if (pr->pr_domain->dom_externalize != NULL) {
2834 error = (*pr->pr_domain->dom_externalize)
2835 (cm, controlp, flags);
2836 } else if (controlp != NULL)
2837 *controlp = cm;
2838 else
2839 m_freem(cm);
2840 if (controlp != NULL) {
2841 while (*controlp != NULL)
2842 controlp = &(*controlp)->m_next;
2843 }
2844 cm = cmn;
2845 }
2846 }
2847 KASSERT(m == NULL || m->m_type == MT_DATA,
2848 ("soreceive_dgram: !data"));
2849 while (m != NULL && uio->uio_resid > 0) {
2850 len = uio->uio_resid;
2851 if (len > m->m_len)
2852 len = m->m_len;
2853 error = uiomove(mtod(m, char *), (int)len, uio);
2854 if (error) {
2855 m_freem(m);
2856 return (error);
2857 }
2858 if (len == m->m_len)
2859 m = m_free(m);
2860 else {
2861 m->m_data += len;
2862 m->m_len -= len;
2863 }
2864 }
2865 if (m != NULL) {
2866 flags |= MSG_TRUNC;
2867 m_freem(m);
2868 }
2869 if (flagsp != NULL)
2870 *flagsp |= flags;
2871 return (0);
2872 }
2873
2874 int
2875 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
2876 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2877 {
2878 int error;
2879
2880 CURVNET_SET(so->so_vnet);
2881 error = so->so_proto->pr_soreceive(so, psa, uio, mp0, controlp, flagsp);
2882 CURVNET_RESTORE();
2883 return (error);
2884 }
2885
2886 int
2887 soshutdown(struct socket *so, int how)
2888 {
2889 struct protosw *pr;
2890 int error, soerror_enotconn;
2891
2892 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR))
2893 return (EINVAL);
2894
2895 soerror_enotconn = 0;
2896 SOCK_LOCK(so);
2897 if ((so->so_state &
2898 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) {
2899 /*
2900 * POSIX mandates us to return ENOTCONN when shutdown(2) is
2901 * invoked on a datagram sockets, however historically we would
2902 * actually tear socket down. This is known to be leveraged by
2903 * some applications to unblock process waiting in recvXXX(2)
2904 * by other process that it shares that socket with. Try to meet
2905 * both backward-compatibility and POSIX requirements by forcing
2906 * ENOTCONN but still asking protocol to perform pru_shutdown().
2907 */
2908 if (so->so_type != SOCK_DGRAM && !SOLISTENING(so)) {
2909 SOCK_UNLOCK(so);
2910 return (ENOTCONN);
2911 }
2912 soerror_enotconn = 1;
2913 }
2914
2915 if (SOLISTENING(so)) {
2916 if (how != SHUT_WR) {
2917 so->so_error = ECONNABORTED;
2918 solisten_wakeup(so); /* unlocks so */
2919 } else {
2920 SOCK_UNLOCK(so);
2921 }
2922 goto done;
2923 }
2924 SOCK_UNLOCK(so);
2925
2926 CURVNET_SET(so->so_vnet);
2927 pr = so->so_proto;
2928 if (pr->pr_flush != NULL)
2929 pr->pr_flush(so, how);
2930 if (how != SHUT_WR)
2931 sorflush(so);
2932 if (how != SHUT_RD) {
2933 error = pr->pr_shutdown(so);
2934 wakeup(&so->so_timeo);
2935 CURVNET_RESTORE();
2936 return ((error == 0 && soerror_enotconn) ? ENOTCONN : error);
2937 }
2938 wakeup(&so->so_timeo);
2939 CURVNET_RESTORE();
2940
2941 done:
2942 return (soerror_enotconn ? ENOTCONN : 0);
2943 }
2944
2945 void
2946 sorflush(struct socket *so)
2947 {
2948 struct protosw *pr;
2949 int error;
2950
2951 VNET_SO_ASSERT(so);
2952
2953 /*
2954 * Dislodge threads currently blocked in receive and wait to acquire
2955 * a lock against other simultaneous readers before clearing the
2956 * socket buffer. Don't let our acquire be interrupted by a signal
2957 * despite any existing socket disposition on interruptable waiting.
2958 */
2959 socantrcvmore(so);
2960
2961 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR);
2962 if (error != 0) {
2963 KASSERT(SOLISTENING(so),
2964 ("%s: soiolock(%p) failed", __func__, so));
2965 return;
2966 }
2967
2968 pr = so->so_proto;
2969 if (pr->pr_flags & PR_RIGHTS) {
2970 MPASS(pr->pr_domain->dom_dispose != NULL);
2971 (*pr->pr_domain->dom_dispose)(so);
2972 } else {
2973 sbrelease(so, SO_RCV);
2974 SOCK_IO_RECV_UNLOCK(so);
2975 }
2976
2977 }
2978
2979 /*
2980 * Wrapper for Socket established helper hook.
2981 * Parameters: socket, context of the hook point, hook id.
2982 */
2983 static int inline
2984 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id)
2985 {
2986 struct socket_hhook_data hhook_data = {
2987 .so = so,
2988 .hctx = hctx,
2989 .m = NULL,
2990 .status = 0
2991 };
2992
2993 CURVNET_SET(so->so_vnet);
2994 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd);
2995 CURVNET_RESTORE();
2996
2997 /* Ugly but needed, since hhooks return void for now */
2998 return (hhook_data.status);
2999 }
3000
3001 /*
3002 * Perhaps this routine, and sooptcopyout(), below, ought to come in an
3003 * additional variant to handle the case where the option value needs to be
3004 * some kind of integer, but not a specific size. In addition to their use
3005 * here, these functions are also called by the protocol-level pr_ctloutput()
3006 * routines.
3007 */
3008 int
3009 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen)
3010 {
3011 size_t valsize;
3012
3013 /*
3014 * If the user gives us more than we wanted, we ignore it, but if we
3015 * don't get the minimum length the caller wants, we return EINVAL.
3016 * On success, sopt->sopt_valsize is set to however much we actually
3017 * retrieved.
3018 */
3019 if ((valsize = sopt->sopt_valsize) < minlen)
3020 return EINVAL;
3021 if (valsize > len)
3022 sopt->sopt_valsize = valsize = len;
3023
3024 if (sopt->sopt_td != NULL)
3025 return (copyin(sopt->sopt_val, buf, valsize));
3026
3027 bcopy(sopt->sopt_val, buf, valsize);
3028 return (0);
3029 }
3030
3031 /*
3032 * Kernel version of setsockopt(2).
3033 *
3034 * XXX: optlen is size_t, not socklen_t
3035 */
3036 int
3037 so_setsockopt(struct socket *so, int level, int optname, void *optval,
3038 size_t optlen)
3039 {
3040 struct sockopt sopt;
3041
3042 sopt.sopt_level = level;
3043 sopt.sopt_name = optname;
3044 sopt.sopt_dir = SOPT_SET;
3045 sopt.sopt_val = optval;
3046 sopt.sopt_valsize = optlen;
3047 sopt.sopt_td = NULL;
3048 return (sosetopt(so, &sopt));
3049 }
3050
3051 int
3052 sosetopt(struct socket *so, struct sockopt *sopt)
3053 {
3054 int error, optval;
3055 struct linger l;
3056 struct timeval tv;
3057 sbintime_t val, *valp;
3058 uint32_t val32;
3059 #ifdef MAC
3060 struct mac extmac;
3061 #endif
3062
3063 CURVNET_SET(so->so_vnet);
3064 error = 0;
3065 if (sopt->sopt_level != SOL_SOCKET) {
3066 if (so->so_proto->pr_ctloutput != NULL)
3067 error = (*so->so_proto->pr_ctloutput)(so, sopt);
3068 else
3069 error = ENOPROTOOPT;
3070 } else {
3071 switch (sopt->sopt_name) {
3072 case SO_ACCEPTFILTER:
3073 error = accept_filt_setopt(so, sopt);
3074 if (error)
3075 goto bad;
3076 break;
3077
3078 case SO_LINGER:
3079 error = sooptcopyin(sopt, &l, sizeof l, sizeof l);
3080 if (error)
3081 goto bad;
3082 if (l.l_linger < 0 ||
3083 l.l_linger > USHRT_MAX ||
3084 l.l_linger > (INT_MAX / hz)) {
3085 error = EDOM;
3086 goto bad;
3087 }
3088 SOCK_LOCK(so);
3089 so->so_linger = l.l_linger;
3090 if (l.l_onoff)
3091 so->so_options |= SO_LINGER;
3092 else
3093 so->so_options &= ~SO_LINGER;
3094 SOCK_UNLOCK(so);
3095 break;
3096
3097 case SO_DEBUG:
3098 case SO_KEEPALIVE:
3099 case SO_DONTROUTE:
3100 case SO_USELOOPBACK:
3101 case SO_BROADCAST:
3102 case SO_REUSEADDR:
3103 case SO_REUSEPORT:
3104 case SO_REUSEPORT_LB:
3105 case SO_OOBINLINE:
3106 case SO_TIMESTAMP:
3107 case SO_BINTIME:
3108 case SO_NOSIGPIPE:
3109 case SO_NO_DDP:
3110 case SO_NO_OFFLOAD:
3111 case SO_RERROR:
3112 error = sooptcopyin(sopt, &optval, sizeof optval,
3113 sizeof optval);
3114 if (error)
3115 goto bad;
3116 SOCK_LOCK(so);
3117 if (optval)
3118 so->so_options |= sopt->sopt_name;
3119 else
3120 so->so_options &= ~sopt->sopt_name;
3121 SOCK_UNLOCK(so);
3122 break;
3123
3124 case SO_SETFIB:
3125 error = sooptcopyin(sopt, &optval, sizeof optval,
3126 sizeof optval);
3127 if (error)
3128 goto bad;
3129
3130 if (optval < 0 || optval >= rt_numfibs) {
3131 error = EINVAL;
3132 goto bad;
3133 }
3134 if (((so->so_proto->pr_domain->dom_family == PF_INET) ||
3135 (so->so_proto->pr_domain->dom_family == PF_INET6) ||
3136 (so->so_proto->pr_domain->dom_family == PF_ROUTE)))
3137 so->so_fibnum = optval;
3138 else
3139 so->so_fibnum = 0;
3140 break;
3141
3142 case SO_USER_COOKIE:
3143 error = sooptcopyin(sopt, &val32, sizeof val32,
3144 sizeof val32);
3145 if (error)
3146 goto bad;
3147 so->so_user_cookie = val32;
3148 break;
3149
3150 case SO_SNDBUF:
3151 case SO_RCVBUF:
3152 case SO_SNDLOWAT:
3153 case SO_RCVLOWAT:
3154 error = so->so_proto->pr_setsbopt(so, sopt);
3155 if (error)
3156 goto bad;
3157 break;
3158
3159 case SO_SNDTIMEO:
3160 case SO_RCVTIMEO:
3161 #ifdef COMPAT_FREEBSD32
3162 if (SV_CURPROC_FLAG(SV_ILP32)) {
3163 struct timeval32 tv32;
3164
3165 error = sooptcopyin(sopt, &tv32, sizeof tv32,
3166 sizeof tv32);
3167 CP(tv32, tv, tv_sec);
3168 CP(tv32, tv, tv_usec);
3169 } else
3170 #endif
3171 error = sooptcopyin(sopt, &tv, sizeof tv,
3172 sizeof tv);
3173 if (error)
3174 goto bad;
3175 if (tv.tv_sec < 0 || tv.tv_usec < 0 ||
3176 tv.tv_usec >= 1000000) {
3177 error = EDOM;
3178 goto bad;
3179 }
3180 if (tv.tv_sec > INT32_MAX)
3181 val = SBT_MAX;
3182 else
3183 val = tvtosbt(tv);
3184 SOCK_LOCK(so);
3185 valp = sopt->sopt_name == SO_SNDTIMEO ?
3186 (SOLISTENING(so) ? &so->sol_sbsnd_timeo :
3187 &so->so_snd.sb_timeo) :
3188 (SOLISTENING(so) ? &so->sol_sbrcv_timeo :
3189 &so->so_rcv.sb_timeo);
3190 *valp = val;
3191 SOCK_UNLOCK(so);
3192 break;
3193
3194 case SO_LABEL:
3195 #ifdef MAC
3196 error = sooptcopyin(sopt, &extmac, sizeof extmac,
3197 sizeof extmac);
3198 if (error)
3199 goto bad;
3200 error = mac_setsockopt_label(sopt->sopt_td->td_ucred,
3201 so, &extmac);
3202 #else
3203 error = EOPNOTSUPP;
3204 #endif
3205 break;
3206
3207 case SO_TS_CLOCK:
3208 error = sooptcopyin(sopt, &optval, sizeof optval,
3209 sizeof optval);
3210 if (error)
3211 goto bad;
3212 if (optval < 0 || optval > SO_TS_CLOCK_MAX) {
3213 error = EINVAL;
3214 goto bad;
3215 }
3216 so->so_ts_clock = optval;
3217 break;
3218
3219 case SO_MAX_PACING_RATE:
3220 error = sooptcopyin(sopt, &val32, sizeof(val32),
3221 sizeof(val32));
3222 if (error)
3223 goto bad;
3224 so->so_max_pacing_rate = val32;
3225 break;
3226
3227 default:
3228 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0)
3229 error = hhook_run_socket(so, sopt,
3230 HHOOK_SOCKET_OPT);
3231 else
3232 error = ENOPROTOOPT;
3233 break;
3234 }
3235 if (error == 0 && so->so_proto->pr_ctloutput != NULL)
3236 (void)(*so->so_proto->pr_ctloutput)(so, sopt);
3237 }
3238 bad:
3239 CURVNET_RESTORE();
3240 return (error);
3241 }
3242
3243 /*
3244 * Helper routine for getsockopt.
3245 */
3246 int
3247 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len)
3248 {
3249 int error;
3250 size_t valsize;
3251
3252 error = 0;
3253
3254 /*
3255 * Documented get behavior is that we always return a value, possibly
3256 * truncated to fit in the user's buffer. Traditional behavior is
3257 * that we always tell the user precisely how much we copied, rather
3258 * than something useful like the total amount we had available for
3259 * her. Note that this interface is not idempotent; the entire
3260 * answer must be generated ahead of time.
3261 */
3262 valsize = min(len, sopt->sopt_valsize);
3263 sopt->sopt_valsize = valsize;
3264 if (sopt->sopt_val != NULL) {
3265 if (sopt->sopt_td != NULL)
3266 error = copyout(buf, sopt->sopt_val, valsize);
3267 else
3268 bcopy(buf, sopt->sopt_val, valsize);
3269 }
3270 return (error);
3271 }
3272
3273 int
3274 sogetopt(struct socket *so, struct sockopt *sopt)
3275 {
3276 int error, optval;
3277 struct linger l;
3278 struct timeval tv;
3279 #ifdef MAC
3280 struct mac extmac;
3281 #endif
3282
3283 CURVNET_SET(so->so_vnet);
3284 error = 0;
3285 if (sopt->sopt_level != SOL_SOCKET) {
3286 if (so->so_proto->pr_ctloutput != NULL)
3287 error = (*so->so_proto->pr_ctloutput)(so, sopt);
3288 else
3289 error = ENOPROTOOPT;
3290 CURVNET_RESTORE();
3291 return (error);
3292 } else {
3293 switch (sopt->sopt_name) {
3294 case SO_ACCEPTFILTER:
3295 error = accept_filt_getopt(so, sopt);
3296 break;
3297
3298 case SO_LINGER:
3299 SOCK_LOCK(so);
3300 l.l_onoff = so->so_options & SO_LINGER;
3301 l.l_linger = so->so_linger;
3302 SOCK_UNLOCK(so);
3303 error = sooptcopyout(sopt, &l, sizeof l);
3304 break;
3305
3306 case SO_USELOOPBACK:
3307 case SO_DONTROUTE:
3308 case SO_DEBUG:
3309 case SO_KEEPALIVE:
3310 case SO_REUSEADDR:
3311 case SO_REUSEPORT:
3312 case SO_REUSEPORT_LB:
3313 case SO_BROADCAST:
3314 case SO_OOBINLINE:
3315 case SO_ACCEPTCONN:
3316 case SO_TIMESTAMP:
3317 case SO_BINTIME:
3318 case SO_NOSIGPIPE:
3319 case SO_NO_DDP:
3320 case SO_NO_OFFLOAD:
3321 case SO_RERROR:
3322 optval = so->so_options & sopt->sopt_name;
3323 integer:
3324 error = sooptcopyout(sopt, &optval, sizeof optval);
3325 break;
3326
3327 case SO_DOMAIN:
3328 optval = so->so_proto->pr_domain->dom_family;
3329 goto integer;
3330
3331 case SO_TYPE:
3332 optval = so->so_type;
3333 goto integer;
3334
3335 case SO_PROTOCOL:
3336 optval = so->so_proto->pr_protocol;
3337 goto integer;
3338
3339 case SO_ERROR:
3340 SOCK_LOCK(so);
3341 if (so->so_error) {
3342 optval = so->so_error;
3343 so->so_error = 0;
3344 } else {
3345 optval = so->so_rerror;
3346 so->so_rerror = 0;
3347 }
3348 SOCK_UNLOCK(so);
3349 goto integer;
3350
3351 case SO_SNDBUF:
3352 optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat :
3353 so->so_snd.sb_hiwat;
3354 goto integer;
3355
3356 case SO_RCVBUF:
3357 optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat :
3358 so->so_rcv.sb_hiwat;
3359 goto integer;
3360
3361 case SO_SNDLOWAT:
3362 optval = SOLISTENING(so) ? so->sol_sbsnd_lowat :
3363 so->so_snd.sb_lowat;
3364 goto integer;
3365
3366 case SO_RCVLOWAT:
3367 optval = SOLISTENING(so) ? so->sol_sbrcv_lowat :
3368 so->so_rcv.sb_lowat;
3369 goto integer;
3370
3371 case SO_SNDTIMEO:
3372 case SO_RCVTIMEO:
3373 SOCK_LOCK(so);
3374 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ?
3375 (SOLISTENING(so) ? so->sol_sbsnd_timeo :
3376 so->so_snd.sb_timeo) :
3377 (SOLISTENING(so) ? so->sol_sbrcv_timeo :
3378 so->so_rcv.sb_timeo));
3379 SOCK_UNLOCK(so);
3380 #ifdef COMPAT_FREEBSD32
3381 if (SV_CURPROC_FLAG(SV_ILP32)) {
3382 struct timeval32 tv32;
3383
3384 CP(tv, tv32, tv_sec);
3385 CP(tv, tv32, tv_usec);
3386 error = sooptcopyout(sopt, &tv32, sizeof tv32);
3387 } else
3388 #endif
3389 error = sooptcopyout(sopt, &tv, sizeof tv);
3390 break;
3391
3392 case SO_LABEL:
3393 #ifdef MAC
3394 error = sooptcopyin(sopt, &extmac, sizeof(extmac),
3395 sizeof(extmac));
3396 if (error)
3397 goto bad;
3398 error = mac_getsockopt_label(sopt->sopt_td->td_ucred,
3399 so, &extmac);
3400 if (error)
3401 goto bad;
3402 error = sooptcopyout(sopt, &extmac, sizeof extmac);
3403 #else
3404 error = EOPNOTSUPP;
3405 #endif
3406 break;
3407
3408 case SO_PEERLABEL:
3409 #ifdef MAC
3410 error = sooptcopyin(sopt, &extmac, sizeof(extmac),
3411 sizeof(extmac));
3412 if (error)
3413 goto bad;
3414 error = mac_getsockopt_peerlabel(
3415 sopt->sopt_td->td_ucred, so, &extmac);
3416 if (error)
3417 goto bad;
3418 error = sooptcopyout(sopt, &extmac, sizeof extmac);
3419 #else
3420 error = EOPNOTSUPP;
3421 #endif
3422 break;
3423
3424 case SO_LISTENQLIMIT:
3425 optval = SOLISTENING(so) ? so->sol_qlimit : 0;
3426 goto integer;
3427
3428 case SO_LISTENQLEN:
3429 optval = SOLISTENING(so) ? so->sol_qlen : 0;
3430 goto integer;
3431
3432 case SO_LISTENINCQLEN:
3433 optval = SOLISTENING(so) ? so->sol_incqlen : 0;
3434 goto integer;
3435
3436 case SO_TS_CLOCK:
3437 optval = so->so_ts_clock;
3438 goto integer;
3439
3440 case SO_MAX_PACING_RATE:
3441 optval = so->so_max_pacing_rate;
3442 goto integer;
3443
3444 default:
3445 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0)
3446 error = hhook_run_socket(so, sopt,
3447 HHOOK_SOCKET_OPT);
3448 else
3449 error = ENOPROTOOPT;
3450 break;
3451 }
3452 }
3453 #ifdef MAC
3454 bad:
3455 #endif
3456 CURVNET_RESTORE();
3457 return (error);
3458 }
3459
3460 int
3461 soopt_getm(struct sockopt *sopt, struct mbuf **mp)
3462 {
3463 struct mbuf *m, *m_prev;
3464 int sopt_size = sopt->sopt_valsize;
3465
3466 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA);
3467 if (m == NULL)
3468 return ENOBUFS;
3469 if (sopt_size > MLEN) {
3470 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT);
3471 if ((m->m_flags & M_EXT) == 0) {
3472 m_free(m);
3473 return ENOBUFS;
3474 }
3475 m->m_len = min(MCLBYTES, sopt_size);
3476 } else {
3477 m->m_len = min(MLEN, sopt_size);
3478 }
3479 sopt_size -= m->m_len;
3480 *mp = m;
3481 m_prev = m;
3482
3483 while (sopt_size) {
3484 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA);
3485 if (m == NULL) {
3486 m_freem(*mp);
3487 return ENOBUFS;
3488 }
3489 if (sopt_size > MLEN) {
3490 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK :
3491 M_NOWAIT);
3492 if ((m->m_flags & M_EXT) == 0) {
3493 m_freem(m);
3494 m_freem(*mp);
3495 return ENOBUFS;
3496 }
3497 m->m_len = min(MCLBYTES, sopt_size);
3498 } else {
3499 m->m_len = min(MLEN, sopt_size);
3500 }
3501 sopt_size -= m->m_len;
3502 m_prev->m_next = m;
3503 m_prev = m;
3504 }
3505 return (0);
3506 }
3507
3508 int
3509 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m)
3510 {
3511 struct mbuf *m0 = m;
3512
3513 if (sopt->sopt_val == NULL)
3514 return (0);
3515 while (m != NULL && sopt->sopt_valsize >= m->m_len) {
3516 if (sopt->sopt_td != NULL) {
3517 int error;
3518
3519 error = copyin(sopt->sopt_val, mtod(m, char *),
3520 m->m_len);
3521 if (error != 0) {
3522 m_freem(m0);
3523 return(error);
3524 }
3525 } else
3526 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len);
3527 sopt->sopt_valsize -= m->m_len;
3528 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
3529 m = m->m_next;
3530 }
3531 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */
3532 panic("ip6_sooptmcopyin");
3533 return (0);
3534 }
3535
3536 int
3537 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m)
3538 {
3539 struct mbuf *m0 = m;
3540 size_t valsize = 0;
3541
3542 if (sopt->sopt_val == NULL)
3543 return (0);
3544 while (m != NULL && sopt->sopt_valsize >= m->m_len) {
3545 if (sopt->sopt_td != NULL) {
3546 int error;
3547
3548 error = copyout(mtod(m, char *), sopt->sopt_val,
3549 m->m_len);
3550 if (error != 0) {
3551 m_freem(m0);
3552 return(error);
3553 }
3554 } else
3555 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len);
3556 sopt->sopt_valsize -= m->m_len;
3557 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
3558 valsize += m->m_len;
3559 m = m->m_next;
3560 }
3561 if (m != NULL) {
3562 /* enough soopt buffer should be given from user-land */
3563 m_freem(m0);
3564 return(EINVAL);
3565 }
3566 sopt->sopt_valsize = valsize;
3567 return (0);
3568 }
3569
3570 /*
3571 * sohasoutofband(): protocol notifies socket layer of the arrival of new
3572 * out-of-band data, which will then notify socket consumers.
3573 */
3574 void
3575 sohasoutofband(struct socket *so)
3576 {
3577
3578 if (so->so_sigio != NULL)
3579 pgsigio(&so->so_sigio, SIGURG, 0);
3580 selwakeuppri(&so->so_rdsel, PSOCK);
3581 }
3582
3583 int
3584 sopoll(struct socket *so, int events, struct ucred *active_cred,
3585 struct thread *td)
3586 {
3587
3588 /*
3589 * We do not need to set or assert curvnet as long as everyone uses
3590 * sopoll_generic().
3591 */
3592 return (so->so_proto->pr_sopoll(so, events, active_cred, td));
3593 }
3594
3595 int
3596 sopoll_generic(struct socket *so, int events, struct ucred *active_cred,
3597 struct thread *td)
3598 {
3599 int revents;
3600
3601 SOCK_LOCK(so);
3602 if (SOLISTENING(so)) {
3603 if (!(events & (POLLIN | POLLRDNORM)))
3604 revents = 0;
3605 else if (!TAILQ_EMPTY(&so->sol_comp))
3606 revents = events & (POLLIN | POLLRDNORM);
3607 else if ((events & POLLINIGNEOF) == 0 && so->so_error)
3608 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP;
3609 else {
3610 selrecord(td, &so->so_rdsel);
3611 revents = 0;
3612 }
3613 } else {
3614 revents = 0;
3615 SOCK_SENDBUF_LOCK(so);
3616 SOCK_RECVBUF_LOCK(so);
3617 if (events & (POLLIN | POLLRDNORM))
3618 if (soreadabledata(so))
3619 revents |= events & (POLLIN | POLLRDNORM);
3620 if (events & (POLLOUT | POLLWRNORM))
3621 if (sowriteable(so))
3622 revents |= events & (POLLOUT | POLLWRNORM);
3623 if (events & (POLLPRI | POLLRDBAND))
3624 if (so->so_oobmark ||
3625 (so->so_rcv.sb_state & SBS_RCVATMARK))
3626 revents |= events & (POLLPRI | POLLRDBAND);
3627 if ((events & POLLINIGNEOF) == 0) {
3628 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
3629 revents |= events & (POLLIN | POLLRDNORM);
3630 if (so->so_snd.sb_state & SBS_CANTSENDMORE)
3631 revents |= POLLHUP;
3632 }
3633 }
3634 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
3635 revents |= events & POLLRDHUP;
3636 if (revents == 0) {
3637 if (events &
3638 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND | POLLRDHUP)) {
3639 selrecord(td, &so->so_rdsel);
3640 so->so_rcv.sb_flags |= SB_SEL;
3641 }
3642 if (events & (POLLOUT | POLLWRNORM)) {
3643 selrecord(td, &so->so_wrsel);
3644 so->so_snd.sb_flags |= SB_SEL;
3645 }
3646 }
3647 SOCK_RECVBUF_UNLOCK(so);
3648 SOCK_SENDBUF_UNLOCK(so);
3649 }
3650 SOCK_UNLOCK(so);
3651 return (revents);
3652 }
3653
3654 int
3655 soo_kqfilter(struct file *fp, struct knote *kn)
3656 {
3657 struct socket *so = kn->kn_fp->f_data;
3658 struct sockbuf *sb;
3659 sb_which which;
3660 struct knlist *knl;
3661
3662 switch (kn->kn_filter) {
3663 case EVFILT_READ:
3664 kn->kn_fop = &soread_filtops;
3665 knl = &so->so_rdsel.si_note;
3666 sb = &so->so_rcv;
3667 which = SO_RCV;
3668 break;
3669 case EVFILT_WRITE:
3670 kn->kn_fop = &sowrite_filtops;
3671 knl = &so->so_wrsel.si_note;
3672 sb = &so->so_snd;
3673 which = SO_SND;
3674 break;
3675 case EVFILT_EMPTY:
3676 kn->kn_fop = &soempty_filtops;
3677 knl = &so->so_wrsel.si_note;
3678 sb = &so->so_snd;
3679 which = SO_SND;
3680 break;
3681 default:
3682 return (EINVAL);
3683 }
3684
3685 SOCK_LOCK(so);
3686 if (SOLISTENING(so)) {
3687 knlist_add(knl, kn, 1);
3688 } else {
3689 SOCK_BUF_LOCK(so, which);
3690 knlist_add(knl, kn, 1);
3691 sb->sb_flags |= SB_KNOTE;
3692 SOCK_BUF_UNLOCK(so, which);
3693 }
3694 SOCK_UNLOCK(so);
3695 return (0);
3696 }
3697
3698 static void
3699 filt_sordetach(struct knote *kn)
3700 {
3701 struct socket *so = kn->kn_fp->f_data;
3702
3703 so_rdknl_lock(so);
3704 knlist_remove(&so->so_rdsel.si_note, kn, 1);
3705 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note))
3706 so->so_rcv.sb_flags &= ~SB_KNOTE;
3707 so_rdknl_unlock(so);
3708 }
3709
3710 /*ARGSUSED*/
3711 static int
3712 filt_soread(struct knote *kn, long hint)
3713 {
3714 struct socket *so;
3715
3716 so = kn->kn_fp->f_data;
3717
3718 if (SOLISTENING(so)) {
3719 SOCK_LOCK_ASSERT(so);
3720 kn->kn_data = so->sol_qlen;
3721 if (so->so_error) {
3722 kn->kn_flags |= EV_EOF;
3723 kn->kn_fflags = so->so_error;
3724 return (1);
3725 }
3726 return (!TAILQ_EMPTY(&so->sol_comp));
3727 }
3728
3729 SOCK_RECVBUF_LOCK_ASSERT(so);
3730
3731 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl;
3732 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
3733 kn->kn_flags |= EV_EOF;
3734 kn->kn_fflags = so->so_error;
3735 return (1);
3736 } else if (so->so_error || so->so_rerror)
3737 return (1);
3738
3739 if (kn->kn_sfflags & NOTE_LOWAT) {
3740 if (kn->kn_data >= kn->kn_sdata)
3741 return (1);
3742 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat)
3743 return (1);
3744
3745 /* This hook returning non-zero indicates an event, not error */
3746 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD));
3747 }
3748
3749 static void
3750 filt_sowdetach(struct knote *kn)
3751 {
3752 struct socket *so = kn->kn_fp->f_data;
3753
3754 so_wrknl_lock(so);
3755 knlist_remove(&so->so_wrsel.si_note, kn, 1);
3756 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note))
3757 so->so_snd.sb_flags &= ~SB_KNOTE;
3758 so_wrknl_unlock(so);
3759 }
3760
3761 /*ARGSUSED*/
3762 static int
3763 filt_sowrite(struct knote *kn, long hint)
3764 {
3765 struct socket *so;
3766
3767 so = kn->kn_fp->f_data;
3768
3769 if (SOLISTENING(so))
3770 return (0);
3771
3772 SOCK_SENDBUF_LOCK_ASSERT(so);
3773 kn->kn_data = sbspace(&so->so_snd);
3774
3775 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE);
3776
3777 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
3778 kn->kn_flags |= EV_EOF;
3779 kn->kn_fflags = so->so_error;
3780 return (1);
3781 } else if (so->so_error) /* temporary udp error */
3782 return (1);
3783 else if (((so->so_state & SS_ISCONNECTED) == 0) &&
3784 (so->so_proto->pr_flags & PR_CONNREQUIRED))
3785 return (0);
3786 else if (kn->kn_sfflags & NOTE_LOWAT)
3787 return (kn->kn_data >= kn->kn_sdata);
3788 else
3789 return (kn->kn_data >= so->so_snd.sb_lowat);
3790 }
3791
3792 static int
3793 filt_soempty(struct knote *kn, long hint)
3794 {
3795 struct socket *so;
3796
3797 so = kn->kn_fp->f_data;
3798
3799 if (SOLISTENING(so))
3800 return (1);
3801
3802 SOCK_SENDBUF_LOCK_ASSERT(so);
3803 kn->kn_data = sbused(&so->so_snd);
3804
3805 if (kn->kn_data == 0)
3806 return (1);
3807 else
3808 return (0);
3809 }
3810
3811 int
3812 socheckuid(struct socket *so, uid_t uid)
3813 {
3814
3815 if (so == NULL)
3816 return (EPERM);
3817 if (so->so_cred->cr_uid != uid)
3818 return (EPERM);
3819 return (0);
3820 }
3821
3822 /*
3823 * These functions are used by protocols to notify the socket layer (and its
3824 * consumers) of state changes in the sockets driven by protocol-side events.
3825 */
3826
3827 /*
3828 * Procedures to manipulate state flags of socket and do appropriate wakeups.
3829 *
3830 * Normal sequence from the active (originating) side is that
3831 * soisconnecting() is called during processing of connect() call, resulting
3832 * in an eventual call to soisconnected() if/when the connection is
3833 * established. When the connection is torn down soisdisconnecting() is
3834 * called during processing of disconnect() call, and soisdisconnected() is
3835 * called when the connection to the peer is totally severed. The semantics
3836 * of these routines are such that connectionless protocols can call
3837 * soisconnected() and soisdisconnected() only, bypassing the in-progress
3838 * calls when setting up a ``connection'' takes no time.
3839 *
3840 * From the passive side, a socket is created with two queues of sockets:
3841 * so_incomp for connections in progress and so_comp for connections already
3842 * made and awaiting user acceptance. As a protocol is preparing incoming
3843 * connections, it creates a socket structure queued on so_incomp by calling
3844 * sonewconn(). When the connection is established, soisconnected() is
3845 * called, and transfers the socket structure to so_comp, making it available
3846 * to accept().
3847 *
3848 * If a socket is closed with sockets on either so_incomp or so_comp, these
3849 * sockets are dropped.
3850 *
3851 * If higher-level protocols are implemented in the kernel, the wakeups done
3852 * here will sometimes cause software-interrupt process scheduling.
3853 */
3854 void
3855 soisconnecting(struct socket *so)
3856 {
3857
3858 SOCK_LOCK(so);
3859 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
3860 so->so_state |= SS_ISCONNECTING;
3861 SOCK_UNLOCK(so);
3862 }
3863
3864 void
3865 soisconnected(struct socket *so)
3866 {
3867 bool last __diagused;
3868
3869 SOCK_LOCK(so);
3870 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
3871 so->so_state |= SS_ISCONNECTED;
3872
3873 if (so->so_qstate == SQ_INCOMP) {
3874 struct socket *head = so->so_listen;
3875 int ret;
3876
3877 KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so));
3878 /*
3879 * Promoting a socket from incomplete queue to complete, we
3880 * need to go through reverse order of locking. We first do
3881 * trylock, and if that doesn't succeed, we go the hard way
3882 * leaving a reference and rechecking consistency after proper
3883 * locking.
3884 */
3885 if (__predict_false(SOLISTEN_TRYLOCK(head) == 0)) {
3886 soref(head);
3887 SOCK_UNLOCK(so);
3888 SOLISTEN_LOCK(head);
3889 SOCK_LOCK(so);
3890 if (__predict_false(head != so->so_listen)) {
3891 /*
3892 * The socket went off the listen queue,
3893 * should be lost race to close(2) of sol.
3894 * The socket is about to soabort().
3895 */
3896 SOCK_UNLOCK(so);
3897 sorele_locked(head);
3898 return;
3899 }
3900 last = refcount_release(&head->so_count);
3901 KASSERT(!last, ("%s: released last reference for %p",
3902 __func__, head));
3903 }
3904 again:
3905 if ((so->so_options & SO_ACCEPTFILTER) == 0) {
3906 TAILQ_REMOVE(&head->sol_incomp, so, so_list);
3907 head->sol_incqlen--;
3908 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
3909 head->sol_qlen++;
3910 so->so_qstate = SQ_COMP;
3911 SOCK_UNLOCK(so);
3912 solisten_wakeup(head); /* unlocks */
3913 } else {
3914 SOCK_RECVBUF_LOCK(so);
3915 soupcall_set(so, SO_RCV,
3916 head->sol_accept_filter->accf_callback,
3917 head->sol_accept_filter_arg);
3918 so->so_options &= ~SO_ACCEPTFILTER;
3919 ret = head->sol_accept_filter->accf_callback(so,
3920 head->sol_accept_filter_arg, M_NOWAIT);
3921 if (ret == SU_ISCONNECTED) {
3922 soupcall_clear(so, SO_RCV);
3923 SOCK_RECVBUF_UNLOCK(so);
3924 goto again;
3925 }
3926 SOCK_RECVBUF_UNLOCK(so);
3927 SOCK_UNLOCK(so);
3928 SOLISTEN_UNLOCK(head);
3929 }
3930 return;
3931 }
3932 SOCK_UNLOCK(so);
3933 wakeup(&so->so_timeo);
3934 sorwakeup(so);
3935 sowwakeup(so);
3936 }
3937
3938 void
3939 soisdisconnecting(struct socket *so)
3940 {
3941
3942 SOCK_LOCK(so);
3943 so->so_state &= ~SS_ISCONNECTING;
3944 so->so_state |= SS_ISDISCONNECTING;
3945
3946 if (!SOLISTENING(so)) {
3947 SOCK_RECVBUF_LOCK(so);
3948 socantrcvmore_locked(so);
3949 SOCK_SENDBUF_LOCK(so);
3950 socantsendmore_locked(so);
3951 }
3952 SOCK_UNLOCK(so);
3953 wakeup(&so->so_timeo);
3954 }
3955
3956 void
3957 soisdisconnected(struct socket *so)
3958 {
3959
3960 SOCK_LOCK(so);
3961
3962 /*
3963 * There is at least one reader of so_state that does not
3964 * acquire socket lock, namely soreceive_generic(). Ensure
3965 * that it never sees all flags that track connection status
3966 * cleared, by ordering the update with a barrier semantic of
3967 * our release thread fence.
3968 */
3969 so->so_state |= SS_ISDISCONNECTED;
3970 atomic_thread_fence_rel();
3971 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
3972
3973 if (!SOLISTENING(so)) {
3974 SOCK_UNLOCK(so);
3975 SOCK_RECVBUF_LOCK(so);
3976 socantrcvmore_locked(so);
3977 SOCK_SENDBUF_LOCK(so);
3978 sbdrop_locked(&so->so_snd, sbused(&so->so_snd));
3979 socantsendmore_locked(so);
3980 } else
3981 SOCK_UNLOCK(so);
3982 wakeup(&so->so_timeo);
3983 }
3984
3985 int
3986 soiolock(struct socket *so, struct sx *sx, int flags)
3987 {
3988 int error;
3989
3990 KASSERT((flags & SBL_VALID) == flags,
3991 ("soiolock: invalid flags %#x", flags));
3992
3993 if ((flags & SBL_WAIT) != 0) {
3994 if ((flags & SBL_NOINTR) != 0) {
3995 sx_xlock(sx);
3996 } else {
3997 error = sx_xlock_sig(sx);
3998 if (error != 0)
3999 return (error);
4000 }
4001 } else if (!sx_try_xlock(sx)) {
4002 return (EWOULDBLOCK);
4003 }
4004
4005 if (__predict_false(SOLISTENING(so))) {
4006 sx_xunlock(sx);
4007 return (ENOTCONN);
4008 }
4009 return (0);
4010 }
4011
4012 void
4013 soiounlock(struct sx *sx)
4014 {
4015 sx_xunlock(sx);
4016 }
4017
4018 /*
4019 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME.
4020 */
4021 struct sockaddr *
4022 sodupsockaddr(const struct sockaddr *sa, int mflags)
4023 {
4024 struct sockaddr *sa2;
4025
4026 sa2 = malloc(sa->sa_len, M_SONAME, mflags);
4027 if (sa2)
4028 bcopy(sa, sa2, sa->sa_len);
4029 return sa2;
4030 }
4031
4032 /*
4033 * Register per-socket destructor.
4034 */
4035 void
4036 sodtor_set(struct socket *so, so_dtor_t *func)
4037 {
4038
4039 SOCK_LOCK_ASSERT(so);
4040 so->so_dtor = func;
4041 }
4042
4043 /*
4044 * Register per-socket buffer upcalls.
4045 */
4046 void
4047 soupcall_set(struct socket *so, sb_which which, so_upcall_t func, void *arg)
4048 {
4049 struct sockbuf *sb;
4050
4051 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4052
4053 switch (which) {
4054 case SO_RCV:
4055 sb = &so->so_rcv;
4056 break;
4057 case SO_SND:
4058 sb = &so->so_snd;
4059 break;
4060 }
4061 SOCK_BUF_LOCK_ASSERT(so, which);
4062 sb->sb_upcall = func;
4063 sb->sb_upcallarg = arg;
4064 sb->sb_flags |= SB_UPCALL;
4065 }
4066
4067 void
4068 soupcall_clear(struct socket *so, sb_which which)
4069 {
4070 struct sockbuf *sb;
4071
4072 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4073
4074 switch (which) {
4075 case SO_RCV:
4076 sb = &so->so_rcv;
4077 break;
4078 case SO_SND:
4079 sb = &so->so_snd;
4080 break;
4081 }
4082 SOCK_BUF_LOCK_ASSERT(so, which);
4083 KASSERT(sb->sb_upcall != NULL,
4084 ("%s: so %p no upcall to clear", __func__, so));
4085 sb->sb_upcall = NULL;
4086 sb->sb_upcallarg = NULL;
4087 sb->sb_flags &= ~SB_UPCALL;
4088 }
4089
4090 void
4091 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg)
4092 {
4093
4094 SOLISTEN_LOCK_ASSERT(so);
4095 so->sol_upcall = func;
4096 so->sol_upcallarg = arg;
4097 }
4098
4099 static void
4100 so_rdknl_lock(void *arg)
4101 {
4102 struct socket *so = arg;
4103
4104 retry:
4105 if (SOLISTENING(so)) {
4106 SOLISTEN_LOCK(so);
4107 } else {
4108 SOCK_RECVBUF_LOCK(so);
4109 if (__predict_false(SOLISTENING(so))) {
4110 SOCK_RECVBUF_UNLOCK(so);
4111 goto retry;
4112 }
4113 }
4114 }
4115
4116 static void
4117 so_rdknl_unlock(void *arg)
4118 {
4119 struct socket *so = arg;
4120
4121 if (SOLISTENING(so))
4122 SOLISTEN_UNLOCK(so);
4123 else
4124 SOCK_RECVBUF_UNLOCK(so);
4125 }
4126
4127 static void
4128 so_rdknl_assert_lock(void *arg, int what)
4129 {
4130 struct socket *so = arg;
4131
4132 if (what == LA_LOCKED) {
4133 if (SOLISTENING(so))
4134 SOLISTEN_LOCK_ASSERT(so);
4135 else
4136 SOCK_RECVBUF_LOCK_ASSERT(so);
4137 } else {
4138 if (SOLISTENING(so))
4139 SOLISTEN_UNLOCK_ASSERT(so);
4140 else
4141 SOCK_RECVBUF_UNLOCK_ASSERT(so);
4142 }
4143 }
4144
4145 static void
4146 so_wrknl_lock(void *arg)
4147 {
4148 struct socket *so = arg;
4149
4150 retry:
4151 if (SOLISTENING(so)) {
4152 SOLISTEN_LOCK(so);
4153 } else {
4154 SOCK_SENDBUF_LOCK(so);
4155 if (__predict_false(SOLISTENING(so))) {
4156 SOCK_SENDBUF_UNLOCK(so);
4157 goto retry;
4158 }
4159 }
4160 }
4161
4162 static void
4163 so_wrknl_unlock(void *arg)
4164 {
4165 struct socket *so = arg;
4166
4167 if (SOLISTENING(so))
4168 SOLISTEN_UNLOCK(so);
4169 else
4170 SOCK_SENDBUF_UNLOCK(so);
4171 }
4172
4173 static void
4174 so_wrknl_assert_lock(void *arg, int what)
4175 {
4176 struct socket *so = arg;
4177
4178 if (what == LA_LOCKED) {
4179 if (SOLISTENING(so))
4180 SOLISTEN_LOCK_ASSERT(so);
4181 else
4182 SOCK_SENDBUF_LOCK_ASSERT(so);
4183 } else {
4184 if (SOLISTENING(so))
4185 SOLISTEN_UNLOCK_ASSERT(so);
4186 else
4187 SOCK_SENDBUF_UNLOCK_ASSERT(so);
4188 }
4189 }
4190
4191 /*
4192 * Create an external-format (``xsocket'') structure using the information in
4193 * the kernel-format socket structure pointed to by so. This is done to
4194 * reduce the spew of irrelevant information over this interface, to isolate
4195 * user code from changes in the kernel structure, and potentially to provide
4196 * information-hiding if we decide that some of this information should be
4197 * hidden from users.
4198 */
4199 void
4200 sotoxsocket(struct socket *so, struct xsocket *xso)
4201 {
4202
4203 bzero(xso, sizeof(*xso));
4204 xso->xso_len = sizeof *xso;
4205 xso->xso_so = (uintptr_t)so;
4206 xso->so_type = so->so_type;
4207 xso->so_options = so->so_options;
4208 xso->so_linger = so->so_linger;
4209 xso->so_state = so->so_state;
4210 xso->so_pcb = (uintptr_t)so->so_pcb;
4211 xso->xso_protocol = so->so_proto->pr_protocol;
4212 xso->xso_family = so->so_proto->pr_domain->dom_family;
4213 xso->so_timeo = so->so_timeo;
4214 xso->so_error = so->so_error;
4215 xso->so_uid = so->so_cred->cr_uid;
4216 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0;
4217 if (SOLISTENING(so)) {
4218 xso->so_qlen = so->sol_qlen;
4219 xso->so_incqlen = so->sol_incqlen;
4220 xso->so_qlimit = so->sol_qlimit;
4221 xso->so_oobmark = 0;
4222 } else {
4223 xso->so_state |= so->so_qstate;
4224 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0;
4225 xso->so_oobmark = so->so_oobmark;
4226 sbtoxsockbuf(&so->so_snd, &xso->so_snd);
4227 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv);
4228 }
4229 }
4230
4231 struct sockbuf *
4232 so_sockbuf_rcv(struct socket *so)
4233 {
4234
4235 return (&so->so_rcv);
4236 }
4237
4238 struct sockbuf *
4239 so_sockbuf_snd(struct socket *so)
4240 {
4241
4242 return (&so->so_snd);
4243 }
4244
4245 int
4246 so_state_get(const struct socket *so)
4247 {
4248
4249 return (so->so_state);
4250 }
4251
4252 void
4253 so_state_set(struct socket *so, int val)
4254 {
4255
4256 so->so_state = val;
4257 }
4258
4259 int
4260 so_options_get(const struct socket *so)
4261 {
4262
4263 return (so->so_options);
4264 }
4265
4266 void
4267 so_options_set(struct socket *so, int val)
4268 {
4269
4270 so->so_options = val;
4271 }
4272
4273 int
4274 so_error_get(const struct socket *so)
4275 {
4276
4277 return (so->so_error);
4278 }
4279
4280 void
4281 so_error_set(struct socket *so, int val)
4282 {
4283
4284 so->so_error = val;
4285 }
4286
4287 int
4288 so_linger_get(const struct socket *so)
4289 {
4290
4291 return (so->so_linger);
4292 }
4293
4294 void
4295 so_linger_set(struct socket *so, int val)
4296 {
4297
4298 KASSERT(val >= 0 && val <= USHRT_MAX && val <= (INT_MAX / hz),
4299 ("%s: val %d out of range", __func__, val));
4300
4301 so->so_linger = val;
4302 }
4303
4304 struct protosw *
4305 so_protosw_get(const struct socket *so)
4306 {
4307
4308 return (so->so_proto);
4309 }
4310
4311 void
4312 so_protosw_set(struct socket *so, struct protosw *val)
4313 {
4314
4315 so->so_proto = val;
4316 }
4317
4318 void
4319 so_sorwakeup(struct socket *so)
4320 {
4321
4322 sorwakeup(so);
4323 }
4324
4325 void
4326 so_sowwakeup(struct socket *so)
4327 {
4328
4329 sowwakeup(so);
4330 }
4331
4332 void
4333 so_sorwakeup_locked(struct socket *so)
4334 {
4335
4336 sorwakeup_locked(so);
4337 }
4338
4339 void
4340 so_sowwakeup_locked(struct socket *so)
4341 {
4342
4343 sowwakeup_locked(so);
4344 }
4345
4346 void
4347 so_lock(struct socket *so)
4348 {
4349
4350 SOCK_LOCK(so);
4351 }
4352
4353 void
4354 so_unlock(struct socket *so)
4355 {
4356
4357 SOCK_UNLOCK(so);
4358 }
Cache object: 90efbbde14c328b3a25a299593303636
|