1 /* $OpenBSD: uipc_socket.c,v 1.299 2023/01/27 21:01:59 mvs Exp $ */
2 /* $NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $ */
3
4 /*
5 * Copyright (c) 1982, 1986, 1988, 1990, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94
33 */
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/proc.h>
38 #include <sys/file.h>
39 #include <sys/filedesc.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/domain.h>
43 #include <sys/event.h>
44 #include <sys/protosw.h>
45 #include <sys/socket.h>
46 #include <sys/unpcb.h>
47 #include <sys/socketvar.h>
48 #include <sys/signalvar.h>
49 #include <sys/pool.h>
50 #include <sys/atomic.h>
51 #include <sys/rwlock.h>
52 #include <sys/time.h>
53 #include <sys/refcnt.h>
54
55 #ifdef DDB
56 #include <machine/db_machdep.h>
57 #endif
58
59 void sbsync(struct sockbuf *, struct mbuf *);
60
61 int sosplice(struct socket *, int, off_t, struct timeval *);
62 void sounsplice(struct socket *, struct socket *, int);
63 void soidle(void *);
64 void sotask(void *);
65 void soreaper(void *);
66 void soput(void *);
67 int somove(struct socket *, int);
68 void sorflush(struct socket *);
69
70 void filt_sordetach(struct knote *kn);
71 int filt_soread(struct knote *kn, long hint);
72 void filt_sowdetach(struct knote *kn);
73 int filt_sowrite(struct knote *kn, long hint);
74 int filt_soexcept(struct knote *kn, long hint);
75 int filt_solisten(struct knote *kn, long hint);
76 int filt_somodify(struct kevent *kev, struct knote *kn);
77 int filt_soprocess(struct knote *kn, struct kevent *kev);
78
79 const struct filterops solisten_filtops = {
80 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
81 .f_attach = NULL,
82 .f_detach = filt_sordetach,
83 .f_event = filt_solisten,
84 .f_modify = filt_somodify,
85 .f_process = filt_soprocess,
86 };
87
88 const struct filterops soread_filtops = {
89 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
90 .f_attach = NULL,
91 .f_detach = filt_sordetach,
92 .f_event = filt_soread,
93 .f_modify = filt_somodify,
94 .f_process = filt_soprocess,
95 };
96
97 const struct filterops sowrite_filtops = {
98 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
99 .f_attach = NULL,
100 .f_detach = filt_sowdetach,
101 .f_event = filt_sowrite,
102 .f_modify = filt_somodify,
103 .f_process = filt_soprocess,
104 };
105
106 const struct filterops soexcept_filtops = {
107 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
108 .f_attach = NULL,
109 .f_detach = filt_sordetach,
110 .f_event = filt_soexcept,
111 .f_modify = filt_somodify,
112 .f_process = filt_soprocess,
113 };
114
115 #ifndef SOMINCONN
116 #define SOMINCONN 80
117 #endif /* SOMINCONN */
118
119 int somaxconn = SOMAXCONN;
120 int sominconn = SOMINCONN;
121
122 struct pool socket_pool;
123 #ifdef SOCKET_SPLICE
124 struct pool sosplice_pool;
125 struct taskq *sosplice_taskq;
126 struct rwlock sosplice_lock = RWLOCK_INITIALIZER("sosplicelk");
127 #endif
128
129 void
130 soinit(void)
131 {
132 pool_init(&socket_pool, sizeof(struct socket), 0, IPL_SOFTNET, 0,
133 "sockpl", NULL);
134 #ifdef SOCKET_SPLICE
135 pool_init(&sosplice_pool, sizeof(struct sosplice), 0, IPL_SOFTNET, 0,
136 "sosppl", NULL);
137 #endif
138 }
139
140 struct socket *
141 soalloc(int wait)
142 {
143 struct socket *so;
144
145 so = pool_get(&socket_pool, (wait == M_WAIT ? PR_WAITOK : PR_NOWAIT) |
146 PR_ZERO);
147 if (so == NULL)
148 return (NULL);
149 rw_init_flags(&so->so_lock, "solock", RWL_DUPOK);
150 refcnt_init(&so->so_refcnt);
151
152 return (so);
153 }
154
155 /*
156 * Socket operation routines.
157 * These routines are called by the routines in
158 * sys_socket.c or from a system process, and
159 * implement the semantics of socket operations by
160 * switching out to the protocol specific routines.
161 */
162 int
163 socreate(int dom, struct socket **aso, int type, int proto)
164 {
165 struct proc *p = curproc; /* XXX */
166 const struct protosw *prp;
167 struct socket *so;
168 int error;
169
170 if (proto)
171 prp = pffindproto(dom, proto, type);
172 else
173 prp = pffindtype(dom, type);
174 if (prp == NULL || prp->pr_usrreqs == NULL)
175 return (EPROTONOSUPPORT);
176 if (prp->pr_type != type)
177 return (EPROTOTYPE);
178 so = soalloc(M_WAIT);
179 klist_init(&so->so_rcv.sb_klist, &socket_klistops, so);
180 klist_init(&so->so_snd.sb_klist, &socket_klistops, so);
181 sigio_init(&so->so_sigio);
182 TAILQ_INIT(&so->so_q0);
183 TAILQ_INIT(&so->so_q);
184 so->so_type = type;
185 if (suser(p) == 0)
186 so->so_state = SS_PRIV;
187 so->so_ruid = p->p_ucred->cr_ruid;
188 so->so_euid = p->p_ucred->cr_uid;
189 so->so_rgid = p->p_ucred->cr_rgid;
190 so->so_egid = p->p_ucred->cr_gid;
191 so->so_cpid = p->p_p->ps_pid;
192 so->so_proto = prp;
193 so->so_snd.sb_timeo_nsecs = INFSLP;
194 so->so_rcv.sb_timeo_nsecs = INFSLP;
195
196 solock(so);
197 error = pru_attach(so, proto, M_WAIT);
198 if (error) {
199 so->so_state |= SS_NOFDREF;
200 /* sofree() calls sounlock(). */
201 sofree(so, 0);
202 return (error);
203 }
204 sounlock(so);
205 *aso = so;
206 return (0);
207 }
208
209 int
210 sobind(struct socket *so, struct mbuf *nam, struct proc *p)
211 {
212 soassertlocked(so);
213 return pru_bind(so, nam, p);
214 }
215
216 int
217 solisten(struct socket *so, int backlog)
218 {
219 int error;
220
221 soassertlocked(so);
222
223 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING|SS_ISDISCONNECTING))
224 return (EINVAL);
225 #ifdef SOCKET_SPLICE
226 if (isspliced(so) || issplicedback(so))
227 return (EOPNOTSUPP);
228 #endif /* SOCKET_SPLICE */
229 error = pru_listen(so);
230 if (error)
231 return (error);
232 if (TAILQ_FIRST(&so->so_q) == NULL)
233 so->so_options |= SO_ACCEPTCONN;
234 if (backlog < 0 || backlog > somaxconn)
235 backlog = somaxconn;
236 if (backlog < sominconn)
237 backlog = sominconn;
238 so->so_qlimit = backlog;
239 return (0);
240 }
241
242 #define SOSP_FREEING_READ 1
243 #define SOSP_FREEING_WRITE 2
244 void
245 sofree(struct socket *so, int keep_lock)
246 {
247 int persocket = solock_persocket(so);
248
249 soassertlocked(so);
250
251 if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) {
252 if (!keep_lock)
253 sounlock(so);
254 return;
255 }
256 if (so->so_head) {
257 struct socket *head = so->so_head;
258
259 /*
260 * We must not decommission a socket that's on the accept(2)
261 * queue. If we do, then accept(2) may hang after select(2)
262 * indicated that the listening socket was ready.
263 */
264 if (so->so_onq == &head->so_q) {
265 if (!keep_lock)
266 sounlock(so);
267 return;
268 }
269
270 if (persocket) {
271 /*
272 * Concurrent close of `head' could
273 * abort `so' due to re-lock.
274 */
275 soref(so);
276 soref(head);
277 sounlock(so);
278 solock(head);
279 solock(so);
280
281 if (so->so_onq != &head->so_q0) {
282 sounlock(head);
283 sounlock(so);
284 sorele(head);
285 sorele(so);
286 return;
287 }
288
289 sorele(head);
290 sorele(so);
291 }
292
293 soqremque(so, 0);
294
295 if (persocket)
296 sounlock(head);
297 }
298
299 if (persocket) {
300 sounlock(so);
301 refcnt_finalize(&so->so_refcnt, "sofinal");
302 solock(so);
303 }
304
305 sigio_free(&so->so_sigio);
306 klist_free(&so->so_rcv.sb_klist);
307 klist_free(&so->so_snd.sb_klist);
308 #ifdef SOCKET_SPLICE
309 if (so->so_sp) {
310 if (issplicedback(so)) {
311 int freeing = SOSP_FREEING_WRITE;
312
313 if (so->so_sp->ssp_soback == so)
314 freeing |= SOSP_FREEING_READ;
315 sounsplice(so->so_sp->ssp_soback, so, freeing);
316 }
317 if (isspliced(so)) {
318 int freeing = SOSP_FREEING_READ;
319
320 if (so == so->so_sp->ssp_socket)
321 freeing |= SOSP_FREEING_WRITE;
322 sounsplice(so, so->so_sp->ssp_socket, freeing);
323 }
324 }
325 #endif /* SOCKET_SPLICE */
326 sbrelease(so, &so->so_snd);
327 sorflush(so);
328 if (!keep_lock)
329 sounlock(so);
330 #ifdef SOCKET_SPLICE
331 if (so->so_sp) {
332 /* Reuse splice idle, sounsplice() has been called before. */
333 timeout_set_proc(&so->so_sp->ssp_idleto, soreaper, so);
334 timeout_add(&so->so_sp->ssp_idleto, 0);
335 } else
336 #endif /* SOCKET_SPLICE */
337 {
338 pool_put(&socket_pool, so);
339 }
340 }
341
342 static inline uint64_t
343 solinger_nsec(struct socket *so)
344 {
345 if (so->so_linger == 0)
346 return INFSLP;
347
348 return SEC_TO_NSEC(so->so_linger);
349 }
350
351 /*
352 * Close a socket on last file table reference removal.
353 * Initiate disconnect if connected.
354 * Free socket when disconnect complete.
355 */
356 int
357 soclose(struct socket *so, int flags)
358 {
359 struct socket *so2;
360 int error = 0;
361
362 solock(so);
363 /* Revoke async IO early. There is a final revocation in sofree(). */
364 sigio_free(&so->so_sigio);
365 if (so->so_state & SS_ISCONNECTED) {
366 if (so->so_pcb == NULL)
367 goto discard;
368 if ((so->so_state & SS_ISDISCONNECTING) == 0) {
369 error = sodisconnect(so);
370 if (error)
371 goto drop;
372 }
373 if (so->so_options & SO_LINGER) {
374 if ((so->so_state & SS_ISDISCONNECTING) &&
375 (flags & MSG_DONTWAIT))
376 goto drop;
377 while (so->so_state & SS_ISCONNECTED) {
378 error = sosleep_nsec(so, &so->so_timeo,
379 PSOCK | PCATCH, "netcls",
380 solinger_nsec(so));
381 if (error)
382 break;
383 }
384 }
385 }
386 drop:
387 if (so->so_pcb) {
388 int error2;
389 error2 = pru_detach(so);
390 if (error == 0)
391 error = error2;
392 }
393 if (so->so_options & SO_ACCEPTCONN) {
394 int persocket = solock_persocket(so);
395
396 if (persocket) {
397 /* Wait concurrent sonewconn() threads. */
398 while (so->so_newconn > 0) {
399 so->so_state |= SS_NEWCONN_WAIT;
400 sosleep_nsec(so, &so->so_newconn, PSOCK,
401 "netlck", INFSLP);
402 }
403 }
404
405 while ((so2 = TAILQ_FIRST(&so->so_q0)) != NULL) {
406 if (persocket)
407 solock(so2);
408 (void) soqremque(so2, 0);
409 if (persocket)
410 sounlock(so);
411 soabort(so2);
412 if (persocket)
413 solock(so);
414 }
415 while ((so2 = TAILQ_FIRST(&so->so_q)) != NULL) {
416 if (persocket)
417 solock(so2);
418 (void) soqremque(so2, 1);
419 if (persocket)
420 sounlock(so);
421 soabort(so2);
422 if (persocket)
423 solock(so);
424 }
425 }
426 discard:
427 if (so->so_state & SS_NOFDREF)
428 panic("soclose NOFDREF: so %p, so_type %d", so, so->so_type);
429 so->so_state |= SS_NOFDREF;
430 /* sofree() calls sounlock(). */
431 sofree(so, 0);
432 return (error);
433 }
434
435 void
436 soabort(struct socket *so)
437 {
438 soassertlocked(so);
439 pru_abort(so);
440 }
441
442 int
443 soaccept(struct socket *so, struct mbuf *nam)
444 {
445 int error = 0;
446
447 soassertlocked(so);
448
449 if ((so->so_state & SS_NOFDREF) == 0)
450 panic("soaccept !NOFDREF: so %p, so_type %d", so, so->so_type);
451 so->so_state &= ~SS_NOFDREF;
452 if ((so->so_state & SS_ISDISCONNECTED) == 0 ||
453 (so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0)
454 error = pru_accept(so, nam);
455 else
456 error = ECONNABORTED;
457 return (error);
458 }
459
460 int
461 soconnect(struct socket *so, struct mbuf *nam)
462 {
463 int error;
464
465 soassertlocked(so);
466
467 if (so->so_options & SO_ACCEPTCONN)
468 return (EOPNOTSUPP);
469 /*
470 * If protocol is connection-based, can only connect once.
471 * Otherwise, if connected, try to disconnect first.
472 * This allows user to disconnect by connecting to, e.g.,
473 * a null address.
474 */
475 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
476 ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
477 (error = sodisconnect(so))))
478 error = EISCONN;
479 else
480 error = pru_connect(so, nam);
481 return (error);
482 }
483
484 int
485 soconnect2(struct socket *so1, struct socket *so2)
486 {
487 int persocket, error;
488
489 if ((persocket = solock_persocket(so1)))
490 solock_pair(so1, so2);
491 else
492 solock(so1);
493
494 error = pru_connect2(so1, so2);
495
496 if (persocket)
497 sounlock(so2);
498 sounlock(so1);
499 return (error);
500 }
501
502 int
503 sodisconnect(struct socket *so)
504 {
505 int error;
506
507 soassertlocked(so);
508
509 if ((so->so_state & SS_ISCONNECTED) == 0)
510 return (ENOTCONN);
511 if (so->so_state & SS_ISDISCONNECTING)
512 return (EALREADY);
513 error = pru_disconnect(so);
514 return (error);
515 }
516
517 int m_getuio(struct mbuf **, int, long, struct uio *);
518
519 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK)
520 /*
521 * Send on a socket.
522 * If send must go all at once and message is larger than
523 * send buffering, then hard error.
524 * Lock against other senders.
525 * If must go all at once and not enough room now, then
526 * inform user that this would block and do nothing.
527 * Otherwise, if nonblocking, send as much as possible.
528 * The data to be sent is described by "uio" if nonzero,
529 * otherwise by the mbuf chain "top" (which must be null
530 * if uio is not). Data provided in mbuf chain must be small
531 * enough to send all at once.
532 *
533 * Returns nonzero on error, timeout or signal; callers
534 * must check for short counts if EINTR/ERESTART are returned.
535 * Data and control buffers are freed on return.
536 */
537 int
538 sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top,
539 struct mbuf *control, int flags)
540 {
541 long space, clen = 0;
542 size_t resid;
543 int error;
544 int atomic = sosendallatonce(so) || top;
545
546 if (uio)
547 resid = uio->uio_resid;
548 else
549 resid = top->m_pkthdr.len;
550 /* MSG_EOR on a SOCK_STREAM socket is invalid. */
551 if (so->so_type == SOCK_STREAM && (flags & MSG_EOR)) {
552 m_freem(top);
553 m_freem(control);
554 return (EINVAL);
555 }
556 if (uio && uio->uio_procp)
557 uio->uio_procp->p_ru.ru_msgsnd++;
558 if (control) {
559 /*
560 * In theory clen should be unsigned (since control->m_len is).
561 * However, space must be signed, as it might be less than 0
562 * if we over-committed, and we must use a signed comparison
563 * of space and clen.
564 */
565 clen = control->m_len;
566 /* reserve extra space for AF_UNIX's internalize */
567 if (so->so_proto->pr_domain->dom_family == AF_UNIX &&
568 clen >= CMSG_ALIGN(sizeof(struct cmsghdr)) &&
569 mtod(control, struct cmsghdr *)->cmsg_type == SCM_RIGHTS)
570 clen = CMSG_SPACE(
571 (clen - CMSG_ALIGN(sizeof(struct cmsghdr))) *
572 (sizeof(struct fdpass) / sizeof(int)));
573 }
574
575 #define snderr(errno) { error = errno; goto release; }
576
577 solock(so);
578 restart:
579 if ((error = sblock(so, &so->so_snd, SBLOCKWAIT(flags))) != 0)
580 goto out;
581 so->so_snd.sb_state |= SS_ISSENDING;
582 do {
583 if (so->so_snd.sb_state & SS_CANTSENDMORE)
584 snderr(EPIPE);
585 if (so->so_error) {
586 error = so->so_error;
587 so->so_error = 0;
588 snderr(error);
589 }
590 if ((so->so_state & SS_ISCONNECTED) == 0) {
591 if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
592 if (!(resid == 0 && clen != 0))
593 snderr(ENOTCONN);
594 } else if (addr == NULL)
595 snderr(EDESTADDRREQ);
596 }
597 space = sbspace(so, &so->so_snd);
598 if (flags & MSG_OOB)
599 space += 1024;
600 if (so->so_proto->pr_domain->dom_family == AF_UNIX) {
601 if (atomic && resid > so->so_snd.sb_hiwat)
602 snderr(EMSGSIZE);
603 } else {
604 if (clen > so->so_snd.sb_hiwat ||
605 (atomic && resid > so->so_snd.sb_hiwat - clen))
606 snderr(EMSGSIZE);
607 }
608 if (space < clen ||
609 (space - clen < resid &&
610 (atomic || space < so->so_snd.sb_lowat))) {
611 if (flags & MSG_DONTWAIT)
612 snderr(EWOULDBLOCK);
613 sbunlock(so, &so->so_snd);
614 error = sbwait(so, &so->so_snd);
615 so->so_snd.sb_state &= ~SS_ISSENDING;
616 if (error)
617 goto out;
618 goto restart;
619 }
620 space -= clen;
621 do {
622 if (uio == NULL) {
623 /*
624 * Data is prepackaged in "top".
625 */
626 resid = 0;
627 if (flags & MSG_EOR)
628 top->m_flags |= M_EOR;
629 } else {
630 sounlock(so);
631 error = m_getuio(&top, atomic, space, uio);
632 solock(so);
633 if (error)
634 goto release;
635 space -= top->m_pkthdr.len;
636 resid = uio->uio_resid;
637 if (flags & MSG_EOR)
638 top->m_flags |= M_EOR;
639 }
640 if (resid == 0)
641 so->so_snd.sb_state &= ~SS_ISSENDING;
642 if (top && so->so_options & SO_ZEROIZE)
643 top->m_flags |= M_ZEROIZE;
644 if (flags & MSG_OOB)
645 error = pru_sendoob(so, top, addr, control);
646 else
647 error = pru_send(so, top, addr, control);
648 clen = 0;
649 control = NULL;
650 top = NULL;
651 if (error)
652 goto release;
653 } while (resid && space > 0);
654 } while (resid);
655
656 release:
657 so->so_snd.sb_state &= ~SS_ISSENDING;
658 sbunlock(so, &so->so_snd);
659 out:
660 sounlock(so);
661 m_freem(top);
662 m_freem(control);
663 return (error);
664 }
665
666 int
667 m_getuio(struct mbuf **mp, int atomic, long space, struct uio *uio)
668 {
669 struct mbuf *m, *top = NULL;
670 struct mbuf **nextp = ⊤
671 u_long len, mlen;
672 size_t resid = uio->uio_resid;
673 int error;
674
675 do {
676 if (top == NULL) {
677 MGETHDR(m, M_WAIT, MT_DATA);
678 mlen = MHLEN;
679 m->m_pkthdr.len = 0;
680 m->m_pkthdr.ph_ifidx = 0;
681 } else {
682 MGET(m, M_WAIT, MT_DATA);
683 mlen = MLEN;
684 }
685 /* chain mbuf together */
686 *nextp = m;
687 nextp = &m->m_next;
688
689 resid = ulmin(resid, space);
690 if (resid >= MINCLSIZE) {
691 MCLGETL(m, M_NOWAIT, ulmin(resid, MAXMCLBYTES));
692 if ((m->m_flags & M_EXT) == 0)
693 MCLGETL(m, M_NOWAIT, MCLBYTES);
694 if ((m->m_flags & M_EXT) == 0)
695 goto nopages;
696 mlen = m->m_ext.ext_size;
697 len = ulmin(mlen, resid);
698 /*
699 * For datagram protocols, leave room
700 * for protocol headers in first mbuf.
701 */
702 if (atomic && m == top && len < mlen - max_hdr)
703 m->m_data += max_hdr;
704 } else {
705 nopages:
706 len = ulmin(mlen, resid);
707 /*
708 * For datagram protocols, leave room
709 * for protocol headers in first mbuf.
710 */
711 if (atomic && m == top && len < mlen - max_hdr)
712 m_align(m, len);
713 }
714
715 error = uiomove(mtod(m, caddr_t), len, uio);
716 if (error) {
717 m_freem(top);
718 return (error);
719 }
720
721 /* adjust counters */
722 resid = uio->uio_resid;
723 space -= len;
724 m->m_len = len;
725 top->m_pkthdr.len += len;
726
727 /* Is there more space and more data? */
728 } while (space > 0 && resid > 0);
729
730 *mp = top;
731 return 0;
732 }
733
734 /*
735 * Following replacement or removal of the first mbuf on the first
736 * mbuf chain of a socket buffer, push necessary state changes back
737 * into the socket buffer so that other consumers see the values
738 * consistently. 'nextrecord' is the callers locally stored value of
739 * the original value of sb->sb_mb->m_nextpkt which must be restored
740 * when the lead mbuf changes. NOTE: 'nextrecord' may be NULL.
741 */
742 void
743 sbsync(struct sockbuf *sb, struct mbuf *nextrecord)
744 {
745
746 /*
747 * First, update for the new value of nextrecord. If necessary,
748 * make it the first record.
749 */
750 if (sb->sb_mb != NULL)
751 sb->sb_mb->m_nextpkt = nextrecord;
752 else
753 sb->sb_mb = nextrecord;
754
755 /*
756 * Now update any dependent socket buffer fields to reflect
757 * the new state. This is an inline of SB_EMPTY_FIXUP, with
758 * the addition of a second clause that takes care of the
759 * case where sb_mb has been updated, but remains the last
760 * record.
761 */
762 if (sb->sb_mb == NULL) {
763 sb->sb_mbtail = NULL;
764 sb->sb_lastrecord = NULL;
765 } else if (sb->sb_mb->m_nextpkt == NULL)
766 sb->sb_lastrecord = sb->sb_mb;
767 }
768
769 /*
770 * Implement receive operations on a socket.
771 * We depend on the way that records are added to the sockbuf
772 * by sbappend*. In particular, each record (mbufs linked through m_next)
773 * must begin with an address if the protocol so specifies,
774 * followed by an optional mbuf or mbufs containing ancillary data,
775 * and then zero or more mbufs of data.
776 * In order to avoid blocking network for the entire time here, we release
777 * the solock() while doing the actual copy to user space.
778 * Although the sockbuf is locked, new data may still be appended,
779 * and thus we must maintain consistency of the sockbuf during that time.
780 *
781 * The caller may receive the data as a single mbuf chain by supplying
782 * an mbuf **mp0 for use in returning the chain. The uio is then used
783 * only for the count in uio_resid.
784 */
785 int
786 soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio,
787 struct mbuf **mp0, struct mbuf **controlp, int *flagsp,
788 socklen_t controllen)
789 {
790 struct mbuf *m, **mp;
791 struct mbuf *cm;
792 u_long len, offset, moff;
793 int flags, error, type, uio_error = 0;
794 const struct protosw *pr = so->so_proto;
795 struct mbuf *nextrecord;
796 size_t resid, orig_resid = uio->uio_resid;
797
798 mp = mp0;
799 if (paddr)
800 *paddr = NULL;
801 if (controlp)
802 *controlp = NULL;
803 if (flagsp)
804 flags = *flagsp &~ MSG_EOR;
805 else
806 flags = 0;
807 if (flags & MSG_OOB) {
808 m = m_get(M_WAIT, MT_DATA);
809 solock(so);
810 error = pru_rcvoob(so, m, flags & MSG_PEEK);
811 sounlock(so);
812 if (error)
813 goto bad;
814 do {
815 error = uiomove(mtod(m, caddr_t),
816 ulmin(uio->uio_resid, m->m_len), uio);
817 m = m_free(m);
818 } while (uio->uio_resid && error == 0 && m);
819 bad:
820 m_freem(m);
821 return (error);
822 }
823 if (mp)
824 *mp = NULL;
825
826 solock_shared(so);
827 restart:
828 if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) {
829 sounlock_shared(so);
830 return (error);
831 }
832
833 m = so->so_rcv.sb_mb;
834 #ifdef SOCKET_SPLICE
835 if (isspliced(so))
836 m = NULL;
837 #endif /* SOCKET_SPLICE */
838 /*
839 * If we have less data than requested, block awaiting more
840 * (subject to any timeout) if:
841 * 1. the current count is less than the low water mark,
842 * 2. MSG_WAITALL is set, and it is possible to do the entire
843 * receive operation at once if we block (resid <= hiwat), or
844 * 3. MSG_DONTWAIT is not set.
845 * If MSG_WAITALL is set but resid is larger than the receive buffer,
846 * we have to do the receive in sections, and thus risk returning
847 * a short count if a timeout or signal occurs after we start.
848 */
849 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 &&
850 so->so_rcv.sb_cc < uio->uio_resid) &&
851 (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
852 ((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) &&
853 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
854 #ifdef DIAGNOSTIC
855 if (m == NULL && so->so_rcv.sb_cc)
856 #ifdef SOCKET_SPLICE
857 if (!isspliced(so))
858 #endif /* SOCKET_SPLICE */
859 panic("receive 1: so %p, so_type %d, sb_cc %lu",
860 so, so->so_type, so->so_rcv.sb_cc);
861 #endif
862 if (so->so_error) {
863 if (m)
864 goto dontblock;
865 error = so->so_error;
866 if ((flags & MSG_PEEK) == 0)
867 so->so_error = 0;
868 goto release;
869 }
870 if (so->so_rcv.sb_state & SS_CANTRCVMORE) {
871 if (m)
872 goto dontblock;
873 else if (so->so_rcv.sb_cc == 0)
874 goto release;
875 }
876 for (; m; m = m->m_next)
877 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
878 m = so->so_rcv.sb_mb;
879 goto dontblock;
880 }
881 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
882 (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
883 error = ENOTCONN;
884 goto release;
885 }
886 if (uio->uio_resid == 0 && controlp == NULL)
887 goto release;
888 if (flags & MSG_DONTWAIT) {
889 error = EWOULDBLOCK;
890 goto release;
891 }
892 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
893 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
894 sbunlock(so, &so->so_rcv);
895 error = sbwait(so, &so->so_rcv);
896 if (error) {
897 sounlock_shared(so);
898 return (error);
899 }
900 goto restart;
901 }
902 dontblock:
903 /*
904 * On entry here, m points to the first record of the socket buffer.
905 * From this point onward, we maintain 'nextrecord' as a cache of the
906 * pointer to the next record in the socket buffer. We must keep the
907 * various socket buffer pointers and local stack versions of the
908 * pointers in sync, pushing out modifications before operations that
909 * may sleep, and re-reading them afterwards.
910 *
911 * Otherwise, we will race with the network stack appending new data
912 * or records onto the socket buffer by using inconsistent/stale
913 * versions of the field, possibly resulting in socket buffer
914 * corruption.
915 */
916 if (uio->uio_procp)
917 uio->uio_procp->p_ru.ru_msgrcv++;
918 KASSERT(m == so->so_rcv.sb_mb);
919 SBLASTRECORDCHK(&so->so_rcv, "soreceive 1");
920 SBLASTMBUFCHK(&so->so_rcv, "soreceive 1");
921 nextrecord = m->m_nextpkt;
922 if (pr->pr_flags & PR_ADDR) {
923 #ifdef DIAGNOSTIC
924 if (m->m_type != MT_SONAME)
925 panic("receive 1a: so %p, so_type %d, m %p, m_type %d",
926 so, so->so_type, m, m->m_type);
927 #endif
928 orig_resid = 0;
929 if (flags & MSG_PEEK) {
930 if (paddr)
931 *paddr = m_copym(m, 0, m->m_len, M_NOWAIT);
932 m = m->m_next;
933 } else {
934 sbfree(so, &so->so_rcv, m);
935 if (paddr) {
936 *paddr = m;
937 so->so_rcv.sb_mb = m->m_next;
938 m->m_next = NULL;
939 m = so->so_rcv.sb_mb;
940 } else {
941 so->so_rcv.sb_mb = m_free(m);
942 m = so->so_rcv.sb_mb;
943 }
944 sbsync(&so->so_rcv, nextrecord);
945 }
946 }
947 while (m && m->m_type == MT_CONTROL && error == 0) {
948 int skip = 0;
949 if (flags & MSG_PEEK) {
950 if (mtod(m, struct cmsghdr *)->cmsg_type ==
951 SCM_RIGHTS) {
952 /* don't leak internalized SCM_RIGHTS msgs */
953 skip = 1;
954 } else if (controlp)
955 *controlp = m_copym(m, 0, m->m_len, M_NOWAIT);
956 m = m->m_next;
957 } else {
958 sbfree(so, &so->so_rcv, m);
959 so->so_rcv.sb_mb = m->m_next;
960 m->m_nextpkt = m->m_next = NULL;
961 cm = m;
962 m = so->so_rcv.sb_mb;
963 sbsync(&so->so_rcv, nextrecord);
964 if (controlp) {
965 if (pr->pr_domain->dom_externalize) {
966 sounlock_shared(so);
967 error =
968 (*pr->pr_domain->dom_externalize)
969 (cm, controllen, flags);
970 solock_shared(so);
971 }
972 *controlp = cm;
973 } else {
974 /*
975 * Dispose of any SCM_RIGHTS message that went
976 * through the read path rather than recv.
977 */
978 if (pr->pr_domain->dom_dispose)
979 pr->pr_domain->dom_dispose(cm);
980 m_free(cm);
981 }
982 }
983 if (m != NULL)
984 nextrecord = so->so_rcv.sb_mb->m_nextpkt;
985 else
986 nextrecord = so->so_rcv.sb_mb;
987 if (controlp && !skip)
988 controlp = &(*controlp)->m_next;
989 orig_resid = 0;
990 }
991
992 /* If m is non-NULL, we have some data to read. */
993 if (m) {
994 type = m->m_type;
995 if (type == MT_OOBDATA)
996 flags |= MSG_OOB;
997 if (m->m_flags & M_BCAST)
998 flags |= MSG_BCAST;
999 if (m->m_flags & M_MCAST)
1000 flags |= MSG_MCAST;
1001 }
1002 SBLASTRECORDCHK(&so->so_rcv, "soreceive 2");
1003 SBLASTMBUFCHK(&so->so_rcv, "soreceive 2");
1004
1005 moff = 0;
1006 offset = 0;
1007 while (m && uio->uio_resid > 0 && error == 0) {
1008 if (m->m_type == MT_OOBDATA) {
1009 if (type != MT_OOBDATA)
1010 break;
1011 } else if (type == MT_OOBDATA) {
1012 break;
1013 } else if (m->m_type == MT_CONTROL) {
1014 /*
1015 * If there is more than one control message in the
1016 * stream, we do a short read. Next can be received
1017 * or disposed by another system call.
1018 */
1019 break;
1020 #ifdef DIAGNOSTIC
1021 } else if (m->m_type != MT_DATA && m->m_type != MT_HEADER) {
1022 panic("receive 3: so %p, so_type %d, m %p, m_type %d",
1023 so, so->so_type, m, m->m_type);
1024 #endif
1025 }
1026 so->so_rcv.sb_state &= ~SS_RCVATMARK;
1027 len = uio->uio_resid;
1028 if (so->so_oobmark && len > so->so_oobmark - offset)
1029 len = so->so_oobmark - offset;
1030 if (len > m->m_len - moff)
1031 len = m->m_len - moff;
1032 /*
1033 * If mp is set, just pass back the mbufs.
1034 * Otherwise copy them out via the uio, then free.
1035 * Sockbuf must be consistent here (points to current mbuf,
1036 * it points to next record) when we drop priority;
1037 * we must note any additions to the sockbuf when we
1038 * block interrupts again.
1039 */
1040 if (mp == NULL && uio_error == 0) {
1041 SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
1042 SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
1043 resid = uio->uio_resid;
1044 sounlock_shared(so);
1045 uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio);
1046 solock_shared(so);
1047 if (uio_error)
1048 uio->uio_resid = resid - len;
1049 } else
1050 uio->uio_resid -= len;
1051 if (len == m->m_len - moff) {
1052 if (m->m_flags & M_EOR)
1053 flags |= MSG_EOR;
1054 if (flags & MSG_PEEK) {
1055 m = m->m_next;
1056 moff = 0;
1057 orig_resid = 0;
1058 } else {
1059 nextrecord = m->m_nextpkt;
1060 sbfree(so, &so->so_rcv, m);
1061 if (mp) {
1062 *mp = m;
1063 mp = &m->m_next;
1064 so->so_rcv.sb_mb = m = m->m_next;
1065 *mp = NULL;
1066 } else {
1067 so->so_rcv.sb_mb = m_free(m);
1068 m = so->so_rcv.sb_mb;
1069 }
1070 /*
1071 * If m != NULL, we also know that
1072 * so->so_rcv.sb_mb != NULL.
1073 */
1074 KASSERT(so->so_rcv.sb_mb == m);
1075 if (m) {
1076 m->m_nextpkt = nextrecord;
1077 if (nextrecord == NULL)
1078 so->so_rcv.sb_lastrecord = m;
1079 } else {
1080 so->so_rcv.sb_mb = nextrecord;
1081 SB_EMPTY_FIXUP(&so->so_rcv);
1082 }
1083 SBLASTRECORDCHK(&so->so_rcv, "soreceive 3");
1084 SBLASTMBUFCHK(&so->so_rcv, "soreceive 3");
1085 }
1086 } else {
1087 if (flags & MSG_PEEK) {
1088 moff += len;
1089 orig_resid = 0;
1090 } else {
1091 if (mp)
1092 *mp = m_copym(m, 0, len, M_WAIT);
1093 m->m_data += len;
1094 m->m_len -= len;
1095 so->so_rcv.sb_cc -= len;
1096 so->so_rcv.sb_datacc -= len;
1097 }
1098 }
1099 if (so->so_oobmark) {
1100 if ((flags & MSG_PEEK) == 0) {
1101 so->so_oobmark -= len;
1102 if (so->so_oobmark == 0) {
1103 so->so_rcv.sb_state |= SS_RCVATMARK;
1104 break;
1105 }
1106 } else {
1107 offset += len;
1108 if (offset == so->so_oobmark)
1109 break;
1110 }
1111 }
1112 if (flags & MSG_EOR)
1113 break;
1114 /*
1115 * If the MSG_WAITALL flag is set (for non-atomic socket),
1116 * we must not quit until "uio->uio_resid == 0" or an error
1117 * termination. If a signal/timeout occurs, return
1118 * with a short count but without error.
1119 * Keep sockbuf locked against other readers.
1120 */
1121 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
1122 !sosendallatonce(so) && !nextrecord) {
1123 if (so->so_rcv.sb_state & SS_CANTRCVMORE ||
1124 so->so_error)
1125 break;
1126 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2");
1127 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2");
1128 error = sbwait(so, &so->so_rcv);
1129 if (error) {
1130 sbunlock(so, &so->so_rcv);
1131 sounlock_shared(so);
1132 return (0);
1133 }
1134 if ((m = so->so_rcv.sb_mb) != NULL)
1135 nextrecord = m->m_nextpkt;
1136 }
1137 }
1138
1139 if (m && pr->pr_flags & PR_ATOMIC) {
1140 flags |= MSG_TRUNC;
1141 if ((flags & MSG_PEEK) == 0)
1142 (void) sbdroprecord(so, &so->so_rcv);
1143 }
1144 if ((flags & MSG_PEEK) == 0) {
1145 if (m == NULL) {
1146 /*
1147 * First part is an inline SB_EMPTY_FIXUP(). Second
1148 * part makes sure sb_lastrecord is up-to-date if
1149 * there is still data in the socket buffer.
1150 */
1151 so->so_rcv.sb_mb = nextrecord;
1152 if (so->so_rcv.sb_mb == NULL) {
1153 so->so_rcv.sb_mbtail = NULL;
1154 so->so_rcv.sb_lastrecord = NULL;
1155 } else if (nextrecord->m_nextpkt == NULL)
1156 so->so_rcv.sb_lastrecord = nextrecord;
1157 }
1158 SBLASTRECORDCHK(&so->so_rcv, "soreceive 4");
1159 SBLASTMBUFCHK(&so->so_rcv, "soreceive 4");
1160 if (pr->pr_flags & PR_WANTRCVD)
1161 pru_rcvd(so);
1162 }
1163 if (orig_resid == uio->uio_resid && orig_resid &&
1164 (flags & MSG_EOR) == 0 &&
1165 (so->so_rcv.sb_state & SS_CANTRCVMORE) == 0) {
1166 sbunlock(so, &so->so_rcv);
1167 goto restart;
1168 }
1169
1170 if (uio_error)
1171 error = uio_error;
1172
1173 if (flagsp)
1174 *flagsp |= flags;
1175 release:
1176 sbunlock(so, &so->so_rcv);
1177 sounlock_shared(so);
1178 return (error);
1179 }
1180
1181 int
1182 soshutdown(struct socket *so, int how)
1183 {
1184 int error = 0;
1185
1186 solock(so);
1187 switch (how) {
1188 case SHUT_RD:
1189 sorflush(so);
1190 break;
1191 case SHUT_RDWR:
1192 sorflush(so);
1193 /* FALLTHROUGH */
1194 case SHUT_WR:
1195 error = pru_shutdown(so);
1196 break;
1197 default:
1198 error = EINVAL;
1199 break;
1200 }
1201 sounlock(so);
1202
1203 return (error);
1204 }
1205
1206 void
1207 sorflush(struct socket *so)
1208 {
1209 struct sockbuf *sb = &so->so_rcv;
1210 struct mbuf *m;
1211 const struct protosw *pr = so->so_proto;
1212 int error;
1213
1214 sb->sb_flags |= SB_NOINTR;
1215 error = sblock(so, sb, M_WAITOK);
1216 /* with SB_NOINTR and M_WAITOK sblock() must not fail */
1217 KASSERT(error == 0);
1218 socantrcvmore(so);
1219 m = sb->sb_mb;
1220 memset(&sb->sb_startzero, 0,
1221 (caddr_t)&sb->sb_endzero - (caddr_t)&sb->sb_startzero);
1222 sb->sb_timeo_nsecs = INFSLP;
1223 sbunlock(so, sb);
1224 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose)
1225 (*pr->pr_domain->dom_dispose)(m);
1226 m_purge(m);
1227 }
1228
1229 #ifdef SOCKET_SPLICE
1230
1231 #define so_splicelen so_sp->ssp_len
1232 #define so_splicemax so_sp->ssp_max
1233 #define so_idletv so_sp->ssp_idletv
1234 #define so_idleto so_sp->ssp_idleto
1235 #define so_splicetask so_sp->ssp_task
1236
1237 int
1238 sosplice(struct socket *so, int fd, off_t max, struct timeval *tv)
1239 {
1240 struct file *fp;
1241 struct socket *sosp;
1242 struct sosplice *sp;
1243 struct taskq *tq;
1244 int error = 0;
1245
1246 soassertlocked(so);
1247
1248 if (sosplice_taskq == NULL) {
1249 rw_enter_write(&sosplice_lock);
1250 if (sosplice_taskq == NULL) {
1251 tq = taskq_create("sosplice", 1, IPL_SOFTNET,
1252 TASKQ_MPSAFE);
1253 /* Ensure the taskq is fully visible to other CPUs. */
1254 membar_producer();
1255 sosplice_taskq = tq;
1256 }
1257 rw_exit_write(&sosplice_lock);
1258 }
1259 if (sosplice_taskq == NULL)
1260 return (ENOMEM);
1261
1262 if ((so->so_proto->pr_flags & PR_SPLICE) == 0)
1263 return (EPROTONOSUPPORT);
1264 if (so->so_options & SO_ACCEPTCONN)
1265 return (EOPNOTSUPP);
1266 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
1267 (so->so_proto->pr_flags & PR_CONNREQUIRED))
1268 return (ENOTCONN);
1269 if (so->so_sp == NULL) {
1270 sp = pool_get(&sosplice_pool, PR_WAITOK | PR_ZERO);
1271 if (so->so_sp == NULL)
1272 so->so_sp = sp;
1273 else
1274 pool_put(&sosplice_pool, sp);
1275 }
1276
1277 /* If no fd is given, unsplice by removing existing link. */
1278 if (fd < 0) {
1279 /* Lock receive buffer. */
1280 if ((error = sblock(so, &so->so_rcv, M_WAITOK)) != 0) {
1281 return (error);
1282 }
1283 if (so->so_sp->ssp_socket)
1284 sounsplice(so, so->so_sp->ssp_socket, 0);
1285 sbunlock(so, &so->so_rcv);
1286 return (0);
1287 }
1288
1289 if (max && max < 0)
1290 return (EINVAL);
1291
1292 if (tv && (tv->tv_sec < 0 || !timerisvalid(tv)))
1293 return (EINVAL);
1294
1295 /* Find sosp, the drain socket where data will be spliced into. */
1296 if ((error = getsock(curproc, fd, &fp)) != 0)
1297 return (error);
1298 sosp = fp->f_data;
1299 if (sosp->so_proto->pr_usrreqs->pru_send !=
1300 so->so_proto->pr_usrreqs->pru_send) {
1301 error = EPROTONOSUPPORT;
1302 goto frele;
1303 }
1304 if (sosp->so_sp == NULL) {
1305 sp = pool_get(&sosplice_pool, PR_WAITOK | PR_ZERO);
1306 if (sosp->so_sp == NULL)
1307 sosp->so_sp = sp;
1308 else
1309 pool_put(&sosplice_pool, sp);
1310 }
1311
1312 /* Lock both receive and send buffer. */
1313 if ((error = sblock(so, &so->so_rcv, M_WAITOK)) != 0) {
1314 goto frele;
1315 }
1316 if ((error = sblock(so, &sosp->so_snd, M_WAITOK)) != 0) {
1317 sbunlock(so, &so->so_rcv);
1318 goto frele;
1319 }
1320
1321 if (so->so_sp->ssp_socket || sosp->so_sp->ssp_soback) {
1322 error = EBUSY;
1323 goto release;
1324 }
1325 if (sosp->so_options & SO_ACCEPTCONN) {
1326 error = EOPNOTSUPP;
1327 goto release;
1328 }
1329 if ((sosp->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) {
1330 error = ENOTCONN;
1331 goto release;
1332 }
1333
1334 /* Splice so and sosp together. */
1335 so->so_sp->ssp_socket = sosp;
1336 sosp->so_sp->ssp_soback = so;
1337 so->so_splicelen = 0;
1338 so->so_splicemax = max;
1339 if (tv)
1340 so->so_idletv = *tv;
1341 else
1342 timerclear(&so->so_idletv);
1343 timeout_set_proc(&so->so_idleto, soidle, so);
1344 task_set(&so->so_splicetask, sotask, so);
1345
1346 /*
1347 * To prevent softnet interrupt from calling somove() while
1348 * we sleep, the socket buffers are not marked as spliced yet.
1349 */
1350 if (somove(so, M_WAIT)) {
1351 so->so_rcv.sb_flags |= SB_SPLICE;
1352 sosp->so_snd.sb_flags |= SB_SPLICE;
1353 }
1354
1355 release:
1356 sbunlock(sosp, &sosp->so_snd);
1357 sbunlock(so, &so->so_rcv);
1358 frele:
1359 /*
1360 * FRELE() must not be called with the socket lock held. It is safe to
1361 * release the lock here as long as no other operation happen on the
1362 * socket when sosplice() returns. The dance could be avoided by
1363 * grabbing the socket lock inside this function.
1364 */
1365 sounlock(so);
1366 FRELE(fp, curproc);
1367 solock(so);
1368 return (error);
1369 }
1370
1371 void
1372 sounsplice(struct socket *so, struct socket *sosp, int freeing)
1373 {
1374 soassertlocked(so);
1375
1376 task_del(sosplice_taskq, &so->so_splicetask);
1377 timeout_del(&so->so_idleto);
1378 sosp->so_snd.sb_flags &= ~SB_SPLICE;
1379 so->so_rcv.sb_flags &= ~SB_SPLICE;
1380 so->so_sp->ssp_socket = sosp->so_sp->ssp_soback = NULL;
1381 /* Do not wakeup a socket that is about to be freed. */
1382 if ((freeing & SOSP_FREEING_READ) == 0 && soreadable(so))
1383 sorwakeup(so);
1384 if ((freeing & SOSP_FREEING_WRITE) == 0 && sowriteable(sosp))
1385 sowwakeup(sosp);
1386 }
1387
1388 void
1389 soidle(void *arg)
1390 {
1391 struct socket *so = arg;
1392
1393 solock(so);
1394 if (so->so_rcv.sb_flags & SB_SPLICE) {
1395 so->so_error = ETIMEDOUT;
1396 sounsplice(so, so->so_sp->ssp_socket, 0);
1397 }
1398 sounlock(so);
1399 }
1400
1401 void
1402 sotask(void *arg)
1403 {
1404 struct socket *so = arg;
1405
1406 solock(so);
1407 if (so->so_rcv.sb_flags & SB_SPLICE) {
1408 /*
1409 * We may not sleep here as sofree() and unsplice() may be
1410 * called from softnet interrupt context. This would remove
1411 * the socket during somove().
1412 */
1413 somove(so, M_DONTWAIT);
1414 }
1415 sounlock(so);
1416
1417 /* Avoid user land starvation. */
1418 yield();
1419 }
1420
1421 /*
1422 * The socket splicing task or idle timeout may sleep while grabbing the net
1423 * lock. As sofree() can be called anytime, sotask() or soidle() could access
1424 * the socket memory of a freed socket after wakeup. So delay the pool_put()
1425 * after all pending socket splicing tasks or timeouts have finished. Do this
1426 * by scheduling it on the same threads.
1427 */
1428 void
1429 soreaper(void *arg)
1430 {
1431 struct socket *so = arg;
1432
1433 /* Reuse splice task, sounsplice() has been called before. */
1434 task_set(&so->so_sp->ssp_task, soput, so);
1435 task_add(sosplice_taskq, &so->so_sp->ssp_task);
1436 }
1437
1438 void
1439 soput(void *arg)
1440 {
1441 struct socket *so = arg;
1442
1443 pool_put(&sosplice_pool, so->so_sp);
1444 pool_put(&socket_pool, so);
1445 }
1446
1447 /*
1448 * Move data from receive buffer of spliced source socket to send
1449 * buffer of drain socket. Try to move as much as possible in one
1450 * big chunk. It is a TCP only implementation.
1451 * Return value 0 means splicing has been finished, 1 continue.
1452 */
1453 int
1454 somove(struct socket *so, int wait)
1455 {
1456 struct socket *sosp = so->so_sp->ssp_socket;
1457 struct mbuf *m, **mp, *nextrecord;
1458 u_long len, off, oobmark;
1459 long space;
1460 int error = 0, maxreached = 0;
1461 unsigned int rcvstate;
1462
1463 soassertlocked(so);
1464
1465 nextpkt:
1466 if (so->so_error) {
1467 error = so->so_error;
1468 goto release;
1469 }
1470 if (sosp->so_snd.sb_state & SS_CANTSENDMORE) {
1471 error = EPIPE;
1472 goto release;
1473 }
1474 if (sosp->so_error && sosp->so_error != ETIMEDOUT &&
1475 sosp->so_error != EFBIG && sosp->so_error != ELOOP) {
1476 error = sosp->so_error;
1477 goto release;
1478 }
1479 if ((sosp->so_state & SS_ISCONNECTED) == 0)
1480 goto release;
1481
1482 /* Calculate how many bytes can be copied now. */
1483 len = so->so_rcv.sb_datacc;
1484 if (so->so_splicemax) {
1485 KASSERT(so->so_splicelen < so->so_splicemax);
1486 if (so->so_splicemax <= so->so_splicelen + len) {
1487 len = so->so_splicemax - so->so_splicelen;
1488 maxreached = 1;
1489 }
1490 }
1491 space = sbspace(sosp, &sosp->so_snd);
1492 if (so->so_oobmark && so->so_oobmark < len &&
1493 so->so_oobmark < space + 1024)
1494 space += 1024;
1495 if (space <= 0) {
1496 maxreached = 0;
1497 goto release;
1498 }
1499 if (space < len) {
1500 maxreached = 0;
1501 if (space < sosp->so_snd.sb_lowat)
1502 goto release;
1503 len = space;
1504 }
1505 sosp->so_snd.sb_state |= SS_ISSENDING;
1506
1507 SBLASTRECORDCHK(&so->so_rcv, "somove 1");
1508 SBLASTMBUFCHK(&so->so_rcv, "somove 1");
1509 m = so->so_rcv.sb_mb;
1510 if (m == NULL)
1511 goto release;
1512 nextrecord = m->m_nextpkt;
1513
1514 /* Drop address and control information not used with splicing. */
1515 if (so->so_proto->pr_flags & PR_ADDR) {
1516 #ifdef DIAGNOSTIC
1517 if (m->m_type != MT_SONAME)
1518 panic("somove soname: so %p, so_type %d, m %p, "
1519 "m_type %d", so, so->so_type, m, m->m_type);
1520 #endif
1521 m = m->m_next;
1522 }
1523 while (m && m->m_type == MT_CONTROL)
1524 m = m->m_next;
1525 if (m == NULL) {
1526 sbdroprecord(so, &so->so_rcv);
1527 if (so->so_proto->pr_flags & PR_WANTRCVD)
1528 pru_rcvd(so);
1529 goto nextpkt;
1530 }
1531
1532 /*
1533 * By splicing sockets connected to localhost, userland might create a
1534 * loop. Dissolve splicing with error if loop is detected by counter.
1535 *
1536 * If we deal with looped broadcast/multicast packet we bail out with
1537 * no error to suppress splice termination.
1538 */
1539 if ((m->m_flags & M_PKTHDR) &&
1540 ((m->m_pkthdr.ph_loopcnt++ >= M_MAXLOOP) ||
1541 ((m->m_flags & M_LOOP) && (m->m_flags & (M_BCAST|M_MCAST))))) {
1542 error = ELOOP;
1543 goto release;
1544 }
1545
1546 if (so->so_proto->pr_flags & PR_ATOMIC) {
1547 if ((m->m_flags & M_PKTHDR) == 0)
1548 panic("somove !PKTHDR: so %p, so_type %d, m %p, "
1549 "m_type %d", so, so->so_type, m, m->m_type);
1550 if (sosp->so_snd.sb_hiwat < m->m_pkthdr.len) {
1551 error = EMSGSIZE;
1552 goto release;
1553 }
1554 if (len < m->m_pkthdr.len)
1555 goto release;
1556 if (m->m_pkthdr.len < len) {
1557 maxreached = 0;
1558 len = m->m_pkthdr.len;
1559 }
1560 /*
1561 * Throw away the name mbuf after it has been assured
1562 * that the whole first record can be processed.
1563 */
1564 m = so->so_rcv.sb_mb;
1565 sbfree(so, &so->so_rcv, m);
1566 so->so_rcv.sb_mb = m_free(m);
1567 sbsync(&so->so_rcv, nextrecord);
1568 }
1569 /*
1570 * Throw away the control mbufs after it has been assured
1571 * that the whole first record can be processed.
1572 */
1573 m = so->so_rcv.sb_mb;
1574 while (m && m->m_type == MT_CONTROL) {
1575 sbfree(so, &so->so_rcv, m);
1576 so->so_rcv.sb_mb = m_free(m);
1577 m = so->so_rcv.sb_mb;
1578 sbsync(&so->so_rcv, nextrecord);
1579 }
1580
1581 SBLASTRECORDCHK(&so->so_rcv, "somove 2");
1582 SBLASTMBUFCHK(&so->so_rcv, "somove 2");
1583
1584 /* Take at most len mbufs out of receive buffer. */
1585 for (off = 0, mp = &m; off <= len && *mp;
1586 off += (*mp)->m_len, mp = &(*mp)->m_next) {
1587 u_long size = len - off;
1588
1589 #ifdef DIAGNOSTIC
1590 if ((*mp)->m_type != MT_DATA && (*mp)->m_type != MT_HEADER)
1591 panic("somove type: so %p, so_type %d, m %p, "
1592 "m_type %d", so, so->so_type, *mp, (*mp)->m_type);
1593 #endif
1594 if ((*mp)->m_len > size) {
1595 /*
1596 * Move only a partial mbuf at maximum splice length or
1597 * if the drain buffer is too small for this large mbuf.
1598 */
1599 if (!maxreached && so->so_snd.sb_datacc > 0) {
1600 len -= size;
1601 break;
1602 }
1603 *mp = m_copym(so->so_rcv.sb_mb, 0, size, wait);
1604 if (*mp == NULL) {
1605 len -= size;
1606 break;
1607 }
1608 so->so_rcv.sb_mb->m_data += size;
1609 so->so_rcv.sb_mb->m_len -= size;
1610 so->so_rcv.sb_cc -= size;
1611 so->so_rcv.sb_datacc -= size;
1612 } else {
1613 *mp = so->so_rcv.sb_mb;
1614 sbfree(so, &so->so_rcv, *mp);
1615 so->so_rcv.sb_mb = (*mp)->m_next;
1616 sbsync(&so->so_rcv, nextrecord);
1617 }
1618 }
1619 *mp = NULL;
1620
1621 SBLASTRECORDCHK(&so->so_rcv, "somove 3");
1622 SBLASTMBUFCHK(&so->so_rcv, "somove 3");
1623 SBCHECK(so, &so->so_rcv);
1624 if (m == NULL)
1625 goto release;
1626 m->m_nextpkt = NULL;
1627 if (m->m_flags & M_PKTHDR) {
1628 m_resethdr(m);
1629 m->m_pkthdr.len = len;
1630 }
1631
1632 /* Send window update to source peer as receive buffer has changed. */
1633 if (so->so_proto->pr_flags & PR_WANTRCVD)
1634 pru_rcvd(so);
1635
1636 /* Receive buffer did shrink by len bytes, adjust oob. */
1637 rcvstate = so->so_rcv.sb_state;
1638 so->so_rcv.sb_state &= ~SS_RCVATMARK;
1639 oobmark = so->so_oobmark;
1640 so->so_oobmark = oobmark > len ? oobmark - len : 0;
1641 if (oobmark) {
1642 if (oobmark == len)
1643 so->so_rcv.sb_state |= SS_RCVATMARK;
1644 if (oobmark >= len)
1645 oobmark = 0;
1646 }
1647
1648 /*
1649 * Handle oob data. If any malloc fails, ignore error.
1650 * TCP urgent data is not very reliable anyway.
1651 */
1652 while (((rcvstate & SS_RCVATMARK) || oobmark) &&
1653 (so->so_options & SO_OOBINLINE)) {
1654 struct mbuf *o = NULL;
1655
1656 if (rcvstate & SS_RCVATMARK) {
1657 o = m_get(wait, MT_DATA);
1658 rcvstate &= ~SS_RCVATMARK;
1659 } else if (oobmark) {
1660 o = m_split(m, oobmark, wait);
1661 if (o) {
1662 error = pru_send(sosp, m, NULL, NULL);
1663 if (error) {
1664 if (sosp->so_snd.sb_state &
1665 SS_CANTSENDMORE)
1666 error = EPIPE;
1667 m_freem(o);
1668 goto release;
1669 }
1670 len -= oobmark;
1671 so->so_splicelen += oobmark;
1672 m = o;
1673 o = m_get(wait, MT_DATA);
1674 }
1675 oobmark = 0;
1676 }
1677 if (o) {
1678 o->m_len = 1;
1679 *mtod(o, caddr_t) = *mtod(m, caddr_t);
1680 error = pru_sendoob(sosp, o, NULL, NULL);
1681 if (error) {
1682 if (sosp->so_snd.sb_state & SS_CANTSENDMORE)
1683 error = EPIPE;
1684 m_freem(m);
1685 goto release;
1686 }
1687 len -= 1;
1688 so->so_splicelen += 1;
1689 if (oobmark) {
1690 oobmark -= 1;
1691 if (oobmark == 0)
1692 rcvstate |= SS_RCVATMARK;
1693 }
1694 m_adj(m, 1);
1695 }
1696 }
1697
1698 /* Append all remaining data to drain socket. */
1699 if (so->so_rcv.sb_cc == 0 || maxreached)
1700 sosp->so_snd.sb_state &= ~SS_ISSENDING;
1701 error = pru_send(sosp, m, NULL, NULL);
1702 if (error) {
1703 if (sosp->so_snd.sb_state & SS_CANTSENDMORE)
1704 error = EPIPE;
1705 goto release;
1706 }
1707 so->so_splicelen += len;
1708
1709 /* Move several packets if possible. */
1710 if (!maxreached && nextrecord)
1711 goto nextpkt;
1712
1713 release:
1714 sosp->so_snd.sb_state &= ~SS_ISSENDING;
1715 if (!error && maxreached && so->so_splicemax == so->so_splicelen)
1716 error = EFBIG;
1717 if (error)
1718 so->so_error = error;
1719 if (((so->so_rcv.sb_state & SS_CANTRCVMORE) &&
1720 so->so_rcv.sb_cc == 0) ||
1721 (sosp->so_snd.sb_state & SS_CANTSENDMORE) ||
1722 maxreached || error) {
1723 sounsplice(so, sosp, 0);
1724 return (0);
1725 }
1726 if (timerisset(&so->so_idletv))
1727 timeout_add_tv(&so->so_idleto, &so->so_idletv);
1728 return (1);
1729 }
1730
1731 #endif /* SOCKET_SPLICE */
1732
1733 void
1734 sorwakeup(struct socket *so)
1735 {
1736 soassertlocked(so);
1737
1738 #ifdef SOCKET_SPLICE
1739 if (so->so_rcv.sb_flags & SB_SPLICE) {
1740 /*
1741 * TCP has a sendbuffer that can handle multiple packets
1742 * at once. So queue the stream a bit to accumulate data.
1743 * The sosplice thread will call somove() later and send
1744 * the packets calling tcp_output() only once.
1745 * In the UDP case, send out the packets immediately.
1746 * Using a thread would make things slower.
1747 */
1748 if (so->so_proto->pr_flags & PR_WANTRCVD)
1749 task_add(sosplice_taskq, &so->so_splicetask);
1750 else
1751 somove(so, M_DONTWAIT);
1752 }
1753 if (isspliced(so))
1754 return;
1755 #endif
1756 sowakeup(so, &so->so_rcv);
1757 if (so->so_upcall)
1758 (*(so->so_upcall))(so, so->so_upcallarg, M_DONTWAIT);
1759 }
1760
1761 void
1762 sowwakeup(struct socket *so)
1763 {
1764 soassertlocked(so);
1765
1766 #ifdef SOCKET_SPLICE
1767 if (so->so_snd.sb_flags & SB_SPLICE)
1768 task_add(sosplice_taskq, &so->so_sp->ssp_soback->so_splicetask);
1769 if (issplicedback(so))
1770 return;
1771 #endif
1772 sowakeup(so, &so->so_snd);
1773 }
1774
1775 int
1776 sosetopt(struct socket *so, int level, int optname, struct mbuf *m)
1777 {
1778 int error = 0;
1779
1780 soassertlocked(so);
1781
1782 if (level != SOL_SOCKET) {
1783 if (so->so_proto->pr_ctloutput) {
1784 error = (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so,
1785 level, optname, m);
1786 return (error);
1787 }
1788 error = ENOPROTOOPT;
1789 } else {
1790 switch (optname) {
1791 case SO_BINDANY:
1792 if ((error = suser(curproc)) != 0) /* XXX */
1793 return (error);
1794 break;
1795 }
1796
1797 switch (optname) {
1798
1799 case SO_LINGER:
1800 if (m == NULL || m->m_len != sizeof (struct linger) ||
1801 mtod(m, struct linger *)->l_linger < 0 ||
1802 mtod(m, struct linger *)->l_linger > SHRT_MAX)
1803 return (EINVAL);
1804 so->so_linger = mtod(m, struct linger *)->l_linger;
1805 /* FALLTHROUGH */
1806
1807 case SO_BINDANY:
1808 case SO_DEBUG:
1809 case SO_KEEPALIVE:
1810 case SO_USELOOPBACK:
1811 case SO_BROADCAST:
1812 case SO_REUSEADDR:
1813 case SO_REUSEPORT:
1814 case SO_OOBINLINE:
1815 case SO_TIMESTAMP:
1816 case SO_ZEROIZE:
1817 if (m == NULL || m->m_len < sizeof (int))
1818 return (EINVAL);
1819 if (*mtod(m, int *))
1820 so->so_options |= optname;
1821 else
1822 so->so_options &= ~optname;
1823 break;
1824
1825 case SO_DONTROUTE:
1826 if (m == NULL || m->m_len < sizeof (int))
1827 return (EINVAL);
1828 if (*mtod(m, int *))
1829 error = EOPNOTSUPP;
1830 break;
1831
1832 case SO_SNDBUF:
1833 case SO_RCVBUF:
1834 case SO_SNDLOWAT:
1835 case SO_RCVLOWAT:
1836 {
1837 u_long cnt;
1838
1839 if (m == NULL || m->m_len < sizeof (int))
1840 return (EINVAL);
1841 cnt = *mtod(m, int *);
1842 if ((long)cnt <= 0)
1843 cnt = 1;
1844 switch (optname) {
1845
1846 case SO_SNDBUF:
1847 if (so->so_snd.sb_state & SS_CANTSENDMORE)
1848 return (EINVAL);
1849 if (sbcheckreserve(cnt, so->so_snd.sb_wat) ||
1850 sbreserve(so, &so->so_snd, cnt))
1851 return (ENOBUFS);
1852 so->so_snd.sb_wat = cnt;
1853 break;
1854
1855 case SO_RCVBUF:
1856 if (so->so_rcv.sb_state & SS_CANTRCVMORE)
1857 return (EINVAL);
1858 if (sbcheckreserve(cnt, so->so_rcv.sb_wat) ||
1859 sbreserve(so, &so->so_rcv, cnt))
1860 return (ENOBUFS);
1861 so->so_rcv.sb_wat = cnt;
1862 break;
1863
1864 case SO_SNDLOWAT:
1865 so->so_snd.sb_lowat =
1866 (cnt > so->so_snd.sb_hiwat) ?
1867 so->so_snd.sb_hiwat : cnt;
1868 break;
1869 case SO_RCVLOWAT:
1870 so->so_rcv.sb_lowat =
1871 (cnt > so->so_rcv.sb_hiwat) ?
1872 so->so_rcv.sb_hiwat : cnt;
1873 break;
1874 }
1875 break;
1876 }
1877
1878 case SO_SNDTIMEO:
1879 case SO_RCVTIMEO:
1880 {
1881 struct timeval tv;
1882 uint64_t nsecs;
1883
1884 if (m == NULL || m->m_len < sizeof (tv))
1885 return (EINVAL);
1886 memcpy(&tv, mtod(m, struct timeval *), sizeof tv);
1887 if (!timerisvalid(&tv))
1888 return (EINVAL);
1889 nsecs = TIMEVAL_TO_NSEC(&tv);
1890 if (nsecs == UINT64_MAX)
1891 return (EDOM);
1892 if (nsecs == 0)
1893 nsecs = INFSLP;
1894 switch (optname) {
1895
1896 case SO_SNDTIMEO:
1897 so->so_snd.sb_timeo_nsecs = nsecs;
1898 break;
1899 case SO_RCVTIMEO:
1900 so->so_rcv.sb_timeo_nsecs = nsecs;
1901 break;
1902 }
1903 break;
1904 }
1905
1906 case SO_RTABLE:
1907 if (so->so_proto->pr_domain &&
1908 so->so_proto->pr_domain->dom_protosw &&
1909 so->so_proto->pr_ctloutput) {
1910 const struct domain *dom =
1911 so->so_proto->pr_domain;
1912
1913 level = dom->dom_protosw->pr_protocol;
1914 error = (*so->so_proto->pr_ctloutput)
1915 (PRCO_SETOPT, so, level, optname, m);
1916 return (error);
1917 }
1918 error = ENOPROTOOPT;
1919 break;
1920
1921 #ifdef SOCKET_SPLICE
1922 case SO_SPLICE:
1923 if (m == NULL) {
1924 error = sosplice(so, -1, 0, NULL);
1925 } else if (m->m_len < sizeof(int)) {
1926 return (EINVAL);
1927 } else if (m->m_len < sizeof(struct splice)) {
1928 error = sosplice(so, *mtod(m, int *), 0, NULL);
1929 } else {
1930 error = sosplice(so,
1931 mtod(m, struct splice *)->sp_fd,
1932 mtod(m, struct splice *)->sp_max,
1933 &mtod(m, struct splice *)->sp_idle);
1934 }
1935 break;
1936 #endif /* SOCKET_SPLICE */
1937
1938 default:
1939 error = ENOPROTOOPT;
1940 break;
1941 }
1942 if (error == 0 && so->so_proto->pr_ctloutput) {
1943 (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so,
1944 level, optname, m);
1945 }
1946 }
1947
1948 return (error);
1949 }
1950
1951 int
1952 sogetopt(struct socket *so, int level, int optname, struct mbuf *m)
1953 {
1954 int error = 0;
1955
1956 if (level != SOL_SOCKET) {
1957 if (so->so_proto->pr_ctloutput) {
1958 m->m_len = 0;
1959
1960 solock(so);
1961 error = (*so->so_proto->pr_ctloutput)(PRCO_GETOPT, so,
1962 level, optname, m);
1963 sounlock(so);
1964 return (error);
1965 } else
1966 return (ENOPROTOOPT);
1967 } else {
1968 m->m_len = sizeof (int);
1969
1970 switch (optname) {
1971
1972 case SO_LINGER:
1973 m->m_len = sizeof (struct linger);
1974 solock_shared(so);
1975 mtod(m, struct linger *)->l_onoff =
1976 so->so_options & SO_LINGER;
1977 mtod(m, struct linger *)->l_linger = so->so_linger;
1978 sounlock_shared(so);
1979 break;
1980
1981 case SO_BINDANY:
1982 case SO_USELOOPBACK:
1983 case SO_DEBUG:
1984 case SO_KEEPALIVE:
1985 case SO_REUSEADDR:
1986 case SO_REUSEPORT:
1987 case SO_BROADCAST:
1988 case SO_OOBINLINE:
1989 case SO_TIMESTAMP:
1990 case SO_ZEROIZE:
1991 *mtod(m, int *) = so->so_options & optname;
1992 break;
1993
1994 case SO_DONTROUTE:
1995 *mtod(m, int *) = 0;
1996 break;
1997
1998 case SO_TYPE:
1999 *mtod(m, int *) = so->so_type;
2000 break;
2001
2002 case SO_ERROR:
2003 solock(so);
2004 *mtod(m, int *) = so->so_error;
2005 so->so_error = 0;
2006 sounlock(so);
2007
2008 break;
2009
2010 case SO_DOMAIN:
2011 *mtod(m, int *) = so->so_proto->pr_domain->dom_family;
2012 break;
2013
2014 case SO_PROTOCOL:
2015 *mtod(m, int *) = so->so_proto->pr_protocol;
2016 break;
2017
2018 case SO_SNDBUF:
2019 *mtod(m, int *) = so->so_snd.sb_hiwat;
2020 break;
2021
2022 case SO_RCVBUF:
2023 *mtod(m, int *) = so->so_rcv.sb_hiwat;
2024 break;
2025
2026 case SO_SNDLOWAT:
2027 *mtod(m, int *) = so->so_snd.sb_lowat;
2028 break;
2029
2030 case SO_RCVLOWAT:
2031 *mtod(m, int *) = so->so_rcv.sb_lowat;
2032 break;
2033
2034 case SO_SNDTIMEO:
2035 case SO_RCVTIMEO:
2036 {
2037 struct sockbuf *sb = (optname == SO_SNDTIMEO ?
2038 &so->so_snd : &so->so_rcv);
2039 struct timeval tv;
2040 uint64_t nsecs;
2041
2042 solock_shared(so);
2043 nsecs = sb->sb_timeo_nsecs;
2044 sounlock_shared(so);
2045
2046 m->m_len = sizeof(struct timeval);
2047 memset(&tv, 0, sizeof(tv));
2048 if (nsecs != INFSLP)
2049 NSEC_TO_TIMEVAL(nsecs, &tv);
2050 memcpy(mtod(m, struct timeval *), &tv, sizeof tv);
2051 break;
2052 }
2053
2054 case SO_RTABLE:
2055 if (so->so_proto->pr_domain &&
2056 so->so_proto->pr_domain->dom_protosw &&
2057 so->so_proto->pr_ctloutput) {
2058 const struct domain *dom =
2059 so->so_proto->pr_domain;
2060
2061 level = dom->dom_protosw->pr_protocol;
2062 solock(so);
2063 error = (*so->so_proto->pr_ctloutput)
2064 (PRCO_GETOPT, so, level, optname, m);
2065 sounlock(so);
2066 if (error)
2067 return (error);
2068 break;
2069 }
2070 return (ENOPROTOOPT);
2071
2072 #ifdef SOCKET_SPLICE
2073 case SO_SPLICE:
2074 {
2075 off_t len;
2076
2077 m->m_len = sizeof(off_t);
2078 solock_shared(so);
2079 len = so->so_sp ? so->so_sp->ssp_len : 0;
2080 sounlock_shared(so);
2081 memcpy(mtod(m, off_t *), &len, sizeof(off_t));
2082 break;
2083 }
2084 #endif /* SOCKET_SPLICE */
2085
2086 case SO_PEERCRED:
2087 if (so->so_proto->pr_protocol == AF_UNIX) {
2088 struct unpcb *unp = sotounpcb(so);
2089
2090 solock(so);
2091 if (unp->unp_flags & UNP_FEIDS) {
2092 m->m_len = sizeof(unp->unp_connid);
2093 memcpy(mtod(m, caddr_t),
2094 &(unp->unp_connid), m->m_len);
2095 sounlock(so);
2096 break;
2097 }
2098 sounlock(so);
2099
2100 return (ENOTCONN);
2101 }
2102 return (EOPNOTSUPP);
2103
2104 default:
2105 return (ENOPROTOOPT);
2106 }
2107 return (0);
2108 }
2109 }
2110
2111 void
2112 sohasoutofband(struct socket *so)
2113 {
2114 pgsigio(&so->so_sigio, SIGURG, 0);
2115 KNOTE(&so->so_rcv.sb_klist, 0);
2116 }
2117
2118 int
2119 soo_kqfilter(struct file *fp, struct knote *kn)
2120 {
2121 struct socket *so = kn->kn_fp->f_data;
2122 struct sockbuf *sb;
2123
2124 solock(so);
2125 switch (kn->kn_filter) {
2126 case EVFILT_READ:
2127 if (so->so_options & SO_ACCEPTCONN)
2128 kn->kn_fop = &solisten_filtops;
2129 else
2130 kn->kn_fop = &soread_filtops;
2131 sb = &so->so_rcv;
2132 break;
2133 case EVFILT_WRITE:
2134 kn->kn_fop = &sowrite_filtops;
2135 sb = &so->so_snd;
2136 break;
2137 case EVFILT_EXCEPT:
2138 kn->kn_fop = &soexcept_filtops;
2139 sb = &so->so_rcv;
2140 break;
2141 default:
2142 sounlock(so);
2143 return (EINVAL);
2144 }
2145
2146 klist_insert_locked(&sb->sb_klist, kn);
2147 sounlock(so);
2148
2149 return (0);
2150 }
2151
2152 void
2153 filt_sordetach(struct knote *kn)
2154 {
2155 struct socket *so = kn->kn_fp->f_data;
2156
2157 klist_remove(&so->so_rcv.sb_klist, kn);
2158 }
2159
2160 int
2161 filt_soread(struct knote *kn, long hint)
2162 {
2163 struct socket *so = kn->kn_fp->f_data;
2164 int rv = 0;
2165
2166 soassertlocked(so);
2167
2168 kn->kn_data = so->so_rcv.sb_cc;
2169 #ifdef SOCKET_SPLICE
2170 if (isspliced(so)) {
2171 rv = 0;
2172 } else
2173 #endif /* SOCKET_SPLICE */
2174 if (so->so_rcv.sb_state & SS_CANTRCVMORE) {
2175 kn->kn_flags |= EV_EOF;
2176 if (kn->kn_flags & __EV_POLL) {
2177 if (so->so_state & SS_ISDISCONNECTED)
2178 kn->kn_flags |= __EV_HUP;
2179 }
2180 kn->kn_fflags = so->so_error;
2181 rv = 1;
2182 } else if (so->so_error) { /* temporary udp error */
2183 rv = 1;
2184 } else if (kn->kn_sfflags & NOTE_LOWAT) {
2185 rv = (kn->kn_data >= kn->kn_sdata);
2186 } else {
2187 rv = (kn->kn_data >= so->so_rcv.sb_lowat);
2188 }
2189
2190 return rv;
2191 }
2192
2193 void
2194 filt_sowdetach(struct knote *kn)
2195 {
2196 struct socket *so = kn->kn_fp->f_data;
2197
2198 klist_remove(&so->so_snd.sb_klist, kn);
2199 }
2200
2201 int
2202 filt_sowrite(struct knote *kn, long hint)
2203 {
2204 struct socket *so = kn->kn_fp->f_data;
2205 int rv;
2206
2207 soassertlocked(so);
2208
2209 kn->kn_data = sbspace(so, &so->so_snd);
2210 if (so->so_snd.sb_state & SS_CANTSENDMORE) {
2211 kn->kn_flags |= EV_EOF;
2212 if (kn->kn_flags & __EV_POLL) {
2213 if (so->so_state & SS_ISDISCONNECTED)
2214 kn->kn_flags |= __EV_HUP;
2215 }
2216 kn->kn_fflags = so->so_error;
2217 rv = 1;
2218 } else if (so->so_error) { /* temporary udp error */
2219 rv = 1;
2220 } else if (((so->so_state & SS_ISCONNECTED) == 0) &&
2221 (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
2222 rv = 0;
2223 } else if (kn->kn_sfflags & NOTE_LOWAT) {
2224 rv = (kn->kn_data >= kn->kn_sdata);
2225 } else {
2226 rv = (kn->kn_data >= so->so_snd.sb_lowat);
2227 }
2228
2229 return (rv);
2230 }
2231
2232 int
2233 filt_soexcept(struct knote *kn, long hint)
2234 {
2235 struct socket *so = kn->kn_fp->f_data;
2236 int rv = 0;
2237
2238 soassertlocked(so);
2239
2240 #ifdef SOCKET_SPLICE
2241 if (isspliced(so)) {
2242 rv = 0;
2243 } else
2244 #endif /* SOCKET_SPLICE */
2245 if (kn->kn_sfflags & NOTE_OOB) {
2246 if (so->so_oobmark || (so->so_rcv.sb_state & SS_RCVATMARK)) {
2247 kn->kn_fflags |= NOTE_OOB;
2248 kn->kn_data -= so->so_oobmark;
2249 rv = 1;
2250 }
2251 }
2252
2253 if (kn->kn_flags & __EV_POLL) {
2254 if (so->so_state & SS_ISDISCONNECTED) {
2255 kn->kn_flags |= __EV_HUP;
2256 rv = 1;
2257 }
2258 }
2259
2260 return rv;
2261 }
2262
2263 int
2264 filt_solisten(struct knote *kn, long hint)
2265 {
2266 struct socket *so = kn->kn_fp->f_data;
2267 int active;
2268
2269 soassertlocked(so);
2270
2271 kn->kn_data = so->so_qlen;
2272 active = (kn->kn_data != 0);
2273
2274 if (kn->kn_flags & (__EV_POLL | __EV_SELECT)) {
2275 if (so->so_state & SS_ISDISCONNECTED) {
2276 kn->kn_flags |= __EV_HUP;
2277 active = 1;
2278 } else {
2279 active = soreadable(so);
2280 }
2281 }
2282
2283 return (active);
2284 }
2285
2286 int
2287 filt_somodify(struct kevent *kev, struct knote *kn)
2288 {
2289 struct socket *so = kn->kn_fp->f_data;
2290 int rv;
2291
2292 solock(so);
2293 rv = knote_modify(kev, kn);
2294 sounlock(so);
2295
2296 return (rv);
2297 }
2298
2299 int
2300 filt_soprocess(struct knote *kn, struct kevent *kev)
2301 {
2302 struct socket *so = kn->kn_fp->f_data;
2303 int rv;
2304
2305 solock(so);
2306 rv = knote_process(kn, kev);
2307 sounlock(so);
2308
2309 return (rv);
2310 }
2311
2312 void
2313 klist_soassertlk(void *arg)
2314 {
2315 struct socket *so = arg;
2316
2317 soassertlocked(so);
2318 }
2319
2320 int
2321 klist_solock(void *arg)
2322 {
2323 struct socket *so = arg;
2324
2325 solock(so);
2326 return (1);
2327 }
2328
2329 void
2330 klist_sounlock(void *arg, int ls)
2331 {
2332 struct socket *so = arg;
2333
2334 sounlock(so);
2335 }
2336
2337 const struct klistops socket_klistops = {
2338 .klo_assertlk = klist_soassertlk,
2339 .klo_lock = klist_solock,
2340 .klo_unlock = klist_sounlock,
2341 };
2342
2343 #ifdef DDB
2344 void
2345 sobuf_print(struct sockbuf *,
2346 int (*)(const char *, ...) __attribute__((__format__(__kprintf__,1,2))));
2347
2348 void
2349 sobuf_print(struct sockbuf *sb,
2350 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2))))
2351 {
2352 (*pr)("\tsb_cc: %lu\n", sb->sb_cc);
2353 (*pr)("\tsb_datacc: %lu\n", sb->sb_datacc);
2354 (*pr)("\tsb_hiwat: %lu\n", sb->sb_hiwat);
2355 (*pr)("\tsb_wat: %lu\n", sb->sb_wat);
2356 (*pr)("\tsb_mbcnt: %lu\n", sb->sb_mbcnt);
2357 (*pr)("\tsb_mbmax: %lu\n", sb->sb_mbmax);
2358 (*pr)("\tsb_lowat: %ld\n", sb->sb_lowat);
2359 (*pr)("\tsb_mb: %p\n", sb->sb_mb);
2360 (*pr)("\tsb_mbtail: %p\n", sb->sb_mbtail);
2361 (*pr)("\tsb_lastrecord: %p\n", sb->sb_lastrecord);
2362 (*pr)("\tsb_sel: ...\n");
2363 (*pr)("\tsb_flags: %i\n", sb->sb_flags);
2364 (*pr)("\tsb_timeo_nsecs: %llu\n", sb->sb_timeo_nsecs);
2365 }
2366
2367 void
2368 so_print(void *v,
2369 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2))))
2370 {
2371 struct socket *so = v;
2372
2373 (*pr)("socket %p\n", so);
2374 (*pr)("so_type: %i\n", so->so_type);
2375 (*pr)("so_options: 0x%04x\n", so->so_options); /* %b */
2376 (*pr)("so_linger: %i\n", so->so_linger);
2377 (*pr)("so_state: 0x%04x\n", so->so_state);
2378 (*pr)("so_pcb: %p\n", so->so_pcb);
2379 (*pr)("so_proto: %p\n", so->so_proto);
2380 (*pr)("so_sigio: %p\n", so->so_sigio.sir_sigio);
2381
2382 (*pr)("so_head: %p\n", so->so_head);
2383 (*pr)("so_onq: %p\n", so->so_onq);
2384 (*pr)("so_q0: @%p first: %p\n", &so->so_q0, TAILQ_FIRST(&so->so_q0));
2385 (*pr)("so_q: @%p first: %p\n", &so->so_q, TAILQ_FIRST(&so->so_q));
2386 (*pr)("so_eq: next: %p\n", TAILQ_NEXT(so, so_qe));
2387 (*pr)("so_q0len: %i\n", so->so_q0len);
2388 (*pr)("so_qlen: %i\n", so->so_qlen);
2389 (*pr)("so_qlimit: %i\n", so->so_qlimit);
2390 (*pr)("so_timeo: %i\n", so->so_timeo);
2391 (*pr)("so_obmark: %lu\n", so->so_oobmark);
2392
2393 (*pr)("so_sp: %p\n", so->so_sp);
2394 if (so->so_sp != NULL) {
2395 (*pr)("\tssp_socket: %p\n", so->so_sp->ssp_socket);
2396 (*pr)("\tssp_soback: %p\n", so->so_sp->ssp_soback);
2397 (*pr)("\tssp_len: %lld\n",
2398 (unsigned long long)so->so_sp->ssp_len);
2399 (*pr)("\tssp_max: %lld\n",
2400 (unsigned long long)so->so_sp->ssp_max);
2401 (*pr)("\tssp_idletv: %lld %ld\n", so->so_sp->ssp_idletv.tv_sec,
2402 so->so_sp->ssp_idletv.tv_usec);
2403 (*pr)("\tssp_idleto: %spending (@%i)\n",
2404 timeout_pending(&so->so_sp->ssp_idleto) ? "" : "not ",
2405 so->so_sp->ssp_idleto.to_time);
2406 }
2407
2408 (*pr)("so_rcv:\n");
2409 sobuf_print(&so->so_rcv, pr);
2410 (*pr)("so_snd:\n");
2411 sobuf_print(&so->so_snd, pr);
2412
2413 (*pr)("so_upcall: %p so_upcallarg: %p\n",
2414 so->so_upcall, so->so_upcallarg);
2415
2416 (*pr)("so_euid: %d so_ruid: %d\n", so->so_euid, so->so_ruid);
2417 (*pr)("so_egid: %d so_rgid: %d\n", so->so_egid, so->so_rgid);
2418 (*pr)("so_cpid: %d\n", so->so_cpid);
2419 }
2420 #endif
Cache object: c14499c50f31a4740fb1e9e5efa2b57a
|