The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/netinet/in_pcb.c

Version: -  FREEBSD  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-2  -  FREEBSD-11-1  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-4  -  FREEBSD-10-3  -  FREEBSD-10-2  -  FREEBSD-10-1  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-3  -  FREEBSD-9-2  -  FREEBSD-9-1  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-4  -  FREEBSD-8-3  -  FREEBSD-8-2  -  FREEBSD-8-1  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-4  -  FREEBSD-7-3  -  FREEBSD-7-2  -  FREEBSD-7-1  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-4  -  FREEBSD-6-3  -  FREEBSD-6-2  -  FREEBSD-6-1  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-5  -  FREEBSD-5-4  -  FREEBSD-5-3  -  FREEBSD-5-2  -  FREEBSD-5-1  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * SPDX-License-Identifier: BSD-3-Clause
    3  *
    4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
    5  *      The Regents of the University of California.
    6  * Copyright (c) 2007-2009 Robert N. M. Watson
    7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
    8  * All rights reserved.
    9  *
   10  * Portions of this software were developed by Robert N. M. Watson under
   11  * contract to Juniper Networks, Inc.
   12  *
   13  * Redistribution and use in source and binary forms, with or without
   14  * modification, are permitted provided that the following conditions
   15  * are met:
   16  * 1. Redistributions of source code must retain the above copyright
   17  *    notice, this list of conditions and the following disclaimer.
   18  * 2. Redistributions in binary form must reproduce the above copyright
   19  *    notice, this list of conditions and the following disclaimer in the
   20  *    documentation and/or other materials provided with the distribution.
   21  * 3. Neither the name of the University nor the names of its contributors
   22  *    may be used to endorse or promote products derived from this software
   23  *    without specific prior written permission.
   24  *
   25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   35  * SUCH DAMAGE.
   36  *
   37  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
   38  */
   39 
   40 #include <sys/cdefs.h>
   41 __FBSDID("$FreeBSD: head/sys/netinet/in_pcb.c 345712 2019-03-29 19:47:42Z jhb $");
   42 
   43 #include "opt_ddb.h"
   44 #include "opt_ipsec.h"
   45 #include "opt_inet.h"
   46 #include "opt_inet6.h"
   47 #include "opt_ratelimit.h"
   48 #include "opt_pcbgroup.h"
   49 #include "opt_rss.h"
   50 
   51 #include <sys/param.h>
   52 #include <sys/systm.h>
   53 #include <sys/lock.h>
   54 #include <sys/malloc.h>
   55 #include <sys/mbuf.h>
   56 #include <sys/callout.h>
   57 #include <sys/eventhandler.h>
   58 #include <sys/domain.h>
   59 #include <sys/protosw.h>
   60 #include <sys/rmlock.h>
   61 #include <sys/smp.h>
   62 #include <sys/socket.h>
   63 #include <sys/socketvar.h>
   64 #include <sys/sockio.h>
   65 #include <sys/priv.h>
   66 #include <sys/proc.h>
   67 #include <sys/refcount.h>
   68 #include <sys/jail.h>
   69 #include <sys/kernel.h>
   70 #include <sys/sysctl.h>
   71 
   72 #ifdef DDB
   73 #include <ddb/ddb.h>
   74 #endif
   75 
   76 #include <vm/uma.h>
   77 
   78 #include <net/if.h>
   79 #include <net/if_var.h>
   80 #include <net/if_types.h>
   81 #include <net/if_llatbl.h>
   82 #include <net/route.h>
   83 #include <net/rss_config.h>
   84 #include <net/vnet.h>
   85 
   86 #if defined(INET) || defined(INET6)
   87 #include <netinet/in.h>
   88 #include <netinet/in_pcb.h>
   89 #include <netinet/ip_var.h>
   90 #include <netinet/tcp_var.h>
   91 #ifdef TCPHPTS
   92 #include <netinet/tcp_hpts.h>
   93 #endif
   94 #include <netinet/udp.h>
   95 #include <netinet/udp_var.h>
   96 #endif
   97 #ifdef INET
   98 #include <netinet/in_var.h>
   99 #endif
  100 #ifdef INET6
  101 #include <netinet/ip6.h>
  102 #include <netinet6/in6_pcb.h>
  103 #include <netinet6/in6_var.h>
  104 #include <netinet6/ip6_var.h>
  105 #endif /* INET6 */
  106 
  107 #include <netipsec/ipsec_support.h>
  108 
  109 #include <security/mac/mac_framework.h>
  110 
  111 #define INPCBLBGROUP_SIZMIN     8
  112 #define INPCBLBGROUP_SIZMAX     256
  113 
  114 static struct callout   ipport_tick_callout;
  115 
  116 /*
  117  * These configure the range of local port addresses assigned to
  118  * "unspecified" outgoing connections/packets/whatever.
  119  */
  120 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;    /* 1023 */
  121 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;    /* 600 */
  122 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;     /* 10000 */
  123 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;       /* 65535 */
  124 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;      /* 49152 */
  125 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;        /* 65535 */
  126 
  127 /*
  128  * Reserved ports accessible only to root. There are significant
  129  * security considerations that must be accounted for when changing these,
  130  * but the security benefits can be great. Please be careful.
  131  */
  132 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;    /* 1023 */
  133 VNET_DEFINE(int, ipport_reservedlow);
  134 
  135 /* Variables dealing with random ephemeral port allocation. */
  136 VNET_DEFINE(int, ipport_randomized) = 1;        /* user controlled via sysctl */
  137 VNET_DEFINE(int, ipport_randomcps) = 10;        /* user controlled via sysctl */
  138 VNET_DEFINE(int, ipport_randomtime) = 45;       /* user controlled via sysctl */
  139 VNET_DEFINE(int, ipport_stoprandom);            /* toggled by ipport_tick */
  140 VNET_DEFINE(int, ipport_tcpallocs);
  141 VNET_DEFINE_STATIC(int, ipport_tcplastcount);
  142 
  143 #define V_ipport_tcplastcount           VNET(ipport_tcplastcount)
  144 
  145 static void     in_pcbremlists(struct inpcb *inp);
  146 #ifdef INET
  147 static struct inpcb     *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
  148                             struct in_addr faddr, u_int fport_arg,
  149                             struct in_addr laddr, u_int lport_arg,
  150                             int lookupflags, struct ifnet *ifp);
  151 
  152 #define RANGECHK(var, min, max) \
  153         if ((var) < (min)) { (var) = (min); } \
  154         else if ((var) > (max)) { (var) = (max); }
  155 
  156 static int
  157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
  158 {
  159         int error;
  160 
  161         error = sysctl_handle_int(oidp, arg1, arg2, req);
  162         if (error == 0) {
  163                 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
  164                 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
  165                 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
  166                 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
  167                 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
  168                 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
  169         }
  170         return (error);
  171 }
  172 
  173 #undef RANGECHK
  174 
  175 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0,
  176     "IP Ports");
  177 
  178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
  179         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
  180         &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", "");
  181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
  182         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
  183         &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", "");
  184 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
  185         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
  186         &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", "");
  187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
  188         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
  189         &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", "");
  190 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
  191         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
  192         &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", "");
  193 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
  194         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
  195         &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", "");
  196 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
  197         CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
  198         &VNET_NAME(ipport_reservedhigh), 0, "");
  199 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
  200         CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
  201 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
  202         CTLFLAG_VNET | CTLFLAG_RW,
  203         &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
  204 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
  205         CTLFLAG_VNET | CTLFLAG_RW,
  206         &VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
  207         "allocations before switching to a sequental one");
  208 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
  209         CTLFLAG_VNET | CTLFLAG_RW,
  210         &VNET_NAME(ipport_randomtime), 0,
  211         "Minimum time to keep sequental port "
  212         "allocation before switching to a random one");
  213 #endif /* INET */
  214 
  215 /*
  216  * in_pcb.c: manage the Protocol Control Blocks.
  217  *
  218  * NOTE: It is assumed that most of these functions will be called with
  219  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
  220  * functions often modify hash chains or addresses in pcbs.
  221  */
  222 
  223 static struct inpcblbgroup *
  224 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
  225     uint16_t port, const union in_dependaddr *addr, int size)
  226 {
  227         struct inpcblbgroup *grp;
  228         size_t bytes;
  229 
  230         bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
  231         grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
  232         if (!grp)
  233                 return (NULL);
  234         grp->il_vflag = vflag;
  235         grp->il_lport = port;
  236         grp->il_dependladdr = *addr;
  237         grp->il_inpsiz = size;
  238         CK_LIST_INSERT_HEAD(hdr, grp, il_list);
  239         return (grp);
  240 }
  241 
  242 static void
  243 in_pcblbgroup_free_deferred(epoch_context_t ctx)
  244 {
  245         struct inpcblbgroup *grp;
  246 
  247         grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
  248         free(grp, M_PCB);
  249 }
  250 
  251 static void
  252 in_pcblbgroup_free(struct inpcblbgroup *grp)
  253 {
  254 
  255         CK_LIST_REMOVE(grp, il_list);
  256         epoch_call(net_epoch_preempt, &grp->il_epoch_ctx,
  257             in_pcblbgroup_free_deferred);
  258 }
  259 
  260 static struct inpcblbgroup *
  261 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
  262     struct inpcblbgroup *old_grp, int size)
  263 {
  264         struct inpcblbgroup *grp;
  265         int i;
  266 
  267         grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,
  268             old_grp->il_lport, &old_grp->il_dependladdr, size);
  269         if (grp == NULL)
  270                 return (NULL);
  271 
  272         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
  273             ("invalid new local group size %d and old local group count %d",
  274              grp->il_inpsiz, old_grp->il_inpcnt));
  275 
  276         for (i = 0; i < old_grp->il_inpcnt; ++i)
  277                 grp->il_inp[i] = old_grp->il_inp[i];
  278         grp->il_inpcnt = old_grp->il_inpcnt;
  279         in_pcblbgroup_free(old_grp);
  280         return (grp);
  281 }
  282 
  283 /*
  284  * PCB at index 'i' is removed from the group. Pull up the ones below il_inp[i]
  285  * and shrink group if possible.
  286  */
  287 static void
  288 in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp,
  289     int i)
  290 {
  291         struct inpcblbgroup *grp, *new_grp;
  292 
  293         grp = *grpp;
  294         for (; i + 1 < grp->il_inpcnt; ++i)
  295                 grp->il_inp[i] = grp->il_inp[i + 1];
  296         grp->il_inpcnt--;
  297 
  298         if (grp->il_inpsiz > INPCBLBGROUP_SIZMIN &&
  299             grp->il_inpcnt <= grp->il_inpsiz / 4) {
  300                 /* Shrink this group. */
  301                 new_grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz / 2);
  302                 if (new_grp != NULL)
  303                         *grpp = new_grp;
  304         }
  305 }
  306 
  307 /*
  308  * Add PCB to load balance group for SO_REUSEPORT_LB option.
  309  */
  310 static int
  311 in_pcbinslbgrouphash(struct inpcb *inp)
  312 {
  313         const static struct timeval interval = { 60, 0 };
  314         static struct timeval lastprint;
  315         struct inpcbinfo *pcbinfo;
  316         struct inpcblbgrouphead *hdr;
  317         struct inpcblbgroup *grp;
  318         uint32_t idx;
  319 
  320         pcbinfo = inp->inp_pcbinfo;
  321 
  322         INP_WLOCK_ASSERT(inp);
  323         INP_HASH_WLOCK_ASSERT(pcbinfo);
  324 
  325         /*
  326          * Don't allow jailed socket to join local group.
  327          */
  328         if (inp->inp_socket != NULL && jailed(inp->inp_socket->so_cred))
  329                 return (0);
  330 
  331 #ifdef INET6
  332         /*
  333          * Don't allow IPv4 mapped INET6 wild socket.
  334          */
  335         if ((inp->inp_vflag & INP_IPV4) &&
  336             inp->inp_laddr.s_addr == INADDR_ANY &&
  337             INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
  338                 return (0);
  339         }
  340 #endif
  341 
  342         idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
  343         hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
  344         CK_LIST_FOREACH(grp, hdr, il_list) {
  345                 if (grp->il_vflag == inp->inp_vflag &&
  346                     grp->il_lport == inp->inp_lport &&
  347                     memcmp(&grp->il_dependladdr,
  348                     &inp->inp_inc.inc_ie.ie_dependladdr,
  349                     sizeof(grp->il_dependladdr)) == 0)
  350                         break;
  351         }
  352         if (grp == NULL) {
  353                 /* Create new load balance group. */
  354                 grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,
  355                     inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
  356                     INPCBLBGROUP_SIZMIN);
  357                 if (grp == NULL)
  358                         return (ENOBUFS);
  359         } else if (grp->il_inpcnt == grp->il_inpsiz) {
  360                 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
  361                         if (ratecheck(&lastprint, &interval))
  362                                 printf("lb group port %d, limit reached\n",
  363                                     ntohs(grp->il_lport));
  364                         return (0);
  365                 }
  366 
  367                 /* Expand this local group. */
  368                 grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
  369                 if (grp == NULL)
  370                         return (ENOBUFS);
  371         }
  372 
  373         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
  374             ("invalid local group size %d and count %d", grp->il_inpsiz,
  375             grp->il_inpcnt));
  376 
  377         grp->il_inp[grp->il_inpcnt] = inp;
  378         grp->il_inpcnt++;
  379         return (0);
  380 }
  381 
  382 /*
  383  * Remove PCB from load balance group.
  384  */
  385 static void
  386 in_pcbremlbgrouphash(struct inpcb *inp)
  387 {
  388         struct inpcbinfo *pcbinfo;
  389         struct inpcblbgrouphead *hdr;
  390         struct inpcblbgroup *grp;
  391         int i;
  392 
  393         pcbinfo = inp->inp_pcbinfo;
  394 
  395         INP_WLOCK_ASSERT(inp);
  396         INP_HASH_WLOCK_ASSERT(pcbinfo);
  397 
  398         hdr = &pcbinfo->ipi_lbgrouphashbase[
  399             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
  400         CK_LIST_FOREACH(grp, hdr, il_list) {
  401                 for (i = 0; i < grp->il_inpcnt; ++i) {
  402                         if (grp->il_inp[i] != inp)
  403                                 continue;
  404 
  405                         if (grp->il_inpcnt == 1) {
  406                                 /* We are the last, free this local group. */
  407                                 in_pcblbgroup_free(grp);
  408                         } else {
  409                                 /* Pull up inpcbs, shrink group if possible. */
  410                                 in_pcblbgroup_reorder(hdr, &grp, i);
  411                         }
  412                         return;
  413                 }
  414         }
  415 }
  416 
  417 /*
  418  * Different protocols initialize their inpcbs differently - giving
  419  * different name to the lock.  But they all are disposed the same.
  420  */
  421 static void
  422 inpcb_fini(void *mem, int size)
  423 {
  424         struct inpcb *inp = mem;
  425 
  426         INP_LOCK_DESTROY(inp);
  427 }
  428 
  429 /*
  430  * Initialize an inpcbinfo -- we should be able to reduce the number of
  431  * arguments in time.
  432  */
  433 void
  434 in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
  435     struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
  436     char *inpcbzone_name, uma_init inpcbzone_init, u_int hashfields)
  437 {
  438 
  439         porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
  440 
  441         INP_INFO_LOCK_INIT(pcbinfo, name);
  442         INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash");     /* XXXRW: argument? */
  443         INP_LIST_LOCK_INIT(pcbinfo, "pcbinfolist");
  444 #ifdef VIMAGE
  445         pcbinfo->ipi_vnet = curvnet;
  446 #endif
  447         pcbinfo->ipi_listhead = listhead;
  448         CK_LIST_INIT(pcbinfo->ipi_listhead);
  449         pcbinfo->ipi_count = 0;
  450         pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
  451             &pcbinfo->ipi_hashmask);
  452         pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
  453             &pcbinfo->ipi_porthashmask);
  454         pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
  455             &pcbinfo->ipi_lbgrouphashmask);
  456 #ifdef PCBGROUP
  457         in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
  458 #endif
  459         pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
  460             NULL, NULL, inpcbzone_init, inpcb_fini, UMA_ALIGN_PTR, 0);
  461         uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
  462         uma_zone_set_warning(pcbinfo->ipi_zone,
  463             "kern.ipc.maxsockets limit reached");
  464 }
  465 
  466 /*
  467  * Destroy an inpcbinfo.
  468  */
  469 void
  470 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
  471 {
  472 
  473         KASSERT(pcbinfo->ipi_count == 0,
  474             ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
  475 
  476         hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
  477         hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
  478             pcbinfo->ipi_porthashmask);
  479         hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
  480             pcbinfo->ipi_lbgrouphashmask);
  481 #ifdef PCBGROUP
  482         in_pcbgroup_destroy(pcbinfo);
  483 #endif
  484         uma_zdestroy(pcbinfo->ipi_zone);
  485         INP_LIST_LOCK_DESTROY(pcbinfo);
  486         INP_HASH_LOCK_DESTROY(pcbinfo);
  487         INP_INFO_LOCK_DESTROY(pcbinfo);
  488 }
  489 
  490 /*
  491  * Allocate a PCB and associate it with the socket.
  492  * On success return with the PCB locked.
  493  */
  494 int
  495 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
  496 {
  497         struct inpcb *inp;
  498         int error;
  499 
  500 #ifdef INVARIANTS
  501         if (pcbinfo == &V_tcbinfo) {
  502                 INP_INFO_RLOCK_ASSERT(pcbinfo);
  503         } else {
  504                 INP_INFO_WLOCK_ASSERT(pcbinfo);
  505         }
  506 #endif
  507 
  508         error = 0;
  509         inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
  510         if (inp == NULL)
  511                 return (ENOBUFS);
  512         bzero(&inp->inp_start_zero, inp_zero_size);
  513         inp->inp_pcbinfo = pcbinfo;
  514         inp->inp_socket = so;
  515         inp->inp_cred = crhold(so->so_cred);
  516         inp->inp_inc.inc_fibnum = so->so_fibnum;
  517 #ifdef MAC
  518         error = mac_inpcb_init(inp, M_NOWAIT);
  519         if (error != 0)
  520                 goto out;
  521         mac_inpcb_create(so, inp);
  522 #endif
  523 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
  524         error = ipsec_init_pcbpolicy(inp);
  525         if (error != 0) {
  526 #ifdef MAC
  527                 mac_inpcb_destroy(inp);
  528 #endif
  529                 goto out;
  530         }
  531 #endif /*IPSEC*/
  532 #ifdef INET6
  533         if (INP_SOCKAF(so) == AF_INET6) {
  534                 inp->inp_vflag |= INP_IPV6PROTO;
  535                 if (V_ip6_v6only)
  536                         inp->inp_flags |= IN6P_IPV6_V6ONLY;
  537         }
  538 #endif
  539         INP_WLOCK(inp);
  540         INP_LIST_WLOCK(pcbinfo);
  541         CK_LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
  542         pcbinfo->ipi_count++;
  543         so->so_pcb = (caddr_t)inp;
  544 #ifdef INET6
  545         if (V_ip6_auto_flowlabel)
  546                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
  547 #endif
  548         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
  549         refcount_init(&inp->inp_refcount, 1);   /* Reference from inpcbinfo */
  550 
  551         /*
  552          * Routes in inpcb's can cache L2 as well; they are guaranteed
  553          * to be cleaned up.
  554          */
  555         inp->inp_route.ro_flags = RT_LLE_CACHE;
  556         INP_LIST_WUNLOCK(pcbinfo);
  557 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
  558 out:
  559         if (error != 0) {
  560                 crfree(inp->inp_cred);
  561                 uma_zfree(pcbinfo->ipi_zone, inp);
  562         }
  563 #endif
  564         return (error);
  565 }
  566 
  567 #ifdef INET
  568 int
  569 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
  570 {
  571         int anonport, error;
  572 
  573         INP_WLOCK_ASSERT(inp);
  574         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
  575 
  576         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
  577                 return (EINVAL);
  578         anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
  579         error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
  580             &inp->inp_lport, cred);
  581         if (error)
  582                 return (error);
  583         if (in_pcbinshash(inp) != 0) {
  584                 inp->inp_laddr.s_addr = INADDR_ANY;
  585                 inp->inp_lport = 0;
  586                 return (EAGAIN);
  587         }
  588         if (anonport)
  589                 inp->inp_flags |= INP_ANONPORT;
  590         return (0);
  591 }
  592 #endif
  593 
  594 /*
  595  * Select a local port (number) to use.
  596  */
  597 #if defined(INET) || defined(INET6)
  598 int
  599 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
  600     struct ucred *cred, int lookupflags)
  601 {
  602         struct inpcbinfo *pcbinfo;
  603         struct inpcb *tmpinp;
  604         unsigned short *lastport;
  605         int count, dorandom, error;
  606         u_short aux, first, last, lport;
  607 #ifdef INET
  608         struct in_addr laddr;
  609 #endif
  610 
  611         pcbinfo = inp->inp_pcbinfo;
  612 
  613         /*
  614          * Because no actual state changes occur here, a global write lock on
  615          * the pcbinfo isn't required.
  616          */
  617         INP_LOCK_ASSERT(inp);
  618         INP_HASH_LOCK_ASSERT(pcbinfo);
  619 
  620         if (inp->inp_flags & INP_HIGHPORT) {
  621                 first = V_ipport_hifirstauto;   /* sysctl */
  622                 last  = V_ipport_hilastauto;
  623                 lastport = &pcbinfo->ipi_lasthi;
  624         } else if (inp->inp_flags & INP_LOWPORT) {
  625                 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
  626                 if (error)
  627                         return (error);
  628                 first = V_ipport_lowfirstauto;  /* 1023 */
  629                 last  = V_ipport_lowlastauto;   /* 600 */
  630                 lastport = &pcbinfo->ipi_lastlow;
  631         } else {
  632                 first = V_ipport_firstauto;     /* sysctl */
  633                 last  = V_ipport_lastauto;
  634                 lastport = &pcbinfo->ipi_lastport;
  635         }
  636         /*
  637          * For UDP(-Lite), use random port allocation as long as the user
  638          * allows it.  For TCP (and as of yet unknown) connections,
  639          * use random port allocation only if the user allows it AND
  640          * ipport_tick() allows it.
  641          */
  642         if (V_ipport_randomized &&
  643                 (!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
  644                 pcbinfo == &V_ulitecbinfo))
  645                 dorandom = 1;
  646         else
  647                 dorandom = 0;
  648         /*
  649          * It makes no sense to do random port allocation if
  650          * we have the only port available.
  651          */
  652         if (first == last)
  653                 dorandom = 0;
  654         /* Make sure to not include UDP(-Lite) packets in the count. */
  655         if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
  656                 V_ipport_tcpallocs++;
  657         /*
  658          * Instead of having two loops further down counting up or down
  659          * make sure that first is always <= last and go with only one
  660          * code path implementing all logic.
  661          */
  662         if (first > last) {
  663                 aux = first;
  664                 first = last;
  665                 last = aux;
  666         }
  667 
  668 #ifdef INET
  669         /* Make the compiler happy. */
  670         laddr.s_addr = 0;
  671         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
  672                 KASSERT(laddrp != NULL, ("%s: laddrp NULL for v4 inp %p",
  673                     __func__, inp));
  674                 laddr = *laddrp;
  675         }
  676 #endif
  677         tmpinp = NULL;  /* Make compiler happy. */
  678         lport = *lportp;
  679 
  680         if (dorandom)
  681                 *lastport = first + (arc4random() % (last - first));
  682 
  683         count = last - first;
  684 
  685         do {
  686                 if (count-- < 0)        /* completely used? */
  687                         return (EADDRNOTAVAIL);
  688                 ++*lastport;
  689                 if (*lastport < first || *lastport > last)
  690                         *lastport = first;
  691                 lport = htons(*lastport);
  692 
  693 #ifdef INET6
  694                 if ((inp->inp_vflag & INP_IPV6) != 0)
  695                         tmpinp = in6_pcblookup_local(pcbinfo,
  696                             &inp->in6p_laddr, lport, lookupflags, cred);
  697 #endif
  698 #if defined(INET) && defined(INET6)
  699                 else
  700 #endif
  701 #ifdef INET
  702                         tmpinp = in_pcblookup_local(pcbinfo, laddr,
  703                             lport, lookupflags, cred);
  704 #endif
  705         } while (tmpinp != NULL);
  706 
  707 #ifdef INET
  708         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4)
  709                 laddrp->s_addr = laddr.s_addr;
  710 #endif
  711         *lportp = lport;
  712 
  713         return (0);
  714 }
  715 
  716 /*
  717  * Return cached socket options.
  718  */
  719 int
  720 inp_so_options(const struct inpcb *inp)
  721 {
  722         int so_options;
  723 
  724         so_options = 0;
  725 
  726         if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0)
  727                 so_options |= SO_REUSEPORT_LB;
  728         if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
  729                 so_options |= SO_REUSEPORT;
  730         if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
  731                 so_options |= SO_REUSEADDR;
  732         return (so_options);
  733 }
  734 #endif /* INET || INET6 */
  735 
  736 /*
  737  * Check if a new BINDMULTI socket is allowed to be created.
  738  *
  739  * ni points to the new inp.
  740  * oi points to the exisitng inp.
  741  *
  742  * This checks whether the existing inp also has BINDMULTI and
  743  * whether the credentials match.
  744  */
  745 int
  746 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
  747 {
  748         /* Check permissions match */
  749         if ((ni->inp_flags2 & INP_BINDMULTI) &&
  750             (ni->inp_cred->cr_uid !=
  751             oi->inp_cred->cr_uid))
  752                 return (0);
  753 
  754         /* Check the existing inp has BINDMULTI set */
  755         if ((ni->inp_flags2 & INP_BINDMULTI) &&
  756             ((oi->inp_flags2 & INP_BINDMULTI) == 0))
  757                 return (0);
  758 
  759         /*
  760          * We're okay - either INP_BINDMULTI isn't set on ni, or
  761          * it is and it matches the checks.
  762          */
  763         return (1);
  764 }
  765 
  766 #ifdef INET
  767 /*
  768  * Set up a bind operation on a PCB, performing port allocation
  769  * as required, but do not actually modify the PCB. Callers can
  770  * either complete the bind by setting inp_laddr/inp_lport and
  771  * calling in_pcbinshash(), or they can just use the resulting
  772  * port and address to authorise the sending of a once-off packet.
  773  *
  774  * On error, the values of *laddrp and *lportp are not changed.
  775  */
  776 int
  777 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
  778     u_short *lportp, struct ucred *cred)
  779 {
  780         struct socket *so = inp->inp_socket;
  781         struct sockaddr_in *sin;
  782         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
  783         struct in_addr laddr;
  784         u_short lport = 0;
  785         int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
  786         int error;
  787 
  788         /*
  789          * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here
  790          * so that we don't have to add to the (already messy) code below.
  791          */
  792         int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
  793 
  794         /*
  795          * No state changes, so read locks are sufficient here.
  796          */
  797         INP_LOCK_ASSERT(inp);
  798         INP_HASH_LOCK_ASSERT(pcbinfo);
  799 
  800         if (CK_STAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
  801                 return (EADDRNOTAVAIL);
  802         laddr.s_addr = *laddrp;
  803         if (nam != NULL && laddr.s_addr != INADDR_ANY)
  804                 return (EINVAL);
  805         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
  806                 lookupflags = INPLOOKUP_WILDCARD;
  807         if (nam == NULL) {
  808                 if ((error = prison_local_ip4(cred, &laddr)) != 0)
  809                         return (error);
  810         } else {
  811                 sin = (struct sockaddr_in *)nam;
  812                 if (nam->sa_len != sizeof (*sin))
  813                         return (EINVAL);
  814 #ifdef notdef
  815                 /*
  816                  * We should check the family, but old programs
  817                  * incorrectly fail to initialize it.
  818                  */
  819                 if (sin->sin_family != AF_INET)
  820                         return (EAFNOSUPPORT);
  821 #endif
  822                 error = prison_local_ip4(cred, &sin->sin_addr);
  823                 if (error)
  824                         return (error);
  825                 if (sin->sin_port != *lportp) {
  826                         /* Don't allow the port to change. */
  827                         if (*lportp != 0)
  828                                 return (EINVAL);
  829                         lport = sin->sin_port;
  830                 }
  831                 /* NB: lport is left as 0 if the port isn't being changed. */
  832                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
  833                         /*
  834                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
  835                          * allow complete duplication of binding if
  836                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
  837                          * and a multicast address is bound on both
  838                          * new and duplicated sockets.
  839                          */
  840                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
  841                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
  842                         /*
  843                          * XXX: How to deal with SO_REUSEPORT_LB here?
  844                          * Treat same as SO_REUSEPORT for now.
  845                          */
  846                         if ((so->so_options &
  847                             (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
  848                                 reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
  849                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
  850                         sin->sin_port = 0;              /* yech... */
  851                         bzero(&sin->sin_zero, sizeof(sin->sin_zero));
  852                         /*
  853                          * Is the address a local IP address?
  854                          * If INP_BINDANY is set, then the socket may be bound
  855                          * to any endpoint address, local or not.
  856                          */
  857                         if ((inp->inp_flags & INP_BINDANY) == 0 &&
  858                             ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
  859                                 return (EADDRNOTAVAIL);
  860                 }
  861                 laddr = sin->sin_addr;
  862                 if (lport) {
  863                         struct inpcb *t;
  864                         struct tcptw *tw;
  865 
  866                         /* GROSS */
  867                         if (ntohs(lport) <= V_ipport_reservedhigh &&
  868                             ntohs(lport) >= V_ipport_reservedlow &&
  869                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
  870                                 return (EACCES);
  871                         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
  872                             priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
  873                                 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
  874                                     lport, INPLOOKUP_WILDCARD, cred);
  875         /*
  876          * XXX
  877          * This entire block sorely needs a rewrite.
  878          */
  879                                 if (t &&
  880                                     ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
  881                                     ((t->inp_flags & INP_TIMEWAIT) == 0) &&
  882                                     (so->so_type != SOCK_STREAM ||
  883                                      ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
  884                                     (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
  885                                      ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
  886                                      (t->inp_flags2 & INP_REUSEPORT) ||
  887                                      (t->inp_flags2 & INP_REUSEPORT_LB) == 0) &&
  888                                     (inp->inp_cred->cr_uid !=
  889                                      t->inp_cred->cr_uid))
  890                                         return (EADDRINUSE);
  891 
  892                                 /*
  893                                  * If the socket is a BINDMULTI socket, then
  894                                  * the credentials need to match and the
  895                                  * original socket also has to have been bound
  896                                  * with BINDMULTI.
  897                                  */
  898                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
  899                                         return (EADDRINUSE);
  900                         }
  901                         t = in_pcblookup_local(pcbinfo, sin->sin_addr,
  902                             lport, lookupflags, cred);
  903                         if (t && (t->inp_flags & INP_TIMEWAIT)) {
  904                                 /*
  905                                  * XXXRW: If an incpb has had its timewait
  906                                  * state recycled, we treat the address as
  907                                  * being in use (for now).  This is better
  908                                  * than a panic, but not desirable.
  909                                  */
  910                                 tw = intotw(t);
  911                                 if (tw == NULL ||
  912                                     ((reuseport & tw->tw_so_options) == 0 &&
  913                                         (reuseport_lb &
  914                                             tw->tw_so_options) == 0)) {
  915                                         return (EADDRINUSE);
  916                                 }
  917                         } else if (t &&
  918                                    ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
  919                                    (reuseport & inp_so_options(t)) == 0 &&
  920                                    (reuseport_lb & inp_so_options(t)) == 0) {
  921 #ifdef INET6
  922                                 if (ntohl(sin->sin_addr.s_addr) !=
  923                                     INADDR_ANY ||
  924                                     ntohl(t->inp_laddr.s_addr) !=
  925                                     INADDR_ANY ||
  926                                     (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
  927                                     (t->inp_vflag & INP_IPV6PROTO) == 0)
  928 #endif
  929                                                 return (EADDRINUSE);
  930                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
  931                                         return (EADDRINUSE);
  932                         }
  933                 }
  934         }
  935         if (*lportp != 0)
  936                 lport = *lportp;
  937         if (lport == 0) {
  938                 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
  939                 if (error != 0)
  940                         return (error);
  941 
  942         }
  943         *laddrp = laddr.s_addr;
  944         *lportp = lport;
  945         return (0);
  946 }
  947 
  948 /*
  949  * Connect from a socket to a specified address.
  950  * Both address and port must be specified in argument sin.
  951  * If don't have a local address for this socket yet,
  952  * then pick one.
  953  */
  954 int
  955 in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
  956     struct ucred *cred, struct mbuf *m)
  957 {
  958         u_short lport, fport;
  959         in_addr_t laddr, faddr;
  960         int anonport, error;
  961 
  962         INP_WLOCK_ASSERT(inp);
  963         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
  964 
  965         lport = inp->inp_lport;
  966         laddr = inp->inp_laddr.s_addr;
  967         anonport = (lport == 0);
  968         error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
  969             NULL, cred);
  970         if (error)
  971                 return (error);
  972 
  973         /* Do the initial binding of the local address if required. */
  974         if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
  975                 inp->inp_lport = lport;
  976                 inp->inp_laddr.s_addr = laddr;
  977                 if (in_pcbinshash(inp) != 0) {
  978                         inp->inp_laddr.s_addr = INADDR_ANY;
  979                         inp->inp_lport = 0;
  980                         return (EAGAIN);
  981                 }
  982         }
  983 
  984         /* Commit the remaining changes. */
  985         inp->inp_lport = lport;
  986         inp->inp_laddr.s_addr = laddr;
  987         inp->inp_faddr.s_addr = faddr;
  988         inp->inp_fport = fport;
  989         in_pcbrehash_mbuf(inp, m);
  990 
  991         if (anonport)
  992                 inp->inp_flags |= INP_ANONPORT;
  993         return (0);
  994 }
  995 
  996 int
  997 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
  998 {
  999 
 1000         return (in_pcbconnect_mbuf(inp, nam, cred, NULL));
 1001 }
 1002 
 1003 /*
 1004  * Do proper source address selection on an unbound socket in case
 1005  * of connect. Take jails into account as well.
 1006  */
 1007 int
 1008 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
 1009     struct ucred *cred)
 1010 {
 1011         struct ifaddr *ifa;
 1012         struct sockaddr *sa;
 1013         struct sockaddr_in *sin;
 1014         struct route sro;
 1015         struct epoch_tracker et;
 1016         int error;
 1017 
 1018         KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
 1019         /*
 1020          * Bypass source address selection and use the primary jail IP
 1021          * if requested.
 1022          */
 1023         if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
 1024                 return (0);
 1025 
 1026         error = 0;
 1027         bzero(&sro, sizeof(sro));
 1028 
 1029         sin = (struct sockaddr_in *)&sro.ro_dst;
 1030         sin->sin_family = AF_INET;
 1031         sin->sin_len = sizeof(struct sockaddr_in);
 1032         sin->sin_addr.s_addr = faddr->s_addr;
 1033 
 1034         /*
 1035          * If route is known our src addr is taken from the i/f,
 1036          * else punt.
 1037          *
 1038          * Find out route to destination.
 1039          */
 1040         if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
 1041                 in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
 1042 
 1043         /*
 1044          * If we found a route, use the address corresponding to
 1045          * the outgoing interface.
 1046          * 
 1047          * Otherwise assume faddr is reachable on a directly connected
 1048          * network and try to find a corresponding interface to take
 1049          * the source address from.
 1050          */
 1051         NET_EPOCH_ENTER(et);
 1052         if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
 1053                 struct in_ifaddr *ia;
 1054                 struct ifnet *ifp;
 1055 
 1056                 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
 1057                                         inp->inp_socket->so_fibnum));
 1058                 if (ia == NULL) {
 1059                         ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
 1060                                                 inp->inp_socket->so_fibnum));
 1061 
 1062                 }
 1063                 if (ia == NULL) {
 1064                         error = ENETUNREACH;
 1065                         goto done;
 1066                 }
 1067 
 1068                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 1069                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 1070                         goto done;
 1071                 }
 1072 
 1073                 ifp = ia->ia_ifp;
 1074                 ia = NULL;
 1075                 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 1076 
 1077                         sa = ifa->ifa_addr;
 1078                         if (sa->sa_family != AF_INET)
 1079                                 continue;
 1080                         sin = (struct sockaddr_in *)sa;
 1081                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 1082                                 ia = (struct in_ifaddr *)ifa;
 1083                                 break;
 1084                         }
 1085                 }
 1086                 if (ia != NULL) {
 1087                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 1088                         goto done;
 1089                 }
 1090 
 1091                 /* 3. As a last resort return the 'default' jail address. */
 1092                 error = prison_get_ip4(cred, laddr);
 1093                 goto done;
 1094         }
 1095 
 1096         /*
 1097          * If the outgoing interface on the route found is not
 1098          * a loopback interface, use the address from that interface.
 1099          * In case of jails do those three steps:
 1100          * 1. check if the interface address belongs to the jail. If so use it.
 1101          * 2. check if we have any address on the outgoing interface
 1102          *    belonging to this jail. If so use it.
 1103          * 3. as a last resort return the 'default' jail address.
 1104          */
 1105         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0) {
 1106                 struct in_ifaddr *ia;
 1107                 struct ifnet *ifp;
 1108 
 1109                 /* If not jailed, use the default returned. */
 1110                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 1111                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 1112                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 1113                         goto done;
 1114                 }
 1115 
 1116                 /* Jailed. */
 1117                 /* 1. Check if the iface address belongs to the jail. */
 1118                 sin = (struct sockaddr_in *)sro.ro_rt->rt_ifa->ifa_addr;
 1119                 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 1120                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 1121                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 1122                         goto done;
 1123                 }
 1124 
 1125                 /*
 1126                  * 2. Check if we have any address on the outgoing interface
 1127                  *    belonging to this jail.
 1128                  */
 1129                 ia = NULL;
 1130                 ifp = sro.ro_rt->rt_ifp;
 1131                 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 1132                         sa = ifa->ifa_addr;
 1133                         if (sa->sa_family != AF_INET)
 1134                                 continue;
 1135                         sin = (struct sockaddr_in *)sa;
 1136                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 1137                                 ia = (struct in_ifaddr *)ifa;
 1138                                 break;
 1139                         }
 1140                 }
 1141                 if (ia != NULL) {
 1142                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 1143                         goto done;
 1144                 }
 1145 
 1146                 /* 3. As a last resort return the 'default' jail address. */
 1147                 error = prison_get_ip4(cred, laddr);
 1148                 goto done;
 1149         }
 1150 
 1151         /*
 1152          * The outgoing interface is marked with 'loopback net', so a route
 1153          * to ourselves is here.
 1154          * Try to find the interface of the destination address and then
 1155          * take the address from there. That interface is not necessarily
 1156          * a loopback interface.
 1157          * In case of jails, check that it is an address of the jail
 1158          * and if we cannot find, fall back to the 'default' jail address.
 1159          */
 1160         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
 1161                 struct sockaddr_in sain;
 1162                 struct in_ifaddr *ia;
 1163 
 1164                 bzero(&sain, sizeof(struct sockaddr_in));
 1165                 sain.sin_family = AF_INET;
 1166                 sain.sin_len = sizeof(struct sockaddr_in);
 1167                 sain.sin_addr.s_addr = faddr->s_addr;
 1168 
 1169                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&sain),
 1170                                         inp->inp_socket->so_fibnum));
 1171                 if (ia == NULL)
 1172                         ia = ifatoia(ifa_ifwithnet(sintosa(&sain), 0,
 1173                                                 inp->inp_socket->so_fibnum));
 1174                 if (ia == NULL)
 1175                         ia = ifatoia(ifa_ifwithaddr(sintosa(&sain)));
 1176 
 1177                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 1178                         if (ia == NULL) {
 1179                                 error = ENETUNREACH;
 1180                                 goto done;
 1181                         }
 1182                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 1183                         goto done;
 1184                 }
 1185 
 1186                 /* Jailed. */
 1187                 if (ia != NULL) {
 1188                         struct ifnet *ifp;
 1189 
 1190                         ifp = ia->ia_ifp;
 1191                         ia = NULL;
 1192                         CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 1193                                 sa = ifa->ifa_addr;
 1194                                 if (sa->sa_family != AF_INET)
 1195                                         continue;
 1196                                 sin = (struct sockaddr_in *)sa;
 1197                                 if (prison_check_ip4(cred,
 1198                                     &sin->sin_addr) == 0) {
 1199                                         ia = (struct in_ifaddr *)ifa;
 1200                                         break;
 1201                                 }
 1202                         }
 1203                         if (ia != NULL) {
 1204                                 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 1205                                 goto done;
 1206                         }
 1207                 }
 1208 
 1209                 /* 3. As a last resort return the 'default' jail address. */
 1210                 error = prison_get_ip4(cred, laddr);
 1211                 goto done;
 1212         }
 1213 
 1214 done:
 1215         NET_EPOCH_EXIT(et);
 1216         if (sro.ro_rt != NULL)
 1217                 RTFREE(sro.ro_rt);
 1218         return (error);
 1219 }
 1220 
 1221 /*
 1222  * Set up for a connect from a socket to the specified address.
 1223  * On entry, *laddrp and *lportp should contain the current local
 1224  * address and port for the PCB; these are updated to the values
 1225  * that should be placed in inp_laddr and inp_lport to complete
 1226  * the connect.
 1227  *
 1228  * On success, *faddrp and *fportp will be set to the remote address
 1229  * and port. These are not updated in the error case.
 1230  *
 1231  * If the operation fails because the connection already exists,
 1232  * *oinpp will be set to the PCB of that connection so that the
 1233  * caller can decide to override it. In all other cases, *oinpp
 1234  * is set to NULL.
 1235  */
 1236 int
 1237 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
 1238     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
 1239     struct inpcb **oinpp, struct ucred *cred)
 1240 {
 1241         struct rm_priotracker in_ifa_tracker;
 1242         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
 1243         struct in_ifaddr *ia;
 1244         struct inpcb *oinp;
 1245         struct in_addr laddr, faddr;
 1246         u_short lport, fport;
 1247         int error;
 1248 
 1249         /*
 1250          * Because a global state change doesn't actually occur here, a read
 1251          * lock is sufficient.
 1252          */
 1253         INP_LOCK_ASSERT(inp);
 1254         INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
 1255 
 1256         if (oinpp != NULL)
 1257                 *oinpp = NULL;
 1258         if (nam->sa_len != sizeof (*sin))
 1259                 return (EINVAL);
 1260         if (sin->sin_family != AF_INET)
 1261                 return (EAFNOSUPPORT);
 1262         if (sin->sin_port == 0)
 1263                 return (EADDRNOTAVAIL);
 1264         laddr.s_addr = *laddrp;
 1265         lport = *lportp;
 1266         faddr = sin->sin_addr;
 1267         fport = sin->sin_port;
 1268 
 1269         if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
 1270                 /*
 1271                  * If the destination address is INADDR_ANY,
 1272                  * use the primary local address.
 1273                  * If the supplied address is INADDR_BROADCAST,
 1274                  * and the primary interface supports broadcast,
 1275                  * choose the broadcast address for that interface.
 1276                  */
 1277                 if (faddr.s_addr == INADDR_ANY) {
 1278                         IN_IFADDR_RLOCK(&in_ifa_tracker);
 1279                         faddr =
 1280                             IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
 1281                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
 1282                         if (cred != NULL &&
 1283                             (error = prison_get_ip4(cred, &faddr)) != 0)
 1284                                 return (error);
 1285                 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
 1286                         IN_IFADDR_RLOCK(&in_ifa_tracker);
 1287                         if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
 1288                             IFF_BROADCAST)
 1289                                 faddr = satosin(&CK_STAILQ_FIRST(
 1290                                     &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
 1291                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
 1292                 }
 1293         }
 1294         if (laddr.s_addr == INADDR_ANY) {
 1295                 error = in_pcbladdr(inp, &faddr, &laddr, cred);
 1296                 /*
 1297                  * If the destination address is multicast and an outgoing
 1298                  * interface has been set as a multicast option, prefer the
 1299                  * address of that interface as our source address.
 1300                  */
 1301                 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
 1302                     inp->inp_moptions != NULL) {
 1303                         struct ip_moptions *imo;
 1304                         struct ifnet *ifp;
 1305 
 1306                         imo = inp->inp_moptions;
 1307                         if (imo->imo_multicast_ifp != NULL) {
 1308                                 ifp = imo->imo_multicast_ifp;
 1309                                 IN_IFADDR_RLOCK(&in_ifa_tracker);
 1310                                 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
 1311                                         if ((ia->ia_ifp == ifp) &&
 1312                                             (cred == NULL ||
 1313                                             prison_check_ip4(cred,
 1314                                             &ia->ia_addr.sin_addr) == 0))
 1315                                                 break;
 1316                                 }
 1317                                 if (ia == NULL)
 1318                                         error = EADDRNOTAVAIL;
 1319                                 else {
 1320                                         laddr = ia->ia_addr.sin_addr;
 1321                                         error = 0;
 1322                                 }
 1323                                 IN_IFADDR_RUNLOCK(&in_ifa_tracker);
 1324                         }
 1325                 }
 1326                 if (error)
 1327                         return (error);
 1328         }
 1329         oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, fport,
 1330             laddr, lport, 0, NULL);
 1331         if (oinp != NULL) {
 1332                 if (oinpp != NULL)
 1333                         *oinpp = oinp;
 1334                 return (EADDRINUSE);
 1335         }
 1336         if (lport == 0) {
 1337                 error = in_pcbbind_setup(inp, NULL, &laddr.s_addr, &lport,
 1338                     cred);
 1339                 if (error)
 1340                         return (error);
 1341         }
 1342         *laddrp = laddr.s_addr;
 1343         *lportp = lport;
 1344         *faddrp = faddr.s_addr;
 1345         *fportp = fport;
 1346         return (0);
 1347 }
 1348 
 1349 void
 1350 in_pcbdisconnect(struct inpcb *inp)
 1351 {
 1352 
 1353         INP_WLOCK_ASSERT(inp);
 1354         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 1355 
 1356         inp->inp_faddr.s_addr = INADDR_ANY;
 1357         inp->inp_fport = 0;
 1358         in_pcbrehash(inp);
 1359 }
 1360 #endif /* INET */
 1361 
 1362 /*
 1363  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
 1364  * For most protocols, this will be invoked immediately prior to calling
 1365  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
 1366  * socket, in which case in_pcbfree() is deferred.
 1367  */
 1368 void
 1369 in_pcbdetach(struct inpcb *inp)
 1370 {
 1371 
 1372         KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
 1373 
 1374 #ifdef RATELIMIT
 1375         if (inp->inp_snd_tag != NULL)
 1376                 in_pcbdetach_txrtlmt(inp);
 1377 #endif
 1378         inp->inp_socket->so_pcb = NULL;
 1379         inp->inp_socket = NULL;
 1380 }
 1381 
 1382 /*
 1383  * in_pcbref() bumps the reference count on an inpcb in order to maintain
 1384  * stability of an inpcb pointer despite the inpcb lock being released.  This
 1385  * is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
 1386  * but where the inpcb lock may already held, or when acquiring a reference
 1387  * via a pcbgroup.
 1388  *
 1389  * in_pcbref() should be used only to provide brief memory stability, and
 1390  * must always be followed by a call to INP_WLOCK() and in_pcbrele() to
 1391  * garbage collect the inpcb if it has been in_pcbfree()'d from another
 1392  * context.  Until in_pcbrele() has returned that the inpcb is still valid,
 1393  * lock and rele are the *only* safe operations that may be performed on the
 1394  * inpcb.
 1395  *
 1396  * While the inpcb will not be freed, releasing the inpcb lock means that the
 1397  * connection's state may change, so the caller should be careful to
 1398  * revalidate any cached state on reacquiring the lock.  Drop the reference
 1399  * using in_pcbrele().
 1400  */
 1401 void
 1402 in_pcbref(struct inpcb *inp)
 1403 {
 1404 
 1405         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
 1406 
 1407         refcount_acquire(&inp->inp_refcount);
 1408 }
 1409 
 1410 /*
 1411  * Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
 1412  * in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
 1413  * return a flag indicating whether or not the inpcb remains valid.  If it is
 1414  * valid, we return with the inpcb lock held.
 1415  *
 1416  * Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
 1417  * reference on an inpcb.  Historically more work was done here (actually, in
 1418  * in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
 1419  * need for the pcbinfo lock in in_pcbrele().  Deferring the free is entirely
 1420  * about memory stability (and continued use of the write lock).
 1421  */
 1422 int
 1423 in_pcbrele_rlocked(struct inpcb *inp)
 1424 {
 1425         struct inpcbinfo *pcbinfo;
 1426 
 1427         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
 1428 
 1429         INP_RLOCK_ASSERT(inp);
 1430 
 1431         if (refcount_release(&inp->inp_refcount) == 0) {
 1432                 /*
 1433                  * If the inpcb has been freed, let the caller know, even if
 1434                  * this isn't the last reference.
 1435                  */
 1436                 if (inp->inp_flags2 & INP_FREED) {
 1437                         INP_RUNLOCK(inp);
 1438                         return (1);
 1439                 }
 1440                 return (0);
 1441         }
 1442         
 1443         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
 1444 #ifdef TCPHPTS
 1445         if (inp->inp_in_hpts || inp->inp_in_input) {
 1446                 struct tcp_hpts_entry *hpts;
 1447                 /*
 1448                  * We should not be on the hpts at 
 1449                  * this point in any form. we must
 1450                  * get the lock to be sure.
 1451                  */
 1452                 hpts = tcp_hpts_lock(inp);
 1453                 if (inp->inp_in_hpts)
 1454                         panic("Hpts:%p inp:%p at free still on hpts",
 1455                               hpts, inp);
 1456                 mtx_unlock(&hpts->p_mtx);
 1457                 hpts = tcp_input_lock(inp);
 1458                 if (inp->inp_in_input) 
 1459                         panic("Hpts:%p inp:%p at free still on input hpts",
 1460                               hpts, inp);
 1461                 mtx_unlock(&hpts->p_mtx);
 1462         }
 1463 #endif
 1464         INP_RUNLOCK(inp);
 1465         pcbinfo = inp->inp_pcbinfo;
 1466         uma_zfree(pcbinfo->ipi_zone, inp);
 1467         return (1);
 1468 }
 1469 
 1470 int
 1471 in_pcbrele_wlocked(struct inpcb *inp)
 1472 {
 1473         struct inpcbinfo *pcbinfo;
 1474 
 1475         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
 1476 
 1477         INP_WLOCK_ASSERT(inp);
 1478 
 1479         if (refcount_release(&inp->inp_refcount) == 0) {
 1480                 /*
 1481                  * If the inpcb has been freed, let the caller know, even if
 1482                  * this isn't the last reference.
 1483                  */
 1484                 if (inp->inp_flags2 & INP_FREED) {
 1485                         INP_WUNLOCK(inp);
 1486                         return (1);
 1487                 }
 1488                 return (0);
 1489         }
 1490 
 1491         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
 1492 #ifdef TCPHPTS
 1493         if (inp->inp_in_hpts || inp->inp_in_input) {
 1494                 struct tcp_hpts_entry *hpts;
 1495                 /*
 1496                  * We should not be on the hpts at 
 1497                  * this point in any form. we must
 1498                  * get the lock to be sure.
 1499                  */
 1500                 hpts = tcp_hpts_lock(inp);
 1501                 if (inp->inp_in_hpts)
 1502                         panic("Hpts:%p inp:%p at free still on hpts",
 1503                               hpts, inp);
 1504                 mtx_unlock(&hpts->p_mtx);
 1505                 hpts = tcp_input_lock(inp);
 1506                 if (inp->inp_in_input) 
 1507                         panic("Hpts:%p inp:%p at free still on input hpts",
 1508                               hpts, inp);
 1509                 mtx_unlock(&hpts->p_mtx);
 1510         }
 1511 #endif
 1512         INP_WUNLOCK(inp);
 1513         pcbinfo = inp->inp_pcbinfo;
 1514         uma_zfree(pcbinfo->ipi_zone, inp);
 1515         return (1);
 1516 }
 1517 
 1518 /*
 1519  * Temporary wrapper.
 1520  */
 1521 int
 1522 in_pcbrele(struct inpcb *inp)
 1523 {
 1524 
 1525         return (in_pcbrele_wlocked(inp));
 1526 }
 1527 
 1528 void
 1529 in_pcblist_rele_rlocked(epoch_context_t ctx)
 1530 {
 1531         struct in_pcblist *il;
 1532         struct inpcb *inp;
 1533         struct inpcbinfo *pcbinfo;
 1534         int i, n;
 1535 
 1536         il = __containerof(ctx, struct in_pcblist, il_epoch_ctx);
 1537         pcbinfo = il->il_pcbinfo;
 1538         n = il->il_count;
 1539         INP_INFO_WLOCK(pcbinfo);
 1540         for (i = 0; i < n; i++) {
 1541                 inp = il->il_inp_list[i];
 1542                 INP_RLOCK(inp);
 1543                 if (!in_pcbrele_rlocked(inp))
 1544                         INP_RUNLOCK(inp);
 1545         }
 1546         INP_INFO_WUNLOCK(pcbinfo);
 1547         free(il, M_TEMP);
 1548 }
 1549 
 1550 static void
 1551 inpcbport_free(epoch_context_t ctx)
 1552 {
 1553         struct inpcbport *phd;
 1554 
 1555         phd = __containerof(ctx, struct inpcbport, phd_epoch_ctx);
 1556         free(phd, M_PCB);
 1557 }
 1558 
 1559 static void
 1560 in_pcbfree_deferred(epoch_context_t ctx)
 1561 {
 1562         struct inpcb *inp;
 1563         int released __unused;
 1564 
 1565         inp = __containerof(ctx, struct inpcb, inp_epoch_ctx);
 1566 
 1567         INP_WLOCK(inp);
 1568         CURVNET_SET(inp->inp_vnet);
 1569 #ifdef INET
 1570         struct ip_moptions *imo = inp->inp_moptions;
 1571         inp->inp_moptions = NULL;
 1572 #endif
 1573         /* XXXRW: Do as much as possible here. */
 1574 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 1575         if (inp->inp_sp != NULL)
 1576                 ipsec_delete_pcbpolicy(inp);
 1577 #endif
 1578 #ifdef INET6
 1579         struct ip6_moptions *im6o = NULL;
 1580         if (inp->inp_vflag & INP_IPV6PROTO) {
 1581                 ip6_freepcbopts(inp->in6p_outputopts);
 1582                 im6o = inp->in6p_moptions;
 1583                 inp->in6p_moptions = NULL;
 1584         }
 1585 #endif
 1586         if (inp->inp_options)
 1587                 (void)m_free(inp->inp_options);
 1588         inp->inp_vflag = 0;
 1589         crfree(inp->inp_cred);
 1590 #ifdef MAC
 1591         mac_inpcb_destroy(inp);
 1592 #endif
 1593         released = in_pcbrele_wlocked(inp);
 1594         MPASS(released);
 1595 #ifdef INET6
 1596         ip6_freemoptions(im6o);
 1597 #endif
 1598 #ifdef INET
 1599         inp_freemoptions(imo);
 1600 #endif  
 1601         CURVNET_RESTORE();
 1602 }
 1603 
 1604 /*
 1605  * Unconditionally schedule an inpcb to be freed by decrementing its
 1606  * reference count, which should occur only after the inpcb has been detached
 1607  * from its socket.  If another thread holds a temporary reference (acquired
 1608  * using in_pcbref()) then the free is deferred until that reference is
 1609  * released using in_pcbrele(), but the inpcb is still unlocked.  Almost all
 1610  * work, including removal from global lists, is done in this context, where
 1611  * the pcbinfo lock is held.
 1612  */
 1613 void
 1614 in_pcbfree(struct inpcb *inp)
 1615 {
 1616         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 1617 
 1618         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
 1619         KASSERT((inp->inp_flags2 & INP_FREED) == 0,
 1620             ("%s: called twice for pcb %p", __func__, inp));
 1621         if (inp->inp_flags2 & INP_FREED) {
 1622                 INP_WUNLOCK(inp);
 1623                 return;
 1624         }
 1625 
 1626 #ifdef INVARIANTS
 1627         if (pcbinfo == &V_tcbinfo) {
 1628                 INP_INFO_LOCK_ASSERT(pcbinfo);
 1629         } else {
 1630                 INP_INFO_WLOCK_ASSERT(pcbinfo);
 1631         }
 1632 #endif
 1633         INP_WLOCK_ASSERT(inp);
 1634         INP_LIST_WLOCK(pcbinfo);
 1635         in_pcbremlists(inp);
 1636         INP_LIST_WUNLOCK(pcbinfo);
 1637         RO_INVALIDATE_CACHE(&inp->inp_route);
 1638         /* mark as destruction in progress */
 1639         inp->inp_flags2 |= INP_FREED;
 1640         INP_WUNLOCK(inp);
 1641         epoch_call(net_epoch_preempt, &inp->inp_epoch_ctx, in_pcbfree_deferred);
 1642 }
 1643 
 1644 /*
 1645  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
 1646  * port reservation, and preventing it from being returned by inpcb lookups.
 1647  *
 1648  * It is used by TCP to mark an inpcb as unused and avoid future packet
 1649  * delivery or event notification when a socket remains open but TCP has
 1650  * closed.  This might occur as a result of a shutdown()-initiated TCP close
 1651  * or a RST on the wire, and allows the port binding to be reused while still
 1652  * maintaining the invariant that so_pcb always points to a valid inpcb until
 1653  * in_pcbdetach().
 1654  *
 1655  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
 1656  * in_pcbnotifyall() and in_pcbpurgeif0()?
 1657  */
 1658 void
 1659 in_pcbdrop(struct inpcb *inp)
 1660 {
 1661 
 1662         INP_WLOCK_ASSERT(inp);
 1663 #ifdef INVARIANTS
 1664         if (inp->inp_socket != NULL && inp->inp_ppcb != NULL)
 1665                 MPASS(inp->inp_refcount > 1);
 1666 #endif
 1667 
 1668         /*
 1669          * XXXRW: Possibly we should protect the setting of INP_DROPPED with
 1670          * the hash lock...?
 1671          */
 1672         inp->inp_flags |= INP_DROPPED;
 1673         if (inp->inp_flags & INP_INHASHLIST) {
 1674                 struct inpcbport *phd = inp->inp_phd;
 1675 
 1676                 INP_HASH_WLOCK(inp->inp_pcbinfo);
 1677                 in_pcbremlbgrouphash(inp);
 1678                 CK_LIST_REMOVE(inp, inp_hash);
 1679                 CK_LIST_REMOVE(inp, inp_portlist);
 1680                 if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
 1681                         CK_LIST_REMOVE(phd, phd_hash);
 1682                         epoch_call(net_epoch_preempt, &phd->phd_epoch_ctx, inpcbport_free);
 1683                 }
 1684                 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
 1685                 inp->inp_flags &= ~INP_INHASHLIST;
 1686 #ifdef PCBGROUP
 1687                 in_pcbgroup_remove(inp);
 1688 #endif
 1689         }
 1690 }
 1691 
 1692 #ifdef INET
 1693 /*
 1694  * Common routines to return the socket addresses associated with inpcbs.
 1695  */
 1696 struct sockaddr *
 1697 in_sockaddr(in_port_t port, struct in_addr *addr_p)
 1698 {
 1699         struct sockaddr_in *sin;
 1700 
 1701         sin = malloc(sizeof *sin, M_SONAME,
 1702                 M_WAITOK | M_ZERO);
 1703         sin->sin_family = AF_INET;
 1704         sin->sin_len = sizeof(*sin);
 1705         sin->sin_addr = *addr_p;
 1706         sin->sin_port = port;
 1707 
 1708         return (struct sockaddr *)sin;
 1709 }
 1710 
 1711 int
 1712 in_getsockaddr(struct socket *so, struct sockaddr **nam)
 1713 {
 1714         struct inpcb *inp;
 1715         struct in_addr addr;
 1716         in_port_t port;
 1717 
 1718         inp = sotoinpcb(so);
 1719         KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
 1720 
 1721         INP_RLOCK(inp);
 1722         port = inp->inp_lport;
 1723         addr = inp->inp_laddr;
 1724         INP_RUNLOCK(inp);
 1725 
 1726         *nam = in_sockaddr(port, &addr);
 1727         return 0;
 1728 }
 1729 
 1730 int
 1731 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
 1732 {
 1733         struct inpcb *inp;
 1734         struct in_addr addr;
 1735         in_port_t port;
 1736 
 1737         inp = sotoinpcb(so);
 1738         KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
 1739 
 1740         INP_RLOCK(inp);
 1741         port = inp->inp_fport;
 1742         addr = inp->inp_faddr;
 1743         INP_RUNLOCK(inp);
 1744 
 1745         *nam = in_sockaddr(port, &addr);
 1746         return 0;
 1747 }
 1748 
 1749 void
 1750 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
 1751     struct inpcb *(*notify)(struct inpcb *, int))
 1752 {
 1753         struct inpcb *inp, *inp_temp;
 1754 
 1755         INP_INFO_WLOCK(pcbinfo);
 1756         CK_LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
 1757                 INP_WLOCK(inp);
 1758 #ifdef INET6
 1759                 if ((inp->inp_vflag & INP_IPV4) == 0) {
 1760                         INP_WUNLOCK(inp);
 1761                         continue;
 1762                 }
 1763 #endif
 1764                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
 1765                     inp->inp_socket == NULL) {
 1766                         INP_WUNLOCK(inp);
 1767                         continue;
 1768                 }
 1769                 if ((*notify)(inp, errno))
 1770                         INP_WUNLOCK(inp);
 1771         }
 1772         INP_INFO_WUNLOCK(pcbinfo);
 1773 }
 1774 
 1775 void
 1776 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
 1777 {
 1778         struct inpcb *inp;
 1779         struct ip_moptions *imo;
 1780         int i, gap;
 1781 
 1782         INP_INFO_WLOCK(pcbinfo);
 1783         CK_LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
 1784                 INP_WLOCK(inp);
 1785                 imo = inp->inp_moptions;
 1786                 if ((inp->inp_vflag & INP_IPV4) &&
 1787                     imo != NULL) {
 1788                         /*
 1789                          * Unselect the outgoing interface if it is being
 1790                          * detached.
 1791                          */
 1792                         if (imo->imo_multicast_ifp == ifp)
 1793                                 imo->imo_multicast_ifp = NULL;
 1794 
 1795                         /*
 1796                          * Drop multicast group membership if we joined
 1797                          * through the interface being detached.
 1798                          *
 1799                          * XXX This can all be deferred to an epoch_call
 1800                          */
 1801                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
 1802                             i++) {
 1803                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
 1804                                         IN_MULTI_LOCK_ASSERT();
 1805                                         in_leavegroup_locked(imo->imo_membership[i], NULL);
 1806                                         gap++;
 1807                                 } else if (gap != 0)
 1808                                         imo->imo_membership[i - gap] =
 1809                                             imo->imo_membership[i];
 1810                         }
 1811                         imo->imo_num_memberships -= gap;
 1812                 }
 1813                 INP_WUNLOCK(inp);
 1814         }
 1815         INP_INFO_WUNLOCK(pcbinfo);
 1816 }
 1817 
 1818 /*
 1819  * Lookup a PCB based on the local address and port.  Caller must hold the
 1820  * hash lock.  No inpcb locks or references are acquired.
 1821  */
 1822 #define INP_LOOKUP_MAPPED_PCB_COST      3
 1823 struct inpcb *
 1824 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
 1825     u_short lport, int lookupflags, struct ucred *cred)
 1826 {
 1827         struct inpcb *inp;
 1828 #ifdef INET6
 1829         int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
 1830 #else
 1831         int matchwild = 3;
 1832 #endif
 1833         int wildcard;
 1834 
 1835         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
 1836             ("%s: invalid lookup flags %d", __func__, lookupflags));
 1837 
 1838         INP_HASH_LOCK_ASSERT(pcbinfo);
 1839 
 1840         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
 1841                 struct inpcbhead *head;
 1842                 /*
 1843                  * Look for an unconnected (wildcard foreign addr) PCB that
 1844                  * matches the local address and port we're looking for.
 1845                  */
 1846                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
 1847                     0, pcbinfo->ipi_hashmask)];
 1848                 CK_LIST_FOREACH(inp, head, inp_hash) {
 1849 #ifdef INET6
 1850                         /* XXX inp locking */
 1851                         if ((inp->inp_vflag & INP_IPV4) == 0)
 1852                                 continue;
 1853 #endif
 1854                         if (inp->inp_faddr.s_addr == INADDR_ANY &&
 1855                             inp->inp_laddr.s_addr == laddr.s_addr &&
 1856                             inp->inp_lport == lport) {
 1857                                 /*
 1858                                  * Found?
 1859                                  */
 1860                                 if (cred == NULL ||
 1861                                     prison_equal_ip4(cred->cr_prison,
 1862                                         inp->inp_cred->cr_prison))
 1863                                         return (inp);
 1864                         }
 1865                 }
 1866                 /*
 1867                  * Not found.
 1868                  */
 1869                 return (NULL);
 1870         } else {
 1871                 struct inpcbporthead *porthash;
 1872                 struct inpcbport *phd;
 1873                 struct inpcb *match = NULL;
 1874                 /*
 1875                  * Best fit PCB lookup.
 1876                  *
 1877                  * First see if this local port is in use by looking on the
 1878                  * port hash list.
 1879                  */
 1880                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
 1881                     pcbinfo->ipi_porthashmask)];
 1882                 CK_LIST_FOREACH(phd, porthash, phd_hash) {
 1883                         if (phd->phd_port == lport)
 1884                                 break;
 1885                 }
 1886                 if (phd != NULL) {
 1887                         /*
 1888                          * Port is in use by one or more PCBs. Look for best
 1889                          * fit.
 1890                          */
 1891                         CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 1892                                 wildcard = 0;
 1893                                 if (cred != NULL &&
 1894                                     !prison_equal_ip4(inp->inp_cred->cr_prison,
 1895                                         cred->cr_prison))
 1896                                         continue;
 1897 #ifdef INET6
 1898                                 /* XXX inp locking */
 1899                                 if ((inp->inp_vflag & INP_IPV4) == 0)
 1900                                         continue;
 1901                                 /*
 1902                                  * We never select the PCB that has
 1903                                  * INP_IPV6 flag and is bound to :: if
 1904                                  * we have another PCB which is bound
 1905                                  * to 0.0.0.0.  If a PCB has the
 1906                                  * INP_IPV6 flag, then we set its cost
 1907                                  * higher than IPv4 only PCBs.
 1908                                  *
 1909                                  * Note that the case only happens
 1910                                  * when a socket is bound to ::, under
 1911                                  * the condition that the use of the
 1912                                  * mapped address is allowed.
 1913                                  */
 1914                                 if ((inp->inp_vflag & INP_IPV6) != 0)
 1915                                         wildcard += INP_LOOKUP_MAPPED_PCB_COST;
 1916 #endif
 1917                                 if (inp->inp_faddr.s_addr != INADDR_ANY)
 1918                                         wildcard++;
 1919                                 if (inp->inp_laddr.s_addr != INADDR_ANY) {
 1920                                         if (laddr.s_addr == INADDR_ANY)
 1921                                                 wildcard++;
 1922                                         else if (inp->inp_laddr.s_addr != laddr.s_addr)
 1923                                                 continue;
 1924                                 } else {
 1925                                         if (laddr.s_addr != INADDR_ANY)
 1926                                                 wildcard++;
 1927                                 }
 1928                                 if (wildcard < matchwild) {
 1929                                         match = inp;
 1930                                         matchwild = wildcard;
 1931                                         if (matchwild == 0)
 1932                                                 break;
 1933                                 }
 1934                         }
 1935                 }
 1936                 return (match);
 1937         }
 1938 }
 1939 #undef INP_LOOKUP_MAPPED_PCB_COST
 1940 
 1941 static struct inpcb *
 1942 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
 1943     const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr,
 1944     uint16_t fport, int lookupflags)
 1945 {
 1946         struct inpcb *local_wild;
 1947         const struct inpcblbgrouphead *hdr;
 1948         struct inpcblbgroup *grp;
 1949         uint32_t idx;
 1950 
 1951         INP_HASH_LOCK_ASSERT(pcbinfo);
 1952 
 1953         hdr = &pcbinfo->ipi_lbgrouphashbase[
 1954             INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
 1955 
 1956         /*
 1957          * Order of socket selection:
 1958          * 1. non-wild.
 1959          * 2. wild (if lookupflags contains INPLOOKUP_WILDCARD).
 1960          *
 1961          * NOTE:
 1962          * - Load balanced group does not contain jailed sockets
 1963          * - Load balanced group does not contain IPv4 mapped INET6 wild sockets
 1964          */
 1965         local_wild = NULL;
 1966         CK_LIST_FOREACH(grp, hdr, il_list) {
 1967 #ifdef INET6
 1968                 if (!(grp->il_vflag & INP_IPV4))
 1969                         continue;
 1970 #endif
 1971                 if (grp->il_lport != lport)
 1972                         continue;
 1973 
 1974                 idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) %
 1975                     grp->il_inpcnt;
 1976                 if (grp->il_laddr.s_addr == laddr->s_addr)
 1977                         return (grp->il_inp[idx]);
 1978                 if (grp->il_laddr.s_addr == INADDR_ANY &&
 1979                     (lookupflags & INPLOOKUP_WILDCARD) != 0)
 1980                         local_wild = grp->il_inp[idx];
 1981         }
 1982         return (local_wild);
 1983 }
 1984 
 1985 #ifdef PCBGROUP
 1986 /*
 1987  * Lookup PCB in hash list, using pcbgroup tables.
 1988  */
 1989 static struct inpcb *
 1990 in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
 1991     struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
 1992     u_int lport_arg, int lookupflags, struct ifnet *ifp)
 1993 {
 1994         struct inpcbhead *head;
 1995         struct inpcb *inp, *tmpinp;
 1996         u_short fport = fport_arg, lport = lport_arg;
 1997         bool locked;
 1998 
 1999         /*
 2000          * First look for an exact match.
 2001          */
 2002         tmpinp = NULL;
 2003         INP_GROUP_LOCK(pcbgroup);
 2004         head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
 2005             pcbgroup->ipg_hashmask)];
 2006         CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
 2007 #ifdef INET6
 2008                 /* XXX inp locking */
 2009                 if ((inp->inp_vflag & INP_IPV4) == 0)
 2010                         continue;
 2011 #endif
 2012                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
 2013                     inp->inp_laddr.s_addr == laddr.s_addr &&
 2014                     inp->inp_fport == fport &&
 2015                     inp->inp_lport == lport) {
 2016                         /*
 2017                          * XXX We should be able to directly return
 2018                          * the inp here, without any checks.
 2019                          * Well unless both bound with SO_REUSEPORT?
 2020                          */
 2021                         if (prison_flag(inp->inp_cred, PR_IP4))
 2022                                 goto found;
 2023                         if (tmpinp == NULL)
 2024                                 tmpinp = inp;
 2025                 }
 2026         }
 2027         if (tmpinp != NULL) {
 2028                 inp = tmpinp;
 2029                 goto found;
 2030         }
 2031 
 2032 #ifdef  RSS
 2033         /*
 2034          * For incoming connections, we may wish to do a wildcard
 2035          * match for an RSS-local socket.
 2036          */
 2037         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
 2038                 struct inpcb *local_wild = NULL, *local_exact = NULL;
 2039 #ifdef INET6
 2040                 struct inpcb *local_wild_mapped = NULL;
 2041 #endif
 2042                 struct inpcb *jail_wild = NULL;
 2043                 struct inpcbhead *head;
 2044                 int injail;
 2045 
 2046                 /*
 2047                  * Order of socket selection - we always prefer jails.
 2048                  *      1. jailed, non-wild.
 2049                  *      2. jailed, wild.
 2050                  *      3. non-jailed, non-wild.
 2051                  *      4. non-jailed, wild.
 2052                  */
 2053 
 2054                 head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
 2055                     lport, 0, pcbgroup->ipg_hashmask)];
 2056                 CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
 2057 #ifdef INET6
 2058                         /* XXX inp locking */
 2059                         if ((inp->inp_vflag & INP_IPV4) == 0)
 2060                                 continue;
 2061 #endif
 2062                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
 2063                             inp->inp_lport != lport)
 2064                                 continue;
 2065 
 2066                         injail = prison_flag(inp->inp_cred, PR_IP4);
 2067                         if (injail) {
 2068                                 if (prison_check_ip4(inp->inp_cred,
 2069                                     &laddr) != 0)
 2070                                         continue;
 2071                         } else {
 2072                                 if (local_exact != NULL)
 2073                                         continue;
 2074                         }
 2075 
 2076                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
 2077                                 if (injail)
 2078                                         goto found;
 2079                                 else
 2080                                         local_exact = inp;
 2081                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
 2082 #ifdef INET6
 2083                                 /* XXX inp locking, NULL check */
 2084                                 if (inp->inp_vflag & INP_IPV6PROTO)
 2085                                         local_wild_mapped = inp;
 2086                                 else
 2087 #endif
 2088                                         if (injail)
 2089                                                 jail_wild = inp;
 2090                                         else
 2091                                                 local_wild = inp;
 2092                         }
 2093                 } /* LIST_FOREACH */
 2094 
 2095                 inp = jail_wild;
 2096                 if (inp == NULL)
 2097                         inp = local_exact;
 2098                 if (inp == NULL)
 2099                         inp = local_wild;
 2100 #ifdef INET6
 2101                 if (inp == NULL)
 2102                         inp = local_wild_mapped;
 2103 #endif
 2104                 if (inp != NULL)
 2105                         goto found;
 2106         }
 2107 #endif
 2108 
 2109         /*
 2110          * Then look for a wildcard match, if requested.
 2111          */
 2112         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
 2113                 struct inpcb *local_wild = NULL, *local_exact = NULL;
 2114 #ifdef INET6
 2115                 struct inpcb *local_wild_mapped = NULL;
 2116 #endif
 2117                 struct inpcb *jail_wild = NULL;
 2118                 struct inpcbhead *head;
 2119                 int injail;
 2120 
 2121                 /*
 2122                  * Order of socket selection - we always prefer jails.
 2123                  *      1. jailed, non-wild.
 2124                  *      2. jailed, wild.
 2125                  *      3. non-jailed, non-wild.
 2126                  *      4. non-jailed, wild.
 2127                  */
 2128                 head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
 2129                     0, pcbinfo->ipi_wildmask)];
 2130                 CK_LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
 2131 #ifdef INET6
 2132                         /* XXX inp locking */
 2133                         if ((inp->inp_vflag & INP_IPV4) == 0)
 2134                                 continue;
 2135 #endif
 2136                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
 2137                             inp->inp_lport != lport)
 2138                                 continue;
 2139 
 2140                         injail = prison_flag(inp->inp_cred, PR_IP4);
 2141                         if (injail) {
 2142                                 if (prison_check_ip4(inp->inp_cred,
 2143                                     &laddr) != 0)
 2144                                         continue;
 2145                         } else {
 2146                                 if (local_exact != NULL)
 2147                                         continue;
 2148                         }
 2149 
 2150                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
 2151                                 if (injail)
 2152                                         goto found;
 2153                                 else
 2154                                         local_exact = inp;
 2155                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
 2156 #ifdef INET6
 2157                                 /* XXX inp locking, NULL check */
 2158                                 if (inp->inp_vflag & INP_IPV6PROTO)
 2159                                         local_wild_mapped = inp;
 2160                                 else
 2161 #endif
 2162                                         if (injail)
 2163                                                 jail_wild = inp;
 2164                                         else
 2165                                                 local_wild = inp;
 2166                         }
 2167                 } /* LIST_FOREACH */
 2168                 inp = jail_wild;
 2169                 if (inp == NULL)
 2170                         inp = local_exact;
 2171                 if (inp == NULL)
 2172                         inp = local_wild;
 2173 #ifdef INET6
 2174                 if (inp == NULL)
 2175                         inp = local_wild_mapped;
 2176 #endif
 2177                 if (inp != NULL)
 2178                         goto found;
 2179         } /* if (lookupflags & INPLOOKUP_WILDCARD) */
 2180         INP_GROUP_UNLOCK(pcbgroup);
 2181         return (NULL);
 2182 
 2183 found:
 2184         if (lookupflags & INPLOOKUP_WLOCKPCB)
 2185                 locked = INP_TRY_WLOCK(inp);
 2186         else if (lookupflags & INPLOOKUP_RLOCKPCB)
 2187                 locked = INP_TRY_RLOCK(inp);
 2188         else
 2189                 panic("%s: locking bug", __func__);
 2190         if (__predict_false(locked && (inp->inp_flags2 & INP_FREED))) {
 2191                 if (lookupflags & INPLOOKUP_WLOCKPCB)
 2192                         INP_WUNLOCK(inp);
 2193                 else
 2194                         INP_RUNLOCK(inp);
 2195                 return (NULL);
 2196         } else if (!locked)
 2197                 in_pcbref(inp);
 2198         INP_GROUP_UNLOCK(pcbgroup);
 2199         if (!locked) {
 2200                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
 2201                         INP_WLOCK(inp);
 2202                         if (in_pcbrele_wlocked(inp))
 2203                                 return (NULL);
 2204                 } else {
 2205                         INP_RLOCK(inp);
 2206                         if (in_pcbrele_rlocked(inp))
 2207                                 return (NULL);
 2208                 }
 2209         }
 2210 #ifdef INVARIANTS
 2211         if (lookupflags & INPLOOKUP_WLOCKPCB)
 2212                 INP_WLOCK_ASSERT(inp);
 2213         else
 2214                 INP_RLOCK_ASSERT(inp);
 2215 #endif
 2216         return (inp);
 2217 }
 2218 #endif /* PCBGROUP */
 2219 
 2220 /*
 2221  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
 2222  * that the caller has locked the hash list, and will not perform any further
 2223  * locking or reference operations on either the hash list or the connection.
 2224  */
 2225 static struct inpcb *
 2226 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
 2227     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
 2228     struct ifnet *ifp)
 2229 {
 2230         struct inpcbhead *head;
 2231         struct inpcb *inp, *tmpinp;
 2232         u_short fport = fport_arg, lport = lport_arg;
 2233 
 2234 #ifdef INVARIANTS
 2235         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
 2236             ("%s: invalid lookup flags %d", __func__, lookupflags));
 2237         if (!mtx_owned(&pcbinfo->ipi_hash_lock))
 2238                 MPASS(in_epoch_verbose(net_epoch_preempt, 1));
 2239 #endif
 2240         /*
 2241          * First look for an exact match.
 2242          */
 2243         tmpinp = NULL;
 2244         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
 2245             pcbinfo->ipi_hashmask)];
 2246         CK_LIST_FOREACH(inp, head, inp_hash) {
 2247 #ifdef INET6
 2248                 /* XXX inp locking */
 2249                 if ((inp->inp_vflag & INP_IPV4) == 0)
 2250                         continue;
 2251 #endif
 2252                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
 2253                     inp->inp_laddr.s_addr == laddr.s_addr &&
 2254                     inp->inp_fport == fport &&
 2255                     inp->inp_lport == lport) {
 2256                         /*
 2257                          * XXX We should be able to directly return
 2258                          * the inp here, without any checks.
 2259                          * Well unless both bound with SO_REUSEPORT?
 2260                          */
 2261                         if (prison_flag(inp->inp_cred, PR_IP4))
 2262                                 return (inp);
 2263                         if (tmpinp == NULL)
 2264                                 tmpinp = inp;
 2265                 }
 2266         }
 2267         if (tmpinp != NULL)
 2268                 return (tmpinp);
 2269 
 2270         /*
 2271          * Then look in lb group (for wildcard match).
 2272          */
 2273         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
 2274                 inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
 2275                     fport, lookupflags);
 2276                 if (inp != NULL)
 2277                         return (inp);
 2278         }
 2279 
 2280         /*
 2281          * Then look for a wildcard match, if requested.
 2282          */
 2283         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
 2284                 struct inpcb *local_wild = NULL, *local_exact = NULL;
 2285 #ifdef INET6
 2286                 struct inpcb *local_wild_mapped = NULL;
 2287 #endif
 2288                 struct inpcb *jail_wild = NULL;
 2289                 int injail;
 2290 
 2291                 /*
 2292                  * Order of socket selection - we always prefer jails.
 2293                  *      1. jailed, non-wild.
 2294                  *      2. jailed, wild.
 2295                  *      3. non-jailed, non-wild.
 2296                  *      4. non-jailed, wild.
 2297                  */
 2298 
 2299                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
 2300                     0, pcbinfo->ipi_hashmask)];
 2301                 CK_LIST_FOREACH(inp, head, inp_hash) {
 2302 #ifdef INET6
 2303                         /* XXX inp locking */
 2304                         if ((inp->inp_vflag & INP_IPV4) == 0)
 2305                                 continue;
 2306 #endif
 2307                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
 2308                             inp->inp_lport != lport)
 2309                                 continue;
 2310 
 2311                         injail = prison_flag(inp->inp_cred, PR_IP4);
 2312                         if (injail) {
 2313                                 if (prison_check_ip4(inp->inp_cred,
 2314                                     &laddr) != 0)
 2315                                         continue;
 2316                         } else {
 2317                                 if (local_exact != NULL)
 2318                                         continue;
 2319                         }
 2320 
 2321                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
 2322                                 if (injail)
 2323                                         return (inp);
 2324                                 else
 2325                                         local_exact = inp;
 2326                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
 2327 #ifdef INET6
 2328                                 /* XXX inp locking, NULL check */
 2329                                 if (inp->inp_vflag & INP_IPV6PROTO)
 2330                                         local_wild_mapped = inp;
 2331                                 else
 2332 #endif
 2333                                         if (injail)
 2334                                                 jail_wild = inp;
 2335                                         else
 2336                                                 local_wild = inp;
 2337                         }
 2338                 } /* LIST_FOREACH */
 2339                 if (jail_wild != NULL)
 2340                         return (jail_wild);
 2341                 if (local_exact != NULL)
 2342                         return (local_exact);
 2343                 if (local_wild != NULL)
 2344                         return (local_wild);
 2345 #ifdef INET6
 2346                 if (local_wild_mapped != NULL)
 2347                         return (local_wild_mapped);
 2348 #endif
 2349         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
 2350 
 2351         return (NULL);
 2352 }
 2353 
 2354 /*
 2355  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
 2356  * hash list lock, and will return the inpcb locked (i.e., requires
 2357  * INPLOOKUP_LOCKPCB).
 2358  */
 2359 static struct inpcb *
 2360 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
 2361     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
 2362     struct ifnet *ifp)
 2363 {
 2364         struct inpcb *inp;
 2365 
 2366         INP_HASH_RLOCK(pcbinfo);
 2367         inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
 2368             (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
 2369         if (inp != NULL) {
 2370                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
 2371                         INP_WLOCK(inp);
 2372                         if (__predict_false(inp->inp_flags2 & INP_FREED)) {
 2373                                 INP_WUNLOCK(inp);
 2374                                 inp = NULL;
 2375                         }
 2376                 } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
 2377                         INP_RLOCK(inp);
 2378                         if (__predict_false(inp->inp_flags2 & INP_FREED)) {
 2379                                 INP_RUNLOCK(inp);
 2380                                 inp = NULL;
 2381                         }
 2382                 } else
 2383                         panic("%s: locking bug", __func__);
 2384 #ifdef INVARIANTS
 2385                 if (inp != NULL) {
 2386                         if (lookupflags & INPLOOKUP_WLOCKPCB)
 2387                                 INP_WLOCK_ASSERT(inp);
 2388                         else
 2389                                 INP_RLOCK_ASSERT(inp);
 2390                 }
 2391 #endif
 2392         }
 2393         INP_HASH_RUNLOCK(pcbinfo);
 2394         return (inp);
 2395 }
 2396 
 2397 /*
 2398  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
 2399  * from which a pre-calculated hash value may be extracted.
 2400  *
 2401  * Possibly more of this logic should be in in_pcbgroup.c.
 2402  */
 2403 struct inpcb *
 2404 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
 2405     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
 2406 {
 2407 #if defined(PCBGROUP) && !defined(RSS)
 2408         struct inpcbgroup *pcbgroup;
 2409 #endif
 2410 
 2411         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
 2412             ("%s: invalid lookup flags %d", __func__, lookupflags));
 2413         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
 2414             ("%s: LOCKPCB not set", __func__));
 2415 
 2416         /*
 2417          * When not using RSS, use connection groups in preference to the
 2418          * reservation table when looking up 4-tuples.  When using RSS, just
 2419          * use the reservation table, due to the cost of the Toeplitz hash
 2420          * in software.
 2421          *
 2422          * XXXRW: This policy belongs in the pcbgroup code, as in principle
 2423          * we could be doing RSS with a non-Toeplitz hash that is affordable
 2424          * in software.
 2425          */
 2426 #if defined(PCBGROUP) && !defined(RSS)
 2427         if (in_pcbgroup_enabled(pcbinfo)) {
 2428                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
 2429                     fport);
 2430                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
 2431                     laddr, lport, lookupflags, ifp));
 2432         }
 2433 #endif
 2434         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
 2435             lookupflags, ifp));
 2436 }
 2437 
 2438 struct inpcb *
 2439 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
 2440     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
 2441     struct ifnet *ifp, struct mbuf *m)
 2442 {
 2443 #ifdef PCBGROUP
 2444         struct inpcbgroup *pcbgroup;
 2445 #endif
 2446 
 2447         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
 2448             ("%s: invalid lookup flags %d", __func__, lookupflags));
 2449         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
 2450             ("%s: LOCKPCB not set", __func__));
 2451 
 2452 #ifdef PCBGROUP
 2453         /*
 2454          * If we can use a hardware-generated hash to look up the connection
 2455          * group, use that connection group to find the inpcb.  Otherwise
 2456          * fall back on a software hash -- or the reservation table if we're
 2457          * using RSS.
 2458          *
 2459          * XXXRW: As above, that policy belongs in the pcbgroup code.
 2460          */
 2461         if (in_pcbgroup_enabled(pcbinfo) &&
 2462             !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
 2463                 pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
 2464                     m->m_pkthdr.flowid);
 2465                 if (pcbgroup != NULL)
 2466                         return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
 2467                             fport, laddr, lport, lookupflags, ifp));
 2468 #ifndef RSS
 2469                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
 2470                     fport);
 2471                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
 2472                     laddr, lport, lookupflags, ifp));
 2473 #endif
 2474         }
 2475 #endif
 2476         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
 2477             lookupflags, ifp));
 2478 }
 2479 #endif /* INET */
 2480 
 2481 /*
 2482  * Insert PCB onto various hash lists.
 2483  */
 2484 static int
 2485 in_pcbinshash_internal(struct inpcb *inp, int do_pcbgroup_update)
 2486 {
 2487         struct inpcbhead *pcbhash;
 2488         struct inpcbporthead *pcbporthash;
 2489         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 2490         struct inpcbport *phd;
 2491         u_int32_t hashkey_faddr;
 2492         int so_options;
 2493 
 2494         INP_WLOCK_ASSERT(inp);
 2495         INP_HASH_WLOCK_ASSERT(pcbinfo);
 2496 
 2497         KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
 2498             ("in_pcbinshash: INP_INHASHLIST"));
 2499 
 2500 #ifdef INET6
 2501         if (inp->inp_vflag & INP_IPV6)
 2502                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
 2503         else
 2504 #endif
 2505         hashkey_faddr = inp->inp_faddr.s_addr;
 2506 
 2507         pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
 2508                  inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
 2509 
 2510         pcbporthash = &pcbinfo->ipi_porthashbase[
 2511             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
 2512 
 2513         /*
 2514          * Add entry to load balance group.
 2515          * Only do this if SO_REUSEPORT_LB is set.
 2516          */
 2517         so_options = inp_so_options(inp);
 2518         if (so_options & SO_REUSEPORT_LB) {
 2519                 int ret = in_pcbinslbgrouphash(inp);
 2520                 if (ret) {
 2521                         /* pcb lb group malloc fail (ret=ENOBUFS). */
 2522                         return (ret);
 2523                 }
 2524         }
 2525 
 2526         /*
 2527          * Go through port list and look for a head for this lport.
 2528          */
 2529         CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
 2530                 if (phd->phd_port == inp->inp_lport)
 2531                         break;
 2532         }
 2533         /*
 2534          * If none exists, malloc one and tack it on.
 2535          */
 2536         if (phd == NULL) {
 2537                 phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
 2538                 if (phd == NULL) {
 2539                         return (ENOBUFS); /* XXX */
 2540                 }
 2541                 bzero(&phd->phd_epoch_ctx, sizeof(struct epoch_context));
 2542                 phd->phd_port = inp->inp_lport;
 2543                 CK_LIST_INIT(&phd->phd_pcblist);
 2544                 CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
 2545         }
 2546         inp->inp_phd = phd;
 2547         CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
 2548         CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
 2549         inp->inp_flags |= INP_INHASHLIST;
 2550 #ifdef PCBGROUP
 2551         if (do_pcbgroup_update)
 2552                 in_pcbgroup_update(inp);
 2553 #endif
 2554         return (0);
 2555 }
 2556 
 2557 /*
 2558  * For now, there are two public interfaces to insert an inpcb into the hash
 2559  * lists -- one that does update pcbgroups, and one that doesn't.  The latter
 2560  * is used only in the TCP syncache, where in_pcbinshash is called before the
 2561  * full 4-tuple is set for the inpcb, and we don't want to install in the
 2562  * pcbgroup until later.
 2563  *
 2564  * XXXRW: This seems like a misfeature.  in_pcbinshash should always update
 2565  * connection groups, and partially initialised inpcbs should not be exposed
 2566  * to either reservation hash tables or pcbgroups.
 2567  */
 2568 int
 2569 in_pcbinshash(struct inpcb *inp)
 2570 {
 2571 
 2572         return (in_pcbinshash_internal(inp, 1));
 2573 }
 2574 
 2575 int
 2576 in_pcbinshash_nopcbgroup(struct inpcb *inp)
 2577 {
 2578 
 2579         return (in_pcbinshash_internal(inp, 0));
 2580 }
 2581 
 2582 /*
 2583  * Move PCB to the proper hash bucket when { faddr, fport } have  been
 2584  * changed. NOTE: This does not handle the case of the lport changing (the
 2585  * hashed port list would have to be updated as well), so the lport must
 2586  * not change after in_pcbinshash() has been called.
 2587  */
 2588 void
 2589 in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
 2590 {
 2591         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 2592         struct inpcbhead *head;
 2593         u_int32_t hashkey_faddr;
 2594 
 2595         INP_WLOCK_ASSERT(inp);
 2596         INP_HASH_WLOCK_ASSERT(pcbinfo);
 2597 
 2598         KASSERT(inp->inp_flags & INP_INHASHLIST,
 2599             ("in_pcbrehash: !INP_INHASHLIST"));
 2600 
 2601 #ifdef INET6
 2602         if (inp->inp_vflag & INP_IPV6)
 2603                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
 2604         else
 2605 #endif
 2606         hashkey_faddr = inp->inp_faddr.s_addr;
 2607 
 2608         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
 2609                 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
 2610 
 2611         CK_LIST_REMOVE(inp, inp_hash);
 2612         CK_LIST_INSERT_HEAD(head, inp, inp_hash);
 2613 
 2614 #ifdef PCBGROUP
 2615         if (m != NULL)
 2616                 in_pcbgroup_update_mbuf(inp, m);
 2617         else
 2618                 in_pcbgroup_update(inp);
 2619 #endif
 2620 }
 2621 
 2622 void
 2623 in_pcbrehash(struct inpcb *inp)
 2624 {
 2625 
 2626         in_pcbrehash_mbuf(inp, NULL);
 2627 }
 2628 
 2629 /*
 2630  * Remove PCB from various lists.
 2631  */
 2632 static void
 2633 in_pcbremlists(struct inpcb *inp)
 2634 {
 2635         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 2636 
 2637 #ifdef INVARIANTS
 2638         if (pcbinfo == &V_tcbinfo) {
 2639                 INP_INFO_RLOCK_ASSERT(pcbinfo);
 2640         } else {
 2641                 INP_INFO_WLOCK_ASSERT(pcbinfo);
 2642         }
 2643 #endif
 2644 
 2645         INP_WLOCK_ASSERT(inp);
 2646         INP_LIST_WLOCK_ASSERT(pcbinfo);
 2647 
 2648         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 2649         if (inp->inp_flags & INP_INHASHLIST) {
 2650                 struct inpcbport *phd = inp->inp_phd;
 2651 
 2652                 INP_HASH_WLOCK(pcbinfo);
 2653 
 2654                 /* XXX: Only do if SO_REUSEPORT_LB set? */
 2655                 in_pcbremlbgrouphash(inp);
 2656 
 2657                 CK_LIST_REMOVE(inp, inp_hash);
 2658                 CK_LIST_REMOVE(inp, inp_portlist);
 2659                 if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
 2660                         CK_LIST_REMOVE(phd, phd_hash);
 2661                         epoch_call(net_epoch_preempt, &phd->phd_epoch_ctx, inpcbport_free);
 2662                 }
 2663                 INP_HASH_WUNLOCK(pcbinfo);
 2664                 inp->inp_flags &= ~INP_INHASHLIST;
 2665         }
 2666         CK_LIST_REMOVE(inp, inp_list);
 2667         pcbinfo->ipi_count--;
 2668 #ifdef PCBGROUP
 2669         in_pcbgroup_remove(inp);
 2670 #endif
 2671 }
 2672 
 2673 /*
 2674  * Check for alternatives when higher level complains
 2675  * about service problems.  For now, invalidate cached
 2676  * routing information.  If the route was created dynamically
 2677  * (by a redirect), time to try a default gateway again.
 2678  */
 2679 void
 2680 in_losing(struct inpcb *inp)
 2681 {
 2682 
 2683         RO_INVALIDATE_CACHE(&inp->inp_route);
 2684         return;
 2685 }
 2686 
 2687 /*
 2688  * A set label operation has occurred at the socket layer, propagate the
 2689  * label change into the in_pcb for the socket.
 2690  */
 2691 void
 2692 in_pcbsosetlabel(struct socket *so)
 2693 {
 2694 #ifdef MAC
 2695         struct inpcb *inp;
 2696 
 2697         inp = sotoinpcb(so);
 2698         KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
 2699 
 2700         INP_WLOCK(inp);
 2701         SOCK_LOCK(so);
 2702         mac_inpcb_sosetlabel(so, inp);
 2703         SOCK_UNLOCK(so);
 2704         INP_WUNLOCK(inp);
 2705 #endif
 2706 }
 2707 
 2708 /*
 2709  * ipport_tick runs once per second, determining if random port allocation
 2710  * should be continued.  If more than ipport_randomcps ports have been
 2711  * allocated in the last second, then we return to sequential port
 2712  * allocation. We return to random allocation only once we drop below
 2713  * ipport_randomcps for at least ipport_randomtime seconds.
 2714  */
 2715 static void
 2716 ipport_tick(void *xtp)
 2717 {
 2718         VNET_ITERATOR_DECL(vnet_iter);
 2719 
 2720         VNET_LIST_RLOCK_NOSLEEP();
 2721         VNET_FOREACH(vnet_iter) {
 2722                 CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS here */
 2723                 if (V_ipport_tcpallocs <=
 2724                     V_ipport_tcplastcount + V_ipport_randomcps) {
 2725                         if (V_ipport_stoprandom > 0)
 2726                                 V_ipport_stoprandom--;
 2727                 } else
 2728                         V_ipport_stoprandom = V_ipport_randomtime;
 2729                 V_ipport_tcplastcount = V_ipport_tcpallocs;
 2730                 CURVNET_RESTORE();
 2731         }
 2732         VNET_LIST_RUNLOCK_NOSLEEP();
 2733         callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
 2734 }
 2735 
 2736 static void
 2737 ip_fini(void *xtp)
 2738 {
 2739 
 2740         callout_stop(&ipport_tick_callout);
 2741 }
 2742 
 2743 /* 
 2744  * The ipport_callout should start running at about the time we attach the
 2745  * inet or inet6 domains.
 2746  */
 2747 static void
 2748 ipport_tick_init(const void *unused __unused)
 2749 {
 2750 
 2751         /* Start ipport_tick. */
 2752         callout_init(&ipport_tick_callout, 1);
 2753         callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
 2754         EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
 2755                 SHUTDOWN_PRI_DEFAULT);
 2756 }
 2757 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, 
 2758     ipport_tick_init, NULL);
 2759 
 2760 void
 2761 inp_wlock(struct inpcb *inp)
 2762 {
 2763 
 2764         INP_WLOCK(inp);
 2765 }
 2766 
 2767 void
 2768 inp_wunlock(struct inpcb *inp)
 2769 {
 2770 
 2771         INP_WUNLOCK(inp);
 2772 }
 2773 
 2774 void
 2775 inp_rlock(struct inpcb *inp)
 2776 {
 2777 
 2778         INP_RLOCK(inp);
 2779 }
 2780 
 2781 void
 2782 inp_runlock(struct inpcb *inp)
 2783 {
 2784 
 2785         INP_RUNLOCK(inp);
 2786 }
 2787 
 2788 #ifdef INVARIANT_SUPPORT
 2789 void
 2790 inp_lock_assert(struct inpcb *inp)
 2791 {
 2792 
 2793         INP_WLOCK_ASSERT(inp);
 2794 }
 2795 
 2796 void
 2797 inp_unlock_assert(struct inpcb *inp)
 2798 {
 2799 
 2800         INP_UNLOCK_ASSERT(inp);
 2801 }
 2802 #endif
 2803 
 2804 void
 2805 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
 2806 {
 2807         struct inpcb *inp;
 2808 
 2809         INP_INFO_WLOCK(&V_tcbinfo);
 2810         CK_LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
 2811                 INP_WLOCK(inp);
 2812                 func(inp, arg);
 2813                 INP_WUNLOCK(inp);
 2814         }
 2815         INP_INFO_WUNLOCK(&V_tcbinfo);
 2816 }
 2817 
 2818 struct socket *
 2819 inp_inpcbtosocket(struct inpcb *inp)
 2820 {
 2821 
 2822         INP_WLOCK_ASSERT(inp);
 2823         return (inp->inp_socket);
 2824 }
 2825 
 2826 struct tcpcb *
 2827 inp_inpcbtotcpcb(struct inpcb *inp)
 2828 {
 2829 
 2830         INP_WLOCK_ASSERT(inp);
 2831         return ((struct tcpcb *)inp->inp_ppcb);
 2832 }
 2833 
 2834 int
 2835 inp_ip_tos_get(const struct inpcb *inp)
 2836 {
 2837 
 2838         return (inp->inp_ip_tos);
 2839 }
 2840 
 2841 void
 2842 inp_ip_tos_set(struct inpcb *inp, int val)
 2843 {
 2844 
 2845         inp->inp_ip_tos = val;
 2846 }
 2847 
 2848 void
 2849 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
 2850     uint32_t *faddr, uint16_t *fp)
 2851 {
 2852 
 2853         INP_LOCK_ASSERT(inp);
 2854         *laddr = inp->inp_laddr.s_addr;
 2855         *faddr = inp->inp_faddr.s_addr;
 2856         *lp = inp->inp_lport;
 2857         *fp = inp->inp_fport;
 2858 }
 2859 
 2860 struct inpcb *
 2861 so_sotoinpcb(struct socket *so)
 2862 {
 2863 
 2864         return (sotoinpcb(so));
 2865 }
 2866 
 2867 struct tcpcb *
 2868 so_sototcpcb(struct socket *so)
 2869 {
 2870 
 2871         return (sototcpcb(so));
 2872 }
 2873 
 2874 /*
 2875  * Create an external-format (``xinpcb'') structure using the information in
 2876  * the kernel-format in_pcb structure pointed to by inp.  This is done to
 2877  * reduce the spew of irrelevant information over this interface, to isolate
 2878  * user code from changes in the kernel structure, and potentially to provide
 2879  * information-hiding if we decide that some of this information should be
 2880  * hidden from users.
 2881  */
 2882 void
 2883 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
 2884 {
 2885 
 2886         bzero(xi, sizeof(*xi));
 2887         xi->xi_len = sizeof(struct xinpcb);
 2888         if (inp->inp_socket)
 2889                 sotoxsocket(inp->inp_socket, &xi->xi_socket);
 2890         bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
 2891         xi->inp_gencnt = inp->inp_gencnt;
 2892         xi->inp_ppcb = (uintptr_t)inp->inp_ppcb;
 2893         xi->inp_flow = inp->inp_flow;
 2894         xi->inp_flowid = inp->inp_flowid;
 2895         xi->inp_flowtype = inp->inp_flowtype;
 2896         xi->inp_flags = inp->inp_flags;
 2897         xi->inp_flags2 = inp->inp_flags2;
 2898         xi->inp_rss_listen_bucket = inp->inp_rss_listen_bucket;
 2899         xi->in6p_cksum = inp->in6p_cksum;
 2900         xi->in6p_hops = inp->in6p_hops;
 2901         xi->inp_ip_tos = inp->inp_ip_tos;
 2902         xi->inp_vflag = inp->inp_vflag;
 2903         xi->inp_ip_ttl = inp->inp_ip_ttl;
 2904         xi->inp_ip_p = inp->inp_ip_p;
 2905         xi->inp_ip_minttl = inp->inp_ip_minttl;
 2906 }
 2907 
 2908 #ifdef DDB
 2909 static void
 2910 db_print_indent(int indent)
 2911 {
 2912         int i;
 2913 
 2914         for (i = 0; i < indent; i++)
 2915                 db_printf(" ");
 2916 }
 2917 
 2918 static void
 2919 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
 2920 {
 2921         char faddr_str[48], laddr_str[48];
 2922 
 2923         db_print_indent(indent);
 2924         db_printf("%s at %p\n", name, inc);
 2925 
 2926         indent += 2;
 2927 
 2928 #ifdef INET6
 2929         if (inc->inc_flags & INC_ISIPV6) {
 2930                 /* IPv6. */
 2931                 ip6_sprintf(laddr_str, &inc->inc6_laddr);
 2932                 ip6_sprintf(faddr_str, &inc->inc6_faddr);
 2933         } else
 2934 #endif
 2935         {
 2936                 /* IPv4. */
 2937                 inet_ntoa_r(inc->inc_laddr, laddr_str);
 2938                 inet_ntoa_r(inc->inc_faddr, faddr_str);
 2939         }
 2940         db_print_indent(indent);
 2941         db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
 2942             ntohs(inc->inc_lport));
 2943         db_print_indent(indent);
 2944         db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
 2945             ntohs(inc->inc_fport));
 2946 }
 2947 
 2948 static void
 2949 db_print_inpflags(int inp_flags)
 2950 {
 2951         int comma;
 2952 
 2953         comma = 0;
 2954         if (inp_flags & INP_RECVOPTS) {
 2955                 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
 2956                 comma = 1;
 2957         }
 2958         if (inp_flags & INP_RECVRETOPTS) {
 2959                 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
 2960                 comma = 1;
 2961         }
 2962         if (inp_flags & INP_RECVDSTADDR) {
 2963                 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
 2964                 comma = 1;
 2965         }
 2966         if (inp_flags & INP_ORIGDSTADDR) {
 2967                 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
 2968                 comma = 1;
 2969         }
 2970         if (inp_flags & INP_HDRINCL) {
 2971                 db_printf("%sINP_HDRINCL", comma ? ", " : "");
 2972                 comma = 1;
 2973         }
 2974         if (inp_flags & INP_HIGHPORT) {
 2975                 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
 2976                 comma = 1;
 2977         }
 2978         if (inp_flags & INP_LOWPORT) {
 2979                 db_printf("%sINP_LOWPORT", comma ? ", " : "");
 2980                 comma = 1;
 2981         }
 2982         if (inp_flags & INP_ANONPORT) {
 2983                 db_printf("%sINP_ANONPORT", comma ? ", " : "");
 2984                 comma = 1;
 2985         }
 2986         if (inp_flags & INP_RECVIF) {
 2987                 db_printf("%sINP_RECVIF", comma ? ", " : "");
 2988                 comma = 1;
 2989         }
 2990         if (inp_flags & INP_MTUDISC) {
 2991                 db_printf("%sINP_MTUDISC", comma ? ", " : "");
 2992                 comma = 1;
 2993         }
 2994         if (inp_flags & INP_RECVTTL) {
 2995                 db_printf("%sINP_RECVTTL", comma ? ", " : "");
 2996                 comma = 1;
 2997         }
 2998         if (inp_flags & INP_DONTFRAG) {
 2999                 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
 3000                 comma = 1;
 3001         }
 3002         if (inp_flags & INP_RECVTOS) {
 3003                 db_printf("%sINP_RECVTOS", comma ? ", " : "");
 3004                 comma = 1;
 3005         }
 3006         if (inp_flags & IN6P_IPV6_V6ONLY) {
 3007                 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
 3008                 comma = 1;
 3009         }
 3010         if (inp_flags & IN6P_PKTINFO) {
 3011                 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
 3012                 comma = 1;
 3013         }
 3014         if (inp_flags & IN6P_HOPLIMIT) {
 3015                 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
 3016                 comma = 1;
 3017         }
 3018         if (inp_flags & IN6P_HOPOPTS) {
 3019                 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
 3020                 comma = 1;
 3021         }
 3022         if (inp_flags & IN6P_DSTOPTS) {
 3023                 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
 3024                 comma = 1;
 3025         }
 3026         if (inp_flags & IN6P_RTHDR) {
 3027                 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
 3028                 comma = 1;
 3029         }
 3030         if (inp_flags & IN6P_RTHDRDSTOPTS) {
 3031                 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
 3032                 comma = 1;
 3033         }
 3034         if (inp_flags & IN6P_TCLASS) {
 3035                 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
 3036                 comma = 1;
 3037         }
 3038         if (inp_flags & IN6P_AUTOFLOWLABEL) {
 3039                 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
 3040                 comma = 1;
 3041         }
 3042         if (inp_flags & INP_TIMEWAIT) {
 3043                 db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
 3044                 comma  = 1;
 3045         }
 3046         if (inp_flags & INP_ONESBCAST) {
 3047                 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
 3048                 comma  = 1;
 3049         }
 3050         if (inp_flags & INP_DROPPED) {
 3051                 db_printf("%sINP_DROPPED", comma ? ", " : "");
 3052                 comma  = 1;
 3053         }
 3054         if (inp_flags & INP_SOCKREF) {
 3055                 db_printf("%sINP_SOCKREF", comma ? ", " : "");
 3056                 comma  = 1;
 3057         }
 3058         if (inp_flags & IN6P_RFC2292) {
 3059                 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
 3060                 comma = 1;
 3061         }
 3062         if (inp_flags & IN6P_MTU) {
 3063                 db_printf("IN6P_MTU%s", comma ? ", " : "");
 3064                 comma = 1;
 3065         }
 3066 }
 3067 
 3068 static void
 3069 db_print_inpvflag(u_char inp_vflag)
 3070 {
 3071         int comma;
 3072 
 3073         comma = 0;
 3074         if (inp_vflag & INP_IPV4) {
 3075                 db_printf("%sINP_IPV4", comma ? ", " : "");
 3076                 comma  = 1;
 3077         }
 3078         if (inp_vflag & INP_IPV6) {
 3079                 db_printf("%sINP_IPV6", comma ? ", " : "");
 3080                 comma  = 1;
 3081         }
 3082         if (inp_vflag & INP_IPV6PROTO) {
 3083                 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
 3084                 comma  = 1;
 3085         }
 3086 }
 3087 
 3088 static void
 3089 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
 3090 {
 3091 
 3092         db_print_indent(indent);
 3093         db_printf("%s at %p\n", name, inp);
 3094 
 3095         indent += 2;
 3096 
 3097         db_print_indent(indent);
 3098         db_printf("inp_flow: 0x%x\n", inp->inp_flow);
 3099 
 3100         db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
 3101 
 3102         db_print_indent(indent);
 3103         db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
 3104             inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
 3105 
 3106         db_print_indent(indent);
 3107         db_printf("inp_label: %p   inp_flags: 0x%x (",
 3108            inp->inp_label, inp->inp_flags);
 3109         db_print_inpflags(inp->inp_flags);
 3110         db_printf(")\n");
 3111 
 3112         db_print_indent(indent);
 3113         db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
 3114             inp->inp_vflag);
 3115         db_print_inpvflag(inp->inp_vflag);
 3116         db_printf(")\n");
 3117 
 3118         db_print_indent(indent);
 3119         db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
 3120             inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
 3121 
 3122         db_print_indent(indent);
 3123 #ifdef INET6
 3124         if (inp->inp_vflag & INP_IPV6) {
 3125                 db_printf("in6p_options: %p   in6p_outputopts: %p   "
 3126                     "in6p_moptions: %p\n", inp->in6p_options,
 3127                     inp->in6p_outputopts, inp->in6p_moptions);
 3128                 db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
 3129                     "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
 3130                     inp->in6p_hops);
 3131         } else
 3132 #endif
 3133         {
 3134                 db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
 3135                     "inp_ip_moptions: %p\n", inp->inp_ip_tos,
 3136                     inp->inp_options, inp->inp_moptions);
 3137         }
 3138 
 3139         db_print_indent(indent);
 3140         db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
 3141             (uintmax_t)inp->inp_gencnt);
 3142 }
 3143 
 3144 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
 3145 {
 3146         struct inpcb *inp;
 3147 
 3148         if (!have_addr) {
 3149                 db_printf("usage: show inpcb <addr>\n");
 3150                 return;
 3151         }
 3152         inp = (struct inpcb *)addr;
 3153 
 3154         db_print_inpcb(inp, "inpcb", 0);
 3155 }
 3156 #endif /* DDB */
 3157 
 3158 #ifdef RATELIMIT
 3159 /*
 3160  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
 3161  * if any.
 3162  */
 3163 int
 3164 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
 3165 {
 3166         union if_snd_tag_modify_params params = {
 3167                 .rate_limit.max_rate = max_pacing_rate,
 3168         };
 3169         struct m_snd_tag *mst;
 3170         struct ifnet *ifp;
 3171         int error;
 3172 
 3173         mst = inp->inp_snd_tag;
 3174         if (mst == NULL)
 3175                 return (EINVAL);
 3176 
 3177         ifp = mst->ifp;
 3178         if (ifp == NULL)
 3179                 return (EINVAL);
 3180 
 3181         if (ifp->if_snd_tag_modify == NULL) {
 3182                 error = EOPNOTSUPP;
 3183         } else {
 3184                 error = ifp->if_snd_tag_modify(mst, &params);
 3185         }
 3186         return (error);
 3187 }
 3188 
 3189 /*
 3190  * Query existing TX rate limit based on the existing
 3191  * "inp->inp_snd_tag", if any.
 3192  */
 3193 int
 3194 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
 3195 {
 3196         union if_snd_tag_query_params params = { };
 3197         struct m_snd_tag *mst;
 3198         struct ifnet *ifp;
 3199         int error;
 3200 
 3201         mst = inp->inp_snd_tag;
 3202         if (mst == NULL)
 3203                 return (EINVAL);
 3204 
 3205         ifp = mst->ifp;
 3206         if (ifp == NULL)
 3207                 return (EINVAL);
 3208 
 3209         if (ifp->if_snd_tag_query == NULL) {
 3210                 error = EOPNOTSUPP;
 3211         } else {
 3212                 error = ifp->if_snd_tag_query(mst, &params);
 3213                 if (error == 0 &&  p_max_pacing_rate != NULL)
 3214                         *p_max_pacing_rate = params.rate_limit.max_rate;
 3215         }
 3216         return (error);
 3217 }
 3218 
 3219 /*
 3220  * Query existing TX queue level based on the existing
 3221  * "inp->inp_snd_tag", if any.
 3222  */
 3223 int
 3224 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
 3225 {
 3226         union if_snd_tag_query_params params = { };
 3227         struct m_snd_tag *mst;
 3228         struct ifnet *ifp;
 3229         int error;
 3230 
 3231         mst = inp->inp_snd_tag;
 3232         if (mst == NULL)
 3233                 return (EINVAL);
 3234 
 3235         ifp = mst->ifp;
 3236         if (ifp == NULL)
 3237                 return (EINVAL);
 3238 
 3239         if (ifp->if_snd_tag_query == NULL)
 3240                 return (EOPNOTSUPP);
 3241 
 3242         error = ifp->if_snd_tag_query(mst, &params);
 3243         if (error == 0 &&  p_txqueue_level != NULL)
 3244                 *p_txqueue_level = params.rate_limit.queue_level;
 3245         return (error);
 3246 }
 3247 
 3248 /*
 3249  * Allocate a new TX rate limit send tag from the network interface
 3250  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
 3251  */
 3252 int
 3253 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
 3254     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate)
 3255 {
 3256         union if_snd_tag_alloc_params params = {
 3257                 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
 3258                     IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
 3259                 .rate_limit.hdr.flowid = flowid,
 3260                 .rate_limit.hdr.flowtype = flowtype,
 3261                 .rate_limit.max_rate = max_pacing_rate,
 3262         };
 3263         int error;
 3264 
 3265         INP_WLOCK_ASSERT(inp);
 3266 
 3267         if (inp->inp_snd_tag != NULL)
 3268                 return (EINVAL);
 3269 
 3270         if (ifp->if_snd_tag_alloc == NULL) {
 3271                 error = EOPNOTSUPP;
 3272         } else {
 3273                 error = ifp->if_snd_tag_alloc(ifp, &params, &inp->inp_snd_tag);
 3274 
 3275                 /*
 3276                  * At success increment the refcount on
 3277                  * the send tag's network interface:
 3278                  */
 3279                 if (error == 0)
 3280                         if_ref(inp->inp_snd_tag->ifp);
 3281         }
 3282         return (error);
 3283 }
 3284 
 3285 /*
 3286  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
 3287  * if any:
 3288  */
 3289 void
 3290 in_pcbdetach_txrtlmt(struct inpcb *inp)
 3291 {
 3292         struct m_snd_tag *mst;
 3293         struct ifnet *ifp;
 3294 
 3295         INP_WLOCK_ASSERT(inp);
 3296 
 3297         mst = inp->inp_snd_tag;
 3298         inp->inp_snd_tag = NULL;
 3299 
 3300         if (mst == NULL)
 3301                 return;
 3302 
 3303         ifp = mst->ifp;
 3304         if (ifp == NULL)
 3305                 return;
 3306 
 3307         /*
 3308          * If the device was detached while we still had reference(s)
 3309          * on the ifp, we assume if_snd_tag_free() was replaced with
 3310          * stubs.
 3311          */
 3312         ifp->if_snd_tag_free(mst);
 3313 
 3314         /* release reference count on network interface */
 3315         if_rele(ifp);
 3316 }
 3317 
 3318 /*
 3319  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
 3320  * is set in the fast path and will attach/detach/modify the TX rate
 3321  * limit send tag based on the socket's so_max_pacing_rate value.
 3322  */
 3323 void
 3324 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
 3325 {
 3326         struct socket *socket;
 3327         uint32_t max_pacing_rate;
 3328         bool did_upgrade;
 3329         int error;
 3330 
 3331         if (inp == NULL)
 3332                 return;
 3333 
 3334         socket = inp->inp_socket;
 3335         if (socket == NULL)
 3336                 return;
 3337 
 3338         if (!INP_WLOCKED(inp)) {
 3339                 /*
 3340                  * NOTE: If the write locking fails, we need to bail
 3341                  * out and use the non-ratelimited ring for the
 3342                  * transmit until there is a new chance to get the
 3343                  * write lock.
 3344                  */
 3345                 if (!INP_TRY_UPGRADE(inp))
 3346                         return;
 3347                 did_upgrade = 1;
 3348         } else {
 3349                 did_upgrade = 0;
 3350         }
 3351 
 3352         /*
 3353          * NOTE: The so_max_pacing_rate value is read unlocked,
 3354          * because atomic updates are not required since the variable
 3355          * is checked at every mbuf we send. It is assumed that the
 3356          * variable read itself will be atomic.
 3357          */
 3358         max_pacing_rate = socket->so_max_pacing_rate;
 3359 
 3360         /*
 3361          * NOTE: When attaching to a network interface a reference is
 3362          * made to ensure the network interface doesn't go away until
 3363          * all ratelimit connections are gone. The network interface
 3364          * pointers compared below represent valid network interfaces,
 3365          * except when comparing towards NULL.
 3366          */
 3367         if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
 3368                 error = 0;
 3369         } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
 3370                 if (inp->inp_snd_tag != NULL)
 3371                         in_pcbdetach_txrtlmt(inp);
 3372                 error = 0;
 3373         } else if (inp->inp_snd_tag == NULL) {
 3374                 /*
 3375                  * In order to utilize packet pacing with RSS, we need
 3376                  * to wait until there is a valid RSS hash before we
 3377                  * can proceed:
 3378                  */
 3379                 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
 3380                         error = EAGAIN;
 3381                 } else {
 3382                         error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
 3383                             mb->m_pkthdr.flowid, max_pacing_rate);
 3384                 }
 3385         } else {
 3386                 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
 3387         }
 3388         if (error == 0 || error == EOPNOTSUPP)
 3389                 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
 3390         if (did_upgrade)
 3391                 INP_DOWNGRADE(inp);
 3392 }
 3393 
 3394 /*
 3395  * Track route changes for TX rate limiting.
 3396  */
 3397 void
 3398 in_pcboutput_eagain(struct inpcb *inp)
 3399 {
 3400         bool did_upgrade;
 3401 
 3402         if (inp == NULL)
 3403                 return;
 3404 
 3405         if (inp->inp_snd_tag == NULL)
 3406                 return;
 3407 
 3408         if (!INP_WLOCKED(inp)) {
 3409                 /*
 3410                  * NOTE: If the write locking fails, we need to bail
 3411                  * out and use the non-ratelimited ring for the
 3412                  * transmit until there is a new chance to get the
 3413                  * write lock.
 3414                  */
 3415                 if (!INP_TRY_UPGRADE(inp))
 3416                         return;
 3417                 did_upgrade = 1;
 3418         } else {
 3419                 did_upgrade = 0;
 3420         }
 3421 
 3422         /* detach rate limiting */
 3423         in_pcbdetach_txrtlmt(inp);
 3424 
 3425         /* make sure new mbuf send tag allocation is made */
 3426         inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
 3427 
 3428         if (did_upgrade)
 3429                 INP_DOWNGRADE(inp);
 3430 }
 3431 #endif /* RATELIMIT */

Cache object: dbbf85f428682c1048b43f44f8da4f7a


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.