FreeBSD/Linux Kernel Cross Reference
sys/dev/wg/if_wg.c

     /* SPDX-License-Identifier: ISC
      *
      * Copyright (C) 2015-2021 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
      * Copyright (C) 2019-2021 Matt Dunwoodie <ncon@noconroy.net>
      * Copyright (c) 2019-2020 Rubicon Communications, LLC (Netgate)
      * Copyright (c) 2021 Kyle Evans <kevans@FreeBSD.org>
      * Copyright (c) 2022 The FreeBSD Foundation
      */
     
    #include "opt_inet.h"
    #include "opt_inet6.h"
    
    #include <sys/param.h>
    #include <sys/systm.h>
    #include <sys/counter.h>
    #include <sys/gtaskqueue.h>
    #include <sys/jail.h>
    #include <sys/kernel.h>
    #include <sys/lock.h>
    #include <sys/mbuf.h>
    #include <sys/module.h>
    #include <sys/nv.h>
    #include <sys/priv.h>
    #include <sys/protosw.h>
    #include <sys/rmlock.h>
    #include <sys/rwlock.h>
    #include <sys/smp.h>
    #include <sys/socket.h>
    #include <sys/socketvar.h>
    #include <sys/sockio.h>
    #include <sys/sysctl.h>
    #include <sys/sx.h>
    #include <machine/_inttypes.h>
    #include <net/bpf.h>
    #include <net/ethernet.h>
    #include <net/if.h>
    #include <net/if_clone.h>
    #include <net/if_types.h>
    #include <net/if_var.h>
    #include <net/netisr.h>
    #include <net/radix.h>
    #include <netinet/in.h>
    #include <netinet6/in6_var.h>
    #include <netinet/ip.h>
    #include <netinet/ip6.h>
    #include <netinet/ip_icmp.h>
    #include <netinet/icmp6.h>
    #include <netinet/udp_var.h>
    #include <netinet6/nd6.h>
    
    #include "wg_noise.h"
    #include "wg_cookie.h"
    #include "version.h"
    #include "if_wg.h"
    
    #define DEFAULT_MTU             (ETHERMTU - 80)
    #define MAX_MTU                 (IF_MAXMTU - 80)
    
    #define MAX_STAGED_PKT          128
    #define MAX_QUEUED_PKT          1024
    #define MAX_QUEUED_PKT_MASK     (MAX_QUEUED_PKT - 1)
    
    #define MAX_QUEUED_HANDSHAKES   4096
    
    #define REKEY_TIMEOUT_JITTER    334 /* 1/3 sec, round for arc4random_uniform */
    #define MAX_TIMER_HANDSHAKES    (90 / REKEY_TIMEOUT)
    #define NEW_HANDSHAKE_TIMEOUT   (REKEY_TIMEOUT + KEEPALIVE_TIMEOUT)
    #define UNDERLOAD_TIMEOUT       1
    
    #define DPRINTF(sc, ...) if (sc->sc_ifp->if_flags & IFF_DEBUG) if_printf(sc->sc_ifp, ##__VA_ARGS__)
    
    /* First byte indicating packet type on the wire */
    #define WG_PKT_INITIATION htole32(1)
    #define WG_PKT_RESPONSE htole32(2)
    #define WG_PKT_COOKIE htole32(3)
    #define WG_PKT_DATA htole32(4)
    
    #define WG_PKT_PADDING          16
    #define WG_KEY_SIZE             32
    
    struct wg_pkt_initiation {
            uint32_t                t;
            uint32_t                s_idx;
            uint8_t                 ue[NOISE_PUBLIC_KEY_LEN];
            uint8_t                 es[NOISE_PUBLIC_KEY_LEN + NOISE_AUTHTAG_LEN];
            uint8_t                 ets[NOISE_TIMESTAMP_LEN + NOISE_AUTHTAG_LEN];
            struct cookie_macs      m;
    };
    
    struct wg_pkt_response {
            uint32_t                t;
            uint32_t                s_idx;
            uint32_t                r_idx;
            uint8_t                 ue[NOISE_PUBLIC_KEY_LEN];
            uint8_t                 en[0 + NOISE_AUTHTAG_LEN];
            struct cookie_macs      m;
    };
    
    struct wg_pkt_cookie {
           uint32_t                t;
           uint32_t                r_idx;
           uint8_t                 nonce[COOKIE_NONCE_SIZE];
           uint8_t                 ec[COOKIE_ENCRYPTED_SIZE];
   };
   
   struct wg_pkt_data {
           uint32_t                t;
           uint32_t                r_idx;
           uint64_t                nonce;
           uint8_t                 buf[];
   };
   
   struct wg_endpoint {
           union {
                   struct sockaddr         r_sa;
                   struct sockaddr_in      r_sin;
   #ifdef INET6
                   struct sockaddr_in6     r_sin6;
   #endif
           } e_remote;
           union {
                   struct in_addr          l_in;
   #ifdef INET6
                   struct in6_pktinfo      l_pktinfo6;
   #define l_in6 l_pktinfo6.ipi6_addr
   #endif
           } e_local;
   };
   
   struct aip_addr {
           uint8_t         length;
           union {
                   uint8_t         bytes[16];
                   uint32_t        ip;
                   uint32_t        ip6[4];
                   struct in_addr  in;
                   struct in6_addr in6;
           };
   };
   
   struct wg_aip {
           struct radix_node        a_nodes[2];
           LIST_ENTRY(wg_aip)       a_entry;
           struct aip_addr          a_addr;
           struct aip_addr          a_mask;
           struct wg_peer          *a_peer;
           sa_family_t              a_af;
   };
   
   struct wg_packet {
           STAILQ_ENTRY(wg_packet)  p_serial;
           STAILQ_ENTRY(wg_packet)  p_parallel;
           struct wg_endpoint       p_endpoint;
           struct noise_keypair    *p_keypair;
           uint64_t                 p_nonce;
           struct mbuf             *p_mbuf;
           int                      p_mtu;
           sa_family_t              p_af;
           enum wg_ring_state {
                   WG_PACKET_UNCRYPTED,
                   WG_PACKET_CRYPTED,
                   WG_PACKET_DEAD,
           }                        p_state;
   };
   
   STAILQ_HEAD(wg_packet_list, wg_packet);
   
   struct wg_queue {
           struct mtx               q_mtx;
           struct wg_packet_list    q_queue;
           size_t                   q_len;
   };
   
   struct wg_peer {
           TAILQ_ENTRY(wg_peer)             p_entry;
           uint64_t                         p_id;
           struct wg_softc                 *p_sc;
   
           struct noise_remote             *p_remote;
           struct cookie_maker              p_cookie;
   
           struct rwlock                    p_endpoint_lock;
           struct wg_endpoint               p_endpoint;
   
           struct wg_queue                  p_stage_queue;
           struct wg_queue                  p_encrypt_serial;
           struct wg_queue                  p_decrypt_serial;
   
           bool                             p_enabled;
           bool                             p_need_another_keepalive;
           uint16_t                         p_persistent_keepalive_interval;
           struct callout                   p_new_handshake;
           struct callout                   p_send_keepalive;
           struct callout                   p_retry_handshake;
           struct callout                   p_zero_key_material;
           struct callout                   p_persistent_keepalive;
   
           struct mtx                       p_handshake_mtx;
           struct timespec                  p_handshake_complete;  /* nanotime */
           int                              p_handshake_retries;
   
           struct grouptask                 p_send;
           struct grouptask                 p_recv;
   
           counter_u64_t                    p_tx_bytes;
           counter_u64_t                    p_rx_bytes;
   
           LIST_HEAD(, wg_aip)              p_aips;
           size_t                           p_aips_num;
   };
   
   struct wg_socket {
           struct socket   *so_so4;
           struct socket   *so_so6;
           uint32_t         so_user_cookie;
           int              so_fibnum;
           in_port_t        so_port;
   };
   
   struct wg_softc {
           LIST_ENTRY(wg_softc)     sc_entry;
           struct ifnet            *sc_ifp;
           int                      sc_flags;
   
           struct ucred            *sc_ucred;
           struct wg_socket         sc_socket;
   
           TAILQ_HEAD(,wg_peer)     sc_peers;
           size_t                   sc_peers_num;
   
           struct noise_local      *sc_local;
           struct cookie_checker    sc_cookie;
   
           struct radix_node_head  *sc_aip4;
           struct radix_node_head  *sc_aip6;
   
           struct grouptask         sc_handshake;
           struct wg_queue          sc_handshake_queue;
   
           struct grouptask        *sc_encrypt;
           struct grouptask        *sc_decrypt;
           struct wg_queue          sc_encrypt_parallel;
           struct wg_queue          sc_decrypt_parallel;
           u_int                    sc_encrypt_last_cpu;
           u_int                    sc_decrypt_last_cpu;
   
           struct sx                sc_lock;
   };
   
   #define WGF_DYING       0x0001
   
   #define MAX_LOOPS       8
   #define MTAG_WGLOOP     0x77676c70 /* wglp */
   #ifndef ENOKEY
   #define ENOKEY  ENOTCAPABLE
   #endif
   
   #define GROUPTASK_DRAIN(gtask)                  \
           gtaskqueue_drain((gtask)->gt_taskqueue, &(gtask)->gt_task)
   
   #define BPF_MTAP2_AF(ifp, m, af) do { \
                   uint32_t __bpf_tap_af = (af); \
                   BPF_MTAP2(ifp, &__bpf_tap_af, sizeof(__bpf_tap_af), m); \
           } while (0)
   
   static int clone_count;
   static uma_zone_t wg_packet_zone;
   static volatile unsigned long peer_counter = 0;
   static const char wgname[] = "wg";
   static unsigned wg_osd_jail_slot;
   
   static struct sx wg_sx;
   SX_SYSINIT(wg_sx, &wg_sx, "wg_sx");
   
   static LIST_HEAD(, wg_softc) wg_list = LIST_HEAD_INITIALIZER(wg_list);
   
   static TASKQGROUP_DEFINE(wg_tqg, mp_ncpus, 1);
   
   MALLOC_DEFINE(M_WG, "WG", "wireguard");
   
   VNET_DEFINE_STATIC(struct if_clone *, wg_cloner);
   
   #define V_wg_cloner     VNET(wg_cloner)
   #define WG_CAPS         IFCAP_LINKSTATE
   
   struct wg_timespec64 {
           uint64_t        tv_sec;
           uint64_t        tv_nsec;
   };
   
   static int wg_socket_init(struct wg_softc *, in_port_t);
   static int wg_socket_bind(struct socket **, struct socket **, in_port_t *);
   static void wg_socket_set(struct wg_softc *, struct socket *, struct socket *);
   static void wg_socket_uninit(struct wg_softc *);
   static int wg_socket_set_sockopt(struct socket *, struct socket *, int, void *, size_t);
   static int wg_socket_set_cookie(struct wg_softc *, uint32_t);
   static int wg_socket_set_fibnum(struct wg_softc *, int);
   static int wg_send(struct wg_softc *, struct wg_endpoint *, struct mbuf *);
   static void wg_timers_enable(struct wg_peer *);
   static void wg_timers_disable(struct wg_peer *);
   static void wg_timers_set_persistent_keepalive(struct wg_peer *, uint16_t);
   static void wg_timers_get_last_handshake(struct wg_peer *, struct wg_timespec64 *);
   static void wg_timers_event_data_sent(struct wg_peer *);
   static void wg_timers_event_data_received(struct wg_peer *);
   static void wg_timers_event_any_authenticated_packet_sent(struct wg_peer *);
   static void wg_timers_event_any_authenticated_packet_received(struct wg_peer *);
   static void wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *);
   static void wg_timers_event_handshake_initiated(struct wg_peer *);
   static void wg_timers_event_handshake_complete(struct wg_peer *);
   static void wg_timers_event_session_derived(struct wg_peer *);
   static void wg_timers_event_want_initiation(struct wg_peer *);
   static void wg_timers_run_send_initiation(struct wg_peer *, bool);
   static void wg_timers_run_retry_handshake(void *);
   static void wg_timers_run_send_keepalive(void *);
   static void wg_timers_run_new_handshake(void *);
   static void wg_timers_run_zero_key_material(void *);
   static void wg_timers_run_persistent_keepalive(void *);
   static int wg_aip_add(struct wg_softc *, struct wg_peer *, sa_family_t, const void *, uint8_t);
   static struct wg_peer *wg_aip_lookup(struct wg_softc *, sa_family_t, void *);
   static void wg_aip_remove_all(struct wg_softc *, struct wg_peer *);
   static struct wg_peer *wg_peer_alloc(struct wg_softc *, const uint8_t [WG_KEY_SIZE]);
   static void wg_peer_free_deferred(struct noise_remote *);
   static void wg_peer_destroy(struct wg_peer *);
   static void wg_peer_destroy_all(struct wg_softc *);
   static void wg_peer_send_buf(struct wg_peer *, uint8_t *, size_t);
   static void wg_send_initiation(struct wg_peer *);
   static void wg_send_response(struct wg_peer *);
   static void wg_send_cookie(struct wg_softc *, struct cookie_macs *, uint32_t, struct wg_endpoint *);
   static void wg_peer_set_endpoint(struct wg_peer *, struct wg_endpoint *);
   static void wg_peer_clear_src(struct wg_peer *);
   static void wg_peer_get_endpoint(struct wg_peer *, struct wg_endpoint *);
   static void wg_send_buf(struct wg_softc *, struct wg_endpoint *, uint8_t *, size_t);
   static void wg_send_keepalive(struct wg_peer *);
   static void wg_handshake(struct wg_softc *, struct wg_packet *);
   static void wg_encrypt(struct wg_softc *, struct wg_packet *);
   static void wg_decrypt(struct wg_softc *, struct wg_packet *);
   static void wg_softc_handshake_receive(struct wg_softc *);
   static void wg_softc_decrypt(struct wg_softc *);
   static void wg_softc_encrypt(struct wg_softc *);
   static void wg_encrypt_dispatch(struct wg_softc *);
   static void wg_decrypt_dispatch(struct wg_softc *);
   static void wg_deliver_out(struct wg_peer *);
   static void wg_deliver_in(struct wg_peer *);
   static struct wg_packet *wg_packet_alloc(struct mbuf *);
   static void wg_packet_free(struct wg_packet *);
   static void wg_queue_init(struct wg_queue *, const char *);
   static void wg_queue_deinit(struct wg_queue *);
   static size_t wg_queue_len(struct wg_queue *);
   static int wg_queue_enqueue_handshake(struct wg_queue *, struct wg_packet *);
   static struct wg_packet *wg_queue_dequeue_handshake(struct wg_queue *);
   static void wg_queue_push_staged(struct wg_queue *, struct wg_packet *);
   static void wg_queue_enlist_staged(struct wg_queue *, struct wg_packet_list *);
   static void wg_queue_delist_staged(struct wg_queue *, struct wg_packet_list *);
   static void wg_queue_purge(struct wg_queue *);
   static int wg_queue_both(struct wg_queue *, struct wg_queue *, struct wg_packet *);
   static struct wg_packet *wg_queue_dequeue_serial(struct wg_queue *);
   static struct wg_packet *wg_queue_dequeue_parallel(struct wg_queue *);
   static bool wg_input(struct mbuf *, int, struct inpcb *, const struct sockaddr *, void *);
   static void wg_peer_send_staged(struct wg_peer *);
   static int wg_clone_create(struct if_clone *ifc, char *name, size_t len,
           struct ifc_data *ifd, struct ifnet **ifpp);
   static void wg_qflush(struct ifnet *);
   static inline int determine_af_and_pullup(struct mbuf **m, sa_family_t *af);
   static int wg_xmit(struct ifnet *, struct mbuf *, sa_family_t, uint32_t);
   static int wg_transmit(struct ifnet *, struct mbuf *);
   static int wg_output(struct ifnet *, struct mbuf *, const struct sockaddr *, struct route *);
   static int wg_clone_destroy(struct if_clone *ifc, struct ifnet *ifp,
           uint32_t flags);
   static bool wgc_privileged(struct wg_softc *);
   static int wgc_get(struct wg_softc *, struct wg_data_io *);
   static int wgc_set(struct wg_softc *, struct wg_data_io *);
   static int wg_up(struct wg_softc *);
   static void wg_down(struct wg_softc *);
   static void wg_reassign(struct ifnet *, struct vnet *, char *unused);
   static void wg_init(void *);
   static int wg_ioctl(struct ifnet *, u_long, caddr_t);
   static void vnet_wg_init(const void *);
   static void vnet_wg_uninit(const void *);
   static int wg_module_init(void);
   static void wg_module_deinit(void);
   
   /* TODO Peer */
   static struct wg_peer *
   wg_peer_alloc(struct wg_softc *sc, const uint8_t pub_key[WG_KEY_SIZE])
   {
           struct wg_peer *peer;
   
           sx_assert(&sc->sc_lock, SX_XLOCKED);
   
           peer = malloc(sizeof(*peer), M_WG, M_WAITOK | M_ZERO);
           peer->p_remote = noise_remote_alloc(sc->sc_local, peer, pub_key);
           peer->p_tx_bytes = counter_u64_alloc(M_WAITOK);
           peer->p_rx_bytes = counter_u64_alloc(M_WAITOK);
           peer->p_id = peer_counter++;
           peer->p_sc = sc;
   
           cookie_maker_init(&peer->p_cookie, pub_key);
   
           rw_init(&peer->p_endpoint_lock, "wg_peer_endpoint");
   
           wg_queue_init(&peer->p_stage_queue, "stageq");
           wg_queue_init(&peer->p_encrypt_serial, "txq");
           wg_queue_init(&peer->p_decrypt_serial, "rxq");
   
           peer->p_enabled = false;
           peer->p_need_another_keepalive = false;
           peer->p_persistent_keepalive_interval = 0;
           callout_init(&peer->p_new_handshake, true);
           callout_init(&peer->p_send_keepalive, true);
           callout_init(&peer->p_retry_handshake, true);
           callout_init(&peer->p_persistent_keepalive, true);
           callout_init(&peer->p_zero_key_material, true);
   
           mtx_init(&peer->p_handshake_mtx, "peer handshake", NULL, MTX_DEF);
           bzero(&peer->p_handshake_complete, sizeof(peer->p_handshake_complete));
           peer->p_handshake_retries = 0;
   
           GROUPTASK_INIT(&peer->p_send, 0, (gtask_fn_t *)wg_deliver_out, peer);
           taskqgroup_attach(qgroup_wg_tqg, &peer->p_send, peer, NULL, NULL, "wg send");
           GROUPTASK_INIT(&peer->p_recv, 0, (gtask_fn_t *)wg_deliver_in, peer);
           taskqgroup_attach(qgroup_wg_tqg, &peer->p_recv, peer, NULL, NULL, "wg recv");
   
           LIST_INIT(&peer->p_aips);
           peer->p_aips_num = 0;
   
           return (peer);
   }
   
   static void
   wg_peer_free_deferred(struct noise_remote *r)
   {
           struct wg_peer *peer = noise_remote_arg(r);
   
           /* While there are no references remaining, we may still have
            * p_{send,recv} executing (think empty queue, but wg_deliver_{in,out}
            * needs to check the queue. We should wait for them and then free. */
           GROUPTASK_DRAIN(&peer->p_recv);
           GROUPTASK_DRAIN(&peer->p_send);
           taskqgroup_detach(qgroup_wg_tqg, &peer->p_recv);
           taskqgroup_detach(qgroup_wg_tqg, &peer->p_send);
   
           wg_queue_deinit(&peer->p_decrypt_serial);
           wg_queue_deinit(&peer->p_encrypt_serial);
           wg_queue_deinit(&peer->p_stage_queue);
   
           counter_u64_free(peer->p_tx_bytes);
           counter_u64_free(peer->p_rx_bytes);
           rw_destroy(&peer->p_endpoint_lock);
           mtx_destroy(&peer->p_handshake_mtx);
   
           cookie_maker_free(&peer->p_cookie);
   
           free(peer, M_WG);
   }
   
   static void
   wg_peer_destroy(struct wg_peer *peer)
   {
           struct wg_softc *sc = peer->p_sc;
           sx_assert(&sc->sc_lock, SX_XLOCKED);
   
           /* Disable remote and timers. This will prevent any new handshakes
            * occuring. */
           noise_remote_disable(peer->p_remote);
           wg_timers_disable(peer);
   
           /* Now we can remove all allowed IPs so no more packets will be routed
            * to the peer. */
           wg_aip_remove_all(sc, peer);
   
           /* Remove peer from the interface, then free. Some references may still
            * exist to p_remote, so noise_remote_free will wait until they're all
            * put to call wg_peer_free_deferred. */
           sc->sc_peers_num--;
           TAILQ_REMOVE(&sc->sc_peers, peer, p_entry);
           DPRINTF(sc, "Peer %" PRIu64 " destroyed\n", peer->p_id);
           noise_remote_free(peer->p_remote, wg_peer_free_deferred);
   }
   
   static void
   wg_peer_destroy_all(struct wg_softc *sc)
   {
           struct wg_peer *peer, *tpeer;
           TAILQ_FOREACH_SAFE(peer, &sc->sc_peers, p_entry, tpeer)
                   wg_peer_destroy(peer);
   }
   
   static void
   wg_peer_set_endpoint(struct wg_peer *peer, struct wg_endpoint *e)
   {
           MPASS(e->e_remote.r_sa.sa_family != 0);
           if (memcmp(e, &peer->p_endpoint, sizeof(*e)) == 0)
                   return;
   
           rw_wlock(&peer->p_endpoint_lock);
           peer->p_endpoint = *e;
           rw_wunlock(&peer->p_endpoint_lock);
   }
   
   static void
   wg_peer_clear_src(struct wg_peer *peer)
   {
           rw_wlock(&peer->p_endpoint_lock);
           bzero(&peer->p_endpoint.e_local, sizeof(peer->p_endpoint.e_local));
           rw_wunlock(&peer->p_endpoint_lock);
   }
   
   static void
   wg_peer_get_endpoint(struct wg_peer *peer, struct wg_endpoint *e)
   {
           rw_rlock(&peer->p_endpoint_lock);
           *e = peer->p_endpoint;
           rw_runlock(&peer->p_endpoint_lock);
   }
   
   /* Allowed IP */
   static int
   wg_aip_add(struct wg_softc *sc, struct wg_peer *peer, sa_family_t af, const void *addr, uint8_t cidr)
   {
           struct radix_node_head  *root;
           struct radix_node       *node;
           struct wg_aip           *aip;
           int                      ret = 0;
   
           aip = malloc(sizeof(*aip), M_WG, M_WAITOK | M_ZERO);
           aip->a_peer = peer;
           aip->a_af = af;
   
           switch (af) {
   #ifdef INET
           case AF_INET:
                   if (cidr > 32) cidr = 32;
                   root = sc->sc_aip4;
                   aip->a_addr.in = *(const struct in_addr *)addr;
                   aip->a_mask.ip = htonl(~((1LL << (32 - cidr)) - 1) & 0xffffffff);
                   aip->a_addr.ip &= aip->a_mask.ip;
                   aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr);
                   break;
   #endif
   #ifdef INET6
           case AF_INET6:
                   if (cidr > 128) cidr = 128;
                   root = sc->sc_aip6;
                   aip->a_addr.in6 = *(const struct in6_addr *)addr;
                   in6_prefixlen2mask(&aip->a_mask.in6, cidr);
                   for (int i = 0; i < 4; i++)
                           aip->a_addr.ip6[i] &= aip->a_mask.ip6[i];
                   aip->a_addr.length = aip->a_mask.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr);
                   break;
   #endif
           default:
                   free(aip, M_WG);
                   return (EAFNOSUPPORT);
           }
   
           RADIX_NODE_HEAD_LOCK(root);
           node = root->rnh_addaddr(&aip->a_addr, &aip->a_mask, &root->rh, aip->a_nodes);
           if (node == aip->a_nodes) {
                   LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry);
                   peer->p_aips_num++;
           } else if (!node)
                   node = root->rnh_lookup(&aip->a_addr, &aip->a_mask, &root->rh);
           if (!node) {
                   free(aip, M_WG);
                   return (ENOMEM);
           } else if (node != aip->a_nodes) {
                   free(aip, M_WG);
                   aip = (struct wg_aip *)node;
                   if (aip->a_peer != peer) {
                           LIST_REMOVE(aip, a_entry);
                           aip->a_peer->p_aips_num--;
                           aip->a_peer = peer;
                           LIST_INSERT_HEAD(&peer->p_aips, aip, a_entry);
                           aip->a_peer->p_aips_num++;
                   }
           }
           RADIX_NODE_HEAD_UNLOCK(root);
           return (ret);
   }
   
   static struct wg_peer *
   wg_aip_lookup(struct wg_softc *sc, sa_family_t af, void *a)
   {
           struct radix_node_head  *root;
           struct radix_node       *node;
           struct wg_peer          *peer;
           struct aip_addr          addr;
           RADIX_NODE_HEAD_RLOCK_TRACKER;
   
           switch (af) {
           case AF_INET:
                   root = sc->sc_aip4;
                   memcpy(&addr.in, a, sizeof(addr.in));
                   addr.length = offsetof(struct aip_addr, in) + sizeof(struct in_addr);
                   break;
           case AF_INET6:
                   root = sc->sc_aip6;
                   memcpy(&addr.in6, a, sizeof(addr.in6));
                   addr.length = offsetof(struct aip_addr, in6) + sizeof(struct in6_addr);
                   break;
           default:
                   return NULL;
           }
   
           RADIX_NODE_HEAD_RLOCK(root);
           node = root->rnh_matchaddr(&addr, &root->rh);
           if (node != NULL) {
                   peer = ((struct wg_aip *)node)->a_peer;
                   noise_remote_ref(peer->p_remote);
           } else {
                   peer = NULL;
           }
           RADIX_NODE_HEAD_RUNLOCK(root);
   
           return (peer);
   }
   
   static void
   wg_aip_remove_all(struct wg_softc *sc, struct wg_peer *peer)
   {
           struct wg_aip           *aip, *taip;
   
           RADIX_NODE_HEAD_LOCK(sc->sc_aip4);
           LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) {
                   if (aip->a_af == AF_INET) {
                           if (sc->sc_aip4->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip4->rh) == NULL)
                                   panic("failed to delete aip %p", aip);
                           LIST_REMOVE(aip, a_entry);
                           peer->p_aips_num--;
                           free(aip, M_WG);
                   }
           }
           RADIX_NODE_HEAD_UNLOCK(sc->sc_aip4);
   
           RADIX_NODE_HEAD_LOCK(sc->sc_aip6);
           LIST_FOREACH_SAFE(aip, &peer->p_aips, a_entry, taip) {
                   if (aip->a_af == AF_INET6) {
                           if (sc->sc_aip6->rnh_deladdr(&aip->a_addr, &aip->a_mask, &sc->sc_aip6->rh) == NULL)
                                   panic("failed to delete aip %p", aip);
                           LIST_REMOVE(aip, a_entry);
                           peer->p_aips_num--;
                           free(aip, M_WG);
                   }
           }
           RADIX_NODE_HEAD_UNLOCK(sc->sc_aip6);
   
           if (!LIST_EMPTY(&peer->p_aips) || peer->p_aips_num != 0)
                   panic("wg_aip_remove_all could not delete all %p", peer);
   }
   
   static int
   wg_socket_init(struct wg_softc *sc, in_port_t port)
   {
           struct ucred *cred = sc->sc_ucred;
           struct socket *so4 = NULL, *so6 = NULL;
           int rc;
   
           sx_assert(&sc->sc_lock, SX_XLOCKED);
   
           if (!cred)
                   return (EBUSY);
   
           /*
            * For socket creation, we use the creds of the thread that created the
            * tunnel rather than the current thread to maintain the semantics that
            * WireGuard has on Linux with network namespaces -- that the sockets
            * are created in their home vnet so that they can be configured and
            * functionally attached to a foreign vnet as the jail's only interface
            * to the network.
            */
   #ifdef INET
           rc = socreate(AF_INET, &so4, SOCK_DGRAM, IPPROTO_UDP, cred, curthread);
           if (rc)
                   goto out;
   
           rc = udp_set_kernel_tunneling(so4, wg_input, NULL, sc);
           /*
            * udp_set_kernel_tunneling can only fail if there is already a tunneling function set.
            * This should never happen with a new socket.
            */
           MPASS(rc == 0);
   #endif
   
   #ifdef INET6
           rc = socreate(AF_INET6, &so6, SOCK_DGRAM, IPPROTO_UDP, cred, curthread);
           if (rc)
                   goto out;
           rc = udp_set_kernel_tunneling(so6, wg_input, NULL, sc);
           MPASS(rc == 0);
   #endif
   
           if (sc->sc_socket.so_user_cookie) {
                   rc = wg_socket_set_sockopt(so4, so6, SO_USER_COOKIE, &sc->sc_socket.so_user_cookie, sizeof(sc->sc_socket.so_user_cookie));
                   if (rc)
                           goto out;
           }
           rc = wg_socket_set_sockopt(so4, so6, SO_SETFIB, &sc->sc_socket.so_fibnum, sizeof(sc->sc_socket.so_fibnum));
           if (rc)
                   goto out;
   
           rc = wg_socket_bind(&so4, &so6, &port);
           if (!rc) {
                   sc->sc_socket.so_port = port;
                   wg_socket_set(sc, so4, so6);
           }
   out:
           if (rc) {
                   if (so4 != NULL)
                           soclose(so4);
                   if (so6 != NULL)
                           soclose(so6);
           }
           return (rc);
   }
   
   static int wg_socket_set_sockopt(struct socket *so4, struct socket *so6, int name, void *val, size_t len)
   {
           int ret4 = 0, ret6 = 0;
           struct sockopt sopt = {
                   .sopt_dir = SOPT_SET,
                   .sopt_level = SOL_SOCKET,
                   .sopt_name = name,
                   .sopt_val = val,
                   .sopt_valsize = len
           };
   
           if (so4)
                   ret4 = sosetopt(so4, &sopt);
           if (so6)
                   ret6 = sosetopt(so6, &sopt);
           return (ret4 ?: ret6);
   }
   
   static int wg_socket_set_cookie(struct wg_softc *sc, uint32_t user_cookie)
   {
           struct wg_socket *so = &sc->sc_socket;
           int ret;
   
           sx_assert(&sc->sc_lock, SX_XLOCKED);
           ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_USER_COOKIE, &user_cookie, sizeof(user_cookie));
           if (!ret)
                   so->so_user_cookie = user_cookie;
           return (ret);
   }
   
   static int wg_socket_set_fibnum(struct wg_softc *sc, int fibnum)
   {
           struct wg_socket *so = &sc->sc_socket;
           int ret;
   
           sx_assert(&sc->sc_lock, SX_XLOCKED);
   
           ret = wg_socket_set_sockopt(so->so_so4, so->so_so6, SO_SETFIB, &fibnum, sizeof(fibnum));
           if (!ret)
                   so->so_fibnum = fibnum;
           return (ret);
   }
   
   static void
   wg_socket_uninit(struct wg_softc *sc)
   {
           wg_socket_set(sc, NULL, NULL);
   }
   
   static void
   wg_socket_set(struct wg_softc *sc, struct socket *new_so4, struct socket *new_so6)
   {
           struct wg_socket *so = &sc->sc_socket;
           struct socket *so4, *so6;
   
           sx_assert(&sc->sc_lock, SX_XLOCKED);
   
           so4 = atomic_load_ptr(&so->so_so4);
           so6 = atomic_load_ptr(&so->so_so6);
           atomic_store_ptr(&so->so_so4, new_so4);
           atomic_store_ptr(&so->so_so6, new_so6);
   
           if (!so4 && !so6)
                   return;
           NET_EPOCH_WAIT();
           if (so4)
                   soclose(so4);
           if (so6)
                   soclose(so6);
   }
   
   static int
   wg_socket_bind(struct socket **in_so4, struct socket **in_so6, in_port_t *requested_port)
   {
           struct socket *so4 = *in_so4, *so6 = *in_so6;
           int ret4 = 0, ret6 = 0;
           in_port_t port = *requested_port;
           struct sockaddr_in sin = {
                   .sin_len = sizeof(struct sockaddr_in),
                   .sin_family = AF_INET,
                   .sin_port = htons(port)
           };
           struct sockaddr_in6 sin6 = {
                   .sin6_len = sizeof(struct sockaddr_in6),
                   .sin6_family = AF_INET6,
                   .sin6_port = htons(port)
           };
   
           if (so4) {
                   ret4 = sobind(so4, (struct sockaddr *)&sin, curthread);
                   if (ret4 && ret4 != EADDRNOTAVAIL)
                           return (ret4);
                   if (!ret4 && !sin.sin_port) {
                           struct sockaddr_in *bound_sin;
                           int ret = so4->so_proto->pr_sockaddr(so4,
                               (struct sockaddr **)&bound_sin);
                           if (ret)
                                   return (ret);
                           port = ntohs(bound_sin->sin_port);
                           sin6.sin6_port = bound_sin->sin_port;
                           free(bound_sin, M_SONAME);
                   }
           }
   
           if (so6) {
                   ret6 = sobind(so6, (struct sockaddr *)&sin6, curthread);
                   if (ret6 && ret6 != EADDRNOTAVAIL)
                           return (ret6);
                   if (!ret6 && !sin6.sin6_port) {
                           struct sockaddr_in6 *bound_sin6;
                           int ret = so6->so_proto->pr_sockaddr(so6,
                               (struct sockaddr **)&bound_sin6);
                           if (ret)
                                   return (ret);
                           port = ntohs(bound_sin6->sin6_port);
                           free(bound_sin6, M_SONAME);
                   }
           }
   
           if (ret4 && ret6)
                   return (ret4);
           *requested_port = port;
           if (ret4 && !ret6 && so4) {
                   soclose(so4);
                   *in_so4 = NULL;
           } else if (ret6 && !ret4 && so6) {
                   soclose(so6);
                   *in_so6 = NULL;
           }
           return (0);
   }
   
   static int
   wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m)
   {
           struct epoch_tracker et;
           struct sockaddr *sa;
           struct wg_socket *so = &sc->sc_socket;
           struct socket *so4, *so6;
           struct mbuf *control = NULL;
           int ret = 0;
           size_t len = m->m_pkthdr.len;
   
           /* Get local control address before locking */
           if (e->e_remote.r_sa.sa_family == AF_INET) {
                   if (e->e_local.l_in.s_addr != INADDR_ANY)
                           control = sbcreatecontrol((caddr_t)&e->e_local.l_in,
                               sizeof(struct in_addr), IP_SENDSRCADDR,
                               IPPROTO_IP, M_NOWAIT);
   #ifdef INET6
           } else if (e->e_remote.r_sa.sa_family == AF_INET6) {
                   if (!IN6_IS_ADDR_UNSPECIFIED(&e->e_local.l_in6))
                           control = sbcreatecontrol((caddr_t)&e->e_local.l_pktinfo6,
                               sizeof(struct in6_pktinfo), IPV6_PKTINFO,
                               IPPROTO_IPV6, M_NOWAIT);
   #endif
           } else {
                   m_freem(m);
                   return (EAFNOSUPPORT);
           }
   
           /* Get remote address */
           sa = &e->e_remote.r_sa;
   
           NET_EPOCH_ENTER(et);
           so4 = atomic_load_ptr(&so->so_so4);
           so6 = atomic_load_ptr(&so->so_so6);
           if (e->e_remote.r_sa.sa_family == AF_INET && so4 != NULL)
                   ret = sosend(so4, sa, NULL, m, control, 0, curthread);
           else if (e->e_remote.r_sa.sa_family == AF_INET6 && so6 != NULL)
                   ret = sosend(so6, sa, NULL, m, control, 0, curthread);
           else {
                   ret = ENOTCONN;
                   m_freem(control);
                   m_freem(m);
           }
           NET_EPOCH_EXIT(et);
           if (ret == 0) {
                   if_inc_counter(sc->sc_ifp, IFCOUNTER_OPACKETS, 1);
                   if_inc_counter(sc->sc_ifp, IFCOUNTER_OBYTES, len);
           }
           return (ret);
   }
   
   static void
   wg_send_buf(struct wg_softc *sc, struct wg_endpoint *e, uint8_t *buf, size_t len)
   {
           struct mbuf     *m;
           int              ret = 0;
           bool             retried = false;
   
   retry:
           m = m_get2(len, M_NOWAIT, MT_DATA, M_PKTHDR);
           if (!m) {
                   ret = ENOMEM;
                   goto out;
           }
           m_copyback(m, 0, len, buf);
   
           if (ret == 0) {
                   ret = wg_send(sc, e, m);
                   /* Retry if we couldn't bind to e->e_local */
                   if (ret == EADDRNOTAVAIL && !retried) {
                           bzero(&e->e_local, sizeof(e->e_local));
                           retried = true;
                           goto retry;
                   }
           } else {
                   ret = wg_send(sc, e, m);
           }
   out:
           if (ret)
                   DPRINTF(sc, "Unable to send packet: %d\n", ret);
   }
   
   /* Timers */
   static void
   wg_timers_enable(struct wg_peer *peer)
   {
           atomic_store_bool(&peer->p_enabled, true);
           wg_timers_run_persistent_keepalive(peer);
   }
   
   static void
   wg_timers_disable(struct wg_peer *peer)
   {
           /* By setting p_enabled = false, then calling NET_EPOCH_WAIT, we can be
            * sure no new handshakes are created after the wait. This is because
            * all callout_resets (scheduling the callout) are guarded by
            * p_enabled. We can be sure all sections that read p_enabled and then
            * optionally call callout_reset are finished as they are surrounded by
            * NET_EPOCH_{ENTER,EXIT}.
            *
            * However, as new callouts may be scheduled during NET_EPOCH_WAIT (but
            * not after), we stop all callouts leaving no callouts active.
            *
            * We should also pull NET_EPOCH_WAIT out of the FOREACH(peer) loops, but the
            * performance impact is acceptable for the time being. */
           atomic_store_bool(&peer->p_enabled, false);
           NET_EPOCH_WAIT();
           atomic_store_bool(&peer->p_need_another_keepalive, false);
   
           callout_stop(&peer->p_new_handshake);
           callout_stop(&peer->p_send_keepalive);
           callout_stop(&peer->p_retry_handshake);
           callout_stop(&peer->p_persistent_keepalive);
           callout_stop(&peer->p_zero_key_material);
   }
   
   static void
   wg_timers_set_persistent_keepalive(struct wg_peer *peer, uint16_t interval)
   {
           struct epoch_tracker et;
           if (interval != peer->p_persistent_keepalive_interval) {
                   atomic_store_16(&peer->p_persistent_keepalive_interval, interval);
                   NET_EPOCH_ENTER(et);
                   if (atomic_load_bool(&peer->p_enabled))
                           wg_timers_run_persistent_keepalive(peer);
                   NET_EPOCH_EXIT(et);
           }
   }
   
   static void
   wg_timers_get_last_handshake(struct wg_peer *peer, struct wg_timespec64 *time)
   {
           mtx_lock(&peer->p_handshake_mtx);
           time->tv_sec = peer->p_handshake_complete.tv_sec;
           time->tv_nsec = peer->p_handshake_complete.tv_nsec;
           mtx_unlock(&peer->p_handshake_mtx);
   }
   
   static void
   wg_timers_event_data_sent(struct wg_peer *peer)
   {
           struct epoch_tracker et;
           NET_EPOCH_ENTER(et);
           if (atomic_load_bool(&peer->p_enabled) &&
               !callout_pending(&peer->p_new_handshake))
                   callout_reset(&peer->p_new_handshake, MSEC_2_TICKS(
                       NEW_HANDSHAKE_TIMEOUT * 1000 +
                       arc4random_uniform(REKEY_TIMEOUT_JITTER)),
                       wg_timers_run_new_handshake, peer);
           NET_EPOCH_EXIT(et);
   }
  
  static void
  wg_timers_event_data_received(struct wg_peer *peer)
  {
          struct epoch_tracker et;
          NET_EPOCH_ENTER(et);
          if (atomic_load_bool(&peer->p_enabled)) {
                  if (!callout_pending(&peer->p_send_keepalive))
                          callout_reset(&peer->p_send_keepalive,
                              MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000),
                              wg_timers_run_send_keepalive, peer);
                  else
                          atomic_store_bool(&peer->p_need_another_keepalive,
                              true);
          }
          NET_EPOCH_EXIT(et);
  }
  
  static void
  wg_timers_event_any_authenticated_packet_sent(struct wg_peer *peer)
  {
          callout_stop(&peer->p_send_keepalive);
  }
  
  static void
  wg_timers_event_any_authenticated_packet_received(struct wg_peer *peer)
  {
          callout_stop(&peer->p_new_handshake);
  }
  
  static void
  wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *peer)
  {
          struct epoch_tracker et;
          uint16_t interval;
          NET_EPOCH_ENTER(et);
          interval = atomic_load_16(&peer->p_persistent_keepalive_interval);
          if (atomic_load_bool(&peer->p_enabled) && interval > 0)
                  callout_reset(&peer->p_persistent_keepalive,
                       MSEC_2_TICKS(interval * 1000),
                       wg_timers_run_persistent_keepalive, peer);
          NET_EPOCH_EXIT(et);
  }
  
  static void
  wg_timers_event_handshake_initiated(struct wg_peer *peer)
  {
          struct epoch_tracker et;
          NET_EPOCH_ENTER(et);
          if (atomic_load_bool(&peer->p_enabled))
                  callout_reset(&peer->p_retry_handshake, MSEC_2_TICKS(
                      REKEY_TIMEOUT * 1000 +
                      arc4random_uniform(REKEY_TIMEOUT_JITTER)),
                      wg_timers_run_retry_handshake, peer);
          NET_EPOCH_EXIT(et);
  }
  
  static void
  wg_timers_event_handshake_complete(struct wg_peer *peer)
  {
          struct epoch_tracker et;
          NET_EPOCH_ENTER(et);
          if (atomic_load_bool(&peer->p_enabled)) {
                  mtx_lock(&peer->p_handshake_mtx);
                  callout_stop(&peer->p_retry_handshake);
                  peer->p_handshake_retries = 0;
                  getnanotime(&peer->p_handshake_complete);
                  mtx_unlock(&peer->p_handshake_mtx);
                  wg_timers_run_send_keepalive(peer);
          }
          NET_EPOCH_EXIT(et);
  }
  
  static void
  wg_timers_event_session_derived(struct wg_peer *peer)
  {
          struct epoch_tracker et;
          NET_EPOCH_ENTER(et);
          if (atomic_load_bool(&peer->p_enabled))
                  callout_reset(&peer->p_zero_key_material,
                      MSEC_2_TICKS(REJECT_AFTER_TIME * 3 * 1000),
                      wg_timers_run_zero_key_material, peer);
          NET_EPOCH_EXIT(et);
  }
  
  static void
  wg_timers_event_want_initiation(struct wg_peer *peer)
  {
          struct epoch_tracker et;
          NET_EPOCH_ENTER(et);
          if (atomic_load_bool(&peer->p_enabled))
                  wg_timers_run_send_initiation(peer, false);
          NET_EPOCH_EXIT(et);
  }
  
  static void
  wg_timers_run_send_initiation(struct wg_peer *peer, bool is_retry)
  {
          if (!is_retry)
                  peer->p_handshake_retries = 0;
          if (noise_remote_initiation_expired(peer->p_remote) == ETIMEDOUT)
                  wg_send_initiation(peer);
  }
  
  static void
  wg_timers_run_retry_handshake(void *_peer)
  {
          struct epoch_tracker et;
          struct wg_peer *peer = _peer;
  
          mtx_lock(&peer->p_handshake_mtx);
          if (peer->p_handshake_retries <= MAX_TIMER_HANDSHAKES) {
                  peer->p_handshake_retries++;
                  mtx_unlock(&peer->p_handshake_mtx);
  
                  DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete "
                      "after %d seconds, retrying (try %d)\n", peer->p_id,
                      REKEY_TIMEOUT, peer->p_handshake_retries + 1);
                  wg_peer_clear_src(peer);
                  wg_timers_run_send_initiation(peer, true);
          } else {
                  mtx_unlock(&peer->p_handshake_mtx);
  
                  DPRINTF(peer->p_sc, "Handshake for peer %" PRIu64 " did not complete "
                      "after %d retries, giving up\n", peer->p_id,
                      MAX_TIMER_HANDSHAKES + 2);
  
                  callout_stop(&peer->p_send_keepalive);
                  wg_queue_purge(&peer->p_stage_queue);
                  NET_EPOCH_ENTER(et);
                  if (atomic_load_bool(&peer->p_enabled) &&
                      !callout_pending(&peer->p_zero_key_material))
                          callout_reset(&peer->p_zero_key_material,
                              MSEC_2_TICKS(REJECT_AFTER_TIME * 3 * 1000),
                              wg_timers_run_zero_key_material, peer);
                  NET_EPOCH_EXIT(et);
          }
  }
  
  static void
  wg_timers_run_send_keepalive(void *_peer)
  {
          struct epoch_tracker et;
          struct wg_peer *peer = _peer;
  
          wg_send_keepalive(peer);
          NET_EPOCH_ENTER(et);
          if (atomic_load_bool(&peer->p_enabled) &&
              atomic_load_bool(&peer->p_need_another_keepalive)) {
                  atomic_store_bool(&peer->p_need_another_keepalive, false);
                  callout_reset(&peer->p_send_keepalive,
                      MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000),
                      wg_timers_run_send_keepalive, peer);
          }
          NET_EPOCH_EXIT(et);
  }
  
  static void
  wg_timers_run_new_handshake(void *_peer)
  {
          struct wg_peer *peer = _peer;
  
          DPRINTF(peer->p_sc, "Retrying handshake with peer %" PRIu64 " because we "
              "stopped hearing back after %d seconds\n",
              peer->p_id, NEW_HANDSHAKE_TIMEOUT);
  
          wg_peer_clear_src(peer);
          wg_timers_run_send_initiation(peer, false);
  }
  
  static void
  wg_timers_run_zero_key_material(void *_peer)
  {
          struct wg_peer *peer = _peer;
  
          DPRINTF(peer->p_sc, "Zeroing out keys for peer %" PRIu64 ", since we "
              "haven't received a new one in %d seconds\n",
              peer->p_id, REJECT_AFTER_TIME * 3);
          noise_remote_keypairs_clear(peer->p_remote);
  }
  
  static void
  wg_timers_run_persistent_keepalive(void *_peer)
  {
          struct wg_peer *peer = _peer;
  
          if (atomic_load_16(&peer->p_persistent_keepalive_interval) > 0)
                  wg_send_keepalive(peer);
  }
  
  /* TODO Handshake */
  static void
  wg_peer_send_buf(struct wg_peer *peer, uint8_t *buf, size_t len)
  {
          struct wg_endpoint endpoint;
  
          counter_u64_add(peer->p_tx_bytes, len);
          wg_timers_event_any_authenticated_packet_traversal(peer);
          wg_timers_event_any_authenticated_packet_sent(peer);
          wg_peer_get_endpoint(peer, &endpoint);
          wg_send_buf(peer->p_sc, &endpoint, buf, len);
  }
  
  static void
  wg_send_initiation(struct wg_peer *peer)
  {
          struct wg_pkt_initiation pkt;
  
          if (noise_create_initiation(peer->p_remote, &pkt.s_idx, pkt.ue,
              pkt.es, pkt.ets) != 0)
                  return;
  
          DPRINTF(peer->p_sc, "Sending handshake initiation to peer %" PRIu64 "\n", peer->p_id);
  
          pkt.t = WG_PKT_INITIATION;
          cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
              sizeof(pkt) - sizeof(pkt.m));
          wg_peer_send_buf(peer, (uint8_t *)&pkt, sizeof(pkt));
          wg_timers_event_handshake_initiated(peer);
  }
  
  static void
  wg_send_response(struct wg_peer *peer)
  {
          struct wg_pkt_response pkt;
  
          if (noise_create_response(peer->p_remote, &pkt.s_idx, &pkt.r_idx,
              pkt.ue, pkt.en) != 0)
                  return;
  
          DPRINTF(peer->p_sc, "Sending handshake response to peer %" PRIu64 "\n", peer->p_id);
  
          wg_timers_event_session_derived(peer);
          pkt.t = WG_PKT_RESPONSE;
          cookie_maker_mac(&peer->p_cookie, &pkt.m, &pkt,
               sizeof(pkt)-sizeof(pkt.m));
          wg_peer_send_buf(peer, (uint8_t*)&pkt, sizeof(pkt));
  }
  
  static void
  wg_send_cookie(struct wg_softc *sc, struct cookie_macs *cm, uint32_t idx,
      struct wg_endpoint *e)
  {
          struct wg_pkt_cookie    pkt;
  
          DPRINTF(sc, "Sending cookie response for denied handshake message\n");
  
          pkt.t = WG_PKT_COOKIE;
          pkt.r_idx = idx;
  
          cookie_checker_create_payload(&sc->sc_cookie, cm, pkt.nonce,
              pkt.ec, &e->e_remote.r_sa);
          wg_send_buf(sc, e, (uint8_t *)&pkt, sizeof(pkt));
  }
  
  static void
  wg_send_keepalive(struct wg_peer *peer)
  {
          struct wg_packet *pkt;
          struct mbuf *m;
  
          if (wg_queue_len(&peer->p_stage_queue) > 0)
                  goto send;
          if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
                  return;
          if ((pkt = wg_packet_alloc(m)) == NULL) {
                  m_freem(m);
                  return;
          }
          wg_queue_push_staged(&peer->p_stage_queue, pkt);
          DPRINTF(peer->p_sc, "Sending keepalive packet to peer %" PRIu64 "\n", peer->p_id);
  send:
          wg_peer_send_staged(peer);
  }
  
  static void
  wg_handshake(struct wg_softc *sc, struct wg_packet *pkt)
  {
          struct wg_pkt_initiation        *init;
          struct wg_pkt_response          *resp;
          struct wg_pkt_cookie            *cook;
          struct wg_endpoint              *e;
          struct wg_peer                  *peer;
          struct mbuf                     *m;
          struct noise_remote             *remote = NULL;
          int                              res;
          bool                             underload = false;
          static sbintime_t                wg_last_underload; /* sbinuptime */
  
          underload = wg_queue_len(&sc->sc_handshake_queue) >= MAX_QUEUED_HANDSHAKES / 8;
          if (underload) {
                  wg_last_underload = getsbinuptime();
          } else if (wg_last_underload) {
                  underload = wg_last_underload + UNDERLOAD_TIMEOUT * SBT_1S > getsbinuptime();
                  if (!underload)
                          wg_last_underload = 0;
          }
  
          m = pkt->p_mbuf;
          e = &pkt->p_endpoint;
  
          if ((pkt->p_mbuf = m = m_pullup(m, m->m_pkthdr.len)) == NULL)
                  goto error;
  
          switch (*mtod(m, uint32_t *)) {
          case WG_PKT_INITIATION:
                  init = mtod(m, struct wg_pkt_initiation *);
  
                  res = cookie_checker_validate_macs(&sc->sc_cookie, &init->m,
                                  init, sizeof(*init) - sizeof(init->m),
                                  underload, &e->e_remote.r_sa,
                                  sc->sc_ifp->if_vnet);
  
                  if (res == EINVAL) {
                          DPRINTF(sc, "Invalid initiation MAC\n");
                          goto error;
                  } else if (res == ECONNREFUSED) {
                          DPRINTF(sc, "Handshake ratelimited\n");
                          goto error;
                  } else if (res == EAGAIN) {
                          wg_send_cookie(sc, &init->m, init->s_idx, e);
                          goto error;
                  } else if (res != 0) {
                          panic("unexpected response: %d\n", res);
                  }
  
                  if (noise_consume_initiation(sc->sc_local, &remote,
                      init->s_idx, init->ue, init->es, init->ets) != 0) {
                          DPRINTF(sc, "Invalid handshake initiation\n");
                          goto error;
                  }
  
                  peer = noise_remote_arg(remote);
  
                  DPRINTF(sc, "Receiving handshake initiation from peer %" PRIu64 "\n", peer->p_id);
  
                  wg_peer_set_endpoint(peer, e);
                  wg_send_response(peer);
                  break;
          case WG_PKT_RESPONSE:
                  resp = mtod(m, struct wg_pkt_response *);
  
                  res = cookie_checker_validate_macs(&sc->sc_cookie, &resp->m,
                                  resp, sizeof(*resp) - sizeof(resp->m),
                                  underload, &e->e_remote.r_sa,
                                  sc->sc_ifp->if_vnet);
  
                  if (res == EINVAL) {
                          DPRINTF(sc, "Invalid response MAC\n");
                          goto error;
                  } else if (res == ECONNREFUSED) {
                          DPRINTF(sc, "Handshake ratelimited\n");
                          goto error;
                  } else if (res == EAGAIN) {
                          wg_send_cookie(sc, &resp->m, resp->s_idx, e);
                          goto error;
                  } else if (res != 0) {
                          panic("unexpected response: %d\n", res);
                  }
  
                  if (noise_consume_response(sc->sc_local, &remote,
                      resp->s_idx, resp->r_idx, resp->ue, resp->en) != 0) {
                          DPRINTF(sc, "Invalid handshake response\n");
                          goto error;
                  }
  
                  peer = noise_remote_arg(remote);
                  DPRINTF(sc, "Receiving handshake response from peer %" PRIu64 "\n", peer->p_id);
  
                  wg_peer_set_endpoint(peer, e);
                  wg_timers_event_session_derived(peer);
                  wg_timers_event_handshake_complete(peer);
                  break;
          case WG_PKT_COOKIE:
                  cook = mtod(m, struct wg_pkt_cookie *);
  
                  if ((remote = noise_remote_index(sc->sc_local, cook->r_idx)) == NULL) {
                          DPRINTF(sc, "Unknown cookie index\n");
                          goto error;
                  }
  
                  peer = noise_remote_arg(remote);
  
                  if (cookie_maker_consume_payload(&peer->p_cookie,
                      cook->nonce, cook->ec) == 0) {
                          DPRINTF(sc, "Receiving cookie response\n");
                  } else {
                          DPRINTF(sc, "Could not decrypt cookie response\n");
                          goto error;
                  }
  
                  goto not_authenticated;
          default:
                  panic("invalid packet in handshake queue");
          }
  
          wg_timers_event_any_authenticated_packet_received(peer);
          wg_timers_event_any_authenticated_packet_traversal(peer);
  
  not_authenticated:
          counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len);
          if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1);
          if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len);
  error:
          if (remote != NULL)
                  noise_remote_put(remote);
          wg_packet_free(pkt);
  }
  
  static void
  wg_softc_handshake_receive(struct wg_softc *sc)
  {
          struct wg_packet *pkt;
          while ((pkt = wg_queue_dequeue_handshake(&sc->sc_handshake_queue)) != NULL)
                  wg_handshake(sc, pkt);
  }
  
  static void
  wg_mbuf_reset(struct mbuf *m)
  {
  
          struct m_tag *t, *tmp;
  
          /*
           * We want to reset the mbuf to a newly allocated state, containing
           * just the packet contents. Unfortunately FreeBSD doesn't seem to
           * offer this anywhere, so we have to make it up as we go. If we can
           * get this in kern/kern_mbuf.c, that would be best.
           *
           * Notice: this may break things unexpectedly but it is better to fail
           *         closed in the extreme case than leak informtion in every
           *         case.
           *
           * With that said, all this attempts to do is remove any extraneous
           * information that could be present.
           */
  
          M_ASSERTPKTHDR(m);
  
          m->m_flags &= ~(M_BCAST|M_MCAST|M_VLANTAG|M_PROMISC|M_PROTOFLAGS);
  
          M_HASHTYPE_CLEAR(m);
  #ifdef NUMA
          m->m_pkthdr.numa_domain = M_NODOM;
  #endif
          SLIST_FOREACH_SAFE(t, &m->m_pkthdr.tags, m_tag_link, tmp) {
                  if ((t->m_tag_id != 0 || t->m_tag_cookie != MTAG_WGLOOP) &&
                      t->m_tag_id != PACKET_TAG_MACLABEL)
                          m_tag_delete(m, t);
          }
  
          KASSERT((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0,
              ("%s: mbuf %p has a send tag", __func__, m));
  
          m->m_pkthdr.csum_flags = 0;
          m->m_pkthdr.PH_per.sixtyfour[0] = 0;
          m->m_pkthdr.PH_loc.sixtyfour[0] = 0;
  }
  
  static inline unsigned int
  calculate_padding(struct wg_packet *pkt)
  {
          unsigned int padded_size, last_unit = pkt->p_mbuf->m_pkthdr.len;
  
          if (__predict_false(!pkt->p_mtu))
                  return (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1);
  
          if (__predict_false(last_unit > pkt->p_mtu))
                  last_unit %= pkt->p_mtu;
  
          padded_size = (last_unit + (WG_PKT_PADDING - 1)) & ~(WG_PKT_PADDING - 1);
          if (pkt->p_mtu < padded_size)
                  padded_size = pkt->p_mtu;
          return padded_size - last_unit;
  }
  
  static void
  wg_encrypt(struct wg_softc *sc, struct wg_packet *pkt)
  {
          static const uint8_t     padding[WG_PKT_PADDING] = { 0 };
          struct wg_pkt_data      *data;
          struct wg_peer          *peer;
          struct noise_remote     *remote;
          struct mbuf             *m;
          uint32_t                 idx;
          unsigned int             padlen;
          enum wg_ring_state       state = WG_PACKET_DEAD;
  
          remote = noise_keypair_remote(pkt->p_keypair);
          peer = noise_remote_arg(remote);
          m = pkt->p_mbuf;
  
          /* Pad the packet */
          padlen = calculate_padding(pkt);
          if (padlen != 0 && !m_append(m, padlen, padding))
                  goto out;
  
          /* Do encryption */
          if (noise_keypair_encrypt(pkt->p_keypair, &idx, pkt->p_nonce, m) != 0)
                  goto out;
  
          /* Put header into packet */
          M_PREPEND(m, sizeof(struct wg_pkt_data), M_NOWAIT);
          if (m == NULL)
                  goto out;
          data = mtod(m, struct wg_pkt_data *);
          data->t = WG_PKT_DATA;
          data->r_idx = idx;
          data->nonce = htole64(pkt->p_nonce);
  
          wg_mbuf_reset(m);
          state = WG_PACKET_CRYPTED;
  out:
          pkt->p_mbuf = m;
          wmb();
          pkt->p_state = state;
          GROUPTASK_ENQUEUE(&peer->p_send);
          noise_remote_put(remote);
  }
  
  static void
  wg_decrypt(struct wg_softc *sc, struct wg_packet *pkt)
  {
          struct wg_peer          *peer, *allowed_peer;
          struct noise_remote     *remote;
          struct mbuf             *m;
          int                      len;
          enum wg_ring_state       state = WG_PACKET_DEAD;
  
          remote = noise_keypair_remote(pkt->p_keypair);
          peer = noise_remote_arg(remote);
          m = pkt->p_mbuf;
  
          /* Read nonce and then adjust to remove the header. */
          pkt->p_nonce = le64toh(mtod(m, struct wg_pkt_data *)->nonce);
          m_adj(m, sizeof(struct wg_pkt_data));
  
          if (noise_keypair_decrypt(pkt->p_keypair, pkt->p_nonce, m) != 0)
                  goto out;
  
          /* A packet with length 0 is a keepalive packet */
          if (__predict_false(m->m_pkthdr.len == 0)) {
                  DPRINTF(sc, "Receiving keepalive packet from peer "
                      "%" PRIu64 "\n", peer->p_id);
                  state = WG_PACKET_CRYPTED;
                  goto out;
          }
  
          /*
           * We can let the network stack handle the intricate validation of the
           * IP header, we just worry about the sizeof and the version, so we can
           * read the source address in wg_aip_lookup.
           */
  
          if (determine_af_and_pullup(&m, &pkt->p_af) == 0) {
                  if (pkt->p_af == AF_INET) {
                          struct ip *ip = mtod(m, struct ip *);
                          allowed_peer = wg_aip_lookup(sc, AF_INET, &ip->ip_src);
                          len = ntohs(ip->ip_len);
                          if (len >= sizeof(struct ip) && len < m->m_pkthdr.len)
                                  m_adj(m, len - m->m_pkthdr.len);
                  } else if (pkt->p_af == AF_INET6) {
                          struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
                          allowed_peer = wg_aip_lookup(sc, AF_INET6, &ip6->ip6_src);
                          len = ntohs(ip6->ip6_plen) + sizeof(struct ip6_hdr);
                          if (len < m->m_pkthdr.len)
                                  m_adj(m, len - m->m_pkthdr.len);
                  } else
                          panic("determine_af_and_pullup returned unexpected value");
          } else {
                  DPRINTF(sc, "Packet is neither ipv4 nor ipv6 from peer %" PRIu64 "\n", peer->p_id);
                  goto out;
          }
  
          /* We only want to compare the address, not dereference, so drop the ref. */
          if (allowed_peer != NULL)
                  noise_remote_put(allowed_peer->p_remote);
  
          if (__predict_false(peer != allowed_peer)) {
                  DPRINTF(sc, "Packet has unallowed src IP from peer %" PRIu64 "\n", peer->p_id);
                  goto out;
          }
  
          wg_mbuf_reset(m);
          state = WG_PACKET_CRYPTED;
  out:
          pkt->p_mbuf = m;
          wmb();
          pkt->p_state = state;
          GROUPTASK_ENQUEUE(&peer->p_recv);
          noise_remote_put(remote);
  }
  
  static void
  wg_softc_decrypt(struct wg_softc *sc)
  {
          struct wg_packet *pkt;
  
          while ((pkt = wg_queue_dequeue_parallel(&sc->sc_decrypt_parallel)) != NULL)
                  wg_decrypt(sc, pkt);
  }
  
  static void
  wg_softc_encrypt(struct wg_softc *sc)
  {
          struct wg_packet *pkt;
  
          while ((pkt = wg_queue_dequeue_parallel(&sc->sc_encrypt_parallel)) != NULL)
                  wg_encrypt(sc, pkt);
  }
  
  static void
  wg_encrypt_dispatch(struct wg_softc *sc)
  {
          /*
           * The update to encrypt_last_cpu is racey such that we may
           * reschedule the task for the same CPU multiple times, but
           * the race doesn't really matter.
           */
          u_int cpu = (sc->sc_encrypt_last_cpu + 1) % mp_ncpus;
          sc->sc_encrypt_last_cpu = cpu;
          GROUPTASK_ENQUEUE(&sc->sc_encrypt[cpu]);
  }
  
  static void
  wg_decrypt_dispatch(struct wg_softc *sc)
  {
          u_int cpu = (sc->sc_decrypt_last_cpu + 1) % mp_ncpus;
          sc->sc_decrypt_last_cpu = cpu;
          GROUPTASK_ENQUEUE(&sc->sc_decrypt[cpu]);
  }
  
  static void
  wg_deliver_out(struct wg_peer *peer)
  {
          struct wg_endpoint       endpoint;
          struct wg_softc         *sc = peer->p_sc;
          struct wg_packet        *pkt;
          struct mbuf             *m;
          int                      rc, len;
  
          wg_peer_get_endpoint(peer, &endpoint);
  
          while ((pkt = wg_queue_dequeue_serial(&peer->p_encrypt_serial)) != NULL) {
                  if (pkt->p_state != WG_PACKET_CRYPTED)
                          goto error;
  
                  m = pkt->p_mbuf;
                  pkt->p_mbuf = NULL;
  
                  len = m->m_pkthdr.len;
  
                  wg_timers_event_any_authenticated_packet_traversal(peer);
                  wg_timers_event_any_authenticated_packet_sent(peer);
                  rc = wg_send(sc, &endpoint, m);
                  if (rc == 0) {
                          if (len > (sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN))
                                  wg_timers_event_data_sent(peer);
                          counter_u64_add(peer->p_tx_bytes, len);
                  } else if (rc == EADDRNOTAVAIL) {
                          wg_peer_clear_src(peer);
                          wg_peer_get_endpoint(peer, &endpoint);
                          goto error;
                  } else {
                          goto error;
                  }
                  wg_packet_free(pkt);
                  if (noise_keep_key_fresh_send(peer->p_remote))
                          wg_timers_event_want_initiation(peer);
                  continue;
  error:
                  if_inc_counter(sc->sc_ifp, IFCOUNTER_OERRORS, 1);
                  wg_packet_free(pkt);
          }
  }
  
  static void
  wg_deliver_in(struct wg_peer *peer)
  {
          struct wg_softc         *sc = peer->p_sc;
          struct ifnet            *ifp = sc->sc_ifp;
          struct wg_packet        *pkt;
          struct mbuf             *m;
          struct epoch_tracker     et;
  
          while ((pkt = wg_queue_dequeue_serial(&peer->p_decrypt_serial)) != NULL) {
                  if (pkt->p_state != WG_PACKET_CRYPTED)
                          goto error;
  
                  m = pkt->p_mbuf;
                  if (noise_keypair_nonce_check(pkt->p_keypair, pkt->p_nonce) != 0)
                          goto error;
  
                  if (noise_keypair_received_with(pkt->p_keypair) == ECONNRESET)
                          wg_timers_event_handshake_complete(peer);
  
                  wg_timers_event_any_authenticated_packet_received(peer);
                  wg_timers_event_any_authenticated_packet_traversal(peer);
                  wg_peer_set_endpoint(peer, &pkt->p_endpoint);
  
                  counter_u64_add(peer->p_rx_bytes, m->m_pkthdr.len +
                      sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN);
                  if_inc_counter(sc->sc_ifp, IFCOUNTER_IPACKETS, 1);
                  if_inc_counter(sc->sc_ifp, IFCOUNTER_IBYTES, m->m_pkthdr.len +
                      sizeof(struct wg_pkt_data) + NOISE_AUTHTAG_LEN);
  
                  if (m->m_pkthdr.len == 0)
                          goto done;
  
                  MPASS(pkt->p_af == AF_INET || pkt->p_af == AF_INET6);
                  pkt->p_mbuf = NULL;
  
                  m->m_pkthdr.rcvif = ifp;
  
                  NET_EPOCH_ENTER(et);
                  BPF_MTAP2_AF(ifp, m, pkt->p_af);
  
                  CURVNET_SET(ifp->if_vnet);
                  M_SETFIB(m, ifp->if_fib);
                  if (pkt->p_af == AF_INET)
                          netisr_dispatch(NETISR_IP, m);
                  if (pkt->p_af == AF_INET6)
                          netisr_dispatch(NETISR_IPV6, m);
                  CURVNET_RESTORE();
                  NET_EPOCH_EXIT(et);
  
                  wg_timers_event_data_received(peer);
  
  done:
                  if (noise_keep_key_fresh_recv(peer->p_remote))
                          wg_timers_event_want_initiation(peer);
                  wg_packet_free(pkt);
                  continue;
  error:
                  if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
                  wg_packet_free(pkt);
          }
  }
  
  static struct wg_packet *
  wg_packet_alloc(struct mbuf *m)
  {
          struct wg_packet *pkt;
  
          if ((pkt = uma_zalloc(wg_packet_zone, M_NOWAIT | M_ZERO)) == NULL)
                  return (NULL);
          pkt->p_mbuf = m;
          return (pkt);
  }
  
  static void
  wg_packet_free(struct wg_packet *pkt)
  {
          if (pkt->p_keypair != NULL)
                  noise_keypair_put(pkt->p_keypair);
          if (pkt->p_mbuf != NULL)
                  m_freem(pkt->p_mbuf);
          uma_zfree(wg_packet_zone, pkt);
  }
  
  static void
  wg_queue_init(struct wg_queue *queue, const char *name)
  {
          mtx_init(&queue->q_mtx, name, NULL, MTX_DEF);
          STAILQ_INIT(&queue->q_queue);
          queue->q_len = 0;
  }
  
  static void
  wg_queue_deinit(struct wg_queue *queue)
  {
          wg_queue_purge(queue);
          mtx_destroy(&queue->q_mtx);
  }
  
  static size_t
  wg_queue_len(struct wg_queue *queue)
  {
          return (queue->q_len);
  }
  
  static int
  wg_queue_enqueue_handshake(struct wg_queue *hs, struct wg_packet *pkt)
  {
          int ret = 0;
          mtx_lock(&hs->q_mtx);
          if (hs->q_len < MAX_QUEUED_HANDSHAKES) {
                  STAILQ_INSERT_TAIL(&hs->q_queue, pkt, p_parallel);
                  hs->q_len++;
          } else {
                  ret = ENOBUFS;
          }
          mtx_unlock(&hs->q_mtx);
          if (ret != 0)
                  wg_packet_free(pkt);
          return (ret);
  }
  
  static struct wg_packet *
  wg_queue_dequeue_handshake(struct wg_queue *hs)
  {
          struct wg_packet *pkt;
          mtx_lock(&hs->q_mtx);
          if ((pkt = STAILQ_FIRST(&hs->q_queue)) != NULL) {
                  STAILQ_REMOVE_HEAD(&hs->q_queue, p_parallel);
                  hs->q_len--;
          }
          mtx_unlock(&hs->q_mtx);
          return (pkt);
  }
  
  static void
  wg_queue_push_staged(struct wg_queue *staged, struct wg_packet *pkt)
  {
          struct wg_packet *old = NULL;
  
          mtx_lock(&staged->q_mtx);
          if (staged->q_len >= MAX_STAGED_PKT) {
                  old = STAILQ_FIRST(&staged->q_queue);
                  STAILQ_REMOVE_HEAD(&staged->q_queue, p_parallel);
                  staged->q_len--;
          }
          STAILQ_INSERT_TAIL(&staged->q_queue, pkt, p_parallel);
          staged->q_len++;
          mtx_unlock(&staged->q_mtx);
  
          if (old != NULL)
                  wg_packet_free(old);
  }
  
  static void
  wg_queue_enlist_staged(struct wg_queue *staged, struct wg_packet_list *list)
  {
          struct wg_packet *pkt, *tpkt;
          STAILQ_FOREACH_SAFE(pkt, list, p_parallel, tpkt)
                  wg_queue_push_staged(staged, pkt);
  }
  
  static void
  wg_queue_delist_staged(struct wg_queue *staged, struct wg_packet_list *list)
  {
          STAILQ_INIT(list);
          mtx_lock(&staged->q_mtx);
          STAILQ_CONCAT(list, &staged->q_queue);
          staged->q_len = 0;
          mtx_unlock(&staged->q_mtx);
  }
  
  static void
  wg_queue_purge(struct wg_queue *staged)
  {
          struct wg_packet_list list;
          struct wg_packet *pkt, *tpkt;
          wg_queue_delist_staged(staged, &list);
          STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt)
                  wg_packet_free(pkt);
  }
  
  static int
  wg_queue_both(struct wg_queue *parallel, struct wg_queue *serial, struct wg_packet *pkt)
  {
          pkt->p_state = WG_PACKET_UNCRYPTED;
  
          mtx_lock(&serial->q_mtx);
          if (serial->q_len < MAX_QUEUED_PKT) {
                  serial->q_len++;
                  STAILQ_INSERT_TAIL(&serial->q_queue, pkt, p_serial);
          } else {
                  mtx_unlock(&serial->q_mtx);
                  wg_packet_free(pkt);
                  return (ENOBUFS);
          }
          mtx_unlock(&serial->q_mtx);
  
          mtx_lock(&parallel->q_mtx);
          if (parallel->q_len < MAX_QUEUED_PKT) {
                  parallel->q_len++;
                  STAILQ_INSERT_TAIL(&parallel->q_queue, pkt, p_parallel);
          } else {
                  mtx_unlock(&parallel->q_mtx);
                  pkt->p_state = WG_PACKET_DEAD;
                  return (ENOBUFS);
          }
          mtx_unlock(&parallel->q_mtx);
  
          return (0);
  }
  
  static struct wg_packet *
  wg_queue_dequeue_serial(struct wg_queue *serial)
  {
          struct wg_packet *pkt = NULL;
          mtx_lock(&serial->q_mtx);
          if (serial->q_len > 0 && STAILQ_FIRST(&serial->q_queue)->p_state != WG_PACKET_UNCRYPTED) {
                  serial->q_len--;
                  pkt = STAILQ_FIRST(&serial->q_queue);
                  STAILQ_REMOVE_HEAD(&serial->q_queue, p_serial);
          }
          mtx_unlock(&serial->q_mtx);
          return (pkt);
  }
  
  static struct wg_packet *
  wg_queue_dequeue_parallel(struct wg_queue *parallel)
  {
          struct wg_packet *pkt = NULL;
          mtx_lock(&parallel->q_mtx);
          if (parallel->q_len > 0) {
                  parallel->q_len--;
                  pkt = STAILQ_FIRST(&parallel->q_queue);
                  STAILQ_REMOVE_HEAD(&parallel->q_queue, p_parallel);
          }
          mtx_unlock(&parallel->q_mtx);
          return (pkt);
  }
  
  static bool
  wg_input(struct mbuf *m, int offset, struct inpcb *inpcb,
      const struct sockaddr *sa, void *_sc)
  {
  #ifdef INET
          const struct sockaddr_in        *sin;
  #endif
  #ifdef INET6
          const struct sockaddr_in6       *sin6;
  #endif
          struct noise_remote             *remote;
          struct wg_pkt_data              *data;
          struct wg_packet                *pkt;
          struct wg_peer                  *peer;
          struct wg_softc                 *sc = _sc;
          struct mbuf                     *defragged;
  
          defragged = m_defrag(m, M_NOWAIT);
          if (defragged)
                  m = defragged;
          m = m_unshare(m, M_NOWAIT);
          if (!m) {
                  if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
                  return true;
          }
  
          /* Caller provided us with `sa`, no need for this header. */
          m_adj(m, offset + sizeof(struct udphdr));
  
          /* Pullup enough to read packet type */
          if ((m = m_pullup(m, sizeof(uint32_t))) == NULL) {
                  if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
                  return true;
          }
  
          if ((pkt = wg_packet_alloc(m)) == NULL) {
                  if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
                  m_freem(m);
                  return true;
          }
  
          /* Save send/recv address and port for later. */
          switch (sa->sa_family) {
  #ifdef INET
          case AF_INET:
                  sin = (const struct sockaddr_in *)sa;
                  pkt->p_endpoint.e_remote.r_sin = sin[0];
                  pkt->p_endpoint.e_local.l_in = sin[1].sin_addr;
                  break;
  #endif
  #ifdef INET6
          case AF_INET6:
                  sin6 = (const struct sockaddr_in6 *)sa;
                  pkt->p_endpoint.e_remote.r_sin6 = sin6[0];
                  pkt->p_endpoint.e_local.l_in6 = sin6[1].sin6_addr;
                  break;
  #endif
          default:
                  goto error;
          }
  
          if ((m->m_pkthdr.len == sizeof(struct wg_pkt_initiation) &&
                  *mtod(m, uint32_t *) == WG_PKT_INITIATION) ||
              (m->m_pkthdr.len == sizeof(struct wg_pkt_response) &&
                  *mtod(m, uint32_t *) == WG_PKT_RESPONSE) ||
              (m->m_pkthdr.len == sizeof(struct wg_pkt_cookie) &&
                  *mtod(m, uint32_t *) == WG_PKT_COOKIE)) {
  
                  if (wg_queue_enqueue_handshake(&sc->sc_handshake_queue, pkt) != 0) {
                          if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
                          DPRINTF(sc, "Dropping handshake packet\n");
                  }
                  GROUPTASK_ENQUEUE(&sc->sc_handshake);
          } else if (m->m_pkthdr.len >= sizeof(struct wg_pkt_data) +
              NOISE_AUTHTAG_LEN && *mtod(m, uint32_t *) == WG_PKT_DATA) {
  
                  /* Pullup whole header to read r_idx below. */
                  if ((pkt->p_mbuf = m_pullup(m, sizeof(struct wg_pkt_data))) == NULL)
                          goto error;
  
                  data = mtod(pkt->p_mbuf, struct wg_pkt_data *);
                  if ((pkt->p_keypair = noise_keypair_lookup(sc->sc_local, data->r_idx)) == NULL)
                          goto error;
  
                  remote = noise_keypair_remote(pkt->p_keypair);
                  peer = noise_remote_arg(remote);
                  if (wg_queue_both(&sc->sc_decrypt_parallel, &peer->p_decrypt_serial, pkt) != 0)
                          if_inc_counter(sc->sc_ifp, IFCOUNTER_IQDROPS, 1);
                  wg_decrypt_dispatch(sc);
                  noise_remote_put(remote);
          } else {
                  goto error;
          }
          return true;
  error:
          if_inc_counter(sc->sc_ifp, IFCOUNTER_IERRORS, 1);
          wg_packet_free(pkt);
          return true;
  }
  
  static void
  wg_peer_send_staged(struct wg_peer *peer)
  {
          struct wg_packet_list    list;
          struct noise_keypair    *keypair;
          struct wg_packet        *pkt, *tpkt;
          struct wg_softc         *sc = peer->p_sc;
  
          wg_queue_delist_staged(&peer->p_stage_queue, &list);
  
          if (STAILQ_EMPTY(&list))
                  return;
  
          if ((keypair = noise_keypair_current(peer->p_remote)) == NULL)
                  goto error;
  
          STAILQ_FOREACH(pkt, &list, p_parallel) {
                  if (noise_keypair_nonce_next(keypair, &pkt->p_nonce) != 0)
                          goto error_keypair;
          }
          STAILQ_FOREACH_SAFE(pkt, &list, p_parallel, tpkt) {
                  pkt->p_keypair = noise_keypair_ref(keypair);
                  if (wg_queue_both(&sc->sc_encrypt_parallel, &peer->p_encrypt_serial, pkt) != 0)
                          if_inc_counter(sc->sc_ifp, IFCOUNTER_OQDROPS, 1);
          }
          wg_encrypt_dispatch(sc);
          noise_keypair_put(keypair);
          return;
  
  error_keypair:
          noise_keypair_put(keypair);
  error:
          wg_queue_enlist_staged(&peer->p_stage_queue, &list);
          wg_timers_event_want_initiation(peer);
  }
  
  static inline void
  xmit_err(struct ifnet *ifp, struct mbuf *m, struct wg_packet *pkt, sa_family_t af)
  {
          if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
          switch (af) {
  #ifdef INET
          case AF_INET:
                  icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0);
                  if (pkt)
                          pkt->p_mbuf = NULL;
                  m = NULL;
                  break;
  #endif
  #ifdef INET6
          case AF_INET6:
                  icmp6_error(m, ICMP6_DST_UNREACH, 0, 0);
                  if (pkt)
                          pkt->p_mbuf = NULL;
                  m = NULL;
                  break;
  #endif
          }
          if (pkt)
                  wg_packet_free(pkt);
          else if (m)
                  m_freem(m);
  }
  
  static int
  wg_xmit(struct ifnet *ifp, struct mbuf *m, sa_family_t af, uint32_t mtu)
  {
          struct wg_packet        *pkt = NULL;
          struct wg_softc         *sc = ifp->if_softc;
          struct wg_peer          *peer;
          int                      rc = 0;
          sa_family_t              peer_af;
  
          /* Work around lifetime issue in the ipv6 mld code. */
          if (__predict_false((ifp->if_flags & IFF_DYING) || !sc)) {
                  rc = ENXIO;
                  goto err_xmit;
          }
  
          if ((pkt = wg_packet_alloc(m)) == NULL) {
                  rc = ENOBUFS;
                  goto err_xmit;
          }
          pkt->p_mtu = mtu;
          pkt->p_af = af;
  
          if (af == AF_INET) {
                  peer = wg_aip_lookup(sc, AF_INET, &mtod(m, struct ip *)->ip_dst);
          } else if (af == AF_INET6) {
                  peer = wg_aip_lookup(sc, AF_INET6, &mtod(m, struct ip6_hdr *)->ip6_dst);
          } else {
                  rc = EAFNOSUPPORT;
                  goto err_xmit;
          }
  
          BPF_MTAP2_AF(ifp, m, pkt->p_af);
  
          if (__predict_false(peer == NULL)) {
                  rc = ENOKEY;
                  goto err_xmit;
          }
  
          if (__predict_false(if_tunnel_check_nesting(ifp, m, MTAG_WGLOOP, MAX_LOOPS))) {
                  DPRINTF(sc, "Packet looped");
                  rc = ELOOP;
                  goto err_peer;
          }
  
          peer_af = peer->p_endpoint.e_remote.r_sa.sa_family;
          if (__predict_false(peer_af != AF_INET && peer_af != AF_INET6)) {
                  DPRINTF(sc, "No valid endpoint has been configured or "
                              "discovered for peer %" PRIu64 "\n", peer->p_id);
                  rc = EHOSTUNREACH;
                  goto err_peer;
          }
  
          wg_queue_push_staged(&peer->p_stage_queue, pkt);
          wg_peer_send_staged(peer);
          noise_remote_put(peer->p_remote);
          return (0);
  
  err_peer:
          noise_remote_put(peer->p_remote);
  err_xmit:
          xmit_err(ifp, m, pkt, af);
          return (rc);
  }
  
  static inline int
  determine_af_and_pullup(struct mbuf **m, sa_family_t *af)
  {
          u_char ipv;
          if ((*m)->m_pkthdr.len >= sizeof(struct ip6_hdr))
                  *m = m_pullup(*m, sizeof(struct ip6_hdr));
          else if ((*m)->m_pkthdr.len >= sizeof(struct ip))
                  *m = m_pullup(*m, sizeof(struct ip));
          else
                  return (EAFNOSUPPORT);
          if (*m == NULL)
                  return (ENOBUFS);
          ipv = mtod(*m, struct ip *)->ip_v;
          if (ipv == 4)
                  *af = AF_INET;
          else if (ipv == 6 && (*m)->m_pkthdr.len >= sizeof(struct ip6_hdr))
                  *af = AF_INET6;
          else
                  return (EAFNOSUPPORT);
          return (0);
  }
  
  static int
  wg_transmit(struct ifnet *ifp, struct mbuf *m)
  {
          sa_family_t af;
          int ret;
          struct mbuf *defragged;
  
          defragged = m_defrag(m, M_NOWAIT);
          if (defragged)
                  m = defragged;
          m = m_unshare(m, M_NOWAIT);
          if (!m) {
                  xmit_err(ifp, m, NULL, AF_UNSPEC);
                  return (ENOBUFS);
          }
  
          ret = determine_af_and_pullup(&m, &af);
          if (ret) {
                  xmit_err(ifp, m, NULL, AF_UNSPEC);
                  return (ret);
          }
          return (wg_xmit(ifp, m, af, ifp->if_mtu));
  }
  
  static int
  wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, struct route *ro)
  {
          sa_family_t parsed_af;
          uint32_t af, mtu;
          int ret;
          struct mbuf *defragged;
  
          if (dst->sa_family == AF_UNSPEC)
                  memcpy(&af, dst->sa_data, sizeof(af));
          else
                  af = dst->sa_family;
          if (af == AF_UNSPEC) {
                  xmit_err(ifp, m, NULL, af);
                  return (EAFNOSUPPORT);
          }
  
          defragged = m_defrag(m, M_NOWAIT);
          if (defragged)
                  m = defragged;
          m = m_unshare(m, M_NOWAIT);
          if (!m) {
                  xmit_err(ifp, m, NULL, AF_UNSPEC);
                  return (ENOBUFS);
          }
  
          ret = determine_af_and_pullup(&m, &parsed_af);
          if (ret) {
                  xmit_err(ifp, m, NULL, AF_UNSPEC);
                  return (ret);
          }
          if (parsed_af != af) {
                  xmit_err(ifp, m, NULL, AF_UNSPEC);
                  return (EAFNOSUPPORT);
          }
          mtu = (ro != NULL && ro->ro_mtu > 0) ? ro->ro_mtu : ifp->if_mtu;
          return (wg_xmit(ifp, m, parsed_af, mtu));
  }
  
  static int
  wg_peer_add(struct wg_softc *sc, const nvlist_t *nvl)
  {
          uint8_t                  public[WG_KEY_SIZE];
          const void *pub_key, *preshared_key = NULL;
          const struct sockaddr *endpoint;
          int err;
          size_t size;
          struct noise_remote *remote;
          struct wg_peer *peer = NULL;
          bool need_insert = false;
  
          sx_assert(&sc->sc_lock, SX_XLOCKED);
  
          if (!nvlist_exists_binary(nvl, "public-key")) {
                  return (EINVAL);
          }
          pub_key = nvlist_get_binary(nvl, "public-key", &size);
          if (size != WG_KEY_SIZE) {
                  return (EINVAL);
          }
          if (noise_local_keys(sc->sc_local, public, NULL) == 0 &&
              bcmp(public, pub_key, WG_KEY_SIZE) == 0) {
                  return (0); // Silently ignored; not actually a failure.
          }
          if ((remote = noise_remote_lookup(sc->sc_local, pub_key)) != NULL)
                  peer = noise_remote_arg(remote);
          if (nvlist_exists_bool(nvl, "remove") &&
                  nvlist_get_bool(nvl, "remove")) {
                  if (remote != NULL) {
                          wg_peer_destroy(peer);
                          noise_remote_put(remote);
                  }
                  return (0);
          }
          if (nvlist_exists_bool(nvl, "replace-allowedips") &&
                  nvlist_get_bool(nvl, "replace-allowedips") &&
              peer != NULL) {
  
                  wg_aip_remove_all(sc, peer);
          }
          if (peer == NULL) {
                  peer = wg_peer_alloc(sc, pub_key);
                  need_insert = true;
          }
          if (nvlist_exists_binary(nvl, "endpoint")) {
                  endpoint = nvlist_get_binary(nvl, "endpoint", &size);
                  if (size > sizeof(peer->p_endpoint.e_remote)) {
                          err = EINVAL;
                          goto out;
                  }
                  memcpy(&peer->p_endpoint.e_remote, endpoint, size);
          }
          if (nvlist_exists_binary(nvl, "preshared-key")) {
                  preshared_key = nvlist_get_binary(nvl, "preshared-key", &size);
                  if (size != WG_KEY_SIZE) {
                          err = EINVAL;
                          goto out;
                  }
                  noise_remote_set_psk(peer->p_remote, preshared_key);
          }
          if (nvlist_exists_number(nvl, "persistent-keepalive-interval")) {
                  uint64_t pki = nvlist_get_number(nvl, "persistent-keepalive-interval");
                  if (pki > UINT16_MAX) {
                          err = EINVAL;
                          goto out;
                  }
                  wg_timers_set_persistent_keepalive(peer, pki);
          }
          if (nvlist_exists_nvlist_array(nvl, "allowed-ips")) {
                  const void *addr;
                  uint64_t cidr;
                  const nvlist_t * const * aipl;
                  size_t allowedip_count;
  
                  aipl = nvlist_get_nvlist_array(nvl, "allowed-ips", &allowedip_count);
                  for (size_t idx = 0; idx < allowedip_count; idx++) {
                          if (!nvlist_exists_number(aipl[idx], "cidr"))
                                  continue;
                          cidr = nvlist_get_number(aipl[idx], "cidr");
                          if (nvlist_exists_binary(aipl[idx], "ipv4")) {
                                  addr = nvlist_get_binary(aipl[idx], "ipv4", &size);
                                  if (addr == NULL || cidr > 32 || size != sizeof(struct in_addr)) {
                                          err = EINVAL;
                                          goto out;
                                  }
                                  if ((err = wg_aip_add(sc, peer, AF_INET, addr, cidr)) != 0)
                                          goto out;
                          } else if (nvlist_exists_binary(aipl[idx], "ipv6")) {
                                  addr = nvlist_get_binary(aipl[idx], "ipv6", &size);
                                  if (addr == NULL || cidr > 128 || size != sizeof(struct in6_addr)) {
                                          err = EINVAL;
                                          goto out;
                                  }
                                  if ((err = wg_aip_add(sc, peer, AF_INET6, addr, cidr)) != 0)
                                          goto out;
                          } else {
                                  continue;
                          }
                  }
          }
          if (need_insert) {
                  if ((err = noise_remote_enable(peer->p_remote)) != 0)
                          goto out;
                  TAILQ_INSERT_TAIL(&sc->sc_peers, peer, p_entry);
                  sc->sc_peers_num++;
                  if (sc->sc_ifp->if_link_state == LINK_STATE_UP)
                          wg_timers_enable(peer);
          }
          if (remote != NULL)
                  noise_remote_put(remote);
          return (0);
  out:
          if (need_insert) /* If we fail, only destroy if it was new. */
                  wg_peer_destroy(peer);
          if (remote != NULL)
                  noise_remote_put(remote);
          return (err);
  }
  
  static int
  wgc_set(struct wg_softc *sc, struct wg_data_io *wgd)
  {
          uint8_t public[WG_KEY_SIZE], private[WG_KEY_SIZE];
          struct ifnet *ifp;
          void *nvlpacked;
          nvlist_t *nvl;
          ssize_t size;
          int err;
  
          ifp = sc->sc_ifp;
          if (wgd->wgd_size == 0 || wgd->wgd_data == NULL)
                  return (EFAULT);
  
          /* Can nvlists be streamed in? It's not nice to impose arbitrary limits like that but
           * there needs to be _some_ limitation. */
          if (wgd->wgd_size >= UINT32_MAX / 2)
                  return (E2BIG);
  
          nvlpacked = malloc(wgd->wgd_size, M_TEMP, M_WAITOK | M_ZERO);
  
          err = copyin(wgd->wgd_data, nvlpacked, wgd->wgd_size);
          if (err)
                  goto out;
          nvl = nvlist_unpack(nvlpacked, wgd->wgd_size, 0);
          if (nvl == NULL) {
                  err = EBADMSG;
                  goto out;
          }
          sx_xlock(&sc->sc_lock);
          if (nvlist_exists_bool(nvl, "replace-peers") &&
                  nvlist_get_bool(nvl, "replace-peers"))
                  wg_peer_destroy_all(sc);
          if (nvlist_exists_number(nvl, "listen-port")) {
                  uint64_t new_port = nvlist_get_number(nvl, "listen-port");
                  if (new_port > UINT16_MAX) {
                          err = EINVAL;
                          goto out_locked;
                  }
                  if (new_port != sc->sc_socket.so_port) {
                          if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
                                  if ((err = wg_socket_init(sc, new_port)) != 0)
                                          goto out_locked;
                          } else
                                  sc->sc_socket.so_port = new_port;
                  }
          }
          if (nvlist_exists_binary(nvl, "private-key")) {
                  const void *key = nvlist_get_binary(nvl, "private-key", &size);
                  if (size != WG_KEY_SIZE) {
                          err = EINVAL;
                          goto out_locked;
                  }
  
                  if (noise_local_keys(sc->sc_local, NULL, private) != 0 ||
                      timingsafe_bcmp(private, key, WG_KEY_SIZE) != 0) {
                          struct wg_peer *peer;
  
                          if (curve25519_generate_public(public, key)) {
                                  /* Peer conflict: remove conflicting peer. */
                                  struct noise_remote *remote;
                                  if ((remote = noise_remote_lookup(sc->sc_local,
                                      public)) != NULL) {
                                          peer = noise_remote_arg(remote);
                                          wg_peer_destroy(peer);
                                          noise_remote_put(remote);
                                  }
                          }
  
                          /*
                           * Set the private key and invalidate all existing
                           * handshakes.
                           */
                          /* Note: we might be removing the private key. */
                          noise_local_private(sc->sc_local, key);
                          if (noise_local_keys(sc->sc_local, NULL, NULL) == 0)
                                  cookie_checker_update(&sc->sc_cookie, public);
                          else
                                  cookie_checker_update(&sc->sc_cookie, NULL);
                  }
          }
          if (nvlist_exists_number(nvl, "user-cookie")) {
                  uint64_t user_cookie = nvlist_get_number(nvl, "user-cookie");
                  if (user_cookie > UINT32_MAX) {
                          err = EINVAL;
                          goto out_locked;
                  }
                  err = wg_socket_set_cookie(sc, user_cookie);
                  if (err)
                          goto out_locked;
          }
          if (nvlist_exists_nvlist_array(nvl, "peers")) {
                  size_t peercount;
                  const nvlist_t * const*nvl_peers;
  
                  nvl_peers = nvlist_get_nvlist_array(nvl, "peers", &peercount);
                  for (int i = 0; i < peercount; i++) {
                          err = wg_peer_add(sc, nvl_peers[i]);
                          if (err != 0)
                                  goto out_locked;
                  }
          }
  
  out_locked:
          sx_xunlock(&sc->sc_lock);
          nvlist_destroy(nvl);
  out:
          zfree(nvlpacked, M_TEMP);
          return (err);
  }
  
  static int
  wgc_get(struct wg_softc *sc, struct wg_data_io *wgd)
  {
          uint8_t public_key[WG_KEY_SIZE] = { 0 };
          uint8_t private_key[WG_KEY_SIZE] = { 0 };
          uint8_t preshared_key[NOISE_SYMMETRIC_KEY_LEN] = { 0 };
          nvlist_t *nvl, *nvl_peer, *nvl_aip, **nvl_peers, **nvl_aips;
          size_t size, peer_count, aip_count, i, j;
          struct wg_timespec64 ts64;
          struct wg_peer *peer;
          struct wg_aip *aip;
          void *packed;
          int err = 0;
  
          nvl = nvlist_create(0);
          if (!nvl)
                  return (ENOMEM);
  
          sx_slock(&sc->sc_lock);
  
          if (sc->sc_socket.so_port != 0)
                  nvlist_add_number(nvl, "listen-port", sc->sc_socket.so_port);
          if (sc->sc_socket.so_user_cookie != 0)
                  nvlist_add_number(nvl, "user-cookie", sc->sc_socket.so_user_cookie);
          if (noise_local_keys(sc->sc_local, public_key, private_key) == 0) {
                  nvlist_add_binary(nvl, "public-key", public_key, WG_KEY_SIZE);
                  if (wgc_privileged(sc))
                          nvlist_add_binary(nvl, "private-key", private_key, WG_KEY_SIZE);
                  explicit_bzero(private_key, sizeof(private_key));
          }
          peer_count = sc->sc_peers_num;
          if (peer_count) {
                  nvl_peers = mallocarray(peer_count, sizeof(void *), M_NVLIST, M_WAITOK | M_ZERO);
                  i = 0;
                  TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) {
                          if (i >= peer_count)
                                  panic("peers changed from under us");
  
                          nvl_peers[i++] = nvl_peer = nvlist_create(0);
                          if (!nvl_peer) {
                                  err = ENOMEM;
                                  goto err_peer;
                          }
  
                          (void)noise_remote_keys(peer->p_remote, public_key, preshared_key);
                          nvlist_add_binary(nvl_peer, "public-key", public_key, sizeof(public_key));
                          if (wgc_privileged(sc))
                                  nvlist_add_binary(nvl_peer, "preshared-key", preshared_key, sizeof(preshared_key));
                          explicit_bzero(preshared_key, sizeof(preshared_key));
                          if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET)
                                  nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in));
                          else if (peer->p_endpoint.e_remote.r_sa.sa_family == AF_INET6)
                                  nvlist_add_binary(nvl_peer, "endpoint", &peer->p_endpoint.e_remote, sizeof(struct sockaddr_in6));
                          wg_timers_get_last_handshake(peer, &ts64);
                          nvlist_add_binary(nvl_peer, "last-handshake-time", &ts64, sizeof(ts64));
                          nvlist_add_number(nvl_peer, "persistent-keepalive-interval", peer->p_persistent_keepalive_interval);
                          nvlist_add_number(nvl_peer, "rx-bytes", counter_u64_fetch(peer->p_rx_bytes));
                          nvlist_add_number(nvl_peer, "tx-bytes", counter_u64_fetch(peer->p_tx_bytes));
  
                          aip_count = peer->p_aips_num;
                          if (aip_count) {
                                  nvl_aips = mallocarray(aip_count, sizeof(void *), M_NVLIST, M_WAITOK | M_ZERO);
                                  j = 0;
                                  LIST_FOREACH(aip, &peer->p_aips, a_entry) {
                                          if (j >= aip_count)
                                                  panic("aips changed from under us");
  
                                          nvl_aips[j++] = nvl_aip = nvlist_create(0);
                                          if (!nvl_aip) {
                                                  err = ENOMEM;
                                                  goto err_aip;
                                          }
                                          if (aip->a_af == AF_INET) {
                                                  nvlist_add_binary(nvl_aip, "ipv4", &aip->a_addr.in, sizeof(aip->a_addr.in));
                                                  nvlist_add_number(nvl_aip, "cidr", bitcount32(aip->a_mask.ip));
                                          }
  #ifdef INET6
                                          else if (aip->a_af == AF_INET6) {
                                                  nvlist_add_binary(nvl_aip, "ipv6", &aip->a_addr.in6, sizeof(aip->a_addr.in6));
                                                  nvlist_add_number(nvl_aip, "cidr", in6_mask2len(&aip->a_mask.in6, NULL));
                                          }
  #endif
                                  }
                                  nvlist_add_nvlist_array(nvl_peer, "allowed-ips", (const nvlist_t *const *)nvl_aips, aip_count);
                          err_aip:
                                  for (j = 0; j < aip_count; ++j)
                                          nvlist_destroy(nvl_aips[j]);
                                  free(nvl_aips, M_NVLIST);
                                  if (err)
                                          goto err_peer;
                          }
                  }
                  nvlist_add_nvlist_array(nvl, "peers", (const nvlist_t * const *)nvl_peers, peer_count);
          err_peer:
                  for (i = 0; i < peer_count; ++i)
                          nvlist_destroy(nvl_peers[i]);
                  free(nvl_peers, M_NVLIST);
                  if (err) {
                          sx_sunlock(&sc->sc_lock);
                          goto err;
                  }
          }
          sx_sunlock(&sc->sc_lock);
          packed = nvlist_pack(nvl, &size);
          if (!packed) {
                  err = ENOMEM;
                  goto err;
          }
          if (!wgd->wgd_size) {
                  wgd->wgd_size = size;
                  goto out;
          }
          if (wgd->wgd_size < size) {
                  err = ENOSPC;
                  goto out;
          }
          err = copyout(packed, wgd->wgd_data, size);
          wgd->wgd_size = size;
  
  out:
          zfree(packed, M_NVLIST);
  err:
          nvlist_destroy(nvl);
          return (err);
  }
  
  static int
  wg_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
  {
          struct wg_data_io *wgd = (struct wg_data_io *)data;
          struct ifreq *ifr = (struct ifreq *)data;
          struct wg_softc *sc;
          int ret = 0;
  
          sx_slock(&wg_sx);
          sc = ifp->if_softc;
          if (!sc) {
                  ret = ENXIO;
                  goto out;
          }
  
          switch (cmd) {
          case SIOCSWG:
                  ret = priv_check(curthread, PRIV_NET_WG);
                  if (ret == 0)
                          ret = wgc_set(sc, wgd);
                  break;
          case SIOCGWG:
                  ret = wgc_get(sc, wgd);
                  break;
          /* Interface IOCTLs */
          case SIOCSIFADDR:
                  /*
                   * This differs from *BSD norms, but is more uniform with how
                   * WireGuard behaves elsewhere.
                   */
                  break;
          case SIOCSIFFLAGS:
                  if (ifp->if_flags & IFF_UP)
                          ret = wg_up(sc);
                  else
                          wg_down(sc);
                  break;
          case SIOCSIFMTU:
                  if (ifr->ifr_mtu <= 0 || ifr->ifr_mtu > MAX_MTU)
                          ret = EINVAL;
                  else
                          ifp->if_mtu = ifr->ifr_mtu;
                  break;
          case SIOCADDMULTI:
          case SIOCDELMULTI:
                  break;
          case SIOCGTUNFIB:
                  ifr->ifr_fib = sc->sc_socket.so_fibnum;
                  break;
          case SIOCSTUNFIB:
                  ret = priv_check(curthread, PRIV_NET_WG);
                  if (ret)
                          break;
                  ret = priv_check(curthread, PRIV_NET_SETIFFIB);
                  if (ret)
                          break;
                  sx_xlock(&sc->sc_lock);
                  ret = wg_socket_set_fibnum(sc, ifr->ifr_fib);
                  sx_xunlock(&sc->sc_lock);
                  break;
          default:
                  ret = ENOTTY;
          }
  
  out:
          sx_sunlock(&wg_sx);
          return (ret);
  }
  
  static int
  wg_up(struct wg_softc *sc)
  {
          struct ifnet *ifp = sc->sc_ifp;
          struct wg_peer *peer;
          int rc = EBUSY;
  
          sx_xlock(&sc->sc_lock);
          /* Jail's being removed, no more wg_up(). */
          if ((sc->sc_flags & WGF_DYING) != 0)
                  goto out;
  
          /* Silent success if we're already running. */
          rc = 0;
          if (ifp->if_drv_flags & IFF_DRV_RUNNING)
                  goto out;
          ifp->if_drv_flags |= IFF_DRV_RUNNING;
  
          rc = wg_socket_init(sc, sc->sc_socket.so_port);
          if (rc == 0) {
                  TAILQ_FOREACH(peer, &sc->sc_peers, p_entry)
                          wg_timers_enable(peer);
                  if_link_state_change(sc->sc_ifp, LINK_STATE_UP);
          } else {
                  ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
                  DPRINTF(sc, "Unable to initialize sockets: %d\n", rc);
          }
  out:
          sx_xunlock(&sc->sc_lock);
          return (rc);
  }
  
  static void
  wg_down(struct wg_softc *sc)
  {
          struct ifnet *ifp = sc->sc_ifp;
          struct wg_peer *peer;
  
          sx_xlock(&sc->sc_lock);
          if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
                  sx_xunlock(&sc->sc_lock);
                  return;
          }
          ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
  
          TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) {
                  wg_queue_purge(&peer->p_stage_queue);
                  wg_timers_disable(peer);
          }
  
          wg_queue_purge(&sc->sc_handshake_queue);
  
          TAILQ_FOREACH(peer, &sc->sc_peers, p_entry) {
                  noise_remote_handshake_clear(peer->p_remote);
                  noise_remote_keypairs_clear(peer->p_remote);
          }
  
          if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN);
          wg_socket_uninit(sc);
  
          sx_xunlock(&sc->sc_lock);
  }
  
  static int
  wg_clone_create(struct if_clone *ifc, char *name, size_t len,
      struct ifc_data *ifd, struct ifnet **ifpp)
  {
          struct wg_softc *sc;
          struct ifnet *ifp;
  
          sc = malloc(sizeof(*sc), M_WG, M_WAITOK | M_ZERO);
  
          sc->sc_local = noise_local_alloc(sc);
  
          sc->sc_encrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO);
  
          sc->sc_decrypt = mallocarray(sizeof(struct grouptask), mp_ncpus, M_WG, M_WAITOK | M_ZERO);
  
          if (!rn_inithead((void **)&sc->sc_aip4, offsetof(struct aip_addr, in) * NBBY))
                  goto free_decrypt;
  
          if (!rn_inithead((void **)&sc->sc_aip6, offsetof(struct aip_addr, in6) * NBBY))
                  goto free_aip4;
  
          atomic_add_int(&clone_count, 1);
          ifp = sc->sc_ifp = if_alloc(IFT_WIREGUARD);
  
          sc->sc_ucred = crhold(curthread->td_ucred);
          sc->sc_socket.so_fibnum = curthread->td_proc->p_fibnum;
          sc->sc_socket.so_port = 0;
  
          TAILQ_INIT(&sc->sc_peers);
          sc->sc_peers_num = 0;
  
          cookie_checker_init(&sc->sc_cookie);
  
          RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip4);
          RADIX_NODE_HEAD_LOCK_INIT(sc->sc_aip6);
  
          GROUPTASK_INIT(&sc->sc_handshake, 0, (gtask_fn_t *)wg_softc_handshake_receive, sc);
          taskqgroup_attach(qgroup_wg_tqg, &sc->sc_handshake, sc, NULL, NULL, "wg tx initiation");
          wg_queue_init(&sc->sc_handshake_queue, "hsq");
  
          for (int i = 0; i < mp_ncpus; i++) {
                  GROUPTASK_INIT(&sc->sc_encrypt[i], 0,
                       (gtask_fn_t *)wg_softc_encrypt, sc);
                  taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_encrypt[i], sc, i, NULL, NULL, "wg encrypt");
                  GROUPTASK_INIT(&sc->sc_decrypt[i], 0,
                      (gtask_fn_t *)wg_softc_decrypt, sc);
                  taskqgroup_attach_cpu(qgroup_wg_tqg, &sc->sc_decrypt[i], sc, i, NULL, NULL, "wg decrypt");
          }
  
          wg_queue_init(&sc->sc_encrypt_parallel, "encp");
          wg_queue_init(&sc->sc_decrypt_parallel, "decp");
  
          sx_init(&sc->sc_lock, "wg softc lock");
  
          ifp->if_softc = sc;
          ifp->if_capabilities = ifp->if_capenable = WG_CAPS;
          if_initname(ifp, wgname, ifd->unit);
  
          if_setmtu(ifp, DEFAULT_MTU);
          ifp->if_flags = IFF_NOARP | IFF_MULTICAST;
          ifp->if_init = wg_init;
          ifp->if_reassign = wg_reassign;
          ifp->if_qflush = wg_qflush;
          ifp->if_transmit = wg_transmit;
          ifp->if_output = wg_output;
          ifp->if_ioctl = wg_ioctl;
          if_attach(ifp);
          bpfattach(ifp, DLT_NULL, sizeof(uint32_t));
  #ifdef INET6
          ND_IFINFO(ifp)->flags &= ~ND6_IFF_AUTO_LINKLOCAL;
          ND_IFINFO(ifp)->flags |= ND6_IFF_NO_DAD;
  #endif
          sx_xlock(&wg_sx);
          LIST_INSERT_HEAD(&wg_list, sc, sc_entry);
          sx_xunlock(&wg_sx);
          *ifpp = ifp;
          return (0);
  free_aip4:
          RADIX_NODE_HEAD_DESTROY(sc->sc_aip4);
          free(sc->sc_aip4, M_RTABLE);
  free_decrypt:
          free(sc->sc_decrypt, M_WG);
          free(sc->sc_encrypt, M_WG);
          noise_local_free(sc->sc_local, NULL);
          free(sc, M_WG);
          return (ENOMEM);
  }
  
  static void
  wg_clone_deferred_free(struct noise_local *l)
  {
          struct wg_softc *sc = noise_local_arg(l);
  
          free(sc, M_WG);
          atomic_add_int(&clone_count, -1);
  }
  
  static int
  wg_clone_destroy(struct if_clone *ifc, struct ifnet *ifp, uint32_t flags)
  {
          struct wg_softc *sc = ifp->if_softc;
          struct ucred *cred;
  
          sx_xlock(&wg_sx);
          ifp->if_softc = NULL;
          sx_xlock(&sc->sc_lock);
          sc->sc_flags |= WGF_DYING;
          cred = sc->sc_ucred;
          sc->sc_ucred = NULL;
          sx_xunlock(&sc->sc_lock);
          LIST_REMOVE(sc, sc_entry);
          sx_xunlock(&wg_sx);
  
          if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN);
          CURVNET_SET(sc->sc_ifp->if_vnet);
          if_purgeaddrs(sc->sc_ifp);
          CURVNET_RESTORE();
  
          sx_xlock(&sc->sc_lock);
          wg_socket_uninit(sc);
          sx_xunlock(&sc->sc_lock);
  
          /*
           * No guarantees that all traffic have passed until the epoch has
           * elapsed with the socket closed.
           */
          NET_EPOCH_WAIT();
  
          taskqgroup_drain_all(qgroup_wg_tqg);
          sx_xlock(&sc->sc_lock);
          wg_peer_destroy_all(sc);
          NET_EPOCH_DRAIN_CALLBACKS();
          sx_xunlock(&sc->sc_lock);
          sx_destroy(&sc->sc_lock);
          taskqgroup_detach(qgroup_wg_tqg, &sc->sc_handshake);
          for (int i = 0; i < mp_ncpus; i++) {
                  taskqgroup_detach(qgroup_wg_tqg, &sc->sc_encrypt[i]);
                  taskqgroup_detach(qgroup_wg_tqg, &sc->sc_decrypt[i]);
          }
          free(sc->sc_encrypt, M_WG);
          free(sc->sc_decrypt, M_WG);
          wg_queue_deinit(&sc->sc_handshake_queue);
          wg_queue_deinit(&sc->sc_encrypt_parallel);
          wg_queue_deinit(&sc->sc_decrypt_parallel);
  
          RADIX_NODE_HEAD_DESTROY(sc->sc_aip4);
          RADIX_NODE_HEAD_DESTROY(sc->sc_aip6);
          rn_detachhead((void **)&sc->sc_aip4);
          rn_detachhead((void **)&sc->sc_aip6);
  
          cookie_checker_free(&sc->sc_cookie);
  
          if (cred != NULL)
                  crfree(cred);
          if_detach(sc->sc_ifp);
          if_free(sc->sc_ifp);
  
          noise_local_free(sc->sc_local, wg_clone_deferred_free);
  
          return (0);
  }
  
  static void
  wg_qflush(struct ifnet *ifp __unused)
  {
  }
  
  /*
   * Privileged information (private-key, preshared-key) are only exported for
   * root and jailed root by default.
   */
  static bool
  wgc_privileged(struct wg_softc *sc)
  {
          struct thread *td;
  
          td = curthread;
          return (priv_check(td, PRIV_NET_WG) == 0);
  }
  
  static void
  wg_reassign(struct ifnet *ifp, struct vnet *new_vnet __unused,
      char *unused __unused)
  {
          struct wg_softc *sc;
  
          sc = ifp->if_softc;
          wg_down(sc);
  }
  
  static void
  wg_init(void *xsc)
  {
          struct wg_softc *sc;
  
          sc = xsc;
          wg_up(sc);
  }
  
  static void
  vnet_wg_init(const void *unused __unused)
  {
          struct if_clone_addreq req = {
                  .create_f = wg_clone_create,
                  .destroy_f = wg_clone_destroy,
                  .flags = IFC_F_AUTOUNIT,
          };
          V_wg_cloner = ifc_attach_cloner(wgname, &req);
  }
  VNET_SYSINIT(vnet_wg_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
               vnet_wg_init, NULL);
  
  static void
  vnet_wg_uninit(const void *unused __unused)
  {
          if (V_wg_cloner)
                  ifc_detach_cloner(V_wg_cloner);
  }
  VNET_SYSUNINIT(vnet_wg_uninit, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY,
                 vnet_wg_uninit, NULL);
  
  static int
  wg_prison_remove(void *obj, void *data __unused)
  {
          const struct prison *pr = obj;
          struct wg_softc *sc;
  
          /*
           * Do a pass through all if_wg interfaces and release creds on any from
           * the jail that are supposed to be going away.  This will, in turn, let
           * the jail die so that we don't end up with Schrödinger's jail.
           */
          sx_slock(&wg_sx);
          LIST_FOREACH(sc, &wg_list, sc_entry) {
                  sx_xlock(&sc->sc_lock);
                  if (!(sc->sc_flags & WGF_DYING) && sc->sc_ucred && sc->sc_ucred->cr_prison == pr) {
                          struct ucred *cred = sc->sc_ucred;
                          DPRINTF(sc, "Creating jail exiting\n");
                          if_link_state_change(sc->sc_ifp, LINK_STATE_DOWN);
                          wg_socket_uninit(sc);
                          sc->sc_ucred = NULL;
                          crfree(cred);
                          sc->sc_flags |= WGF_DYING;
                  }
                  sx_xunlock(&sc->sc_lock);
          }
          sx_sunlock(&wg_sx);
  
          return (0);
  }
  
  #ifdef SELFTESTS
  #include "selftest/allowedips.c"
  static bool wg_run_selftests(void)
  {
          bool ret = true;
          ret &= wg_allowedips_selftest();
          ret &= noise_counter_selftest();
          ret &= cookie_selftest();
          return ret;
  }
  #else
  static inline bool wg_run_selftests(void) { return true; }
  #endif
  
  static int
  wg_module_init(void)
  {
          int ret = ENOMEM;
  
          osd_method_t methods[PR_MAXMETHOD] = {
                  [PR_METHOD_REMOVE] = wg_prison_remove,
          };
  
          if ((wg_packet_zone = uma_zcreate("wg packet", sizeof(struct wg_packet),
               NULL, NULL, NULL, NULL, 0, 0)) == NULL)
                  goto free_none;
          ret = crypto_init();
          if (ret != 0)
                  goto free_zone;
          if (cookie_init() != 0)
                  goto free_crypto;
  
          wg_osd_jail_slot = osd_jail_register(NULL, methods);
  
          ret = ENOTRECOVERABLE;
          if (!wg_run_selftests())
                  goto free_all;
  
          return (0);
  
  free_all:
          osd_jail_deregister(wg_osd_jail_slot);
          cookie_deinit();
  free_crypto:
          crypto_deinit();
  free_zone:
          uma_zdestroy(wg_packet_zone);
  free_none:
          return (ret);
  }
  
  static void
  wg_module_deinit(void)
  {
          VNET_ITERATOR_DECL(vnet_iter);
          VNET_LIST_RLOCK();
          VNET_FOREACH(vnet_iter) {
                  struct if_clone *clone = VNET_VNET(vnet_iter, wg_cloner);
                  if (clone) {
                          ifc_detach_cloner(clone);
                          VNET_VNET(vnet_iter, wg_cloner) = NULL;
                  }
          }
          VNET_LIST_RUNLOCK();
          NET_EPOCH_WAIT();
          MPASS(LIST_EMPTY(&wg_list));
          osd_jail_deregister(wg_osd_jail_slot);
          cookie_deinit();
          crypto_deinit();
          uma_zdestroy(wg_packet_zone);
  }
  
  static int
  wg_module_event_handler(module_t mod, int what, void *arg)
  {
          switch (what) {
                  case MOD_LOAD:
                          return wg_module_init();
                  case MOD_UNLOAD:
                          wg_module_deinit();
                          break;
                  default:
                          return (EOPNOTSUPP);
          }
          return (0);
  }
  
  static moduledata_t wg_moduledata = {
          wgname,
          wg_module_event_handler,
          NULL
  };
  
  DECLARE_MODULE(wg, wg_moduledata, SI_SUB_PSEUDO, SI_ORDER_ANY);
  MODULE_VERSION(wg, WIREGUARD_VERSION);
  MODULE_DEPEND(wg, crypto, 1, 1, 1);

Cache object: d3dce68896173684fd4029ab11fe48a5