The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/net/iflib.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /*-
    2  * Copyright (c) 2014-2017, Matthew Macy <mmacy@nextbsd.org>
    3  * All rights reserved.
    4  *
    5  * Redistribution and use in source and binary forms, with or without
    6  * modification, are permitted provided that the following conditions are met:
    7  *
    8  *  1. Redistributions of source code must retain the above copyright notice,
    9  *     this list of conditions and the following disclaimer.
   10  *
   11  *  2. Neither the name of Matthew Macy nor the names of its
   12  *     contributors may be used to endorse or promote products derived from
   13  *     this software without specific prior written permission.
   14  *
   15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   16  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   18  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
   19  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   20  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   21  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   22  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   23  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   24  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   25  * POSSIBILITY OF SUCH DAMAGE.
   26  */
   27 
   28 #include <sys/cdefs.h>
   29 __FBSDID("$FreeBSD: releng/11.2/sys/net/iflib.c 333503 2018-05-11 20:40:26Z shurd $");
   30 
   31 #include "opt_inet.h"
   32 #include "opt_inet6.h"
   33 #include "opt_acpi.h"
   34 #include "opt_sched.h"
   35 
   36 #include <sys/param.h>
   37 #include <sys/types.h>
   38 #include <sys/bus.h>
   39 #include <sys/eventhandler.h>
   40 #include <sys/sockio.h>
   41 #include <sys/kernel.h>
   42 #include <sys/lock.h>
   43 #include <sys/mutex.h>
   44 #include <sys/module.h>
   45 #include <sys/kobj.h>
   46 #include <sys/rman.h>
   47 #include <sys/sbuf.h>
   48 #include <sys/smp.h>
   49 #include <sys/socket.h>
   50 #include <sys/sysctl.h>
   51 #include <sys/syslog.h>
   52 #include <sys/taskqueue.h>
   53 #include <sys/limits.h>
   54 
   55 
   56 #include <net/if.h>
   57 #include <net/if_var.h>
   58 #include <net/if_types.h>
   59 #include <net/if_media.h>
   60 #include <net/bpf.h>
   61 #include <net/ethernet.h>
   62 #include <net/mp_ring.h>
   63 #include <net/vnet.h>
   64 
   65 #include <netinet/in.h>
   66 #include <netinet/in_pcb.h>
   67 #include <netinet/tcp_lro.h>
   68 #include <netinet/in_systm.h>
   69 #include <netinet/if_ether.h>
   70 #include <netinet/ip.h>
   71 #include <netinet/ip6.h>
   72 #include <netinet/tcp.h>
   73 #include <netinet/ip_var.h>
   74 #include <netinet6/ip6_var.h>
   75 
   76 #include <machine/bus.h>
   77 #include <machine/in_cksum.h>
   78 
   79 #include <vm/vm.h>
   80 #include <vm/pmap.h>
   81 
   82 #include <dev/led/led.h>
   83 #include <dev/pci/pcireg.h>
   84 #include <dev/pci/pcivar.h>
   85 #include <dev/pci/pci_private.h>
   86 
   87 #include <net/iflib.h>
   88 
   89 #include "ifdi_if.h"
   90 
   91 #if defined(__i386__) || defined(__amd64__)
   92 #include <sys/memdesc.h>
   93 #include <machine/bus.h>
   94 #include <machine/md_var.h>
   95 #include <machine/specialreg.h>
   96 #include <x86/include/busdma_impl.h>
   97 #include <x86/iommu/busdma_dmar.h>
   98 #endif
   99 
  100 #include <sys/bitstring.h>
  101 /*
  102  * enable accounting of every mbuf as it comes in to and goes out of
  103  * iflib's software descriptor references
  104  */
  105 #define MEMORY_LOGGING 0
  106 /*
  107  * Enable mbuf vectors for compressing long mbuf chains
  108  */
  109 
  110 /*
  111  * NB:
  112  * - Prefetching in tx cleaning should perhaps be a tunable. The distance ahead
  113  *   we prefetch needs to be determined by the time spent in m_free vis a vis
  114  *   the cost of a prefetch. This will of course vary based on the workload:
  115  *      - NFLX's m_free path is dominated by vm-based M_EXT manipulation which
  116  *        is quite expensive, thus suggesting very little prefetch.
  117  *      - small packet forwarding which is just returning a single mbuf to
  118  *        UMA will typically be very fast vis a vis the cost of a memory
  119  *        access.
  120  */
  121 
  122 
  123 /*
  124  * File organization:
  125  *  - private structures
  126  *  - iflib private utility functions
  127  *  - ifnet functions
  128  *  - vlan registry and other exported functions
  129  *  - iflib public core functions
  130  *
  131  *
  132  */
  133 static MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library");
  134 
  135 struct iflib_txq;
  136 typedef struct iflib_txq *iflib_txq_t;
  137 struct iflib_rxq;
  138 typedef struct iflib_rxq *iflib_rxq_t;
  139 struct iflib_fl;
  140 typedef struct iflib_fl *iflib_fl_t;
  141 
  142 static void iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid);
  143 
  144 typedef struct iflib_filter_info {
  145         driver_filter_t *ifi_filter;
  146         void *ifi_filter_arg;
  147         struct grouptask *ifi_task;
  148         void *ifi_ctx;
  149 } *iflib_filter_info_t;
  150 
  151 struct iflib_ctx {
  152         KOBJ_FIELDS;
  153    /*
  154    * Pointer to hardware driver's softc
  155    */
  156         void *ifc_softc;
  157         device_t ifc_dev;
  158         if_t ifc_ifp;
  159 
  160         cpuset_t ifc_cpus;
  161         if_shared_ctx_t ifc_sctx;
  162         struct if_softc_ctx ifc_softc_ctx;
  163 
  164         struct mtx ifc_mtx;
  165 
  166         uint16_t ifc_nhwtxqs;
  167 
  168         iflib_txq_t ifc_txqs;
  169         iflib_rxq_t ifc_rxqs;
  170         uint32_t ifc_if_flags;
  171         uint32_t ifc_flags;
  172         uint32_t ifc_max_fl_buf_size;
  173         int ifc_in_detach;
  174 
  175         int ifc_link_state;
  176         int ifc_link_irq;
  177         int ifc_watchdog_events;
  178         struct cdev *ifc_led_dev;
  179         struct resource *ifc_msix_mem;
  180 
  181         struct if_irq ifc_legacy_irq;
  182         struct grouptask ifc_admin_task;
  183         struct grouptask ifc_vflr_task;
  184         struct iflib_filter_info ifc_filter_info;
  185         struct ifmedia  ifc_media;
  186 
  187         struct sysctl_oid *ifc_sysctl_node;
  188         uint16_t ifc_sysctl_ntxqs;
  189         uint16_t ifc_sysctl_nrxqs;
  190         uint16_t ifc_sysctl_qs_eq_override;
  191         uint16_t ifc_sysctl_rx_budget;
  192 
  193         qidx_t ifc_sysctl_ntxds[8];
  194         qidx_t ifc_sysctl_nrxds[8];
  195         struct if_txrx ifc_txrx;
  196 #define isc_txd_encap  ifc_txrx.ift_txd_encap
  197 #define isc_txd_flush  ifc_txrx.ift_txd_flush
  198 #define isc_txd_credits_update  ifc_txrx.ift_txd_credits_update
  199 #define isc_rxd_available ifc_txrx.ift_rxd_available
  200 #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get
  201 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
  202 #define isc_rxd_flush ifc_txrx.ift_rxd_flush
  203 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
  204 #define isc_rxd_refill ifc_txrx.ift_rxd_refill
  205 #define isc_legacy_intr ifc_txrx.ift_legacy_intr
  206         eventhandler_tag ifc_vlan_attach_event;
  207         eventhandler_tag ifc_vlan_detach_event;
  208         uint8_t ifc_mac[ETHER_ADDR_LEN];
  209         char ifc_mtx_name[16];
  210 };
  211 
  212 
  213 void *
  214 iflib_get_softc(if_ctx_t ctx)
  215 {
  216 
  217         return (ctx->ifc_softc);
  218 }
  219 
  220 device_t
  221 iflib_get_dev(if_ctx_t ctx)
  222 {
  223 
  224         return (ctx->ifc_dev);
  225 }
  226 
  227 if_t
  228 iflib_get_ifp(if_ctx_t ctx)
  229 {
  230 
  231         return (ctx->ifc_ifp);
  232 }
  233 
  234 struct ifmedia *
  235 iflib_get_media(if_ctx_t ctx)
  236 {
  237 
  238         return (&ctx->ifc_media);
  239 }
  240 
  241 void
  242 iflib_set_mac(if_ctx_t ctx, uint8_t mac[ETHER_ADDR_LEN])
  243 {
  244 
  245         bcopy(mac, ctx->ifc_mac, ETHER_ADDR_LEN);
  246 }
  247 
  248 if_softc_ctx_t
  249 iflib_get_softc_ctx(if_ctx_t ctx)
  250 {
  251 
  252         return (&ctx->ifc_softc_ctx);
  253 }
  254 
  255 if_shared_ctx_t
  256 iflib_get_sctx(if_ctx_t ctx)
  257 {
  258 
  259         return (ctx->ifc_sctx);
  260 }
  261 
  262 #define IP_ALIGNED(m) ((((uintptr_t)(m)->m_data) & 0x3) == 0x2)
  263 #define CACHE_PTR_INCREMENT (CACHE_LINE_SIZE/sizeof(void*))
  264 #define CACHE_PTR_NEXT(ptr) ((void *)(((uintptr_t)(ptr)+CACHE_LINE_SIZE-1) & (CACHE_LINE_SIZE-1)))
  265 
  266 #define LINK_ACTIVE(ctx) ((ctx)->ifc_link_state == LINK_STATE_UP)
  267 #define CTX_IS_VF(ctx) ((ctx)->ifc_sctx->isc_flags & IFLIB_IS_VF)
  268 
  269 #define RX_SW_DESC_MAP_CREATED  (1 << 0)
  270 #define TX_SW_DESC_MAP_CREATED  (1 << 1)
  271 #define RX_SW_DESC_INUSE        (1 << 3)
  272 #define TX_SW_DESC_MAPPED       (1 << 4)
  273 
  274 #define M_TOOBIG                M_PROTO1
  275 
  276 typedef struct iflib_sw_rx_desc_array {
  277         bus_dmamap_t    *ifsd_map;         /* bus_dma maps for packet */
  278         struct mbuf     **ifsd_m;           /* pkthdr mbufs */
  279         caddr_t         *ifsd_cl;          /* direct cluster pointer for rx */
  280         uint8_t         *ifsd_flags;
  281 } iflib_rxsd_array_t;
  282 
  283 typedef struct iflib_sw_tx_desc_array {
  284         bus_dmamap_t    *ifsd_map;         /* bus_dma maps for packet */
  285         struct mbuf    **ifsd_m;           /* pkthdr mbufs */
  286         uint8_t         *ifsd_flags;
  287 } if_txsd_vec_t;
  288 
  289 
  290 /* magic number that should be high enough for any hardware */
  291 #define IFLIB_MAX_TX_SEGS               128
  292 /* bnxt supports 64 with hardware LRO enabled */
  293 #define IFLIB_MAX_RX_SEGS               64
  294 #define IFLIB_RX_COPY_THRESH            128
  295 #define IFLIB_MAX_RX_REFRESH            32
  296 /* The minimum descriptors per second before we start coalescing */
  297 #define IFLIB_MIN_DESC_SEC              16384
  298 #define IFLIB_DEFAULT_TX_UPDATE_FREQ    16
  299 #define IFLIB_QUEUE_IDLE                0
  300 #define IFLIB_QUEUE_HUNG                1
  301 #define IFLIB_QUEUE_WORKING             2
  302 /* maximum number of txqs that can share an rx interrupt */
  303 #define IFLIB_MAX_TX_SHARED_INTR        4
  304 
  305 /* this should really scale with ring size - this is a fairly arbitrary value */
  306 #define TX_BATCH_SIZE                   32
  307 
  308 #define IFLIB_RESTART_BUDGET            8
  309 
  310 #define IFC_LEGACY              0x001
  311 #define IFC_QFLUSH              0x002
  312 #define IFC_MULTISEG            0x004
  313 #define IFC_DMAR                0x008
  314 #define IFC_SC_ALLOCATED        0x010
  315 #define IFC_INIT_DONE           0x020
  316 #define IFC_PREFETCH            0x040
  317 #define IFC_DO_RESET            0x080
  318 #define IFC_CHECK_HUNG          0x100
  319 
  320 #define CSUM_OFFLOAD            (CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP| \
  321                                  CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \
  322                                  CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP)
  323 struct iflib_txq {
  324         qidx_t          ift_in_use;
  325         qidx_t          ift_cidx;
  326         qidx_t          ift_cidx_processed;
  327         qidx_t          ift_pidx;
  328         uint8_t         ift_gen;
  329         uint8_t         ift_br_offset;
  330         uint16_t        ift_npending;
  331         uint16_t        ift_db_pending;
  332         uint16_t        ift_rs_pending;
  333         /* implicit pad */
  334         uint8_t         ift_txd_size[8];
  335         uint64_t        ift_processed;
  336         uint64_t        ift_cleaned;
  337         uint64_t        ift_cleaned_prev;
  338 #if MEMORY_LOGGING
  339         uint64_t        ift_enqueued;
  340         uint64_t        ift_dequeued;
  341 #endif
  342         uint64_t        ift_no_tx_dma_setup;
  343         uint64_t        ift_no_desc_avail;
  344         uint64_t        ift_mbuf_defrag_failed;
  345         uint64_t        ift_mbuf_defrag;
  346         uint64_t        ift_map_failed;
  347         uint64_t        ift_txd_encap_efbig;
  348         uint64_t        ift_pullups;
  349 
  350         struct mtx      ift_mtx;
  351         struct mtx      ift_db_mtx;
  352 
  353         /* constant values */
  354         if_ctx_t        ift_ctx;
  355         struct ifmp_ring        *ift_br;
  356         struct grouptask        ift_task;
  357         qidx_t          ift_size;
  358         uint16_t        ift_id;
  359         struct callout  ift_timer;
  360 
  361         if_txsd_vec_t   ift_sds;
  362         uint8_t         ift_qstatus;
  363         uint8_t         ift_closed;
  364         uint8_t         ift_update_freq;
  365         struct iflib_filter_info ift_filter_info;
  366         bus_dma_tag_t           ift_desc_tag;
  367         bus_dma_tag_t           ift_tso_desc_tag;
  368         iflib_dma_info_t        ift_ifdi;
  369 #define MTX_NAME_LEN 16
  370         char                    ift_mtx_name[MTX_NAME_LEN];
  371         char                    ift_db_mtx_name[MTX_NAME_LEN];
  372         bus_dma_segment_t       ift_segs[IFLIB_MAX_TX_SEGS]  __aligned(CACHE_LINE_SIZE);
  373 #ifdef IFLIB_DIAGNOSTICS
  374         uint64_t ift_cpu_exec_count[256];
  375 #endif
  376 } __aligned(CACHE_LINE_SIZE);
  377 
  378 struct iflib_fl {
  379         qidx_t          ifl_cidx;
  380         qidx_t          ifl_pidx;
  381         qidx_t          ifl_credits;
  382         uint8_t         ifl_gen;
  383         uint8_t         ifl_rxd_size;
  384 #if MEMORY_LOGGING
  385         uint64_t        ifl_m_enqueued;
  386         uint64_t        ifl_m_dequeued;
  387         uint64_t        ifl_cl_enqueued;
  388         uint64_t        ifl_cl_dequeued;
  389 #endif
  390         /* implicit pad */
  391 
  392         bitstr_t        *ifl_rx_bitmap;
  393         qidx_t          ifl_fragidx;
  394         /* constant */
  395         qidx_t          ifl_size;
  396         uint16_t        ifl_buf_size;
  397         uint16_t        ifl_cltype;
  398         uma_zone_t      ifl_zone;
  399         iflib_rxsd_array_t      ifl_sds;
  400         iflib_rxq_t     ifl_rxq;
  401         uint8_t         ifl_id;
  402         bus_dma_tag_t           ifl_desc_tag;
  403         iflib_dma_info_t        ifl_ifdi;
  404         uint64_t        ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE);
  405         caddr_t         ifl_vm_addrs[IFLIB_MAX_RX_REFRESH];
  406         qidx_t  ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH];
  407 }  __aligned(CACHE_LINE_SIZE);
  408 
  409 static inline qidx_t
  410 get_inuse(int size, qidx_t cidx, qidx_t pidx, uint8_t gen)
  411 {
  412         qidx_t used;
  413 
  414         if (pidx > cidx)
  415                 used = pidx - cidx;
  416         else if (pidx < cidx)
  417                 used = size - cidx + pidx;
  418         else if (gen == 0 && pidx == cidx)
  419                 used = 0;
  420         else if (gen == 1 && pidx == cidx)
  421                 used = size;
  422         else
  423                 panic("bad state");
  424 
  425         return (used);
  426 }
  427 
  428 #define TXQ_AVAIL(txq) (txq->ift_size - get_inuse(txq->ift_size, txq->ift_cidx, txq->ift_pidx, txq->ift_gen))
  429 
  430 #define IDXDIFF(head, tail, wrap) \
  431         ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
  432 
  433 struct iflib_rxq {
  434         /* If there is a separate completion queue -
  435          * these are the cq cidx and pidx. Otherwise
  436          * these are unused.
  437          */
  438         qidx_t          ifr_size;
  439         qidx_t          ifr_cq_cidx;
  440         qidx_t          ifr_cq_pidx;
  441         uint8_t         ifr_cq_gen;
  442         uint8_t         ifr_fl_offset;
  443 
  444         if_ctx_t        ifr_ctx;
  445         iflib_fl_t      ifr_fl;
  446         uint64_t        ifr_rx_irq;
  447         uint16_t        ifr_id;
  448         uint8_t         ifr_lro_enabled;
  449         uint8_t         ifr_nfl;
  450         uint8_t         ifr_ntxqirq;
  451         uint8_t         ifr_txqid[IFLIB_MAX_TX_SHARED_INTR];
  452         struct lro_ctrl                 ifr_lc;
  453         struct grouptask        ifr_task;
  454         struct iflib_filter_info ifr_filter_info;
  455         iflib_dma_info_t                ifr_ifdi;
  456 
  457         /* dynamically allocate if any drivers need a value substantially larger than this */
  458         struct if_rxd_frag      ifr_frags[IFLIB_MAX_RX_SEGS] __aligned(CACHE_LINE_SIZE);
  459 #ifdef IFLIB_DIAGNOSTICS
  460         uint64_t ifr_cpu_exec_count[256];
  461 #endif
  462 }  __aligned(CACHE_LINE_SIZE);
  463 
  464 typedef struct if_rxsd {
  465         caddr_t *ifsd_cl;
  466         struct mbuf **ifsd_m;
  467         iflib_fl_t ifsd_fl;
  468         qidx_t ifsd_cidx;
  469 } *if_rxsd_t;
  470 
  471 /* multiple of word size */
  472 #ifdef __LP64__
  473 #define PKT_INFO_SIZE   6
  474 #define RXD_INFO_SIZE   5
  475 #define PKT_TYPE uint64_t
  476 #else
  477 #define PKT_INFO_SIZE   11
  478 #define RXD_INFO_SIZE   8
  479 #define PKT_TYPE uint32_t
  480 #endif
  481 #define PKT_LOOP_BOUND  ((PKT_INFO_SIZE/3)*3)
  482 #define RXD_LOOP_BOUND  ((RXD_INFO_SIZE/4)*4)
  483 
  484 typedef struct if_pkt_info_pad {
  485         PKT_TYPE pkt_val[PKT_INFO_SIZE];
  486 } *if_pkt_info_pad_t;
  487 typedef struct if_rxd_info_pad {
  488         PKT_TYPE rxd_val[RXD_INFO_SIZE];
  489 } *if_rxd_info_pad_t;
  490 
  491 CTASSERT(sizeof(struct if_pkt_info_pad) == sizeof(struct if_pkt_info));
  492 CTASSERT(sizeof(struct if_rxd_info_pad) == sizeof(struct if_rxd_info));
  493 
  494 
  495 static inline void
  496 pkt_info_zero(if_pkt_info_t pi)
  497 {
  498         if_pkt_info_pad_t pi_pad;
  499 
  500         pi_pad = (if_pkt_info_pad_t)pi;
  501         pi_pad->pkt_val[0] = 0; pi_pad->pkt_val[1] = 0; pi_pad->pkt_val[2] = 0;
  502         pi_pad->pkt_val[3] = 0; pi_pad->pkt_val[4] = 0; pi_pad->pkt_val[5] = 0;
  503 #ifndef __LP64__
  504         pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0;
  505         pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0;
  506 #endif  
  507 }
  508 
  509 static inline void
  510 rxd_info_zero(if_rxd_info_t ri)
  511 {
  512         if_rxd_info_pad_t ri_pad;
  513         int i;
  514 
  515         ri_pad = (if_rxd_info_pad_t)ri;
  516         for (i = 0; i < RXD_LOOP_BOUND; i += 4) {
  517                 ri_pad->rxd_val[i] = 0;
  518                 ri_pad->rxd_val[i+1] = 0;
  519                 ri_pad->rxd_val[i+2] = 0;
  520                 ri_pad->rxd_val[i+3] = 0;
  521         }
  522 #ifdef __LP64__
  523         ri_pad->rxd_val[RXD_INFO_SIZE-1] = 0;
  524 #endif
  525 }
  526 
  527 /*
  528  * Only allow a single packet to take up most 1/nth of the tx ring
  529  */
  530 #define MAX_SINGLE_PACKET_FRACTION 12
  531 #define IF_BAD_DMA (bus_addr_t)-1
  532 
  533 #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING))
  534 
  535 #define CTX_LOCK_INIT(_sc, _name)  mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF)
  536 
  537 #define CTX_LOCK(ctx) mtx_lock(&(ctx)->ifc_mtx)
  538 #define CTX_UNLOCK(ctx) mtx_unlock(&(ctx)->ifc_mtx)
  539 #define CTX_LOCK_DESTROY(ctx) mtx_destroy(&(ctx)->ifc_mtx)
  540 
  541 
  542 #define CALLOUT_LOCK(txq)       mtx_lock(&txq->ift_mtx)
  543 #define CALLOUT_UNLOCK(txq)     mtx_unlock(&txq->ift_mtx)
  544 
  545 
  546 /* Our boot-time initialization hook */
  547 static int      iflib_module_event_handler(module_t, int, void *);
  548 
  549 static moduledata_t iflib_moduledata = {
  550         "iflib",
  551         iflib_module_event_handler,
  552         NULL
  553 };
  554 
  555 DECLARE_MODULE(iflib, iflib_moduledata, SI_SUB_INIT_IF, SI_ORDER_ANY);
  556 MODULE_VERSION(iflib, 1);
  557 
  558 MODULE_DEPEND(iflib, pci, 1, 1, 1);
  559 MODULE_DEPEND(iflib, ether, 1, 1, 1);
  560 
  561 TASKQGROUP_DEFINE(if_io_tqg, mp_ncpus, 1);
  562 TASKQGROUP_DEFINE(if_config_tqg, 1, 1);
  563 
  564 #ifndef IFLIB_DEBUG_COUNTERS
  565 #ifdef INVARIANTS
  566 #define IFLIB_DEBUG_COUNTERS 1
  567 #else
  568 #define IFLIB_DEBUG_COUNTERS 0
  569 #endif /* !INVARIANTS */
  570 #endif
  571 
  572 static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0,
  573                    "iflib driver parameters");
  574 
  575 /*
  576  * XXX need to ensure that this can't accidentally cause the head to be moved backwards 
  577  */
  578 static int iflib_min_tx_latency = 0;
  579 SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW,
  580                    &iflib_min_tx_latency, 0, "minimize transmit latency at the possible expense of throughput");
  581 static int iflib_no_tx_batch = 0;
  582 SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW,
  583                    &iflib_no_tx_batch, 0, "minimize transmit latency at the possible expense of throughput");
  584 
  585 
  586 #if IFLIB_DEBUG_COUNTERS
  587 
  588 static int iflib_tx_seen;
  589 static int iflib_tx_sent;
  590 static int iflib_tx_encap;
  591 static int iflib_rx_allocs;
  592 static int iflib_fl_refills;
  593 static int iflib_fl_refills_large;
  594 static int iflib_tx_frees;
  595 
  596 SYSCTL_INT(_net_iflib, OID_AUTO, tx_seen, CTLFLAG_RD,
  597                    &iflib_tx_seen, 0, "# tx mbufs seen");
  598 SYSCTL_INT(_net_iflib, OID_AUTO, tx_sent, CTLFLAG_RD,
  599                    &iflib_tx_sent, 0, "# tx mbufs sent");
  600 SYSCTL_INT(_net_iflib, OID_AUTO, tx_encap, CTLFLAG_RD,
  601                    &iflib_tx_encap, 0, "# tx mbufs encapped");
  602 SYSCTL_INT(_net_iflib, OID_AUTO, tx_frees, CTLFLAG_RD,
  603                    &iflib_tx_frees, 0, "# tx frees");
  604 SYSCTL_INT(_net_iflib, OID_AUTO, rx_allocs, CTLFLAG_RD,
  605                    &iflib_rx_allocs, 0, "# rx allocations");
  606 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills, CTLFLAG_RD,
  607                    &iflib_fl_refills, 0, "# refills");
  608 SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills_large, CTLFLAG_RD,
  609                    &iflib_fl_refills_large, 0, "# large refills");
  610 
  611 
  612 static int iflib_txq_drain_flushing;
  613 static int iflib_txq_drain_oactive;
  614 static int iflib_txq_drain_notready;
  615 static int iflib_txq_drain_encapfail;
  616 
  617 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_flushing, CTLFLAG_RD,
  618                    &iflib_txq_drain_flushing, 0, "# drain flushes");
  619 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_oactive, CTLFLAG_RD,
  620                    &iflib_txq_drain_oactive, 0, "# drain oactives");
  621 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_notready, CTLFLAG_RD,
  622                    &iflib_txq_drain_notready, 0, "# drain notready");
  623 SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_encapfail, CTLFLAG_RD,
  624                    &iflib_txq_drain_encapfail, 0, "# drain encap fails");
  625 
  626 
  627 static int iflib_encap_load_mbuf_fail;
  628 static int iflib_encap_pad_mbuf_fail;
  629 static int iflib_encap_txq_avail_fail;
  630 static int iflib_encap_txd_encap_fail;
  631 
  632 SYSCTL_INT(_net_iflib, OID_AUTO, encap_load_mbuf_fail, CTLFLAG_RD,
  633                    &iflib_encap_load_mbuf_fail, 0, "# busdma load failures");
  634 SYSCTL_INT(_net_iflib, OID_AUTO, encap_pad_mbuf_fail, CTLFLAG_RD,
  635                    &iflib_encap_pad_mbuf_fail, 0, "# runt frame pad failures");
  636 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txq_avail_fail, CTLFLAG_RD,
  637                    &iflib_encap_txq_avail_fail, 0, "# txq avail failures");
  638 SYSCTL_INT(_net_iflib, OID_AUTO, encap_txd_encap_fail, CTLFLAG_RD,
  639                    &iflib_encap_txd_encap_fail, 0, "# driver encap failures");
  640 
  641 static int iflib_task_fn_rxs;
  642 static int iflib_rx_intr_enables;
  643 static int iflib_fast_intrs;
  644 static int iflib_intr_link;
  645 static int iflib_intr_msix; 
  646 static int iflib_rx_unavail;
  647 static int iflib_rx_ctx_inactive;
  648 static int iflib_rx_zero_len;
  649 static int iflib_rx_if_input;
  650 static int iflib_rx_mbuf_null;
  651 static int iflib_rxd_flush;
  652 
  653 static int iflib_verbose_debug;
  654 
  655 SYSCTL_INT(_net_iflib, OID_AUTO, intr_link, CTLFLAG_RD,
  656                    &iflib_intr_link, 0, "# intr link calls");
  657 SYSCTL_INT(_net_iflib, OID_AUTO, intr_msix, CTLFLAG_RD,
  658                    &iflib_intr_msix, 0, "# intr msix calls");
  659 SYSCTL_INT(_net_iflib, OID_AUTO, task_fn_rx, CTLFLAG_RD,
  660                    &iflib_task_fn_rxs, 0, "# task_fn_rx calls");
  661 SYSCTL_INT(_net_iflib, OID_AUTO, rx_intr_enables, CTLFLAG_RD,
  662                    &iflib_rx_intr_enables, 0, "# rx intr enables");
  663 SYSCTL_INT(_net_iflib, OID_AUTO, fast_intrs, CTLFLAG_RD,
  664                    &iflib_fast_intrs, 0, "# fast_intr calls");
  665 SYSCTL_INT(_net_iflib, OID_AUTO, rx_unavail, CTLFLAG_RD,
  666                    &iflib_rx_unavail, 0, "# times rxeof called with no available data");
  667 SYSCTL_INT(_net_iflib, OID_AUTO, rx_ctx_inactive, CTLFLAG_RD,
  668                    &iflib_rx_ctx_inactive, 0, "# times rxeof called with inactive context");
  669 SYSCTL_INT(_net_iflib, OID_AUTO, rx_zero_len, CTLFLAG_RD,
  670                    &iflib_rx_zero_len, 0, "# times rxeof saw zero len mbuf");
  671 SYSCTL_INT(_net_iflib, OID_AUTO, rx_if_input, CTLFLAG_RD,
  672                    &iflib_rx_if_input, 0, "# times rxeof called if_input");
  673 SYSCTL_INT(_net_iflib, OID_AUTO, rx_mbuf_null, CTLFLAG_RD,
  674                    &iflib_rx_mbuf_null, 0, "# times rxeof got null mbuf");
  675 SYSCTL_INT(_net_iflib, OID_AUTO, rxd_flush, CTLFLAG_RD,
  676                  &iflib_rxd_flush, 0, "# times rxd_flush called");
  677 SYSCTL_INT(_net_iflib, OID_AUTO, verbose_debug, CTLFLAG_RW,
  678                    &iflib_verbose_debug, 0, "enable verbose debugging");
  679 
  680 #define DBG_COUNTER_INC(name) atomic_add_int(&(iflib_ ## name), 1)
  681 static void
  682 iflib_debug_reset(void)
  683 {
  684         iflib_tx_seen = iflib_tx_sent = iflib_tx_encap = iflib_rx_allocs =
  685                 iflib_fl_refills = iflib_fl_refills_large = iflib_tx_frees =
  686                 iflib_txq_drain_flushing = iflib_txq_drain_oactive =
  687                 iflib_txq_drain_notready = iflib_txq_drain_encapfail =
  688                 iflib_encap_load_mbuf_fail = iflib_encap_pad_mbuf_fail =
  689                 iflib_encap_txq_avail_fail = iflib_encap_txd_encap_fail =
  690                 iflib_task_fn_rxs = iflib_rx_intr_enables = iflib_fast_intrs =
  691                 iflib_intr_link = iflib_intr_msix = iflib_rx_unavail =
  692                 iflib_rx_ctx_inactive = iflib_rx_zero_len = iflib_rx_if_input =
  693                 iflib_rx_mbuf_null = iflib_rxd_flush = 0;
  694 }
  695 
  696 #else
  697 #define DBG_COUNTER_INC(name)
  698 static void iflib_debug_reset(void) {}
  699 #endif
  700 
  701 
  702 
  703 #define IFLIB_DEBUG 0
  704 
  705 static void iflib_tx_structures_free(if_ctx_t ctx);
  706 static void iflib_rx_structures_free(if_ctx_t ctx);
  707 static int iflib_queues_alloc(if_ctx_t ctx);
  708 static int iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq);
  709 static int iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget);
  710 static int iflib_qset_structures_setup(if_ctx_t ctx);
  711 static int iflib_msix_init(if_ctx_t ctx);
  712 static int iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filterarg, int *rid, char *str);
  713 static void iflib_txq_check_drain(iflib_txq_t txq, int budget);
  714 static uint32_t iflib_txq_can_drain(struct ifmp_ring *);
  715 static int iflib_register(if_ctx_t);
  716 static void iflib_init_locked(if_ctx_t ctx);
  717 static void iflib_add_device_sysctl_pre(if_ctx_t ctx);
  718 static void iflib_add_device_sysctl_post(if_ctx_t ctx);
  719 static void iflib_ifmp_purge(iflib_txq_t txq);
  720 static void _iflib_pre_assert(if_softc_ctx_t scctx);
  721 static void iflib_stop(if_ctx_t ctx);
  722 static void iflib_if_init_locked(if_ctx_t ctx);
  723 #ifndef __NO_STRICT_ALIGNMENT
  724 static struct mbuf * iflib_fixup_rx(struct mbuf *m);
  725 #endif
  726 
  727 #ifdef DEV_NETMAP
  728 #include <sys/selinfo.h>
  729 #include <net/netmap.h>
  730 #include <dev/netmap/netmap_kern.h>
  731 
  732 MODULE_DEPEND(iflib, netmap, 1, 1, 1);
  733 
  734 static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init);
  735 
  736 /*
  737  * device-specific sysctl variables:
  738  *
  739  * iflib_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
  740  *      During regular operations the CRC is stripped, but on some
  741  *      hardware reception of frames not multiple of 64 is slower,
  742  *      so using crcstrip=0 helps in benchmarks.
  743  *
  744  * iflib_rx_miss, iflib_rx_miss_bufs:
  745  *      count packets that might be missed due to lost interrupts.
  746  */
  747 SYSCTL_DECL(_dev_netmap);
  748 /*
  749  * The xl driver by default strips CRCs and we do not override it.
  750  */
  751 
  752 int iflib_crcstrip = 1;
  753 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_crcstrip,
  754     CTLFLAG_RW, &iflib_crcstrip, 1, "strip CRC on rx frames");
  755 
  756 int iflib_rx_miss, iflib_rx_miss_bufs;
  757 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss,
  758     CTLFLAG_RW, &iflib_rx_miss, 0, "potentially missed rx intr");
  759 SYSCTL_INT(_dev_netmap, OID_AUTO, iflib_rx_miss_bufs,
  760     CTLFLAG_RW, &iflib_rx_miss_bufs, 0, "potentially missed rx intr bufs");
  761 
  762 /*
  763  * Register/unregister. We are already under netmap lock.
  764  * Only called on the first register or the last unregister.
  765  */
  766 static int
  767 iflib_netmap_register(struct netmap_adapter *na, int onoff)
  768 {
  769         struct ifnet *ifp = na->ifp;
  770         if_ctx_t ctx = ifp->if_softc;
  771         int status;
  772 
  773         CTX_LOCK(ctx);
  774         IFDI_INTR_DISABLE(ctx);
  775 
  776         /* Tell the stack that the interface is no longer active */
  777         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
  778 
  779         if (!CTX_IS_VF(ctx))
  780                 IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip);
  781 
  782         /* enable or disable flags and callbacks in na and ifp */
  783         if (onoff) {
  784                 nm_set_native_flags(na);
  785         } else {
  786                 nm_clear_native_flags(na);
  787         }
  788         iflib_stop(ctx);
  789         iflib_init_locked(ctx);
  790         IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ?
  791         status = ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1;
  792         if (status)
  793                 nm_clear_native_flags(na);
  794         CTX_UNLOCK(ctx);
  795         return (status);
  796 }
  797 
  798 static int
  799 netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init)
  800 {
  801         struct netmap_adapter *na = kring->na;
  802         u_int const lim = kring->nkr_num_slots - 1;
  803         u_int head = kring->rhead;
  804         struct netmap_ring *ring = kring->ring;
  805         bus_dmamap_t *map;
  806         struct if_rxd_update iru;
  807         if_ctx_t ctx = rxq->ifr_ctx;
  808         iflib_fl_t fl = &rxq->ifr_fl[0];
  809         uint32_t refill_pidx, nic_i;
  810 
  811         if (nm_i == head && __predict_true(!init))
  812                 return 0;
  813         iru_init(&iru, rxq, 0 /* flid */);
  814         map = fl->ifl_sds.ifsd_map;
  815         refill_pidx = netmap_idx_k2n(kring, nm_i);
  816         /*
  817          * IMPORTANT: we must leave one free slot in the ring,
  818          * so move head back by one unit
  819          */
  820         head = nm_prev(head, lim);
  821         while (nm_i != head) {
  822                 for (int tmp_pidx = 0; tmp_pidx < IFLIB_MAX_RX_REFRESH && nm_i != head; tmp_pidx++) {
  823                         struct netmap_slot *slot = &ring->slot[nm_i];
  824                         void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[tmp_pidx]);
  825                         uint32_t nic_i_dma = refill_pidx;
  826                         nic_i = netmap_idx_k2n(kring, nm_i);
  827 
  828                         MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH);
  829 
  830                         if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
  831                                 return netmap_ring_reinit(kring);
  832 
  833                         fl->ifl_vm_addrs[tmp_pidx] = addr;
  834                         if (__predict_false(init) && map) {
  835                                 netmap_load_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr);
  836                         } else if (map && (slot->flags & NS_BUF_CHANGED)) {
  837                                 /* buffer has changed, reload map */
  838                                 netmap_reload_map(na, fl->ifl_ifdi->idi_tag, map[nic_i], addr);
  839                         }
  840                         slot->flags &= ~NS_BUF_CHANGED;
  841 
  842                         nm_i = nm_next(nm_i, lim);
  843                         fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim);
  844                         if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1)
  845                                 continue;
  846 
  847                         iru.iru_pidx = refill_pidx;
  848                         iru.iru_count = tmp_pidx+1;
  849                         ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
  850 
  851                         refill_pidx = nic_i;
  852                         if (map == NULL)
  853                                 continue;
  854 
  855                         for (int n = 0; n < iru.iru_count; n++) {
  856                                 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, map[nic_i_dma],
  857                                                 BUS_DMASYNC_PREREAD);
  858                                 /* XXX - change this to not use the netmap func*/
  859                                 nic_i_dma = nm_next(nic_i_dma, lim);
  860                         }
  861                 }
  862         }
  863         kring->nr_hwcur = head;
  864 
  865         if (map)
  866                 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
  867                                 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
  868         ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i);
  869         return (0);
  870 }
  871 
  872 /*
  873  * Reconcile kernel and user view of the transmit ring.
  874  *
  875  * All information is in the kring.
  876  * Userspace wants to send packets up to the one before kring->rhead,
  877  * kernel knows kring->nr_hwcur is the first unsent packet.
  878  *
  879  * Here we push packets out (as many as possible), and possibly
  880  * reclaim buffers from previously completed transmission.
  881  *
  882  * The caller (netmap) guarantees that there is only one instance
  883  * running at any time. Any interference with other driver
  884  * methods should be handled by the individual drivers.
  885  */
  886 static int
  887 iflib_netmap_txsync(struct netmap_kring *kring, int flags)
  888 {
  889         struct netmap_adapter *na = kring->na;
  890         struct ifnet *ifp = na->ifp;
  891         struct netmap_ring *ring = kring->ring;
  892         u_int nm_i;     /* index into the netmap ring */
  893         u_int nic_i;    /* index into the NIC ring */
  894         u_int n;
  895         u_int const lim = kring->nkr_num_slots - 1;
  896         u_int const head = kring->rhead;
  897         struct if_pkt_info pi;
  898 
  899         /*
  900          * interrupts on every tx packet are expensive so request
  901          * them every half ring, or where NS_REPORT is set
  902          */
  903         u_int report_frequency = kring->nkr_num_slots >> 1;
  904         /* device-specific */
  905         if_ctx_t ctx = ifp->if_softc;
  906         iflib_txq_t txq = &ctx->ifc_txqs[kring->ring_id];
  907 
  908         if (txq->ift_sds.ifsd_map)
  909                 bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
  910                                 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
  911 
  912 
  913         /*
  914          * First part: process new packets to send.
  915          * nm_i is the current index in the netmap ring,
  916          * nic_i is the corresponding index in the NIC ring.
  917          *
  918          * If we have packets to send (nm_i != head)
  919          * iterate over the netmap ring, fetch length and update
  920          * the corresponding slot in the NIC ring. Some drivers also
  921          * need to update the buffer's physical address in the NIC slot
  922          * even NS_BUF_CHANGED is not set (PNMB computes the addresses).
  923          *
  924          * The netmap_reload_map() calls is especially expensive,
  925          * even when (as in this case) the tag is 0, so do only
  926          * when the buffer has actually changed.
  927          *
  928          * If possible do not set the report/intr bit on all slots,
  929          * but only a few times per ring or when NS_REPORT is set.
  930          *
  931          * Finally, on 10G and faster drivers, it might be useful
  932          * to prefetch the next slot and txr entry.
  933          */
  934 
  935         nm_i = netmap_idx_n2k(kring, kring->nr_hwcur);
  936         pkt_info_zero(&pi);
  937         pi.ipi_segs = txq->ift_segs;
  938         pi.ipi_qsidx = kring->ring_id;
  939         if (nm_i != head) {     /* we have new packets to send */
  940                 nic_i = netmap_idx_k2n(kring, nm_i);
  941 
  942                 __builtin_prefetch(&ring->slot[nm_i]);
  943                 __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
  944                 if (txq->ift_sds.ifsd_map)
  945                         __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
  946 
  947                 for (n = 0; nm_i != head; n++) {
  948                         struct netmap_slot *slot = &ring->slot[nm_i];
  949                         u_int len = slot->len;
  950                         uint64_t paddr;
  951                         void *addr = PNMB(na, slot, &paddr);
  952                         int flags = (slot->flags & NS_REPORT ||
  953                                 nic_i == 0 || nic_i == report_frequency) ?
  954                                 IPI_TX_INTR : 0;
  955 
  956                         /* device-specific */
  957                         pi.ipi_len = len;
  958                         pi.ipi_segs[0].ds_addr = paddr;
  959                         pi.ipi_segs[0].ds_len = len;
  960                         pi.ipi_nsegs = 1;
  961                         pi.ipi_ndescs = 0;
  962                         pi.ipi_pidx = nic_i;
  963                         pi.ipi_flags = flags;
  964 
  965                         /* Fill the slot in the NIC ring. */
  966                         ctx->isc_txd_encap(ctx->ifc_softc, &pi);
  967 
  968                         /* prefetch for next round */
  969                         __builtin_prefetch(&ring->slot[nm_i + 1]);
  970                         __builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
  971                         if (txq->ift_sds.ifsd_map) {
  972                                 __builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
  973 
  974                                 NM_CHECK_ADDR_LEN(na, addr, len);
  975 
  976                                 if (slot->flags & NS_BUF_CHANGED) {
  977                                         /* buffer has changed, reload map */
  978                                         netmap_reload_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[nic_i], addr);
  979                                 }
  980                                 /* make sure changes to the buffer are synced */
  981                                 bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_sds.ifsd_map[nic_i],
  982                                                 BUS_DMASYNC_PREWRITE);
  983                         }
  984                         slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
  985                         nm_i = nm_next(nm_i, lim);
  986                         nic_i = nm_next(nic_i, lim);
  987                 }
  988                 kring->nr_hwcur = head;
  989 
  990                 /* synchronize the NIC ring */
  991                 if (txq->ift_sds.ifsd_map)
  992                         bus_dmamap_sync(txq->ift_desc_tag, txq->ift_ifdi->idi_map,
  993                                                 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
  994 
  995                 /* (re)start the tx unit up to slot nic_i (excluded) */
  996                 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, nic_i);
  997         }
  998 
  999         /*
 1000          * Second part: reclaim buffers for completed transmissions.
 1001          */
 1002         if (iflib_tx_credits_update(ctx, txq)) {
 1003                 /* some tx completed, increment avail */
 1004                 nic_i = txq->ift_cidx_processed;
 1005                 kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
 1006         }
 1007         return (0);
 1008 }
 1009 
 1010 /*
 1011  * Reconcile kernel and user view of the receive ring.
 1012  * Same as for the txsync, this routine must be efficient.
 1013  * The caller guarantees a single invocations, but races against
 1014  * the rest of the driver should be handled here.
 1015  *
 1016  * On call, kring->rhead is the first packet that userspace wants
 1017  * to keep, and kring->rcur is the wakeup point.
 1018  * The kernel has previously reported packets up to kring->rtail.
 1019  *
 1020  * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
 1021  * of whether or not we received an interrupt.
 1022  */
 1023 static int
 1024 iflib_netmap_rxsync(struct netmap_kring *kring, int flags)
 1025 {
 1026         struct netmap_adapter *na = kring->na;
 1027         struct netmap_ring *ring = kring->ring;
 1028         uint32_t nm_i;  /* index into the netmap ring */
 1029         uint32_t nic_i; /* index into the NIC ring */
 1030         u_int i, n;
 1031         u_int const lim = kring->nkr_num_slots - 1;
 1032         u_int const head = netmap_idx_n2k(kring, kring->rhead);
 1033         int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
 1034         struct if_rxd_info ri;
 1035 
 1036         struct ifnet *ifp = na->ifp;
 1037         if_ctx_t ctx = ifp->if_softc;
 1038         iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id];
 1039         iflib_fl_t fl = rxq->ifr_fl;
 1040         if (head > lim)
 1041                 return netmap_ring_reinit(kring);
 1042 
 1043         /* XXX check sync modes */
 1044         for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) {
 1045                 if (fl->ifl_sds.ifsd_map == NULL)
 1046                         continue;
 1047                 bus_dmamap_sync(rxq->ifr_fl[i].ifl_desc_tag, fl->ifl_ifdi->idi_map,
 1048                                 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 1049         }
 1050         /*
 1051          * First part: import newly received packets.
 1052          *
 1053          * nm_i is the index of the next free slot in the netmap ring,
 1054          * nic_i is the index of the next received packet in the NIC ring,
 1055          * and they may differ in case if_init() has been called while
 1056          * in netmap mode. For the receive ring we have
 1057          *
 1058          *      nic_i = rxr->next_check;
 1059          *      nm_i = kring->nr_hwtail (previous)
 1060          * and
 1061          *      nm_i == (nic_i + kring->nkr_hwofs) % ring_size
 1062          *
 1063          * rxr->next_check is set to 0 on a ring reinit
 1064          */
 1065         if (netmap_no_pendintr || force_update) {
 1066                 int crclen = iflib_crcstrip ? 0 : 4;
 1067                 int error, avail;
 1068                 uint16_t slot_flags = kring->nkr_slot_flags;
 1069 
 1070                 for (i = 0; i < rxq->ifr_nfl; i++) {
 1071                         fl = &rxq->ifr_fl[i];
 1072                         nic_i = fl->ifl_cidx;
 1073                         nm_i = netmap_idx_n2k(kring, nic_i);
 1074                         avail = iflib_rxd_avail(ctx, rxq, nic_i, USHRT_MAX);
 1075                         for (n = 0; avail > 0; n++, avail--) {
 1076                                 rxd_info_zero(&ri);
 1077                                 ri.iri_frags = rxq->ifr_frags;
 1078                                 ri.iri_qsidx = kring->ring_id;
 1079                                 ri.iri_ifp = ctx->ifc_ifp;
 1080                                 ri.iri_cidx = nic_i;
 1081 
 1082                                 error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
 1083                                 ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen;
 1084                                 ring->slot[nm_i].flags = slot_flags;
 1085                                 if (fl->ifl_sds.ifsd_map)
 1086                                         bus_dmamap_sync(fl->ifl_ifdi->idi_tag,
 1087                                                         fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD);
 1088                                 nm_i = nm_next(nm_i, lim);
 1089                                 nic_i = nm_next(nic_i, lim);
 1090                         }
 1091                         if (n) { /* update the state variables */
 1092                                 if (netmap_no_pendintr && !force_update) {
 1093                                         /* diagnostics */
 1094                                         iflib_rx_miss ++;
 1095                                         iflib_rx_miss_bufs += n;
 1096                                 }
 1097                                 fl->ifl_cidx = nic_i;
 1098                                 kring->nr_hwtail = netmap_idx_k2n(kring, nm_i);
 1099                         }
 1100                         kring->nr_kflags &= ~NKR_PENDINTR;
 1101                 }
 1102         }
 1103         /*
 1104          * Second part: skip past packets that userspace has released.
 1105          * (kring->nr_hwcur to head excluded),
 1106          * and make the buffers available for reception.
 1107          * As usual nm_i is the index in the netmap ring,
 1108          * nic_i is the index in the NIC ring, and
 1109          * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
 1110          */
 1111         /* XXX not sure how this will work with multiple free lists */
 1112         nm_i = netmap_idx_n2k(kring, kring->nr_hwcur);
 1113 
 1114         return (netmap_fl_refill(rxq, kring, nm_i, false));
 1115 }
 1116 
 1117 static int
 1118 iflib_netmap_attach(if_ctx_t ctx)
 1119 {
 1120         struct netmap_adapter na;
 1121         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 1122 
 1123         bzero(&na, sizeof(na));
 1124 
 1125         na.ifp = ctx->ifc_ifp;
 1126         na.na_flags = NAF_BDG_MAYSLEEP;
 1127         MPASS(ctx->ifc_softc_ctx.isc_ntxqsets);
 1128         MPASS(ctx->ifc_softc_ctx.isc_nrxqsets);
 1129 
 1130         na.num_tx_desc = scctx->isc_ntxd[0];
 1131         na.num_rx_desc = scctx->isc_nrxd[0];
 1132         na.nm_txsync = iflib_netmap_txsync;
 1133         na.nm_rxsync = iflib_netmap_rxsync;
 1134         na.nm_register = iflib_netmap_register;
 1135         na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets;
 1136         na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets;
 1137         return (netmap_attach(&na));
 1138 }
 1139 
 1140 static void
 1141 iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq)
 1142 {
 1143         struct netmap_adapter *na = NA(ctx->ifc_ifp);
 1144         struct netmap_slot *slot;
 1145 
 1146         slot = netmap_reset(na, NR_TX, txq->ift_id, 0);
 1147         if (slot == NULL)
 1148                 return;
 1149         if (txq->ift_sds.ifsd_map == NULL)
 1150                 return;
 1151 
 1152         for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) {
 1153 
 1154                 /*
 1155                  * In netmap mode, set the map for the packet buffer.
 1156                  * NOTE: Some drivers (not this one) also need to set
 1157                  * the physical buffer address in the NIC ring.
 1158                  * netmap_idx_n2k() maps a nic index, i, into the corresponding
 1159                  * netmap slot index, si
 1160                  */
 1161                 int si = netmap_idx_n2k(&na->tx_rings[txq->ift_id], i);
 1162                 netmap_load_map(na, txq->ift_desc_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si));
 1163         }
 1164 }
 1165 
 1166 static void
 1167 iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq)
 1168 {
 1169         struct netmap_adapter *na = NA(ctx->ifc_ifp);
 1170         struct netmap_kring *kring = &na->rx_rings[rxq->ifr_id];
 1171         struct netmap_slot *slot;
 1172         uint32_t nm_i;
 1173 
 1174         slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0);
 1175         if (slot == NULL)
 1176                 return;
 1177         nm_i = netmap_idx_n2k(kring, 0);
 1178         netmap_fl_refill(rxq, kring, nm_i, true);
 1179 }
 1180 
 1181 #define iflib_netmap_detach(ifp) netmap_detach(ifp)
 1182 
 1183 #else
 1184 #define iflib_netmap_txq_init(ctx, txq)
 1185 #define iflib_netmap_rxq_init(ctx, rxq)
 1186 #define iflib_netmap_detach(ifp)
 1187 
 1188 #define iflib_netmap_attach(ctx) (0)
 1189 #define netmap_rx_irq(ifp, qid, budget) (0)
 1190 #define netmap_tx_irq(ifp, qid) do {} while (0)
 1191 
 1192 #endif
 1193 
 1194 #if defined(__i386__) || defined(__amd64__)
 1195 static __inline void
 1196 prefetch(void *x)
 1197 {
 1198         __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
 1199 }
 1200 static __inline void
 1201 prefetch2cachelines(void *x)
 1202 {
 1203         __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
 1204 #if (CACHE_LINE_SIZE < 128)
 1205         __asm volatile("prefetcht0 %0" :: "m" (*(((unsigned long *)x)+CACHE_LINE_SIZE/(sizeof(unsigned long)))));
 1206 #endif
 1207 }
 1208 #else
 1209 #define prefetch(x)
 1210 #define prefetch2cachelines(x)
 1211 #endif
 1212 
 1213 static void
 1214 iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid)
 1215 {
 1216         iflib_fl_t fl;
 1217 
 1218         fl = &rxq->ifr_fl[flid];
 1219         iru->iru_paddrs = fl->ifl_bus_addrs;
 1220         iru->iru_vaddrs = &fl->ifl_vm_addrs[0];
 1221         iru->iru_idxs = fl->ifl_rxd_idxs;
 1222         iru->iru_qsidx = rxq->ifr_id;
 1223         iru->iru_buf_size = fl->ifl_buf_size;
 1224         iru->iru_flidx = fl->ifl_id;
 1225 }
 1226 
 1227 static void
 1228 _iflib_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
 1229 {
 1230         if (err)
 1231                 return;
 1232         *(bus_addr_t *) arg = segs[0].ds_addr;
 1233 }
 1234 
 1235 int
 1236 iflib_dma_alloc(if_ctx_t ctx, int size, iflib_dma_info_t dma, int mapflags)
 1237 {
 1238         int err;
 1239         if_shared_ctx_t sctx = ctx->ifc_sctx;
 1240         device_t dev = ctx->ifc_dev;
 1241 
 1242         KASSERT(sctx->isc_q_align != 0, ("alignment value not initialized"));
 1243 
 1244         err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
 1245                                 sctx->isc_q_align, 0,   /* alignment, bounds */
 1246                                 BUS_SPACE_MAXADDR,      /* lowaddr */
 1247                                 BUS_SPACE_MAXADDR,      /* highaddr */
 1248                                 NULL, NULL,             /* filter, filterarg */
 1249                                 size,                   /* maxsize */
 1250                                 1,                      /* nsegments */
 1251                                 size,                   /* maxsegsize */
 1252                                 BUS_DMA_ALLOCNOW,       /* flags */
 1253                                 NULL,                   /* lockfunc */
 1254                                 NULL,                   /* lockarg */
 1255                                 &dma->idi_tag);
 1256         if (err) {
 1257                 device_printf(dev,
 1258                     "%s: bus_dma_tag_create failed: %d\n",
 1259                     __func__, err);
 1260                 goto fail_0;
 1261         }
 1262 
 1263         err = bus_dmamem_alloc(dma->idi_tag, (void**) &dma->idi_vaddr,
 1264             BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->idi_map);
 1265         if (err) {
 1266                 device_printf(dev,
 1267                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
 1268                     __func__, (uintmax_t)size, err);
 1269                 goto fail_1;
 1270         }
 1271 
 1272         dma->idi_paddr = IF_BAD_DMA;
 1273         err = bus_dmamap_load(dma->idi_tag, dma->idi_map, dma->idi_vaddr,
 1274             size, _iflib_dmamap_cb, &dma->idi_paddr, mapflags | BUS_DMA_NOWAIT);
 1275         if (err || dma->idi_paddr == IF_BAD_DMA) {
 1276                 device_printf(dev,
 1277                     "%s: bus_dmamap_load failed: %d\n",
 1278                     __func__, err);
 1279                 goto fail_2;
 1280         }
 1281 
 1282         dma->idi_size = size;
 1283         return (0);
 1284 
 1285 fail_2:
 1286         bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
 1287 fail_1:
 1288         bus_dma_tag_destroy(dma->idi_tag);
 1289 fail_0:
 1290         dma->idi_tag = NULL;
 1291 
 1292         return (err);
 1293 }
 1294 
 1295 int
 1296 iflib_dma_alloc_multi(if_ctx_t ctx, int *sizes, iflib_dma_info_t *dmalist, int mapflags, int count)
 1297 {
 1298         int i, err;
 1299         iflib_dma_info_t *dmaiter;
 1300 
 1301         dmaiter = dmalist;
 1302         for (i = 0; i < count; i++, dmaiter++) {
 1303                 if ((err = iflib_dma_alloc(ctx, sizes[i], *dmaiter, mapflags)) != 0)
 1304                         break;
 1305         }
 1306         if (err)
 1307                 iflib_dma_free_multi(dmalist, i);
 1308         return (err);
 1309 }
 1310 
 1311 void
 1312 iflib_dma_free(iflib_dma_info_t dma)
 1313 {
 1314         if (dma->idi_tag == NULL)
 1315                 return;
 1316         if (dma->idi_paddr != IF_BAD_DMA) {
 1317                 bus_dmamap_sync(dma->idi_tag, dma->idi_map,
 1318                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 1319                 bus_dmamap_unload(dma->idi_tag, dma->idi_map);
 1320                 dma->idi_paddr = IF_BAD_DMA;
 1321         }
 1322         if (dma->idi_vaddr != NULL) {
 1323                 bus_dmamem_free(dma->idi_tag, dma->idi_vaddr, dma->idi_map);
 1324                 dma->idi_vaddr = NULL;
 1325         }
 1326         bus_dma_tag_destroy(dma->idi_tag);
 1327         dma->idi_tag = NULL;
 1328 }
 1329 
 1330 void
 1331 iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count)
 1332 {
 1333         int i;
 1334         iflib_dma_info_t *dmaiter = dmalist;
 1335 
 1336         for (i = 0; i < count; i++, dmaiter++)
 1337                 iflib_dma_free(*dmaiter);
 1338 }
 1339 
 1340 #ifdef EARLY_AP_STARTUP
 1341 static const int iflib_started = 1;
 1342 #else
 1343 /*
 1344  * We used to abuse the smp_started flag to decide if the queues have been
 1345  * fully initialized (by late taskqgroup_adjust() calls in a SYSINIT()).
 1346  * That gave bad races, since the SYSINIT() runs strictly after smp_started
 1347  * is set.  Run a SYSINIT() strictly after that to just set a usable
 1348  * completion flag.
 1349  */
 1350 
 1351 static int iflib_started;
 1352 
 1353 static void
 1354 iflib_record_started(void *arg)
 1355 {
 1356         iflib_started = 1;
 1357 }
 1358 
 1359 SYSINIT(iflib_record_started, SI_SUB_SMP + 1, SI_ORDER_FIRST,
 1360         iflib_record_started, NULL);
 1361 #endif
 1362 
 1363 static int
 1364 iflib_fast_intr(void *arg)
 1365 {
 1366         iflib_filter_info_t info = arg;
 1367         struct grouptask *gtask = info->ifi_task;
 1368         if (!iflib_started)
 1369                 return (FILTER_HANDLED);
 1370 
 1371         DBG_COUNTER_INC(fast_intrs);
 1372         if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
 1373                 return (FILTER_HANDLED);
 1374 
 1375         GROUPTASK_ENQUEUE(gtask);
 1376         return (FILTER_HANDLED);
 1377 }
 1378 
 1379 static int
 1380 iflib_fast_intr_rxtx(void *arg)
 1381 {
 1382         iflib_filter_info_t info = arg;
 1383         struct grouptask *gtask = info->ifi_task;
 1384         iflib_rxq_t rxq = (iflib_rxq_t)info->ifi_ctx;
 1385         if_ctx_t ctx;
 1386         int i, cidx;
 1387 
 1388         if (!iflib_started)
 1389                 return (FILTER_HANDLED);
 1390 
 1391         DBG_COUNTER_INC(fast_intrs);
 1392         if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
 1393                 return (FILTER_HANDLED);
 1394 
 1395         for (i = 0; i < rxq->ifr_ntxqirq; i++) {
 1396                 qidx_t txqid = rxq->ifr_txqid[i];
 1397 
 1398                 ctx = rxq->ifr_ctx;
 1399 
 1400                 if (!ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) {
 1401                         IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid);
 1402                         continue;
 1403                 }
 1404                 GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
 1405         }
 1406         if (ctx->ifc_sctx->isc_flags & IFLIB_HAS_RXCQ)
 1407                 cidx = rxq->ifr_cq_cidx;
 1408         else
 1409                 cidx = rxq->ifr_fl[0].ifl_cidx;
 1410         if (iflib_rxd_avail(ctx, rxq, cidx, 1))
 1411                 GROUPTASK_ENQUEUE(gtask);
 1412         else
 1413                 IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
 1414         return (FILTER_HANDLED);
 1415 }
 1416 
 1417 
 1418 static int
 1419 iflib_fast_intr_ctx(void *arg)
 1420 {
 1421         iflib_filter_info_t info = arg;
 1422         struct grouptask *gtask = info->ifi_task;
 1423 
 1424         if (!iflib_started)
 1425                 return (FILTER_HANDLED);
 1426 
 1427         DBG_COUNTER_INC(fast_intrs);
 1428         if (info->ifi_filter != NULL && info->ifi_filter(info->ifi_filter_arg) == FILTER_HANDLED)
 1429                 return (FILTER_HANDLED);
 1430 
 1431         GROUPTASK_ENQUEUE(gtask);
 1432         return (FILTER_HANDLED);
 1433 }
 1434 
 1435 static int
 1436 _iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
 1437         driver_filter_t filter, driver_intr_t handler, void *arg,
 1438                                  char *name)
 1439 {
 1440         int rc, flags;
 1441         struct resource *res;
 1442         void *tag = NULL;
 1443         device_t dev = ctx->ifc_dev;
 1444 
 1445         flags = RF_ACTIVE;
 1446         if (ctx->ifc_flags & IFC_LEGACY)
 1447                 flags |= RF_SHAREABLE;
 1448         MPASS(rid < 512);
 1449         irq->ii_rid = rid;
 1450         res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq->ii_rid, flags);
 1451         if (res == NULL) {
 1452                 device_printf(dev,
 1453                     "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
 1454                 return (ENOMEM);
 1455         }
 1456         irq->ii_res = res;
 1457         KASSERT(filter == NULL || handler == NULL, ("filter and handler can't both be non-NULL"));
 1458         rc = bus_setup_intr(dev, res, INTR_MPSAFE | INTR_TYPE_NET,
 1459                                                 filter, handler, arg, &tag);
 1460         if (rc != 0) {
 1461                 device_printf(dev,
 1462                     "failed to setup interrupt for rid %d, name %s: %d\n",
 1463                                           rid, name ? name : "unknown", rc);
 1464                 return (rc);
 1465         } else if (name)
 1466                 bus_describe_intr(dev, res, tag, "%s", name);
 1467 
 1468         irq->ii_tag = tag;
 1469         return (0);
 1470 }
 1471 
 1472 
 1473 /*********************************************************************
 1474  *
 1475  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
 1476  *  the information needed to transmit a packet on the wire. This is
 1477  *  called only once at attach, setup is done every reset.
 1478  *
 1479  **********************************************************************/
 1480 
 1481 static int
 1482 iflib_txsd_alloc(iflib_txq_t txq)
 1483 {
 1484         if_ctx_t ctx = txq->ift_ctx;
 1485         if_shared_ctx_t sctx = ctx->ifc_sctx;
 1486         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 1487         device_t dev = ctx->ifc_dev;
 1488         int err, nsegments, ntsosegments;
 1489 
 1490         nsegments = scctx->isc_tx_nsegments;
 1491         ntsosegments = scctx->isc_tx_tso_segments_max;
 1492         MPASS(scctx->isc_ntxd[0] > 0);
 1493         MPASS(scctx->isc_ntxd[txq->ift_br_offset] > 0);
 1494         MPASS(nsegments > 0);
 1495         MPASS(ntsosegments > 0);
 1496         /*
 1497          * Setup DMA descriptor areas.
 1498          */
 1499         if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
 1500                                1, 0,                    /* alignment, bounds */
 1501                                BUS_SPACE_MAXADDR,       /* lowaddr */
 1502                                BUS_SPACE_MAXADDR,       /* highaddr */
 1503                                NULL, NULL,              /* filter, filterarg */
 1504                                sctx->isc_tx_maxsize,            /* maxsize */
 1505                                nsegments,       /* nsegments */
 1506                                sctx->isc_tx_maxsegsize, /* maxsegsize */
 1507                                0,                       /* flags */
 1508                                NULL,                    /* lockfunc */
 1509                                NULL,                    /* lockfuncarg */
 1510                                &txq->ift_desc_tag))) {
 1511                 device_printf(dev,"Unable to allocate TX DMA tag: %d\n", err);
 1512                 device_printf(dev,"maxsize: %ju nsegments: %d maxsegsize: %ju\n",
 1513                     (uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize);
 1514                 goto fail;
 1515         }
 1516         if ((err = bus_dma_tag_create(bus_get_dma_tag(dev),
 1517                                1, 0,                    /* alignment, bounds */
 1518                                BUS_SPACE_MAXADDR,       /* lowaddr */
 1519                                BUS_SPACE_MAXADDR,       /* highaddr */
 1520                                NULL, NULL,              /* filter, filterarg */
 1521                                scctx->isc_tx_tso_size_max,              /* maxsize */
 1522                                ntsosegments,    /* nsegments */
 1523                                scctx->isc_tx_tso_segsize_max,   /* maxsegsize */
 1524                                0,                       /* flags */
 1525                                NULL,                    /* lockfunc */
 1526                                NULL,                    /* lockfuncarg */
 1527                                &txq->ift_tso_desc_tag))) {
 1528                 device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err);
 1529 
 1530                 goto fail;
 1531         }
 1532         if (!(txq->ift_sds.ifsd_flags =
 1533             (uint8_t *) malloc(sizeof(uint8_t) *
 1534             scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 1535                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 1536                 err = ENOMEM;
 1537                 goto fail;
 1538         }
 1539         if (!(txq->ift_sds.ifsd_m =
 1540             (struct mbuf **) malloc(sizeof(struct mbuf *) *
 1541             scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 1542                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 1543                 err = ENOMEM;
 1544                 goto fail;
 1545         }
 1546 
 1547         /* Create the descriptor buffer dma maps */
 1548 #if defined(ACPI_DMAR) || (! (defined(__i386__) || defined(__amd64__)))
 1549         if ((ctx->ifc_flags & IFC_DMAR) == 0)
 1550                 return (0);
 1551 
 1552         if (!(txq->ift_sds.ifsd_map =
 1553             (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 1554                 device_printf(dev, "Unable to allocate tx_buffer map memory\n");
 1555                 err = ENOMEM;
 1556                 goto fail;
 1557         }
 1558 
 1559         for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) {
 1560                 err = bus_dmamap_create(txq->ift_desc_tag, 0, &txq->ift_sds.ifsd_map[i]);
 1561                 if (err != 0) {
 1562                         device_printf(dev, "Unable to create TX DMA map\n");
 1563                         goto fail;
 1564                 }
 1565         }
 1566 #endif
 1567         return (0);
 1568 fail:
 1569         /* We free all, it handles case where we are in the middle */
 1570         iflib_tx_structures_free(ctx);
 1571         return (err);
 1572 }
 1573 
 1574 static void
 1575 iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i)
 1576 {
 1577         bus_dmamap_t map;
 1578 
 1579         map = NULL;
 1580         if (txq->ift_sds.ifsd_map != NULL)
 1581                 map = txq->ift_sds.ifsd_map[i];
 1582         if (map != NULL) {
 1583                 bus_dmamap_unload(txq->ift_desc_tag, map);
 1584                 bus_dmamap_destroy(txq->ift_desc_tag, map);
 1585                 txq->ift_sds.ifsd_map[i] = NULL;
 1586         }
 1587 }
 1588 
 1589 static void
 1590 iflib_txq_destroy(iflib_txq_t txq)
 1591 {
 1592         if_ctx_t ctx = txq->ift_ctx;
 1593 
 1594         for (int i = 0; i < txq->ift_size; i++)
 1595                 iflib_txsd_destroy(ctx, txq, i);
 1596         if (txq->ift_sds.ifsd_map != NULL) {
 1597                 free(txq->ift_sds.ifsd_map, M_IFLIB);
 1598                 txq->ift_sds.ifsd_map = NULL;
 1599         }
 1600         if (txq->ift_sds.ifsd_m != NULL) {
 1601                 free(txq->ift_sds.ifsd_m, M_IFLIB);
 1602                 txq->ift_sds.ifsd_m = NULL;
 1603         }
 1604         if (txq->ift_sds.ifsd_flags != NULL) {
 1605                 free(txq->ift_sds.ifsd_flags, M_IFLIB);
 1606                 txq->ift_sds.ifsd_flags = NULL;
 1607         }
 1608         if (txq->ift_desc_tag != NULL) {
 1609                 bus_dma_tag_destroy(txq->ift_desc_tag);
 1610                 txq->ift_desc_tag = NULL;
 1611         }
 1612         if (txq->ift_tso_desc_tag != NULL) {
 1613                 bus_dma_tag_destroy(txq->ift_tso_desc_tag);
 1614                 txq->ift_tso_desc_tag = NULL;
 1615         }
 1616 }
 1617 
 1618 static void
 1619 iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i)
 1620 {
 1621         struct mbuf **mp;
 1622 
 1623         mp = &txq->ift_sds.ifsd_m[i];
 1624         if (*mp == NULL)
 1625                 return;
 1626 
 1627         if (txq->ift_sds.ifsd_map != NULL) {
 1628                 bus_dmamap_sync(txq->ift_desc_tag,
 1629                                 txq->ift_sds.ifsd_map[i],
 1630                                 BUS_DMASYNC_POSTWRITE);
 1631                 bus_dmamap_unload(txq->ift_desc_tag,
 1632                                   txq->ift_sds.ifsd_map[i]);
 1633         }
 1634         m_free(*mp);
 1635         DBG_COUNTER_INC(tx_frees);
 1636         *mp = NULL;
 1637 }
 1638 
 1639 static int
 1640 iflib_txq_setup(iflib_txq_t txq)
 1641 {
 1642         if_ctx_t ctx = txq->ift_ctx;
 1643         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 1644         iflib_dma_info_t di;
 1645         int i;
 1646 
 1647         /* Set number of descriptors available */
 1648         txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 1649         /* XXX make configurable */
 1650         txq->ift_update_freq = IFLIB_DEFAULT_TX_UPDATE_FREQ;
 1651 
 1652         /* Reset indices */
 1653         txq->ift_cidx_processed = 0;
 1654         txq->ift_pidx = txq->ift_cidx = txq->ift_npending = 0;
 1655         txq->ift_size = scctx->isc_ntxd[txq->ift_br_offset];
 1656 
 1657         for (i = 0, di = txq->ift_ifdi; i < ctx->ifc_nhwtxqs; i++, di++)
 1658                 bzero((void *)di->idi_vaddr, di->idi_size);
 1659 
 1660         IFDI_TXQ_SETUP(ctx, txq->ift_id);
 1661         for (i = 0, di = txq->ift_ifdi; i < ctx->ifc_nhwtxqs; i++, di++)
 1662                 bus_dmamap_sync(di->idi_tag, di->idi_map,
 1663                                                 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 1664         return (0);
 1665 }
 1666 
 1667 /*********************************************************************
 1668  *
 1669  *  Allocate memory for rx_buffer structures. Since we use one
 1670  *  rx_buffer per received packet, the maximum number of rx_buffer's
 1671  *  that we'll need is equal to the number of receive descriptors
 1672  *  that we've allocated.
 1673  *
 1674  **********************************************************************/
 1675 static int
 1676 iflib_rxsd_alloc(iflib_rxq_t rxq)
 1677 {
 1678         if_ctx_t ctx = rxq->ifr_ctx;
 1679         if_shared_ctx_t sctx = ctx->ifc_sctx;
 1680         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 1681         device_t dev = ctx->ifc_dev;
 1682         iflib_fl_t fl;
 1683         int                     err;
 1684 
 1685         MPASS(scctx->isc_nrxd[0] > 0);
 1686         MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0);
 1687 
 1688         fl = rxq->ifr_fl;
 1689         for (int i = 0; i <  rxq->ifr_nfl; i++, fl++) {
 1690                 fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */
 1691                 err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
 1692                                          1, 0,                  /* alignment, bounds */
 1693                                          BUS_SPACE_MAXADDR,     /* lowaddr */
 1694                                          BUS_SPACE_MAXADDR,     /* highaddr */
 1695                                          NULL, NULL,            /* filter, filterarg */
 1696                                          sctx->isc_rx_maxsize,  /* maxsize */
 1697                                          sctx->isc_rx_nsegments,        /* nsegments */
 1698                                          sctx->isc_rx_maxsegsize,       /* maxsegsize */
 1699                                          0,                     /* flags */
 1700                                          NULL,                  /* lockfunc */
 1701                                          NULL,                  /* lockarg */
 1702                                          &fl->ifl_desc_tag);
 1703                 if (err) {
 1704                         device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
 1705                                 __func__, err);
 1706                         goto fail;
 1707                 }
 1708                 if (!(fl->ifl_sds.ifsd_flags =
 1709                       (uint8_t *) malloc(sizeof(uint8_t) *
 1710                                          scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 1711                         device_printf(dev, "Unable to allocate tx_buffer memory\n");
 1712                         err = ENOMEM;
 1713                         goto fail;
 1714                 }
 1715                 if (!(fl->ifl_sds.ifsd_m =
 1716                       (struct mbuf **) malloc(sizeof(struct mbuf *) *
 1717                                               scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 1718                         device_printf(dev, "Unable to allocate tx_buffer memory\n");
 1719                         err = ENOMEM;
 1720                         goto fail;
 1721                 }
 1722                 if (!(fl->ifl_sds.ifsd_cl =
 1723                       (caddr_t *) malloc(sizeof(caddr_t) *
 1724                                               scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 1725                         device_printf(dev, "Unable to allocate tx_buffer memory\n");
 1726                         err = ENOMEM;
 1727                         goto fail;
 1728                 }
 1729 
 1730                 /* Create the descriptor buffer dma maps */
 1731 #if defined(ACPI_DMAR) || (! (defined(__i386__) || defined(__amd64__)))
 1732                 if ((ctx->ifc_flags & IFC_DMAR) == 0)
 1733                         continue;
 1734 
 1735                 if (!(fl->ifl_sds.ifsd_map =
 1736                       (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_nrxd[rxq->ifr_fl_offset], M_IFLIB, M_NOWAIT | M_ZERO))) {
 1737                         device_printf(dev, "Unable to allocate tx_buffer map memory\n");
 1738                         err = ENOMEM;
 1739                         goto fail;
 1740                 }
 1741 
 1742                 for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) {
 1743                         err = bus_dmamap_create(fl->ifl_desc_tag, 0, &fl->ifl_sds.ifsd_map[i]);
 1744                         if (err != 0) {
 1745                                 device_printf(dev, "Unable to create RX buffer DMA map\n");
 1746                                 goto fail;
 1747                         }
 1748                 }
 1749 #endif
 1750         }
 1751         return (0);
 1752 
 1753 fail:
 1754         iflib_rx_structures_free(ctx);
 1755         return (err);
 1756 }
 1757 
 1758 
 1759 /*
 1760  * Internal service routines
 1761  */
 1762 
 1763 struct rxq_refill_cb_arg {
 1764         int               error;
 1765         bus_dma_segment_t seg;
 1766         int               nseg;
 1767 };
 1768 
 1769 static void
 1770 _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 1771 {
 1772         struct rxq_refill_cb_arg *cb_arg = arg;
 1773 
 1774         cb_arg->error = error;
 1775         cb_arg->seg = segs[0];
 1776         cb_arg->nseg = nseg;
 1777 }
 1778 
 1779 
 1780 #ifdef ACPI_DMAR
 1781 #define IS_DMAR(ctx) (ctx->ifc_flags & IFC_DMAR)
 1782 #else
 1783 #define IS_DMAR(ctx) (0)
 1784 #endif
 1785 
 1786 /**
 1787  *      rxq_refill - refill an rxq  free-buffer list
 1788  *      @ctx: the iflib context
 1789  *      @rxq: the free-list to refill
 1790  *      @n: the number of new buffers to allocate
 1791  *
 1792  *      (Re)populate an rxq free-buffer list with up to @n new packet buffers.
 1793  *      The caller must assure that @n does not exceed the queue's capacity.
 1794  */
 1795 static void
 1796 _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count)
 1797 {
 1798         struct mbuf *m;
 1799         int idx, frag_idx = fl->ifl_fragidx;
 1800         int pidx = fl->ifl_pidx;
 1801         caddr_t cl, *sd_cl;
 1802         struct mbuf **sd_m;
 1803         uint8_t *sd_flags;
 1804         struct if_rxd_update iru;
 1805         bus_dmamap_t *sd_map;
 1806         int n, i = 0;
 1807         uint64_t bus_addr;
 1808         int err;
 1809         qidx_t credits;
 1810 
 1811         sd_m = fl->ifl_sds.ifsd_m;
 1812         sd_map = fl->ifl_sds.ifsd_map;
 1813         sd_cl = fl->ifl_sds.ifsd_cl;
 1814         sd_flags = fl->ifl_sds.ifsd_flags;
 1815         idx = pidx;
 1816         credits = fl->ifl_credits;
 1817 
 1818         n  = count;
 1819         MPASS(n > 0);
 1820         MPASS(credits + n <= fl->ifl_size);
 1821 
 1822         if (pidx < fl->ifl_cidx)
 1823                 MPASS(pidx + n <= fl->ifl_cidx);
 1824         if (pidx == fl->ifl_cidx && (credits < fl->ifl_size))
 1825                 MPASS(fl->ifl_gen == 0);
 1826         if (pidx > fl->ifl_cidx)
 1827                 MPASS(n <= fl->ifl_size - pidx + fl->ifl_cidx);
 1828 
 1829         DBG_COUNTER_INC(fl_refills);
 1830         if (n > 8)
 1831                 DBG_COUNTER_INC(fl_refills_large);
 1832         iru_init(&iru, fl->ifl_rxq, fl->ifl_id);
 1833         while (n--) {
 1834                 /*
 1835                  * We allocate an uninitialized mbuf + cluster, mbuf is
 1836                  * initialized after rx.
 1837                  *
 1838                  * If the cluster is still set then we know a minimum sized packet was received
 1839                  */
 1840                 bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size,  &frag_idx);
 1841                 if ((frag_idx < 0) || (frag_idx >= fl->ifl_size))
 1842                         bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx);
 1843                 if ((cl = sd_cl[frag_idx]) == NULL) {
 1844                        if ((cl = sd_cl[frag_idx] = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL)
 1845                                 break;
 1846 #if MEMORY_LOGGING
 1847                         fl->ifl_cl_enqueued++;
 1848 #endif
 1849                 }
 1850                 if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) {
 1851                         break;
 1852                 }
 1853 #if MEMORY_LOGGING
 1854                 fl->ifl_m_enqueued++;
 1855 #endif
 1856 
 1857                 DBG_COUNTER_INC(rx_allocs);
 1858 #if defined(__i386__) || defined(__amd64__)
 1859                 if (!IS_DMAR(ctx)) {
 1860                         bus_addr = pmap_kextract((vm_offset_t)cl);
 1861                 } else
 1862 #endif
 1863                 {
 1864                         struct rxq_refill_cb_arg cb_arg;
 1865                         iflib_rxq_t q;
 1866 
 1867                         cb_arg.error = 0;
 1868                         q = fl->ifl_rxq;
 1869                         MPASS(sd_map != NULL);
 1870                         MPASS(sd_map[frag_idx] != NULL);
 1871                         err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx],
 1872                          cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0);
 1873                         bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx],
 1874                                         BUS_DMASYNC_PREREAD);
 1875 
 1876                         if (err != 0 || cb_arg.error) {
 1877                                 /*
 1878                                  * !zone_pack ?
 1879                                  */
 1880                                 if (fl->ifl_zone == zone_pack)
 1881                                         uma_zfree(fl->ifl_zone, cl);
 1882                                 m_free(m);
 1883                                 n = 0;
 1884                                 goto done;
 1885                         }
 1886                         bus_addr = cb_arg.seg.ds_addr;
 1887                 }
 1888                 bit_set(fl->ifl_rx_bitmap, frag_idx);
 1889                 sd_flags[frag_idx] |= RX_SW_DESC_INUSE;
 1890 
 1891                 MPASS(sd_m[frag_idx] == NULL);
 1892                 sd_cl[frag_idx] = cl;
 1893                 sd_m[frag_idx] = m;
 1894                 fl->ifl_rxd_idxs[i] = frag_idx;
 1895                 fl->ifl_bus_addrs[i] = bus_addr;
 1896                 fl->ifl_vm_addrs[i] = cl;
 1897                 credits++;
 1898                 i++;
 1899                 MPASS(credits <= fl->ifl_size);
 1900                 if (++idx == fl->ifl_size) {
 1901                         fl->ifl_gen = 1;
 1902                         idx = 0;
 1903                 }
 1904                 if (n == 0 || i == IFLIB_MAX_RX_REFRESH) {
 1905                         iru.iru_pidx = pidx;
 1906                         iru.iru_count = i;
 1907                         ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
 1908                         i = 0;
 1909                         pidx = idx;
 1910                         fl->ifl_pidx = idx;
 1911                         fl->ifl_credits = credits;
 1912                 }
 1913 
 1914         }
 1915 done:
 1916         if (i) {
 1917                 iru.iru_pidx = pidx;
 1918                 iru.iru_count = i;
 1919                 ctx->isc_rxd_refill(ctx->ifc_softc, &iru);
 1920                 fl->ifl_pidx = idx;
 1921                 fl->ifl_credits = credits;
 1922         }
 1923         DBG_COUNTER_INC(rxd_flush);
 1924         if (fl->ifl_pidx == 0)
 1925                 pidx = fl->ifl_size - 1;
 1926         else
 1927                 pidx = fl->ifl_pidx - 1;
 1928 
 1929         if (sd_map)
 1930                 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 1931                                 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 1932         ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx);
 1933         fl->ifl_fragidx = frag_idx;
 1934 }
 1935 
 1936 static __inline void
 1937 __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max)
 1938 {
 1939         /* we avoid allowing pidx to catch up with cidx as it confuses ixl */
 1940         int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1;
 1941 #ifdef INVARIANTS
 1942         int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1;
 1943 #endif
 1944 
 1945         MPASS(fl->ifl_credits <= fl->ifl_size);
 1946         MPASS(reclaimable == delta);
 1947 
 1948         if (reclaimable > 0)
 1949                 _iflib_fl_refill(ctx, fl, min(max, reclaimable));
 1950 }
 1951 
 1952 static void
 1953 iflib_fl_bufs_free(iflib_fl_t fl)
 1954 {
 1955         iflib_dma_info_t idi = fl->ifl_ifdi;
 1956         uint32_t i;
 1957 
 1958         for (i = 0; i < fl->ifl_size; i++) {
 1959                 struct mbuf **sd_m = &fl->ifl_sds.ifsd_m[i];
 1960                 uint8_t *sd_flags = &fl->ifl_sds.ifsd_flags[i];
 1961                 caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i];
 1962 
 1963                 if (*sd_flags & RX_SW_DESC_INUSE) {
 1964                         if (fl->ifl_sds.ifsd_map != NULL) {
 1965                                 bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i];
 1966                                 bus_dmamap_unload(fl->ifl_desc_tag, sd_map);
 1967                                 if (fl->ifl_rxq->ifr_ctx->ifc_in_detach)
 1968                                         bus_dmamap_destroy(fl->ifl_desc_tag, sd_map);
 1969                         }
 1970                         if (*sd_m != NULL) {
 1971                                 m_init(*sd_m, M_NOWAIT, MT_DATA, 0);
 1972                                 uma_zfree(zone_mbuf, *sd_m);
 1973                         }
 1974                         if (*sd_cl != NULL)
 1975                                 uma_zfree(fl->ifl_zone, *sd_cl);
 1976                         *sd_flags = 0;
 1977                 } else {
 1978                         MPASS(*sd_cl == NULL);
 1979                         MPASS(*sd_m == NULL);
 1980                 }
 1981 #if MEMORY_LOGGING
 1982                 fl->ifl_m_dequeued++;
 1983                 fl->ifl_cl_dequeued++;
 1984 #endif
 1985                 *sd_cl = NULL;
 1986                 *sd_m = NULL;
 1987         }
 1988 #ifdef INVARIANTS
 1989         for (i = 0; i < fl->ifl_size; i++) {
 1990                 MPASS(fl->ifl_sds.ifsd_flags[i] == 0);
 1991                 MPASS(fl->ifl_sds.ifsd_cl[i] == NULL);
 1992                 MPASS(fl->ifl_sds.ifsd_m[i] == NULL);
 1993         }
 1994 #endif
 1995         /*
 1996          * Reset free list values
 1997          */
 1998         fl->ifl_credits = fl->ifl_cidx = fl->ifl_pidx = fl->ifl_gen = fl->ifl_fragidx = 0;
 1999         bzero(idi->idi_vaddr, idi->idi_size);
 2000 }
 2001 
 2002 /*********************************************************************
 2003  *
 2004  *  Initialize a receive ring and its buffers.
 2005  *
 2006  **********************************************************************/
 2007 static int
 2008 iflib_fl_setup(iflib_fl_t fl)
 2009 {
 2010         iflib_rxq_t rxq = fl->ifl_rxq;
 2011         if_ctx_t ctx = rxq->ifr_ctx;
 2012         if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 2013 
 2014         bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1);
 2015         /*
 2016         ** Free current RX buffer structs and their mbufs
 2017         */
 2018         iflib_fl_bufs_free(fl);
 2019         /* Now replenish the mbufs */
 2020         MPASS(fl->ifl_credits == 0);
 2021         /*
 2022          * XXX don't set the max_frame_size to larger
 2023          * than the hardware can handle
 2024          */
 2025         if (sctx->isc_max_frame_size <= 2048)
 2026                 fl->ifl_buf_size = MCLBYTES;
 2027 #ifndef CONTIGMALLOC_WORKS
 2028         else
 2029                 fl->ifl_buf_size = MJUMPAGESIZE;
 2030 #else
 2031         else if (sctx->isc_max_frame_size <= 4096)
 2032                 fl->ifl_buf_size = MJUMPAGESIZE;
 2033         else if (sctx->isc_max_frame_size <= 9216)
 2034                 fl->ifl_buf_size = MJUM9BYTES;
 2035         else
 2036                 fl->ifl_buf_size = MJUM16BYTES;
 2037 #endif
 2038         if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size)
 2039                 ctx->ifc_max_fl_buf_size = fl->ifl_buf_size;
 2040         fl->ifl_cltype = m_gettype(fl->ifl_buf_size);
 2041         fl->ifl_zone = m_getzone(fl->ifl_buf_size);
 2042 
 2043 
 2044         /* avoid pre-allocating zillions of clusters to an idle card
 2045          * potentially speeding up attach
 2046          */
 2047         _iflib_fl_refill(ctx, fl, min(128, fl->ifl_size));
 2048         MPASS(min(128, fl->ifl_size) == fl->ifl_credits);
 2049         if (min(128, fl->ifl_size) != fl->ifl_credits)
 2050                 return (ENOBUFS);
 2051         /*
 2052          * handle failure
 2053          */
 2054         MPASS(rxq != NULL);
 2055         MPASS(fl->ifl_ifdi != NULL);
 2056         bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 2057             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 2058         return (0);
 2059 }
 2060 
 2061 /*********************************************************************
 2062  *
 2063  *  Free receive ring data structures
 2064  *
 2065  **********************************************************************/
 2066 static void
 2067 iflib_rx_sds_free(iflib_rxq_t rxq)
 2068 {
 2069         iflib_fl_t fl;
 2070         int i;
 2071 
 2072         if (rxq->ifr_fl != NULL) {
 2073                 for (i = 0; i < rxq->ifr_nfl; i++) {
 2074                         fl = &rxq->ifr_fl[i];
 2075                         if (fl->ifl_desc_tag != NULL) {
 2076                                 bus_dma_tag_destroy(fl->ifl_desc_tag);
 2077                                 fl->ifl_desc_tag = NULL;
 2078                         }
 2079                         free(fl->ifl_sds.ifsd_m, M_IFLIB);
 2080                         free(fl->ifl_sds.ifsd_cl, M_IFLIB);
 2081                         /* XXX destroy maps first */
 2082                         free(fl->ifl_sds.ifsd_map, M_IFLIB);
 2083                         fl->ifl_sds.ifsd_m = NULL;
 2084                         fl->ifl_sds.ifsd_cl = NULL;
 2085                         fl->ifl_sds.ifsd_map = NULL;
 2086                 }
 2087                 free(rxq->ifr_fl, M_IFLIB);
 2088                 rxq->ifr_fl = NULL;
 2089                 rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
 2090         }
 2091 }
 2092 
 2093 /*
 2094  * MI independent logic
 2095  *
 2096  */
 2097 static void
 2098 iflib_timer(void *arg)
 2099 {
 2100         iflib_txq_t txq = arg;
 2101         if_ctx_t ctx = txq->ift_ctx;
 2102         if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 2103 
 2104         if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
 2105                 return;
 2106         /*
 2107         ** Check on the state of the TX queue(s), this
 2108         ** can be done without the lock because its RO
 2109         ** and the HUNG state will be static if set.
 2110         */
 2111         IFDI_TIMER(ctx, txq->ift_id);
 2112         if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) &&
 2113             ((txq->ift_cleaned_prev == txq->ift_cleaned) ||
 2114              (sctx->isc_pause_frames == 0)))
 2115                 goto hung;
 2116 
 2117         if (ifmp_ring_is_stalled(txq->ift_br))
 2118                 txq->ift_qstatus = IFLIB_QUEUE_HUNG;
 2119         txq->ift_cleaned_prev = txq->ift_cleaned;
 2120         /* handle any laggards */
 2121         if (txq->ift_db_pending)
 2122                 GROUPTASK_ENQUEUE(&txq->ift_task);
 2123 
 2124         sctx->isc_pause_frames = 0;
 2125         if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) 
 2126                 callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
 2127         return;
 2128 hung:
 2129         CTX_LOCK(ctx);
 2130         if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 2131         device_printf(ctx->ifc_dev,  "TX(%d) desc avail = %d, pidx = %d\n",
 2132                                   txq->ift_id, TXQ_AVAIL(txq), txq->ift_pidx);
 2133 
 2134         IFDI_WATCHDOG_RESET(ctx);
 2135         ctx->ifc_watchdog_events++;
 2136 
 2137         ctx->ifc_flags |= IFC_DO_RESET;
 2138         iflib_admin_intr_deferred(ctx);
 2139         CTX_UNLOCK(ctx);
 2140 }
 2141 
 2142 static void
 2143 iflib_init_locked(if_ctx_t ctx)
 2144 {
 2145         if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 2146         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 2147         if_t ifp = ctx->ifc_ifp;
 2148         iflib_fl_t fl;
 2149         iflib_txq_t txq;
 2150         iflib_rxq_t rxq;
 2151         int i, j, tx_ip_csum_flags, tx_ip6_csum_flags;
 2152 
 2153 
 2154         if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 2155         IFDI_INTR_DISABLE(ctx);
 2156 
 2157         tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP);
 2158         tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP);
 2159         /* Set hardware offload abilities */
 2160         if_clearhwassist(ifp);
 2161         if (if_getcapenable(ifp) & IFCAP_TXCSUM)
 2162                 if_sethwassistbits(ifp, tx_ip_csum_flags, 0);
 2163         if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
 2164                 if_sethwassistbits(ifp,  tx_ip6_csum_flags, 0);
 2165         if (if_getcapenable(ifp) & IFCAP_TSO4)
 2166                 if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
 2167         if (if_getcapenable(ifp) & IFCAP_TSO6)
 2168                 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
 2169 
 2170         for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) {
 2171                 CALLOUT_LOCK(txq);
 2172                 callout_stop(&txq->ift_timer);
 2173                 CALLOUT_UNLOCK(txq);
 2174                 iflib_netmap_txq_init(ctx, txq);
 2175         }
 2176 #ifdef INVARIANTS
 2177         i = if_getdrvflags(ifp);
 2178 #endif
 2179         IFDI_INIT(ctx);
 2180         MPASS(if_getdrvflags(ifp) == i);
 2181         for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) {
 2182                 /* XXX this should really be done on a per-queue basis */
 2183                 if (if_getcapenable(ifp) & IFCAP_NETMAP) {
 2184                         MPASS(rxq->ifr_id == i);
 2185                         iflib_netmap_rxq_init(ctx, rxq);
 2186                         continue;
 2187                 }
 2188                 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
 2189                         if (iflib_fl_setup(fl)) {
 2190                                 device_printf(ctx->ifc_dev, "freelist setup failed - check cluster settings\n");
 2191                                 goto done;
 2192                         }
 2193                 }
 2194         }
 2195         done:
 2196         if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
 2197         IFDI_INTR_ENABLE(ctx);
 2198         txq = ctx->ifc_txqs;
 2199         for (i = 0; i < sctx->isc_ntxqsets; i++, txq++)
 2200                 callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq,
 2201                         txq->ift_timer.c_cpu);
 2202 }
 2203 
 2204 static int
 2205 iflib_media_change(if_t ifp)
 2206 {
 2207         if_ctx_t ctx = if_getsoftc(ifp);
 2208         int err;
 2209 
 2210         CTX_LOCK(ctx);
 2211         if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0)
 2212                 iflib_init_locked(ctx);
 2213         CTX_UNLOCK(ctx);
 2214         return (err);
 2215 }
 2216 
 2217 static void
 2218 iflib_media_status(if_t ifp, struct ifmediareq *ifmr)
 2219 {
 2220         if_ctx_t ctx = if_getsoftc(ifp);
 2221 
 2222         CTX_LOCK(ctx);
 2223         IFDI_UPDATE_ADMIN_STATUS(ctx);
 2224         IFDI_MEDIA_STATUS(ctx, ifmr);
 2225         CTX_UNLOCK(ctx);
 2226 }
 2227 
 2228 static void
 2229 iflib_stop(if_ctx_t ctx)
 2230 {
 2231         iflib_txq_t txq = ctx->ifc_txqs;
 2232         iflib_rxq_t rxq = ctx->ifc_rxqs;
 2233         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 2234         iflib_dma_info_t di;
 2235         iflib_fl_t fl;
 2236         int i, j;
 2237 
 2238         /* Tell the stack that the interface is no longer active */
 2239         if_setdrvflagbits(ctx->ifc_ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
 2240 
 2241         IFDI_INTR_DISABLE(ctx);
 2242         DELAY(1000);
 2243         IFDI_STOP(ctx);
 2244         DELAY(1000);
 2245 
 2246         iflib_debug_reset();
 2247         /* Wait for current tx queue users to exit to disarm watchdog timer. */
 2248         for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) {
 2249                 /* make sure all transmitters have completed before proceeding XXX */
 2250 
 2251                 CALLOUT_LOCK(txq);
 2252                 callout_stop(&txq->ift_timer);
 2253                 CALLOUT_UNLOCK(txq);
 2254 
 2255                 /* clean any enqueued buffers */
 2256                 iflib_ifmp_purge(txq);
 2257                 /* Free any existing tx buffers. */
 2258                 for (j = 0; j < txq->ift_size; j++) {
 2259                         iflib_txsd_free(ctx, txq, j);
 2260                 }
 2261                 txq->ift_processed = txq->ift_cleaned = txq->ift_cidx_processed = 0;
 2262                 txq->ift_in_use = txq->ift_gen = txq->ift_cidx = txq->ift_pidx = txq->ift_no_desc_avail = 0;
 2263                 txq->ift_closed = txq->ift_mbuf_defrag = txq->ift_mbuf_defrag_failed = 0;
 2264                 txq->ift_no_tx_dma_setup = txq->ift_txd_encap_efbig = txq->ift_map_failed = 0;
 2265                 txq->ift_pullups = 0;
 2266                 ifmp_ring_reset_stats(txq->ift_br);
 2267                 for (j = 0, di = txq->ift_ifdi; j < ctx->ifc_nhwtxqs; j++, di++)
 2268                         bzero((void *)di->idi_vaddr, di->idi_size);
 2269         }
 2270         for (i = 0; i < scctx->isc_nrxqsets; i++, rxq++) {
 2271                 /* make sure all transmitters have completed before proceeding XXX */
 2272 
 2273                 for (j = 0, di = rxq->ifr_ifdi; j < rxq->ifr_nfl; j++, di++)
 2274                         bzero((void *)di->idi_vaddr, di->idi_size);
 2275                 /* also resets the free lists pidx/cidx */
 2276                 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
 2277                         iflib_fl_bufs_free(fl);
 2278         }
 2279 }
 2280 
 2281 static inline caddr_t
 2282 calc_next_rxd(iflib_fl_t fl, int cidx)
 2283 {
 2284         qidx_t size;
 2285         int nrxd;
 2286         caddr_t start, end, cur, next;
 2287 
 2288         nrxd = fl->ifl_size;
 2289         size = fl->ifl_rxd_size;
 2290         start = fl->ifl_ifdi->idi_vaddr;
 2291 
 2292         if (__predict_false(size == 0))
 2293                 return (start);
 2294         cur = start + size*cidx;
 2295         end = start + size*nrxd;
 2296         next = CACHE_PTR_NEXT(cur);
 2297         return (next < end ? next : start);
 2298 }
 2299 
 2300 static inline void
 2301 prefetch_pkts(iflib_fl_t fl, int cidx)
 2302 {
 2303         int nextptr;
 2304         int nrxd = fl->ifl_size;
 2305         caddr_t next_rxd;
 2306 
 2307 
 2308         nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd-1);
 2309         prefetch(&fl->ifl_sds.ifsd_m[nextptr]);
 2310         prefetch(&fl->ifl_sds.ifsd_cl[nextptr]);
 2311         next_rxd = calc_next_rxd(fl, cidx);
 2312         prefetch(next_rxd);
 2313         prefetch(fl->ifl_sds.ifsd_m[(cidx + 1) & (nrxd-1)]);
 2314         prefetch(fl->ifl_sds.ifsd_m[(cidx + 2) & (nrxd-1)]);
 2315         prefetch(fl->ifl_sds.ifsd_m[(cidx + 3) & (nrxd-1)]);
 2316         prefetch(fl->ifl_sds.ifsd_m[(cidx + 4) & (nrxd-1)]);
 2317         prefetch(fl->ifl_sds.ifsd_cl[(cidx + 1) & (nrxd-1)]);
 2318         prefetch(fl->ifl_sds.ifsd_cl[(cidx + 2) & (nrxd-1)]);
 2319         prefetch(fl->ifl_sds.ifsd_cl[(cidx + 3) & (nrxd-1)]);
 2320         prefetch(fl->ifl_sds.ifsd_cl[(cidx + 4) & (nrxd-1)]);
 2321 }
 2322 
 2323 static void
 2324 rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int unload, if_rxsd_t sd)
 2325 {
 2326         int flid, cidx;
 2327         bus_dmamap_t map;
 2328         iflib_fl_t fl;
 2329         iflib_dma_info_t di;
 2330         int next;
 2331 
 2332         map = NULL;
 2333         flid = irf->irf_flid;
 2334         cidx = irf->irf_idx;
 2335         fl = &rxq->ifr_fl[flid];
 2336         sd->ifsd_fl = fl;
 2337         sd->ifsd_cidx = cidx;
 2338         sd->ifsd_m = &fl->ifl_sds.ifsd_m[cidx];
 2339         sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx];
 2340         fl->ifl_credits--;
 2341 #if MEMORY_LOGGING
 2342         fl->ifl_m_dequeued++;
 2343 #endif
 2344         if (rxq->ifr_ctx->ifc_flags & IFC_PREFETCH)
 2345                 prefetch_pkts(fl, cidx);
 2346         if (fl->ifl_sds.ifsd_map != NULL) {
 2347                 next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1);
 2348                 prefetch(&fl->ifl_sds.ifsd_map[next]);
 2349                 map = fl->ifl_sds.ifsd_map[cidx];
 2350                 di = fl->ifl_ifdi;
 2351                 next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1);
 2352                 prefetch(&fl->ifl_sds.ifsd_flags[next]);
 2353                 bus_dmamap_sync(di->idi_tag, di->idi_map,
 2354                                 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 2355 
 2356         /* not valid assert if bxe really does SGE from non-contiguous elements */
 2357                 MPASS(fl->ifl_cidx == cidx);
 2358                 if (unload)
 2359                         bus_dmamap_unload(fl->ifl_desc_tag, map);
 2360         }
 2361         fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1);
 2362         if (__predict_false(fl->ifl_cidx == 0))
 2363                 fl->ifl_gen = 0;
 2364         if (map != NULL)
 2365                 bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map,
 2366                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 2367         bit_clear(fl->ifl_rx_bitmap, cidx);
 2368 }
 2369 
 2370 static struct mbuf *
 2371 assemble_segments(iflib_rxq_t rxq, if_rxd_info_t ri, if_rxsd_t sd)
 2372 {
 2373         int i, padlen , flags;
 2374         struct mbuf *m, *mh, *mt;
 2375         caddr_t cl;
 2376 
 2377         i = 0;
 2378         mh = NULL;
 2379         do {
 2380                 rxd_frag_to_sd(rxq, &ri->iri_frags[i], TRUE, sd);
 2381 
 2382                 MPASS(*sd->ifsd_cl != NULL);
 2383                 MPASS(*sd->ifsd_m != NULL);
 2384 
 2385                 /* Don't include zero-length frags */
 2386                 if (ri->iri_frags[i].irf_len == 0) {
 2387                         /* XXX we can save the cluster here, but not the mbuf */
 2388                         m_init(*sd->ifsd_m, M_NOWAIT, MT_DATA, 0);
 2389                         m_free(*sd->ifsd_m);
 2390                         *sd->ifsd_m = NULL;
 2391                         continue;
 2392                 }
 2393                 m = *sd->ifsd_m;
 2394                 *sd->ifsd_m = NULL;
 2395                 if (mh == NULL) {
 2396                         flags = M_PKTHDR|M_EXT;
 2397                         mh = mt = m;
 2398                         padlen = ri->iri_pad;
 2399                 } else {
 2400                         flags = M_EXT;
 2401                         mt->m_next = m;
 2402                         mt = m;
 2403                         /* assuming padding is only on the first fragment */
 2404                         padlen = 0;
 2405                 }
 2406                 cl = *sd->ifsd_cl;
 2407                 *sd->ifsd_cl = NULL;
 2408 
 2409                 /* Can these two be made one ? */
 2410                 m_init(m, M_NOWAIT, MT_DATA, flags);
 2411                 m_cljset(m, cl, sd->ifsd_fl->ifl_cltype);
 2412                 /*
 2413                  * These must follow m_init and m_cljset
 2414                  */
 2415                 m->m_data += padlen;
 2416                 ri->iri_len -= padlen;
 2417                 m->m_len = ri->iri_frags[i].irf_len;
 2418         } while (++i < ri->iri_nfrags);
 2419 
 2420         return (mh);
 2421 }
 2422 
 2423 /*
 2424  * Process one software descriptor
 2425  */
 2426 static struct mbuf *
 2427 iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri)
 2428 {
 2429         struct if_rxsd sd;
 2430         struct mbuf *m;
 2431 
 2432         /* should I merge this back in now that the two paths are basically duplicated? */
 2433         if (ri->iri_nfrags == 1 &&
 2434             ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) {
 2435                 rxd_frag_to_sd(rxq, &ri->iri_frags[0], FALSE, &sd);
 2436                 m = *sd.ifsd_m;
 2437                 *sd.ifsd_m = NULL;
 2438                 m_init(m, M_NOWAIT, MT_DATA, M_PKTHDR);
 2439 #ifndef __NO_STRICT_ALIGNMENT
 2440                 if (!IP_ALIGNED(m))
 2441                         m->m_data += 2;
 2442 #endif
 2443                 memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len);
 2444                 m->m_len = ri->iri_frags[0].irf_len;
 2445        } else {
 2446                 m = assemble_segments(rxq, ri, &sd);
 2447         }
 2448         m->m_pkthdr.len = ri->iri_len;
 2449         m->m_pkthdr.rcvif = ri->iri_ifp;
 2450         m->m_flags |= ri->iri_flags;
 2451         m->m_pkthdr.ether_vtag = ri->iri_vtag;
 2452         m->m_pkthdr.flowid = ri->iri_flowid;
 2453         M_HASHTYPE_SET(m, ri->iri_rsstype);
 2454         m->m_pkthdr.csum_flags = ri->iri_csum_flags;
 2455         m->m_pkthdr.csum_data = ri->iri_csum_data;
 2456         return (m);
 2457 }
 2458 
 2459 #if defined(INET6) || defined(INET)
 2460 static void
 2461 iflib_get_ip_forwarding(struct lro_ctrl *lc, bool *v4, bool *v6)
 2462 {
 2463         CURVNET_SET(lc->ifp->if_vnet);
 2464 #if defined(INET6)
 2465         *v6 = VNET(ip6_forwarding);
 2466 #endif
 2467 #if defined(INET)
 2468         *v4 = VNET(ipforwarding);
 2469 #endif
 2470         CURVNET_RESTORE();
 2471 }
 2472 
 2473 /*
 2474  * Returns true if it's possible this packet could be LROed.
 2475  * if it returns false, it is guaranteed that tcp_lro_rx()
 2476  * would not return zero.
 2477  */
 2478 static bool
 2479 iflib_check_lro_possible(struct mbuf *m, bool v4_forwarding, bool v6_forwarding)
 2480 {
 2481         struct ether_header *eh;
 2482         uint16_t eh_type;
 2483 
 2484         eh = mtod(m, struct ether_header *);
 2485         eh_type = ntohs(eh->ether_type);
 2486         switch (eh_type) {
 2487 #if defined(INET6)
 2488                 case ETHERTYPE_IPV6:
 2489                         return !v6_forwarding;
 2490 #endif
 2491 #if defined (INET)
 2492                 case ETHERTYPE_IP:
 2493                         return !v4_forwarding;
 2494 #endif
 2495         }
 2496 
 2497         return false;
 2498 }
 2499 #else
 2500 static void
 2501 iflib_get_ip_forwarding(struct lro_ctrl *lc __unused, bool *v4 __unused, bool *v6 __unused)
 2502 {
 2503 }
 2504 #endif
 2505 
 2506 static bool
 2507 iflib_rxeof(iflib_rxq_t rxq, qidx_t budget)
 2508 {
 2509         if_ctx_t ctx = rxq->ifr_ctx;
 2510         if_shared_ctx_t sctx = ctx->ifc_sctx;
 2511         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 2512         int avail, i;
 2513         qidx_t *cidxp;
 2514         struct if_rxd_info ri;
 2515         int err, budget_left, rx_bytes, rx_pkts;
 2516         iflib_fl_t fl;
 2517         struct ifnet *ifp;
 2518         int lro_enabled;
 2519         bool lro_possible = false;
 2520         bool v4_forwarding, v6_forwarding;
 2521 
 2522         /*
 2523          * XXX early demux data packets so that if_input processing only handles
 2524          * acks in interrupt context
 2525          */
 2526         struct mbuf *m, *mh, *mt, *mf;
 2527 
 2528         ifp = ctx->ifc_ifp;
 2529         mh = mt = NULL;
 2530         MPASS(budget > 0);
 2531         rx_pkts = rx_bytes = 0;
 2532         if (sctx->isc_flags & IFLIB_HAS_RXCQ)
 2533                 cidxp = &rxq->ifr_cq_cidx;
 2534         else
 2535                 cidxp = &rxq->ifr_fl[0].ifl_cidx;
 2536         if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) {
 2537                 for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
 2538                         __iflib_fl_refill_lt(ctx, fl, budget + 8);
 2539                 DBG_COUNTER_INC(rx_unavail);
 2540                 return (false);
 2541         }
 2542 
 2543         for (budget_left = budget; (budget_left > 0) && (avail > 0); budget_left--, avail--) {
 2544                 if (__predict_false(!CTX_ACTIVE(ctx))) {
 2545                         DBG_COUNTER_INC(rx_ctx_inactive);
 2546                         break;
 2547                 }
 2548                 /*
 2549                  * Reset client set fields to their default values
 2550                  */
 2551                 rxd_info_zero(&ri);
 2552                 ri.iri_qsidx = rxq->ifr_id;
 2553                 ri.iri_cidx = *cidxp;
 2554                 ri.iri_ifp = ifp;
 2555                 ri.iri_frags = rxq->ifr_frags;
 2556                 err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri);
 2557 
 2558                 if (err)
 2559                         goto err;
 2560                 if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
 2561                         *cidxp = ri.iri_cidx;
 2562                         /* Update our consumer index */
 2563                         /* XXX NB: shurd - check if this is still safe */
 2564                         while (rxq->ifr_cq_cidx >= scctx->isc_nrxd[0]) {
 2565                                 rxq->ifr_cq_cidx -= scctx->isc_nrxd[0];
 2566                                 rxq->ifr_cq_gen = 0;
 2567                         }
 2568                         /* was this only a completion queue message? */
 2569                         if (__predict_false(ri.iri_nfrags == 0))
 2570                                 continue;
 2571                 }
 2572                 MPASS(ri.iri_nfrags != 0);
 2573                 MPASS(ri.iri_len != 0);
 2574 
 2575                 /* will advance the cidx on the corresponding free lists */
 2576                 m = iflib_rxd_pkt_get(rxq, &ri);
 2577                 if (avail == 0 && budget_left)
 2578                         avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget_left);
 2579 
 2580                 if (__predict_false(m == NULL)) {
 2581                         DBG_COUNTER_INC(rx_mbuf_null);
 2582                         continue;
 2583                 }
 2584                 /* imm_pkt: -- cxgb */
 2585                 if (mh == NULL)
 2586                         mh = mt = m;
 2587                 else {
 2588                         mt->m_nextpkt = m;
 2589                         mt = m;
 2590                 }
 2591         }
 2592         /* make sure that we can refill faster than drain */
 2593         for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++)
 2594                 __iflib_fl_refill_lt(ctx, fl, budget + 8);
 2595 
 2596         lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO);
 2597         if (lro_enabled)
 2598                 iflib_get_ip_forwarding(&rxq->ifr_lc, &v4_forwarding, &v6_forwarding);
 2599         mt = mf = NULL;
 2600         while (mh != NULL) {
 2601                 m = mh;
 2602                 mh = mh->m_nextpkt;
 2603                 m->m_nextpkt = NULL;
 2604 #ifndef __NO_STRICT_ALIGNMENT
 2605                 if (!IP_ALIGNED(m) && (m = iflib_fixup_rx(m)) == NULL)
 2606                         continue;
 2607 #endif
 2608                 rx_bytes += m->m_pkthdr.len;
 2609                 rx_pkts++;
 2610 #if defined(INET6) || defined(INET)
 2611                 if (lro_enabled) {
 2612                         if (!lro_possible) {
 2613                                 lro_possible = iflib_check_lro_possible(m, v4_forwarding, v6_forwarding);
 2614                                 if (lro_possible && mf != NULL) {
 2615                                         ifp->if_input(ifp, mf);
 2616                                         DBG_COUNTER_INC(rx_if_input);
 2617                                         mt = mf = NULL;
 2618                                 }
 2619                         }
 2620                         if ((m->m_pkthdr.csum_flags & (CSUM_L4_CALC|CSUM_L4_VALID)) ==
 2621                             (CSUM_L4_CALC|CSUM_L4_VALID)) {
 2622                                 if (lro_possible && tcp_lro_rx(&rxq->ifr_lc, m, 0) == 0)
 2623                                         continue;
 2624                         }
 2625                 }
 2626 #endif
 2627                 if (lro_possible) {
 2628                         ifp->if_input(ifp, m);
 2629                         DBG_COUNTER_INC(rx_if_input);
 2630                         continue;
 2631                 }
 2632 
 2633                 if (mf == NULL)
 2634                         mf = m;
 2635                 if (mt != NULL)
 2636                         mt->m_nextpkt = m;
 2637                 mt = m;
 2638         }
 2639         if (mf != NULL) {
 2640                 ifp->if_input(ifp, mf);
 2641                 DBG_COUNTER_INC(rx_if_input);
 2642         }
 2643 
 2644         if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes);
 2645         if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts);
 2646 
 2647         /*
 2648          * Flush any outstanding LRO work
 2649          */
 2650 #if defined(INET6) || defined(INET)
 2651         tcp_lro_flush_all(&rxq->ifr_lc);
 2652 #endif
 2653         if (avail)
 2654                 return true;
 2655         return (iflib_rxd_avail(ctx, rxq, *cidxp, 1));
 2656 err:
 2657         CTX_LOCK(ctx);
 2658         ctx->ifc_flags |= IFC_DO_RESET;
 2659         iflib_admin_intr_deferred(ctx);
 2660         CTX_UNLOCK(ctx);
 2661         return (false);
 2662 }
 2663 
 2664 #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1)
 2665 static inline qidx_t
 2666 txq_max_db_deferred(iflib_txq_t txq, qidx_t in_use)
 2667 {
 2668         qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
 2669         qidx_t minthresh = txq->ift_size / 8;
 2670         if (in_use > 4*minthresh)
 2671                 return (notify_count);
 2672         if (in_use > 2*minthresh)
 2673                 return (notify_count >> 1);
 2674         if (in_use > minthresh)
 2675                 return (notify_count >> 3);
 2676         return (0);
 2677 }
 2678 
 2679 static inline qidx_t
 2680 txq_max_rs_deferred(iflib_txq_t txq)
 2681 {
 2682         qidx_t notify_count = TXD_NOTIFY_COUNT(txq);
 2683         qidx_t minthresh = txq->ift_size / 8;
 2684         if (txq->ift_in_use > 4*minthresh)
 2685                 return (notify_count);
 2686         if (txq->ift_in_use > 2*minthresh)
 2687                 return (notify_count >> 1);
 2688         if (txq->ift_in_use > minthresh)
 2689                 return (notify_count >> 2);
 2690         return (2);
 2691 }
 2692 
 2693 #define M_CSUM_FLAGS(m) ((m)->m_pkthdr.csum_flags)
 2694 #define M_HAS_VLANTAG(m) (m->m_flags & M_VLANTAG)
 2695 
 2696 #define TXQ_MAX_DB_DEFERRED(txq, in_use) txq_max_db_deferred((txq), (in_use))
 2697 #define TXQ_MAX_RS_DEFERRED(txq) txq_max_rs_deferred(txq)
 2698 #define TXQ_MAX_DB_CONSUMED(size) (size >> 4)
 2699 
 2700 /* forward compatibility for cxgb */
 2701 #define FIRST_QSET(ctx) 0
 2702 #define NTXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_ntxqsets)
 2703 #define NRXQSETS(ctx) ((ctx)->ifc_softc_ctx.isc_nrxqsets)
 2704 #define QIDX(ctx, m) ((((m)->m_pkthdr.flowid & ctx->ifc_softc_ctx.isc_rss_table_mask) % NTXQSETS(ctx)) + FIRST_QSET(ctx))
 2705 #define DESC_RECLAIMABLE(q) ((int)((q)->ift_processed - (q)->ift_cleaned - (q)->ift_ctx->ifc_softc_ctx.isc_tx_nsegments))
 2706 
 2707 /* XXX we should be setting this to something other than zero */
 2708 #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh)
 2709 #define MAX_TX_DESC(ctx) ((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max)
 2710 
 2711 static inline bool
 2712 iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring, qidx_t in_use)
 2713 {
 2714         qidx_t dbval, max;
 2715         bool rang;
 2716 
 2717         rang = false;
 2718         max = TXQ_MAX_DB_DEFERRED(txq, in_use);
 2719         if (ring || txq->ift_db_pending >= max) {
 2720                 dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx;
 2721                 ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval);
 2722                 txq->ift_db_pending = txq->ift_npending = 0;
 2723                 rang = true;
 2724         }
 2725         return (rang);
 2726 }
 2727 
 2728 #ifdef PKT_DEBUG
 2729 static void
 2730 print_pkt(if_pkt_info_t pi)
 2731 {
 2732         printf("pi len:  %d qsidx: %d nsegs: %d ndescs: %d flags: %x pidx: %d\n",
 2733                pi->ipi_len, pi->ipi_qsidx, pi->ipi_nsegs, pi->ipi_ndescs, pi->ipi_flags, pi->ipi_pidx);
 2734         printf("pi new_pidx: %d csum_flags: %lx tso_segsz: %d mflags: %x vtag: %d\n",
 2735                pi->ipi_new_pidx, pi->ipi_csum_flags, pi->ipi_tso_segsz, pi->ipi_mflags, pi->ipi_vtag);
 2736         printf("pi etype: %d ehdrlen: %d ip_hlen: %d ipproto: %d\n",
 2737                pi->ipi_etype, pi->ipi_ehdrlen, pi->ipi_ip_hlen, pi->ipi_ipproto);
 2738 }
 2739 #endif
 2740 
 2741 #define IS_TSO4(pi) ((pi)->ipi_csum_flags & CSUM_IP_TSO)
 2742 #define IS_TSO6(pi) ((pi)->ipi_csum_flags & CSUM_IP6_TSO)
 2743 
 2744 static int
 2745 iflib_parse_header(iflib_txq_t txq, if_pkt_info_t pi, struct mbuf **mp)
 2746 {
 2747         if_shared_ctx_t sctx = txq->ift_ctx->ifc_sctx;
 2748         struct ether_vlan_header *eh;
 2749         struct mbuf *m, *n;
 2750 
 2751         n = m = *mp;
 2752         if ((sctx->isc_flags & IFLIB_NEED_SCRATCH) &&
 2753             M_WRITABLE(m) == 0) {
 2754                 if ((m = m_dup(m, M_NOWAIT)) == NULL) {
 2755                         return (ENOMEM);
 2756                 } else {
 2757                         m_freem(*mp);
 2758                         n = *mp = m;
 2759                 }
 2760         }
 2761 
 2762         /*
 2763          * Determine where frame payload starts.
 2764          * Jump over vlan headers if already present,
 2765          * helpful for QinQ too.
 2766          */
 2767         if (__predict_false(m->m_len < sizeof(*eh))) {
 2768                 txq->ift_pullups++;
 2769                 if (__predict_false((m = m_pullup(m, sizeof(*eh))) == NULL))
 2770                         return (ENOMEM);
 2771         }
 2772         eh = mtod(m, struct ether_vlan_header *);
 2773         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 2774                 pi->ipi_etype = ntohs(eh->evl_proto);
 2775                 pi->ipi_ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 2776         } else {
 2777                 pi->ipi_etype = ntohs(eh->evl_encap_proto);
 2778                 pi->ipi_ehdrlen = ETHER_HDR_LEN;
 2779         }
 2780 
 2781         switch (pi->ipi_etype) {
 2782 #ifdef INET
 2783         case ETHERTYPE_IP:
 2784         {
 2785                 struct ip *ip = NULL;
 2786                 struct tcphdr *th = NULL;
 2787                 int minthlen;
 2788 
 2789                 minthlen = min(m->m_pkthdr.len, pi->ipi_ehdrlen + sizeof(*ip) + sizeof(*th));
 2790                 if (__predict_false(m->m_len < minthlen)) {
 2791                         /*
 2792                          * if this code bloat is causing too much of a hit
 2793                          * move it to a separate function and mark it noinline
 2794                          */
 2795                         if (m->m_len == pi->ipi_ehdrlen) {
 2796                                 n = m->m_next;
 2797                                 MPASS(n);
 2798                                 if (n->m_len >= sizeof(*ip))  {
 2799                                         ip = (struct ip *)n->m_data;
 2800                                         if (n->m_len >= (ip->ip_hl << 2) + sizeof(*th))
 2801                                                 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
 2802                                 } else {
 2803                                         txq->ift_pullups++;
 2804                                         if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
 2805                                                 return (ENOMEM);
 2806                                         ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
 2807                                 }
 2808                         } else {
 2809                                 txq->ift_pullups++;
 2810                                 if (__predict_false((m = m_pullup(m, minthlen)) == NULL))
 2811                                         return (ENOMEM);
 2812                                 ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
 2813                                 if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
 2814                                         th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
 2815                         }
 2816                 } else {
 2817                         ip = (struct ip *)(m->m_data + pi->ipi_ehdrlen);
 2818                         if (m->m_len >= (ip->ip_hl << 2) + sizeof(*th))
 2819                                 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
 2820                 }
 2821                 pi->ipi_ip_hlen = ip->ip_hl << 2;
 2822                 pi->ipi_ipproto = ip->ip_p;
 2823                 pi->ipi_flags |= IPI_TX_IPV4;
 2824 
 2825                 if ((sctx->isc_flags & IFLIB_NEED_ZERO_CSUM) && (pi->ipi_csum_flags & CSUM_IP))
 2826                        ip->ip_sum = 0;
 2827 
 2828                 if (IS_TSO4(pi)) {
 2829                         if (pi->ipi_ipproto == IPPROTO_TCP) {
 2830                                 if (__predict_false(th == NULL)) {
 2831                                         txq->ift_pullups++;
 2832                                         if (__predict_false((m = m_pullup(m, (ip->ip_hl << 2) + sizeof(*th))) == NULL))
 2833                                                 return (ENOMEM);
 2834                                         th = (struct tcphdr *)((caddr_t)ip + pi->ipi_ip_hlen);
 2835                                 }
 2836                                 pi->ipi_tcp_hflags = th->th_flags;
 2837                                 pi->ipi_tcp_hlen = th->th_off << 2;
 2838                                 pi->ipi_tcp_seq = th->th_seq;
 2839                         }
 2840                         if (__predict_false(ip->ip_p != IPPROTO_TCP))
 2841                                 return (ENXIO);
 2842                         th->th_sum = in_pseudo(ip->ip_src.s_addr,
 2843                                                ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 2844                         pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
 2845                         if (sctx->isc_flags & IFLIB_TSO_INIT_IP) {
 2846                                 ip->ip_sum = 0;
 2847                                 ip->ip_len = htons(pi->ipi_ip_hlen + pi->ipi_tcp_hlen + pi->ipi_tso_segsz);
 2848                         }
 2849                 }
 2850                 break;
 2851         }
 2852 #endif
 2853 #ifdef INET6
 2854         case ETHERTYPE_IPV6:
 2855         {
 2856                 struct ip6_hdr *ip6 = (struct ip6_hdr *)(m->m_data + pi->ipi_ehdrlen);
 2857                 struct tcphdr *th;
 2858                 pi->ipi_ip_hlen = sizeof(struct ip6_hdr);
 2859 
 2860                 if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) {
 2861                         if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr))) == NULL))
 2862                                 return (ENOMEM);
 2863                 }
 2864                 th = (struct tcphdr *)((caddr_t)ip6 + pi->ipi_ip_hlen);
 2865 
 2866                 /* XXX-BZ this will go badly in case of ext hdrs. */
 2867                 pi->ipi_ipproto = ip6->ip6_nxt;
 2868                 pi->ipi_flags |= IPI_TX_IPV6;
 2869 
 2870                 if (IS_TSO6(pi)) {
 2871                         if (pi->ipi_ipproto == IPPROTO_TCP) {
 2872                                 if (__predict_false(m->m_len < pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) {
 2873                                         if (__predict_false((m = m_pullup(m, pi->ipi_ehdrlen + sizeof(struct ip6_hdr) + sizeof(struct tcphdr))) == NULL))
 2874                                                 return (ENOMEM);
 2875                                 }
 2876                                 pi->ipi_tcp_hflags = th->th_flags;
 2877                                 pi->ipi_tcp_hlen = th->th_off << 2;
 2878                         }
 2879 
 2880                         if (__predict_false(ip6->ip6_nxt != IPPROTO_TCP))
 2881                                 return (ENXIO);
 2882                         /*
 2883                          * The corresponding flag is set by the stack in the IPv4
 2884                          * TSO case, but not in IPv6 (at least in FreeBSD 10.2).
 2885                          * So, set it here because the rest of the flow requires it.
 2886                          */
 2887                         pi->ipi_csum_flags |= CSUM_TCP_IPV6;
 2888                         th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
 2889                         pi->ipi_tso_segsz = m->m_pkthdr.tso_segsz;
 2890                 }
 2891                 break;
 2892         }
 2893 #endif
 2894         default:
 2895                 pi->ipi_csum_flags &= ~CSUM_OFFLOAD;
 2896                 pi->ipi_ip_hlen = 0;
 2897                 break;
 2898         }
 2899         *mp = m;
 2900 
 2901         return (0);
 2902 }
 2903 
 2904 static  __noinline  struct mbuf *
 2905 collapse_pkthdr(struct mbuf *m0)
 2906 {
 2907         struct mbuf *m, *m_next, *tmp;
 2908 
 2909         m = m0;
 2910         m_next = m->m_next;
 2911         while (m_next != NULL && m_next->m_len == 0) {
 2912                 m = m_next;
 2913                 m->m_next = NULL;
 2914                 m_free(m);
 2915                 m_next = m_next->m_next;
 2916         }
 2917         m = m0;
 2918         m->m_next = m_next;
 2919         if ((m_next->m_flags & M_EXT) == 0) {
 2920                 m = m_defrag(m, M_NOWAIT);
 2921         } else {
 2922                 tmp = m_next->m_next;
 2923                 memcpy(m_next, m, MPKTHSIZE);
 2924                 m = m_next;
 2925                 m->m_next = tmp;
 2926         }
 2927         return (m);
 2928 }
 2929 
 2930 /*
 2931  * If dodgy hardware rejects the scatter gather chain we've handed it
 2932  * we'll need to remove the mbuf chain from ifsg_m[] before we can add the
 2933  * m_defrag'd mbufs
 2934  */
 2935 static __noinline struct mbuf *
 2936 iflib_remove_mbuf(iflib_txq_t txq)
 2937 {
 2938         int ntxd, i, pidx;
 2939         struct mbuf *m, *mh, **ifsd_m;
 2940 
 2941         pidx = txq->ift_pidx;
 2942         ifsd_m = txq->ift_sds.ifsd_m;
 2943         ntxd = txq->ift_size;
 2944         mh = m = ifsd_m[pidx];
 2945         ifsd_m[pidx] = NULL;
 2946 #if MEMORY_LOGGING
 2947         txq->ift_dequeued++;
 2948 #endif
 2949         i = 1;
 2950 
 2951         while (m) {
 2952                 ifsd_m[(pidx + i) & (ntxd -1)] = NULL;
 2953 #if MEMORY_LOGGING
 2954                 txq->ift_dequeued++;
 2955 #endif
 2956                 m = m->m_next;
 2957                 i++;
 2958         }
 2959         return (mh);
 2960 }
 2961 
 2962 static int
 2963 iflib_busdma_load_mbuf_sg(iflib_txq_t txq, bus_dma_tag_t tag, bus_dmamap_t map,
 2964                           struct mbuf **m0, bus_dma_segment_t *segs, int *nsegs,
 2965                           int max_segs, int flags)
 2966 {
 2967         if_ctx_t ctx;
 2968         if_shared_ctx_t         sctx;
 2969         if_softc_ctx_t          scctx;
 2970         int i, next, pidx, err, ntxd, count;
 2971         struct mbuf *m, *tmp, **ifsd_m;
 2972 
 2973         m = *m0;
 2974 
 2975         /*
 2976          * Please don't ever do this
 2977          */
 2978         if (__predict_false(m->m_len == 0))
 2979                 *m0 = m = collapse_pkthdr(m);
 2980 
 2981         ctx = txq->ift_ctx;
 2982         sctx = ctx->ifc_sctx;
 2983         scctx = &ctx->ifc_softc_ctx;
 2984         ifsd_m = txq->ift_sds.ifsd_m;
 2985         ntxd = txq->ift_size;
 2986         pidx = txq->ift_pidx;
 2987         if (map != NULL) {
 2988                 uint8_t *ifsd_flags = txq->ift_sds.ifsd_flags;
 2989 
 2990                 err = bus_dmamap_load_mbuf_sg(tag, map,
 2991                                               *m0, segs, nsegs, BUS_DMA_NOWAIT);
 2992                 if (err)
 2993                         return (err);
 2994                 ifsd_flags[pidx] |= TX_SW_DESC_MAPPED;
 2995                 count = 0;
 2996                 m = *m0;
 2997                 do {
 2998                         if (__predict_false(m->m_len <= 0)) {
 2999                                 tmp = m;
 3000                                 m = m->m_next;
 3001                                 tmp->m_next = NULL;
 3002                                 m_free(tmp);
 3003                                 continue;
 3004                         }
 3005                         m = m->m_next;
 3006                         count++;
 3007                 } while (m != NULL);
 3008                 if (count > *nsegs) {
 3009                         ifsd_m[pidx] = *m0;
 3010                         ifsd_m[pidx]->m_flags |= M_TOOBIG;
 3011                         return (0);
 3012                 }
 3013                 m = *m0;
 3014                 count = 0;
 3015                 do {
 3016                         next = (pidx + count) & (ntxd-1);
 3017                         MPASS(ifsd_m[next] == NULL);
 3018                         ifsd_m[next] = m;
 3019                         count++;
 3020                         tmp = m;
 3021                         m = m->m_next;
 3022                 } while (m != NULL);
 3023         } else {
 3024                 int buflen, sgsize, maxsegsz, max_sgsize;
 3025                 vm_offset_t vaddr;
 3026                 vm_paddr_t curaddr;
 3027 
 3028                 count = i = 0;
 3029                 m = *m0;
 3030                 if (m->m_pkthdr.csum_flags & CSUM_TSO)
 3031                         maxsegsz = scctx->isc_tx_tso_segsize_max;
 3032                 else
 3033                         maxsegsz = sctx->isc_tx_maxsegsize;
 3034 
 3035                 do {
 3036                         if (__predict_false(m->m_len <= 0)) {
 3037                                 tmp = m;
 3038                                 m = m->m_next;
 3039                                 tmp->m_next = NULL;
 3040                                 m_free(tmp);
 3041                                 continue;
 3042                         }
 3043                         buflen = m->m_len;
 3044                         vaddr = (vm_offset_t)m->m_data;
 3045                         /*
 3046                          * see if we can't be smarter about physically
 3047                          * contiguous mappings
 3048                          */
 3049                         next = (pidx + count) & (ntxd-1);
 3050                         MPASS(ifsd_m[next] == NULL);
 3051 #if MEMORY_LOGGING
 3052                         txq->ift_enqueued++;
 3053 #endif
 3054                         ifsd_m[next] = m;
 3055                         while (buflen > 0) {
 3056                                 if (i >= max_segs)
 3057                                         goto err;
 3058                                 max_sgsize = MIN(buflen, maxsegsz);
 3059                                 curaddr = pmap_kextract(vaddr);
 3060                                 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK);
 3061                                 sgsize = MIN(sgsize, max_sgsize);
 3062                                 segs[i].ds_addr = curaddr;
 3063                                 segs[i].ds_len = sgsize;
 3064                                 vaddr += sgsize;
 3065                                 buflen -= sgsize;
 3066                                 i++;
 3067                         }
 3068                         count++;
 3069                         tmp = m;
 3070                         m = m->m_next;
 3071                 } while (m != NULL);
 3072                 *nsegs = i;
 3073         }
 3074         return (0);
 3075 err:
 3076         *m0 = iflib_remove_mbuf(txq);
 3077         return (EFBIG);
 3078 }
 3079 
 3080 static inline caddr_t
 3081 calc_next_txd(iflib_txq_t txq, int cidx, uint8_t qid)
 3082 {
 3083         qidx_t size;
 3084         int ntxd;
 3085         caddr_t start, end, cur, next;
 3086 
 3087         ntxd = txq->ift_size;
 3088         size = txq->ift_txd_size[qid];
 3089         start = txq->ift_ifdi[qid].idi_vaddr;
 3090 
 3091         if (__predict_false(size == 0))
 3092                 return (start);
 3093         cur = start + size*cidx;
 3094         end = start + size*ntxd;
 3095         next = CACHE_PTR_NEXT(cur);
 3096         return (next < end ? next : start);
 3097 }
 3098 
 3099 /*
 3100  * Pad an mbuf to ensure a minimum ethernet frame size.
 3101  * min_frame_size is the frame size (less CRC) to pad the mbuf to
 3102  */
 3103 static __noinline int
 3104 iflib_ether_pad(device_t dev, struct mbuf **m_head, uint16_t min_frame_size)
 3105 {
 3106         /*
 3107          * 18 is enough bytes to pad an ARP packet to 46 bytes, and
 3108          * and ARP message is the smallest common payload I can think of
 3109          */
 3110         static char pad[18];    /* just zeros */
 3111         int n;
 3112         struct mbuf *new_head;
 3113 
 3114         if (!M_WRITABLE(*m_head)) {
 3115                 new_head = m_dup(*m_head, M_NOWAIT);
 3116                 if (new_head == NULL) {
 3117                         m_freem(*m_head);
 3118                         device_printf(dev, "cannot pad short frame, m_dup() failed");
 3119                         DBG_COUNTER_INC(encap_pad_mbuf_fail);
 3120                         return ENOMEM;
 3121                 }
 3122                 m_freem(*m_head);
 3123                 *m_head = new_head;
 3124         }
 3125 
 3126         for (n = min_frame_size - (*m_head)->m_pkthdr.len;
 3127              n > 0; n -= sizeof(pad))
 3128                 if (!m_append(*m_head, min(n, sizeof(pad)), pad))
 3129                         break;
 3130 
 3131         if (n > 0) {
 3132                 m_freem(*m_head);
 3133                 device_printf(dev, "cannot pad short frame\n");
 3134                 DBG_COUNTER_INC(encap_pad_mbuf_fail);
 3135                 return (ENOBUFS);
 3136         }
 3137 
 3138         return 0;
 3139 }
 3140 
 3141 static int
 3142 iflib_encap(iflib_txq_t txq, struct mbuf **m_headp)
 3143 {
 3144         if_ctx_t                ctx;
 3145         if_shared_ctx_t         sctx;
 3146         if_softc_ctx_t          scctx;
 3147         bus_dma_segment_t       *segs;
 3148         struct mbuf             *m_head;
 3149         void                    *next_txd;
 3150         bus_dmamap_t            map;
 3151         struct if_pkt_info      pi;
 3152         int remap = 0;
 3153         int err, nsegs, ndesc, max_segs, pidx, cidx, next, ntxd;
 3154         bus_dma_tag_t desc_tag;
 3155 
 3156         segs = txq->ift_segs;
 3157         ctx = txq->ift_ctx;
 3158         sctx = ctx->ifc_sctx;
 3159         scctx = &ctx->ifc_softc_ctx;
 3160         segs = txq->ift_segs;
 3161         ntxd = txq->ift_size;
 3162         m_head = *m_headp;
 3163         map = NULL;
 3164 
 3165         /*
 3166          * If we're doing TSO the next descriptor to clean may be quite far ahead
 3167          */
 3168         cidx = txq->ift_cidx;
 3169         pidx = txq->ift_pidx;
 3170         if (ctx->ifc_flags & IFC_PREFETCH) {
 3171                 next = (cidx + CACHE_PTR_INCREMENT) & (ntxd-1);
 3172                 if (!(ctx->ifc_flags & IFLIB_HAS_TXCQ)) {
 3173                         next_txd = calc_next_txd(txq, cidx, 0);
 3174                         prefetch(next_txd);
 3175                 }
 3176 
 3177                 /* prefetch the next cache line of mbuf pointers and flags */
 3178                 prefetch(&txq->ift_sds.ifsd_m[next]);
 3179                 if (txq->ift_sds.ifsd_map != NULL) {
 3180                         prefetch(&txq->ift_sds.ifsd_map[next]);
 3181                         next = (cidx + CACHE_LINE_SIZE) & (ntxd-1);
 3182                         prefetch(&txq->ift_sds.ifsd_flags[next]);
 3183                 }
 3184         } else if (txq->ift_sds.ifsd_map != NULL)
 3185                 map = txq->ift_sds.ifsd_map[pidx];
 3186 
 3187         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
 3188                 desc_tag = txq->ift_tso_desc_tag;
 3189                 max_segs = scctx->isc_tx_tso_segments_max;
 3190         } else {
 3191                 desc_tag = txq->ift_desc_tag;
 3192                 max_segs = scctx->isc_tx_nsegments;
 3193         }
 3194         if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) &&
 3195             __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) {
 3196                 err = iflib_ether_pad(ctx->ifc_dev, m_headp, scctx->isc_min_frame_size);
 3197                 if (err)
 3198                         return err;
 3199         }
 3200         m_head = *m_headp;
 3201 
 3202         pkt_info_zero(&pi);
 3203         pi.ipi_mflags = (m_head->m_flags & (M_VLANTAG|M_BCAST|M_MCAST));
 3204         pi.ipi_pidx = pidx;
 3205         pi.ipi_qsidx = txq->ift_id;
 3206         pi.ipi_len = m_head->m_pkthdr.len;
 3207         pi.ipi_csum_flags = m_head->m_pkthdr.csum_flags;
 3208         pi.ipi_vtag = (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
 3209 
 3210         /* deliberate bitwise OR to make one condition */
 3211         if (__predict_true((pi.ipi_csum_flags | pi.ipi_vtag))) {
 3212                 if (__predict_false((err = iflib_parse_header(txq, &pi, m_headp)) != 0))
 3213                         return (err);
 3214                 m_head = *m_headp;
 3215         }
 3216 
 3217 retry:
 3218         err = iflib_busdma_load_mbuf_sg(txq, desc_tag, map, m_headp, segs, &nsegs, max_segs, BUS_DMA_NOWAIT);
 3219 defrag:
 3220         if (__predict_false(err)) {
 3221                 switch (err) {
 3222                 case EFBIG:
 3223                         /* try collapse once and defrag once */
 3224                         if (remap == 0)
 3225                                 m_head = m_collapse(*m_headp, M_NOWAIT, max_segs);
 3226                         if (remap == 1)
 3227                                 m_head = m_defrag(*m_headp, M_NOWAIT);
 3228                         remap++;
 3229                         if (__predict_false(m_head == NULL))
 3230                                 goto defrag_failed;
 3231                         txq->ift_mbuf_defrag++;
 3232                         *m_headp = m_head;
 3233                         goto retry;
 3234                         break;
 3235                 case ENOMEM:
 3236                         txq->ift_no_tx_dma_setup++;
 3237                         break;
 3238                 default:
 3239                         txq->ift_no_tx_dma_setup++;
 3240                         m_freem(*m_headp);
 3241                         DBG_COUNTER_INC(tx_frees);
 3242                         *m_headp = NULL;
 3243                         break;
 3244                 }
 3245                 txq->ift_map_failed++;
 3246                 DBG_COUNTER_INC(encap_load_mbuf_fail);
 3247                 return (err);
 3248         }
 3249 
 3250         /*
 3251          * XXX assumes a 1 to 1 relationship between segments and
 3252          *        descriptors - this does not hold true on all drivers, e.g.
 3253          *        cxgb
 3254          */
 3255         if (__predict_false(nsegs + 2 > TXQ_AVAIL(txq))) {
 3256                 txq->ift_no_desc_avail++;
 3257                 if (map != NULL)
 3258                         bus_dmamap_unload(desc_tag, map);
 3259                 DBG_COUNTER_INC(encap_txq_avail_fail);
 3260                 if ((txq->ift_task.gt_task.ta_flags & TASK_ENQUEUED) == 0)
 3261                         GROUPTASK_ENQUEUE(&txq->ift_task);
 3262                 return (ENOBUFS);
 3263         }
 3264         /*
 3265          * On Intel cards we can greatly reduce the number of TX interrupts
 3266          * we see by only setting report status on every Nth descriptor.
 3267          * However, this also means that the driver will need to keep track
 3268          * of the descriptors that RS was set on to check them for the DD bit.
 3269          */
 3270         txq->ift_rs_pending += nsegs + 1;
 3271         if (txq->ift_rs_pending > TXQ_MAX_RS_DEFERRED(txq) ||
 3272              iflib_no_tx_batch || (TXQ_AVAIL(txq) - nsegs) <= MAX_TX_DESC(ctx) + 2) {
 3273                 pi.ipi_flags |= IPI_TX_INTR;
 3274                 txq->ift_rs_pending = 0;
 3275         }
 3276 
 3277         pi.ipi_segs = segs;
 3278         pi.ipi_nsegs = nsegs;
 3279 
 3280         MPASS(pidx >= 0 && pidx < txq->ift_size);
 3281 #ifdef PKT_DEBUG
 3282         print_pkt(&pi);
 3283 #endif
 3284         if (map != NULL)
 3285                 bus_dmamap_sync(desc_tag, map, BUS_DMASYNC_PREWRITE);
 3286         if ((err = ctx->isc_txd_encap(ctx->ifc_softc, &pi)) == 0) {
 3287                 if (map != NULL)
 3288                         bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map,
 3289                                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 3290                 DBG_COUNTER_INC(tx_encap);
 3291                 MPASS(pi.ipi_new_pidx < txq->ift_size);
 3292 
 3293                 ndesc = pi.ipi_new_pidx - pi.ipi_pidx;
 3294                 if (pi.ipi_new_pidx < pi.ipi_pidx) {
 3295                         ndesc += txq->ift_size;
 3296                         txq->ift_gen = 1;
 3297                 }
 3298                 /*
 3299                  * drivers can need as many as 
 3300                  * two sentinels
 3301                  */
 3302                 MPASS(ndesc <= pi.ipi_nsegs + 2);
 3303                 MPASS(pi.ipi_new_pidx != pidx);
 3304                 MPASS(ndesc > 0);
 3305                 txq->ift_in_use += ndesc;
 3306 
 3307                 /*
 3308                  * We update the last software descriptor again here because there may
 3309                  * be a sentinel and/or there may be more mbufs than segments
 3310                  */
 3311                 txq->ift_pidx = pi.ipi_new_pidx;
 3312                 txq->ift_npending += pi.ipi_ndescs;
 3313         } else if (__predict_false(err == EFBIG && remap < 2)) {
 3314                 *m_headp = m_head = iflib_remove_mbuf(txq);
 3315                 remap = 1;
 3316                 txq->ift_txd_encap_efbig++;
 3317                 goto defrag;
 3318         } else
 3319                 DBG_COUNTER_INC(encap_txd_encap_fail);
 3320         return (err);
 3321 
 3322 defrag_failed:
 3323         txq->ift_mbuf_defrag_failed++;
 3324         txq->ift_map_failed++;
 3325         m_freem(*m_headp);
 3326         DBG_COUNTER_INC(tx_frees);
 3327         *m_headp = NULL;
 3328         return (ENOMEM);
 3329 }
 3330 
 3331 static void
 3332 iflib_tx_desc_free(iflib_txq_t txq, int n)
 3333 {
 3334         int hasmap;
 3335         uint32_t qsize, cidx, mask, gen;
 3336         struct mbuf *m, **ifsd_m;
 3337         uint8_t *ifsd_flags;
 3338         bus_dmamap_t *ifsd_map;
 3339         bool do_prefetch;
 3340 
 3341         cidx = txq->ift_cidx;
 3342         gen = txq->ift_gen;
 3343         qsize = txq->ift_size;
 3344         mask = qsize-1;
 3345         hasmap = txq->ift_sds.ifsd_map != NULL;
 3346         ifsd_flags = txq->ift_sds.ifsd_flags;
 3347         ifsd_m = txq->ift_sds.ifsd_m;
 3348         ifsd_map = txq->ift_sds.ifsd_map;
 3349         do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH);
 3350 
 3351         while (n--) {
 3352                 if (do_prefetch) {
 3353                         prefetch(ifsd_m[(cidx + 3) & mask]);
 3354                         prefetch(ifsd_m[(cidx + 4) & mask]);
 3355                 }
 3356                 if (ifsd_m[cidx] != NULL) {
 3357                         prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]);
 3358                         prefetch(&ifsd_flags[(cidx + CACHE_PTR_INCREMENT) & mask]);
 3359                         if (hasmap && (ifsd_flags[cidx] & TX_SW_DESC_MAPPED)) {
 3360                                 /*
 3361                                  * does it matter if it's not the TSO tag? If so we'll
 3362                                  * have to add the type to flags
 3363                                  */
 3364                                 bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[cidx]);
 3365                                 ifsd_flags[cidx] &= ~TX_SW_DESC_MAPPED;
 3366                         }
 3367                         if ((m = ifsd_m[cidx]) != NULL) {
 3368                                 /* XXX we don't support any drivers that batch packets yet */
 3369                                 MPASS(m->m_nextpkt == NULL);
 3370                                 /* if the number of clusters exceeds the number of segments
 3371                                  * there won't be space on the ring to save a pointer to each
 3372                                  * cluster so we simply free the list here
 3373                                  */
 3374                                 if (m->m_flags & M_TOOBIG) {
 3375                                         m_freem(m);
 3376                                 } else {
 3377                                         m_free(m);
 3378                                 }
 3379                                 ifsd_m[cidx] = NULL;
 3380 #if MEMORY_LOGGING
 3381                                 txq->ift_dequeued++;
 3382 #endif
 3383                                 DBG_COUNTER_INC(tx_frees);
 3384                         }
 3385                 }
 3386                 if (__predict_false(++cidx == qsize)) {
 3387                         cidx = 0;
 3388                         gen = 0;
 3389                 }
 3390         }
 3391         txq->ift_cidx = cidx;
 3392         txq->ift_gen = gen;
 3393 }
 3394 
 3395 static __inline int
 3396 iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh)
 3397 {
 3398         int reclaim;
 3399         if_ctx_t ctx = txq->ift_ctx;
 3400 
 3401         KASSERT(thresh >= 0, ("invalid threshold to reclaim"));
 3402         MPASS(thresh /*+ MAX_TX_DESC(txq->ift_ctx) */ < txq->ift_size);
 3403 
 3404         /*
 3405          * Need a rate-limiting check so that this isn't called every time
 3406          */
 3407         iflib_tx_credits_update(ctx, txq);
 3408         reclaim = DESC_RECLAIMABLE(txq);
 3409 
 3410         if (reclaim <= thresh /* + MAX_TX_DESC(txq->ift_ctx) */) {
 3411 #ifdef INVARIANTS
 3412                 if (iflib_verbose_debug) {
 3413                         printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__,
 3414                                txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments,
 3415                                reclaim, thresh);
 3416 
 3417                 }
 3418 #endif
 3419                 return (0);
 3420         }
 3421         iflib_tx_desc_free(txq, reclaim);
 3422         txq->ift_cleaned += reclaim;
 3423         txq->ift_in_use -= reclaim;
 3424 
 3425         return (reclaim);
 3426 }
 3427 
 3428 static struct mbuf **
 3429 _ring_peek_one(struct ifmp_ring *r, int cidx, int offset, int remaining)
 3430 {
 3431         int next, size;
 3432         struct mbuf **items;
 3433 
 3434         size = r->size;
 3435         next = (cidx + CACHE_PTR_INCREMENT) & (size-1);
 3436         items = __DEVOLATILE(struct mbuf **, &r->items[0]);
 3437 
 3438         prefetch(items[(cidx + offset) & (size-1)]);
 3439         if (remaining > 1) {
 3440                 prefetch2cachelines(&items[next]);
 3441                 prefetch2cachelines(items[(cidx + offset + 1) & (size-1)]);
 3442                 prefetch2cachelines(items[(cidx + offset + 2) & (size-1)]);
 3443                 prefetch2cachelines(items[(cidx + offset + 3) & (size-1)]);
 3444         }
 3445         return (__DEVOLATILE(struct mbuf **, &r->items[(cidx + offset) & (size-1)]));
 3446 }
 3447 
 3448 static void
 3449 iflib_txq_check_drain(iflib_txq_t txq, int budget)
 3450 {
 3451 
 3452         ifmp_ring_check_drainage(txq->ift_br, budget);
 3453 }
 3454 
 3455 static uint32_t
 3456 iflib_txq_can_drain(struct ifmp_ring *r)
 3457 {
 3458         iflib_txq_t txq = r->cookie;
 3459         if_ctx_t ctx = txq->ift_ctx;
 3460 
 3461         return ((TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) ||
 3462                 ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false));
 3463 }
 3464 
 3465 static uint32_t
 3466 iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
 3467 {
 3468         iflib_txq_t txq = r->cookie;
 3469         if_ctx_t ctx = txq->ift_ctx;
 3470         struct ifnet *ifp = ctx->ifc_ifp;
 3471         struct mbuf **mp, *m;
 3472         int i, count, consumed, pkt_sent, bytes_sent, mcast_sent, avail;
 3473         int reclaimed, err, in_use_prev, desc_used;
 3474         bool do_prefetch, ring, rang;
 3475 
 3476         if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) ||
 3477                             !LINK_ACTIVE(ctx))) {
 3478                 DBG_COUNTER_INC(txq_drain_notready);
 3479                 return (0);
 3480         }
 3481         reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx));
 3482         rang = iflib_txd_db_check(ctx, txq, reclaimed, txq->ift_in_use);
 3483         avail = IDXDIFF(pidx, cidx, r->size);
 3484         if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) {
 3485                 DBG_COUNTER_INC(txq_drain_flushing);
 3486                 for (i = 0; i < avail; i++) {
 3487                         m_free(r->items[(cidx + i) & (r->size-1)]);
 3488                         r->items[(cidx + i) & (r->size-1)] = NULL;
 3489                 }
 3490                 return (avail);
 3491         }
 3492 
 3493         if (__predict_false(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
 3494                 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 3495                 CALLOUT_LOCK(txq);
 3496                 callout_stop(&txq->ift_timer);
 3497                 CALLOUT_UNLOCK(txq);
 3498                 DBG_COUNTER_INC(txq_drain_oactive);
 3499                 return (0);
 3500         }
 3501         if (reclaimed)
 3502                 txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 3503         consumed = mcast_sent = bytes_sent = pkt_sent = 0;
 3504         count = MIN(avail, TX_BATCH_SIZE);
 3505 #ifdef INVARIANTS
 3506         if (iflib_verbose_debug)
 3507                 printf("%s avail=%d ifc_flags=%x txq_avail=%d ", __FUNCTION__,
 3508                        avail, ctx->ifc_flags, TXQ_AVAIL(txq));
 3509 #endif
 3510         do_prefetch = (ctx->ifc_flags & IFC_PREFETCH);
 3511         avail = TXQ_AVAIL(txq);
 3512         for (desc_used = i = 0; i < count && avail > MAX_TX_DESC(ctx) + 2; i++) {
 3513                 int pidx_prev, rem = do_prefetch ? count - i : 0;
 3514 
 3515                 mp = _ring_peek_one(r, cidx, i, rem);
 3516                 MPASS(mp != NULL && *mp != NULL);
 3517                 if (__predict_false(*mp == (struct mbuf *)txq)) {
 3518                         consumed++;
 3519                         reclaimed++;
 3520                         continue;
 3521                 }
 3522                 in_use_prev = txq->ift_in_use;
 3523                 pidx_prev = txq->ift_pidx;
 3524                 err = iflib_encap(txq, mp);
 3525                 if (__predict_false(err)) {
 3526                         DBG_COUNTER_INC(txq_drain_encapfail);
 3527                         /* no room - bail out */
 3528                         if (err == ENOBUFS)
 3529                                 break;
 3530                         consumed++;
 3531                         DBG_COUNTER_INC(txq_drain_encapfail);
 3532                         /* we can't send this packet - skip it */
 3533                         continue;
 3534                 }
 3535                 consumed++;
 3536                 pkt_sent++;
 3537                 m = *mp;
 3538                 DBG_COUNTER_INC(tx_sent);
 3539                 bytes_sent += m->m_pkthdr.len;
 3540                 mcast_sent += !!(m->m_flags & M_MCAST);
 3541                 avail = TXQ_AVAIL(txq);
 3542 
 3543                 txq->ift_db_pending += (txq->ift_in_use - in_use_prev);
 3544                 desc_used += (txq->ift_in_use - in_use_prev);
 3545                 ETHER_BPF_MTAP(ifp, m);
 3546                 if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING)))
 3547                         break;
 3548                 rang = iflib_txd_db_check(ctx, txq, false, in_use_prev);
 3549         }
 3550 
 3551         /* deliberate use of bitwise or to avoid gratuitous short-circuit */
 3552         ring = rang ? false  : (iflib_min_tx_latency | err) || (TXQ_AVAIL(txq) < MAX_TX_DESC(ctx));
 3553         iflib_txd_db_check(ctx, txq, ring, txq->ift_in_use);
 3554         if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent);
 3555         if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent);
 3556         if (mcast_sent)
 3557                 if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent);
 3558 #ifdef INVARIANTS
 3559         if (iflib_verbose_debug)
 3560                 printf("consumed=%d\n", consumed);
 3561 #endif
 3562         return (consumed);
 3563 }
 3564 
 3565 static uint32_t
 3566 iflib_txq_drain_always(struct ifmp_ring *r)
 3567 {
 3568         return (1);
 3569 }
 3570 
 3571 static uint32_t
 3572 iflib_txq_drain_free(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx)
 3573 {
 3574         int i, avail;
 3575         struct mbuf **mp;
 3576         iflib_txq_t txq;
 3577 
 3578         txq = r->cookie;
 3579 
 3580         txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 3581         CALLOUT_LOCK(txq);
 3582         callout_stop(&txq->ift_timer);
 3583         CALLOUT_UNLOCK(txq);
 3584 
 3585         avail = IDXDIFF(pidx, cidx, r->size);
 3586         for (i = 0; i < avail; i++) {
 3587                 mp = _ring_peek_one(r, cidx, i, avail - i);
 3588                 if (__predict_false(*mp == (struct mbuf *)txq))
 3589                         continue;
 3590                 m_freem(*mp);
 3591         }
 3592         MPASS(ifmp_ring_is_stalled(r) == 0);
 3593         return (avail);
 3594 }
 3595 
 3596 static void
 3597 iflib_ifmp_purge(iflib_txq_t txq)
 3598 {
 3599         struct ifmp_ring *r;
 3600 
 3601         r = txq->ift_br;
 3602         r->drain = iflib_txq_drain_free;
 3603         r->can_drain = iflib_txq_drain_always;
 3604 
 3605         ifmp_ring_check_drainage(r, r->size);
 3606 
 3607         r->drain = iflib_txq_drain;
 3608         r->can_drain = iflib_txq_can_drain;
 3609 }
 3610 
 3611 static void
 3612 _task_fn_tx(void *context)
 3613 {
 3614         iflib_txq_t txq = context;
 3615         if_ctx_t ctx = txq->ift_ctx;
 3616         struct ifnet *ifp = ctx->ifc_ifp;
 3617         int rc;
 3618 
 3619 #ifdef IFLIB_DIAGNOSTICS
 3620         txq->ift_cpu_exec_count[curcpu]++;
 3621 #endif
 3622         if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
 3623                 return;
 3624         if (if_getcapenable(ifp) & IFCAP_NETMAP) {
 3625                 if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false))
 3626                         netmap_tx_irq(ifp, txq->ift_id);
 3627                 IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
 3628                 return;
 3629         }
 3630         if (txq->ift_db_pending)
 3631                 ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE);
 3632         ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
 3633         if (ctx->ifc_flags & IFC_LEGACY)
 3634                 IFDI_INTR_ENABLE(ctx);
 3635         else {
 3636                 rc = IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id);
 3637                 KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver"));
 3638         }
 3639 }
 3640 
 3641 static void
 3642 _task_fn_rx(void *context)
 3643 {
 3644         iflib_rxq_t rxq = context;
 3645         if_ctx_t ctx = rxq->ifr_ctx;
 3646         bool more;
 3647         int rc;
 3648         uint16_t budget;
 3649 
 3650 #ifdef IFLIB_DIAGNOSTICS
 3651         rxq->ifr_cpu_exec_count[curcpu]++;
 3652 #endif
 3653         DBG_COUNTER_INC(task_fn_rxs);
 3654         if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
 3655                 return;
 3656         more = true;
 3657 #ifdef DEV_NETMAP
 3658         if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) {
 3659                 u_int work = 0;
 3660                 if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work)) {
 3661                         more = false;
 3662                 }
 3663         }
 3664 #endif
 3665         budget = ctx->ifc_sysctl_rx_budget;
 3666         if (budget == 0)
 3667                 budget = 16;    /* XXX */
 3668         if (more == false || (more = iflib_rxeof(rxq, budget)) == false) {
 3669                 if (ctx->ifc_flags & IFC_LEGACY)
 3670                         IFDI_INTR_ENABLE(ctx);
 3671                 else {
 3672                         DBG_COUNTER_INC(rx_intr_enables);
 3673                         rc = IFDI_RX_QUEUE_INTR_ENABLE(ctx, rxq->ifr_id);
 3674                         KASSERT(rc != ENOTSUP, ("MSI-X support requires queue_intr_enable, but not implemented in driver"));
 3675                 }
 3676         }
 3677         if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)))
 3678                 return;
 3679         if (more)
 3680                 GROUPTASK_ENQUEUE(&rxq->ifr_task);
 3681 }
 3682 
 3683 static void
 3684 _task_fn_admin(void *context)
 3685 {
 3686         if_ctx_t ctx = context;
 3687         if_softc_ctx_t sctx = &ctx->ifc_softc_ctx;
 3688         iflib_txq_t txq;
 3689         int i;
 3690 
 3691         if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) {
 3692                 if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_OACTIVE)) {
 3693                         return;
 3694                 }
 3695         }
 3696 
 3697         CTX_LOCK(ctx);
 3698         for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) {
 3699                 CALLOUT_LOCK(txq);
 3700                 callout_stop(&txq->ift_timer);
 3701                 CALLOUT_UNLOCK(txq);
 3702         }
 3703         IFDI_UPDATE_ADMIN_STATUS(ctx);
 3704         for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
 3705                 callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, txq->ift_timer.c_cpu);
 3706         IFDI_LINK_INTR_ENABLE(ctx);
 3707         if (ctx->ifc_flags & IFC_DO_RESET) {
 3708                 ctx->ifc_flags &= ~IFC_DO_RESET;
 3709                 iflib_if_init_locked(ctx);
 3710         }
 3711         CTX_UNLOCK(ctx);
 3712 
 3713         if (LINK_ACTIVE(ctx) == 0)
 3714                 return;
 3715         for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++)
 3716                 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
 3717 }
 3718 
 3719 
 3720 static void
 3721 _task_fn_iov(void *context)
 3722 {
 3723         if_ctx_t ctx = context;
 3724 
 3725         if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))
 3726                 return;
 3727 
 3728         CTX_LOCK(ctx);
 3729         IFDI_VFLR_HANDLE(ctx);
 3730         CTX_UNLOCK(ctx);
 3731 }
 3732 
 3733 static int
 3734 iflib_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
 3735 {
 3736         int err;
 3737         if_int_delay_info_t info;
 3738         if_ctx_t ctx;
 3739 
 3740         info = (if_int_delay_info_t)arg1;
 3741         ctx = info->iidi_ctx;
 3742         info->iidi_req = req;
 3743         info->iidi_oidp = oidp;
 3744         CTX_LOCK(ctx);
 3745         err = IFDI_SYSCTL_INT_DELAY(ctx, info);
 3746         CTX_UNLOCK(ctx);
 3747         return (err);
 3748 }
 3749 
 3750 /*********************************************************************
 3751  *
 3752  *  IFNET FUNCTIONS
 3753  *
 3754  **********************************************************************/
 3755 
 3756 static void
 3757 iflib_if_init_locked(if_ctx_t ctx)
 3758 {
 3759         iflib_stop(ctx);
 3760         iflib_init_locked(ctx);
 3761 }
 3762 
 3763 
 3764 static void
 3765 iflib_if_init(void *arg)
 3766 {
 3767         if_ctx_t ctx = arg;
 3768 
 3769         CTX_LOCK(ctx);
 3770         iflib_if_init_locked(ctx);
 3771         CTX_UNLOCK(ctx);
 3772 }
 3773 
 3774 static int
 3775 iflib_if_transmit(if_t ifp, struct mbuf *m)
 3776 {
 3777         if_ctx_t        ctx = if_getsoftc(ifp);
 3778 
 3779         iflib_txq_t txq;
 3780         int err, qidx;
 3781 
 3782         if (__predict_false((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !LINK_ACTIVE(ctx))) {
 3783                 DBG_COUNTER_INC(tx_frees);
 3784                 m_freem(m);
 3785                 return (ENOBUFS);
 3786         }
 3787 
 3788         MPASS(m->m_nextpkt == NULL);
 3789         qidx = 0;
 3790         if ((NTXQSETS(ctx) > 1) && M_HASHTYPE_GET(m))
 3791                 qidx = QIDX(ctx, m);
 3792         /*
 3793          * XXX calculate buf_ring based on flowid (divvy up bits?)
 3794          */
 3795         txq = &ctx->ifc_txqs[qidx];
 3796 
 3797 #ifdef DRIVER_BACKPRESSURE
 3798         if (txq->ift_closed) {
 3799                 while (m != NULL) {
 3800                         next = m->m_nextpkt;
 3801                         m->m_nextpkt = NULL;
 3802                         m_freem(m);
 3803                         m = next;
 3804                 }
 3805                 return (ENOBUFS);
 3806         }
 3807 #endif
 3808 #ifdef notyet
 3809         qidx = count = 0;
 3810         mp = marr;
 3811         next = m;
 3812         do {
 3813                 count++;
 3814                 next = next->m_nextpkt;
 3815         } while (next != NULL);
 3816 
 3817         if (count > nitems(marr))
 3818                 if ((mp = malloc(count*sizeof(struct mbuf *), M_IFLIB, M_NOWAIT)) == NULL) {
 3819                         /* XXX check nextpkt */
 3820                         m_freem(m);
 3821                         /* XXX simplify for now */
 3822                         DBG_COUNTER_INC(tx_frees);
 3823                         return (ENOBUFS);
 3824                 }
 3825         for (next = m, i = 0; next != NULL; i++) {
 3826                 mp[i] = next;
 3827                 next = next->m_nextpkt;
 3828                 mp[i]->m_nextpkt = NULL;
 3829         }
 3830 #endif
 3831         DBG_COUNTER_INC(tx_seen);
 3832         err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE);
 3833 
 3834         GROUPTASK_ENQUEUE(&txq->ift_task);
 3835         if (err) {
 3836                 /* support forthcoming later */
 3837 #ifdef DRIVER_BACKPRESSURE
 3838                 txq->ift_closed = TRUE;
 3839 #endif
 3840                 ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE);
 3841                 m_freem(m);
 3842         }
 3843 
 3844         return (err);
 3845 }
 3846 
 3847 static void
 3848 iflib_if_qflush(if_t ifp)
 3849 {
 3850         if_ctx_t ctx = if_getsoftc(ifp);
 3851         iflib_txq_t txq = ctx->ifc_txqs;
 3852         int i;
 3853 
 3854         CTX_LOCK(ctx);
 3855         ctx->ifc_flags |= IFC_QFLUSH;
 3856         CTX_UNLOCK(ctx);
 3857         for (i = 0; i < NTXQSETS(ctx); i++, txq++)
 3858                 while (!(ifmp_ring_is_idle(txq->ift_br) || ifmp_ring_is_stalled(txq->ift_br)))
 3859                         iflib_txq_check_drain(txq, 0);
 3860         CTX_LOCK(ctx);
 3861         ctx->ifc_flags &= ~IFC_QFLUSH;
 3862         CTX_UNLOCK(ctx);
 3863 
 3864         if_qflush(ifp);
 3865 }
 3866 
 3867 
 3868 #define IFCAP_FLAGS (IFCAP_TXCSUM_IPV6 | IFCAP_RXCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \
 3869                      IFCAP_TSO4 | IFCAP_TSO6 | IFCAP_VLAN_HWTAGGING | IFCAP_HWSTATS | \
 3870                      IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO)
 3871 
 3872 static int
 3873 iflib_if_ioctl(if_t ifp, u_long command, caddr_t data)
 3874 {
 3875         if_ctx_t ctx = if_getsoftc(ifp);
 3876         struct ifreq    *ifr = (struct ifreq *)data;
 3877 #if defined(INET) || defined(INET6)
 3878         struct ifaddr   *ifa = (struct ifaddr *)data;
 3879 #endif
 3880         bool            avoid_reset = FALSE;
 3881         int             err = 0, reinit = 0, bits;
 3882 
 3883         switch (command) {
 3884         case SIOCSIFADDR:
 3885 #ifdef INET
 3886                 if (ifa->ifa_addr->sa_family == AF_INET)
 3887                         avoid_reset = TRUE;
 3888 #endif
 3889 #ifdef INET6
 3890                 if (ifa->ifa_addr->sa_family == AF_INET6)
 3891                         avoid_reset = TRUE;
 3892 #endif
 3893                 /*
 3894                 ** Calling init results in link renegotiation,
 3895                 ** so we avoid doing it when possible.
 3896                 */
 3897                 if (avoid_reset) {
 3898                         if_setflagbits(ifp, IFF_UP,0);
 3899                         if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING))
 3900                                 reinit = 1;
 3901 #ifdef INET
 3902                         if (!(if_getflags(ifp) & IFF_NOARP))
 3903                                 arp_ifinit(ifp, ifa);
 3904 #endif
 3905                 } else
 3906                         err = ether_ioctl(ifp, command, data);
 3907                 break;
 3908         case SIOCSIFMTU:
 3909                 CTX_LOCK(ctx);
 3910                 if (ifr->ifr_mtu == if_getmtu(ifp)) {
 3911                         CTX_UNLOCK(ctx);
 3912                         break;
 3913                 }
 3914                 bits = if_getdrvflags(ifp);
 3915                 /* stop the driver and free any clusters before proceeding */
 3916                 iflib_stop(ctx);
 3917 
 3918                 if ((err = IFDI_MTU_SET(ctx, ifr->ifr_mtu)) == 0) {
 3919                         if (ifr->ifr_mtu > ctx->ifc_max_fl_buf_size)
 3920                                 ctx->ifc_flags |= IFC_MULTISEG;
 3921                         else
 3922                                 ctx->ifc_flags &= ~IFC_MULTISEG;
 3923                         err = if_setmtu(ifp, ifr->ifr_mtu);
 3924                 }
 3925                 iflib_init_locked(ctx);
 3926                 if_setdrvflags(ifp, bits);
 3927                 CTX_UNLOCK(ctx);
 3928                 break;
 3929         case SIOCSIFFLAGS:
 3930                 CTX_LOCK(ctx);
 3931                 if (if_getflags(ifp) & IFF_UP) {
 3932                         if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 3933                                 if ((if_getflags(ifp) ^ ctx->ifc_if_flags) &
 3934                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 3935                                         err = IFDI_PROMISC_SET(ctx, if_getflags(ifp));
 3936                                 }
 3937                         } else
 3938                                 reinit = 1;
 3939                 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 3940                         iflib_stop(ctx);
 3941                 }
 3942                 ctx->ifc_if_flags = if_getflags(ifp);
 3943                 CTX_UNLOCK(ctx);
 3944                 break;
 3945         case SIOCADDMULTI:
 3946         case SIOCDELMULTI:
 3947                 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
 3948                         CTX_LOCK(ctx);
 3949                         IFDI_INTR_DISABLE(ctx);
 3950                         IFDI_MULTI_SET(ctx);
 3951                         IFDI_INTR_ENABLE(ctx);
 3952                         CTX_UNLOCK(ctx);
 3953                 }
 3954                 break;
 3955         case SIOCSIFMEDIA:
 3956                 CTX_LOCK(ctx);
 3957                 IFDI_MEDIA_SET(ctx);
 3958                 CTX_UNLOCK(ctx);
 3959                 /* falls thru */
 3960         case SIOCGIFMEDIA:
 3961         case SIOCGIFXMEDIA:
 3962                 err = ifmedia_ioctl(ifp, ifr, &ctx->ifc_media, command);
 3963                 break;
 3964         case SIOCGI2C:
 3965         {
 3966                 struct ifi2creq i2c;
 3967 
 3968                 err = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
 3969                 if (err != 0)
 3970                         break;
 3971                 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
 3972                         err = EINVAL;
 3973                         break;
 3974                 }
 3975                 if (i2c.len > sizeof(i2c.data)) {
 3976                         err = EINVAL;
 3977                         break;
 3978                 }
 3979 
 3980                 if ((err = IFDI_I2C_REQ(ctx, &i2c)) == 0)
 3981                         err = copyout(&i2c, ifr_data_get_ptr(ifr),
 3982                             sizeof(i2c));
 3983                 break;
 3984         }
 3985         case SIOCSIFCAP:
 3986         {
 3987                 int mask, setmask;
 3988 
 3989                 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
 3990                 setmask = 0;
 3991 #ifdef TCP_OFFLOAD
 3992                 setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6);
 3993 #endif
 3994                 setmask |= (mask & IFCAP_FLAGS);
 3995 
 3996                 if (setmask  & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
 3997                         setmask |= (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6);
 3998                 if ((mask & IFCAP_WOL) &&
 3999                     (if_getcapabilities(ifp) & IFCAP_WOL) != 0)
 4000                         setmask |= (mask & (IFCAP_WOL_MCAST|IFCAP_WOL_MAGIC));
 4001                 if_vlancap(ifp);
 4002                 /*
 4003                  * want to ensure that traffic has stopped before we change any of the flags
 4004                  */
 4005                 if (setmask) {
 4006                         CTX_LOCK(ctx);
 4007                         bits = if_getdrvflags(ifp);
 4008                         if (bits & IFF_DRV_RUNNING)
 4009                                 iflib_stop(ctx);
 4010                         if_togglecapenable(ifp, setmask);
 4011                         if (bits & IFF_DRV_RUNNING)
 4012                                 iflib_init_locked(ctx);
 4013                         if_setdrvflags(ifp, bits);
 4014                         CTX_UNLOCK(ctx);
 4015                 }
 4016                 break;
 4017             }
 4018         case SIOCGPRIVATE_0:
 4019         case SIOCSDRVSPEC:
 4020         case SIOCGDRVSPEC:
 4021                 CTX_LOCK(ctx);
 4022                 err = IFDI_PRIV_IOCTL(ctx, command, data);
 4023                 CTX_UNLOCK(ctx);
 4024                 break;
 4025         default:
 4026                 err = ether_ioctl(ifp, command, data);
 4027                 break;
 4028         }
 4029         if (reinit)
 4030                 iflib_if_init(ctx);
 4031         return (err);
 4032 }
 4033 
 4034 static uint64_t
 4035 iflib_if_get_counter(if_t ifp, ift_counter cnt)
 4036 {
 4037         if_ctx_t ctx = if_getsoftc(ifp);
 4038 
 4039         return (IFDI_GET_COUNTER(ctx, cnt));
 4040 }
 4041 
 4042 /*********************************************************************
 4043  *
 4044  *  OTHER FUNCTIONS EXPORTED TO THE STACK
 4045  *
 4046  **********************************************************************/
 4047 
 4048 static void
 4049 iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag)
 4050 {
 4051         if_ctx_t ctx = if_getsoftc(ifp);
 4052 
 4053         if ((void *)ctx != arg)
 4054                 return;
 4055 
 4056         if ((vtag == 0) || (vtag > 4095))
 4057                 return;
 4058 
 4059         CTX_LOCK(ctx);
 4060         IFDI_VLAN_REGISTER(ctx, vtag);
 4061         /* Re-init to load the changes */
 4062         if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
 4063                 iflib_if_init_locked(ctx);
 4064         CTX_UNLOCK(ctx);
 4065 }
 4066 
 4067 static void
 4068 iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag)
 4069 {
 4070         if_ctx_t ctx = if_getsoftc(ifp);
 4071 
 4072         if ((void *)ctx != arg)
 4073                 return;
 4074 
 4075         if ((vtag == 0) || (vtag > 4095))
 4076                 return;
 4077 
 4078         CTX_LOCK(ctx);
 4079         IFDI_VLAN_UNREGISTER(ctx, vtag);
 4080         /* Re-init to load the changes */
 4081         if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
 4082                 iflib_if_init_locked(ctx);
 4083         CTX_UNLOCK(ctx);
 4084 }
 4085 
 4086 static void
 4087 iflib_led_func(void *arg, int onoff)
 4088 {
 4089         if_ctx_t ctx = arg;
 4090 
 4091         CTX_LOCK(ctx);
 4092         IFDI_LED_FUNC(ctx, onoff);
 4093         CTX_UNLOCK(ctx);
 4094 }
 4095 
 4096 /*********************************************************************
 4097  *
 4098  *  BUS FUNCTION DEFINITIONS
 4099  *
 4100  **********************************************************************/
 4101 
 4102 int
 4103 iflib_device_probe(device_t dev)
 4104 {
 4105         pci_vendor_info_t *ent;
 4106 
 4107         uint16_t        pci_vendor_id, pci_device_id;
 4108         uint16_t        pci_subvendor_id, pci_subdevice_id;
 4109         uint16_t        pci_rev_id;
 4110         if_shared_ctx_t sctx;
 4111 
 4112         if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
 4113                 return (ENOTSUP);
 4114 
 4115         pci_vendor_id = pci_get_vendor(dev);
 4116         pci_device_id = pci_get_device(dev);
 4117         pci_subvendor_id = pci_get_subvendor(dev);
 4118         pci_subdevice_id = pci_get_subdevice(dev);
 4119         pci_rev_id = pci_get_revid(dev);
 4120         if (sctx->isc_parse_devinfo != NULL)
 4121                 sctx->isc_parse_devinfo(&pci_device_id, &pci_subvendor_id, &pci_subdevice_id, &pci_rev_id);
 4122 
 4123         ent = sctx->isc_vendor_info;
 4124         while (ent->pvi_vendor_id != 0) {
 4125                 if (pci_vendor_id != ent->pvi_vendor_id) {
 4126                         ent++;
 4127                         continue;
 4128                 }
 4129                 if ((pci_device_id == ent->pvi_device_id) &&
 4130                     ((pci_subvendor_id == ent->pvi_subvendor_id) ||
 4131                      (ent->pvi_subvendor_id == 0)) &&
 4132                     ((pci_subdevice_id == ent->pvi_subdevice_id) ||
 4133                      (ent->pvi_subdevice_id == 0)) &&
 4134                     ((pci_rev_id == ent->pvi_rev_id) ||
 4135                      (ent->pvi_rev_id == 0))) {
 4136 
 4137                         device_set_desc_copy(dev, ent->pvi_name);
 4138                         /* this needs to be changed to zero if the bus probing code
 4139                          * ever stops re-probing on best match because the sctx
 4140                          * may have its values over written by register calls
 4141                          * in subsequent probes
 4142                          */
 4143                         return (BUS_PROBE_DEFAULT);
 4144                 }
 4145                 ent++;
 4146         }
 4147         return (ENXIO);
 4148 }
 4149 
 4150 int
 4151 iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ctxp)
 4152 {
 4153         int err, rid, msix, msix_bar;
 4154         if_ctx_t ctx;
 4155         if_t ifp;
 4156         if_softc_ctx_t scctx;
 4157         int i;
 4158         uint16_t main_txq;
 4159         uint16_t main_rxq;
 4160 
 4161 
 4162         ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO);
 4163 
 4164         if (sc == NULL) {
 4165                 sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO);
 4166                 device_set_softc(dev, ctx);
 4167                 ctx->ifc_flags |= IFC_SC_ALLOCATED;
 4168         }
 4169 
 4170         ctx->ifc_sctx = sctx;
 4171         ctx->ifc_dev = dev;
 4172         ctx->ifc_softc = sc;
 4173 
 4174         if ((err = iflib_register(ctx)) != 0) {
 4175                 device_printf(dev, "iflib_register failed %d\n", err);
 4176                 return (err);
 4177         }
 4178         iflib_add_device_sysctl_pre(ctx);
 4179 
 4180         scctx = &ctx->ifc_softc_ctx;
 4181         ifp = ctx->ifc_ifp;
 4182         ctx->ifc_nhwtxqs = sctx->isc_ntxqs;
 4183 
 4184         /*
 4185          * XXX sanity check that ntxd & nrxd are a power of 2
 4186          */
 4187         if (ctx->ifc_sysctl_ntxqs != 0)
 4188                 scctx->isc_ntxqsets = ctx->ifc_sysctl_ntxqs;
 4189         if (ctx->ifc_sysctl_nrxqs != 0)
 4190                 scctx->isc_nrxqsets = ctx->ifc_sysctl_nrxqs;
 4191 
 4192         for (i = 0; i < sctx->isc_ntxqs; i++) {
 4193                 if (ctx->ifc_sysctl_ntxds[i] != 0)
 4194                         scctx->isc_ntxd[i] = ctx->ifc_sysctl_ntxds[i];
 4195                 else
 4196                         scctx->isc_ntxd[i] = sctx->isc_ntxd_default[i];
 4197         }
 4198 
 4199         for (i = 0; i < sctx->isc_nrxqs; i++) {
 4200                 if (ctx->ifc_sysctl_nrxds[i] != 0)
 4201                         scctx->isc_nrxd[i] = ctx->ifc_sysctl_nrxds[i];
 4202                 else
 4203                         scctx->isc_nrxd[i] = sctx->isc_nrxd_default[i];
 4204         }
 4205 
 4206         for (i = 0; i < sctx->isc_nrxqs; i++) {
 4207                 if (scctx->isc_nrxd[i] < sctx->isc_nrxd_min[i]) {
 4208                         device_printf(dev, "nrxd%d: %d less than nrxd_min %d - resetting to min\n",
 4209                                       i, scctx->isc_nrxd[i], sctx->isc_nrxd_min[i]);
 4210                         scctx->isc_nrxd[i] = sctx->isc_nrxd_min[i];
 4211                 }
 4212                 if (scctx->isc_nrxd[i] > sctx->isc_nrxd_max[i]) {
 4213                         device_printf(dev, "nrxd%d: %d greater than nrxd_max %d - resetting to max\n",
 4214                                       i, scctx->isc_nrxd[i], sctx->isc_nrxd_max[i]);
 4215                         scctx->isc_nrxd[i] = sctx->isc_nrxd_max[i];
 4216                 }
 4217         }
 4218 
 4219         for (i = 0; i < sctx->isc_ntxqs; i++) {
 4220                 if (scctx->isc_ntxd[i] < sctx->isc_ntxd_min[i]) {
 4221                         device_printf(dev, "ntxd%d: %d less than ntxd_min %d - resetting to min\n",
 4222                                       i, scctx->isc_ntxd[i], sctx->isc_ntxd_min[i]);
 4223                         scctx->isc_ntxd[i] = sctx->isc_ntxd_min[i];
 4224                 }
 4225                 if (scctx->isc_ntxd[i] > sctx->isc_ntxd_max[i]) {
 4226                         device_printf(dev, "ntxd%d: %d greater than ntxd_max %d - resetting to max\n",
 4227                                       i, scctx->isc_ntxd[i], sctx->isc_ntxd_max[i]);
 4228                         scctx->isc_ntxd[i] = sctx->isc_ntxd_max[i];
 4229                 }
 4230         }
 4231 
 4232         if ((err = IFDI_ATTACH_PRE(ctx)) != 0) {
 4233                 device_printf(dev, "IFDI_ATTACH_PRE failed %d\n", err);
 4234                 return (err);
 4235         }
 4236         _iflib_pre_assert(scctx);
 4237         ctx->ifc_txrx = *scctx->isc_txrx;
 4238 
 4239 #ifdef INVARIANTS
 4240         MPASS(scctx->isc_capenable);
 4241         if (scctx->isc_capenable & IFCAP_TXCSUM)
 4242                 MPASS(scctx->isc_tx_csum_flags);
 4243 #endif
 4244 
 4245         if_setcapabilities(ifp, scctx->isc_capenable | IFCAP_HWSTATS);
 4246         if_setcapenable(ifp, scctx->isc_capenable | IFCAP_HWSTATS);
 4247 
 4248         if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets))
 4249                 scctx->isc_ntxqsets = scctx->isc_ntxqsets_max;
 4250         if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets))
 4251                 scctx->isc_nrxqsets = scctx->isc_nrxqsets_max;
 4252 
 4253 #ifdef ACPI_DMAR
 4254         if (dmar_get_dma_tag(device_get_parent(dev), dev) != NULL)
 4255                 ctx->ifc_flags |= IFC_DMAR;
 4256 #elif !(defined(__i386__) || defined(__amd64__))
 4257         /* set unconditionally for !x86 */
 4258         ctx->ifc_flags |= IFC_DMAR;
 4259 #endif
 4260 
 4261         msix_bar = scctx->isc_msix_bar;
 4262         main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0;
 4263         main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0;
 4264 
 4265         /* XXX change for per-queue sizes */
 4266         device_printf(dev, "using %d tx descriptors and %d rx descriptors\n",
 4267                       scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]);
 4268         for (i = 0; i < sctx->isc_nrxqs; i++) {
 4269                 if (!powerof2(scctx->isc_nrxd[i])) {
 4270                         /* round down instead? */
 4271                         device_printf(dev, "# rx descriptors must be a power of 2\n");
 4272                         err = EINVAL;
 4273                         goto fail;
 4274                 }
 4275         }
 4276         for (i = 0; i < sctx->isc_ntxqs; i++) {
 4277                 if (!powerof2(scctx->isc_ntxd[i])) {
 4278                         device_printf(dev,
 4279                             "# tx descriptors must be a power of 2");
 4280                         err = EINVAL;
 4281                         goto fail;
 4282                 }
 4283         }
 4284 
 4285         if (scctx->isc_tx_nsegments > scctx->isc_ntxd[main_txq] /
 4286             MAX_SINGLE_PACKET_FRACTION)
 4287                 scctx->isc_tx_nsegments = max(1, scctx->isc_ntxd[main_txq] /
 4288                     MAX_SINGLE_PACKET_FRACTION);
 4289         if (scctx->isc_tx_tso_segments_max > scctx->isc_ntxd[main_txq] /
 4290             MAX_SINGLE_PACKET_FRACTION)
 4291                 scctx->isc_tx_tso_segments_max = max(1,
 4292                     scctx->isc_ntxd[main_txq] / MAX_SINGLE_PACKET_FRACTION);
 4293 
 4294         /*
 4295          * Protect the stack against modern hardware
 4296          */
 4297         if (scctx->isc_tx_tso_size_max > FREEBSD_TSO_SIZE_MAX)
 4298                 scctx->isc_tx_tso_size_max = FREEBSD_TSO_SIZE_MAX;
 4299 
 4300         /* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */
 4301         ifp->if_hw_tsomaxsegcount = scctx->isc_tx_tso_segments_max;
 4302         ifp->if_hw_tsomax = scctx->isc_tx_tso_size_max;
 4303         ifp->if_hw_tsomaxsegsize = scctx->isc_tx_tso_segsize_max;
 4304         if (scctx->isc_rss_table_size == 0)
 4305                 scctx->isc_rss_table_size = 64;
 4306         scctx->isc_rss_table_mask = scctx->isc_rss_table_size-1;
 4307 
 4308         GROUPTASK_INIT(&ctx->ifc_admin_task, 0, _task_fn_admin, ctx);
 4309         /* XXX format name */
 4310         taskqgroup_attach(qgroup_if_config_tqg, &ctx->ifc_admin_task, ctx, -1, "admin");
 4311 
 4312         /* Set up cpu set.  If it fails, use the set of all CPUs. */
 4313         if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) {
 4314                 device_printf(dev, "Unable to fetch CPU list\n");
 4315                 CPU_COPY(&all_cpus, &ctx->ifc_cpus);
 4316         }
 4317         MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0);
 4318 
 4319         /*
 4320         ** Now setup MSI or MSI/X, should
 4321         ** return us the number of supported
 4322         ** vectors. (Will be 1 for MSI)
 4323         */
 4324         if (sctx->isc_flags & IFLIB_SKIP_MSIX) {
 4325                 msix = scctx->isc_vectors;
 4326         } else if (scctx->isc_msix_bar != 0)
 4327                /*
 4328                 * The simple fact that isc_msix_bar is not 0 does not mean we
 4329                 * we have a good value there that is known to work.
 4330                 */
 4331                 msix = iflib_msix_init(ctx);
 4332         else {
 4333                 scctx->isc_vectors = 1;
 4334                 scctx->isc_ntxqsets = 1;
 4335                 scctx->isc_nrxqsets = 1;
 4336                 scctx->isc_intr = IFLIB_INTR_LEGACY;
 4337                 msix = 0;
 4338         }
 4339         /* Get memory for the station queues */
 4340         if ((err = iflib_queues_alloc(ctx))) {
 4341                 device_printf(dev, "Unable to allocate queue memory\n");
 4342                 goto fail;
 4343         }
 4344 
 4345         if ((err = iflib_qset_structures_setup(ctx)))
 4346                 goto fail_queues;
 4347         /*
 4348          * Group taskqueues aren't properly set up until SMP is started,
 4349          * so we disable interrupts until we can handle them post
 4350          * SI_SUB_SMP.
 4351          *
 4352          * XXX: disabling interrupts doesn't actually work, at least for
 4353          * the non-MSI case.  When they occur before SI_SUB_SMP completes,
 4354          * we do null handling and depend on this not causing too large an
 4355          * interrupt storm.
 4356          */
 4357         IFDI_INTR_DISABLE(ctx);
 4358         if (msix > 1 && (err = IFDI_MSIX_INTR_ASSIGN(ctx, msix)) != 0) {
 4359                 device_printf(dev, "IFDI_MSIX_INTR_ASSIGN failed %d\n", err);
 4360                 goto fail_intr_free;
 4361         }
 4362         if (msix <= 1) {
 4363                 rid = 0;
 4364                 if (scctx->isc_intr == IFLIB_INTR_MSI) {
 4365                         MPASS(msix == 1);
 4366                         rid = 1;
 4367                 }
 4368                 if ((err = iflib_legacy_setup(ctx, ctx->isc_legacy_intr, ctx->ifc_softc, &rid, "irq0")) != 0) {
 4369                         device_printf(dev, "iflib_legacy_setup failed %d\n", err);
 4370                         goto fail_intr_free;
 4371                 }
 4372         }
 4373         ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac);
 4374         if ((err = IFDI_ATTACH_POST(ctx)) != 0) {
 4375                 device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err);
 4376                 goto fail_detach;
 4377         }
 4378         if ((err = iflib_netmap_attach(ctx))) {
 4379                 device_printf(ctx->ifc_dev, "netmap attach failed: %d\n", err);
 4380                 goto fail_detach;
 4381         }
 4382         *ctxp = ctx;
 4383 
 4384         if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter);
 4385         iflib_add_device_sysctl_post(ctx);
 4386         return (0);
 4387 fail_detach:
 4388         ether_ifdetach(ctx->ifc_ifp);
 4389 fail_intr_free:
 4390         if (scctx->isc_intr == IFLIB_INTR_MSIX || scctx->isc_intr == IFLIB_INTR_MSI)
 4391                 pci_release_msi(ctx->ifc_dev);
 4392 fail_queues:
 4393         iflib_tx_structures_free(ctx);
 4394         iflib_rx_structures_free(ctx);
 4395 fail:
 4396         IFDI_DETACH(ctx);
 4397         return (err);
 4398 }
 4399 
 4400 int
 4401 iflib_device_attach(device_t dev)
 4402 {
 4403         if_ctx_t ctx;
 4404         if_shared_ctx_t sctx;
 4405 
 4406         if ((sctx = DEVICE_REGISTER(dev)) == NULL || sctx->isc_magic != IFLIB_MAGIC)
 4407                 return (ENOTSUP);
 4408 
 4409         pci_enable_busmaster(dev);
 4410 
 4411         return (iflib_device_register(dev, NULL, sctx, &ctx));
 4412 }
 4413 
 4414 int
 4415 iflib_device_deregister(if_ctx_t ctx)
 4416 {
 4417         if_t ifp = ctx->ifc_ifp;
 4418         iflib_txq_t txq;
 4419         iflib_rxq_t rxq;
 4420         device_t dev = ctx->ifc_dev;
 4421         int i, j;
 4422         struct taskqgroup *tqg;
 4423         iflib_fl_t fl;
 4424 
 4425         /* Make sure VLANS are not using driver */
 4426         if (if_vlantrunkinuse(ifp)) {
 4427                 device_printf(dev,"Vlan in use, detach first\n");
 4428                 return (EBUSY);
 4429         }
 4430 
 4431         CTX_LOCK(ctx);
 4432         ctx->ifc_in_detach = 1;
 4433         iflib_stop(ctx);
 4434         CTX_UNLOCK(ctx);
 4435 
 4436         /* Unregister VLAN events */
 4437         if (ctx->ifc_vlan_attach_event != NULL)
 4438                 EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event);
 4439         if (ctx->ifc_vlan_detach_event != NULL)
 4440                 EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event);
 4441 
 4442         iflib_netmap_detach(ifp);
 4443         ether_ifdetach(ifp);
 4444         /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/
 4445         CTX_LOCK_DESTROY(ctx);
 4446         if (ctx->ifc_led_dev != NULL)
 4447                 led_destroy(ctx->ifc_led_dev);
 4448         /* XXX drain any dependent tasks */
 4449         tqg = qgroup_if_io_tqg;
 4450         for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) {
 4451                 callout_drain(&txq->ift_timer);
 4452                 if (txq->ift_task.gt_uniq != NULL)
 4453                         taskqgroup_detach(tqg, &txq->ift_task);
 4454         }
 4455         for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) {
 4456                 if (rxq->ifr_task.gt_uniq != NULL)
 4457                         taskqgroup_detach(tqg, &rxq->ifr_task);
 4458 
 4459                 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++)
 4460                         free(fl->ifl_rx_bitmap, M_IFLIB);
 4461                         
 4462         }
 4463         tqg = qgroup_if_config_tqg;
 4464         if (ctx->ifc_admin_task.gt_uniq != NULL)
 4465                 taskqgroup_detach(tqg, &ctx->ifc_admin_task);
 4466         if (ctx->ifc_vflr_task.gt_uniq != NULL)
 4467                 taskqgroup_detach(tqg, &ctx->ifc_vflr_task);
 4468 
 4469         IFDI_DETACH(ctx);
 4470         device_set_softc(ctx->ifc_dev, NULL);
 4471         if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) {
 4472                 pci_release_msi(dev);
 4473         }
 4474         if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) {
 4475                 iflib_irq_free(ctx, &ctx->ifc_legacy_irq);
 4476         }
 4477         if (ctx->ifc_msix_mem != NULL) {
 4478                 bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY,
 4479                         ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem);
 4480                 ctx->ifc_msix_mem = NULL;
 4481         }
 4482 
 4483         bus_generic_detach(dev);
 4484         if_free(ifp);
 4485 
 4486         iflib_tx_structures_free(ctx);
 4487         iflib_rx_structures_free(ctx);
 4488         if (ctx->ifc_flags & IFC_SC_ALLOCATED)
 4489                 free(ctx->ifc_softc, M_IFLIB);
 4490         free(ctx, M_IFLIB);
 4491         return (0);
 4492 }
 4493 
 4494 
 4495 int
 4496 iflib_device_detach(device_t dev)
 4497 {
 4498         if_ctx_t ctx = device_get_softc(dev);
 4499 
 4500         return (iflib_device_deregister(ctx));
 4501 }
 4502 
 4503 int
 4504 iflib_device_suspend(device_t dev)
 4505 {
 4506         if_ctx_t ctx = device_get_softc(dev);
 4507 
 4508         CTX_LOCK(ctx);
 4509         IFDI_SUSPEND(ctx);
 4510         CTX_UNLOCK(ctx);
 4511 
 4512         return bus_generic_suspend(dev);
 4513 }
 4514 int
 4515 iflib_device_shutdown(device_t dev)
 4516 {
 4517         if_ctx_t ctx = device_get_softc(dev);
 4518 
 4519         CTX_LOCK(ctx);
 4520         IFDI_SHUTDOWN(ctx);
 4521         CTX_UNLOCK(ctx);
 4522 
 4523         return bus_generic_suspend(dev);
 4524 }
 4525 
 4526 
 4527 int
 4528 iflib_device_resume(device_t dev)
 4529 {
 4530         if_ctx_t ctx = device_get_softc(dev);
 4531         iflib_txq_t txq = ctx->ifc_txqs;
 4532 
 4533         CTX_LOCK(ctx);
 4534         IFDI_RESUME(ctx);
 4535         iflib_init_locked(ctx);
 4536         CTX_UNLOCK(ctx);
 4537         for (int i = 0; i < NTXQSETS(ctx); i++, txq++)
 4538                 iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET);
 4539 
 4540         return (bus_generic_resume(dev));
 4541 }
 4542 
 4543 int
 4544 iflib_device_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
 4545 {
 4546         int error;
 4547         if_ctx_t ctx = device_get_softc(dev);
 4548 
 4549         CTX_LOCK(ctx);
 4550         error = IFDI_IOV_INIT(ctx, num_vfs, params);
 4551         CTX_UNLOCK(ctx);
 4552 
 4553         return (error);
 4554 }
 4555 
 4556 void
 4557 iflib_device_iov_uninit(device_t dev)
 4558 {
 4559         if_ctx_t ctx = device_get_softc(dev);
 4560 
 4561         CTX_LOCK(ctx);
 4562         IFDI_IOV_UNINIT(ctx);
 4563         CTX_UNLOCK(ctx);
 4564 }
 4565 
 4566 int
 4567 iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
 4568 {
 4569         int error;
 4570         if_ctx_t ctx = device_get_softc(dev);
 4571 
 4572         CTX_LOCK(ctx);
 4573         error = IFDI_IOV_VF_ADD(ctx, vfnum, params);
 4574         CTX_UNLOCK(ctx);
 4575 
 4576         return (error);
 4577 }
 4578 
 4579 /*********************************************************************
 4580  *
 4581  *  MODULE FUNCTION DEFINITIONS
 4582  *
 4583  **********************************************************************/
 4584 
 4585 /*
 4586  * - Start a fast taskqueue thread for each core
 4587  * - Start a taskqueue for control operations
 4588  */
 4589 static int
 4590 iflib_module_init(void)
 4591 {
 4592         return (0);
 4593 }
 4594 
 4595 static int
 4596 iflib_module_event_handler(module_t mod, int what, void *arg)
 4597 {
 4598         int err;
 4599 
 4600         switch (what) {
 4601         case MOD_LOAD:
 4602                 if ((err = iflib_module_init()) != 0)
 4603                         return (err);
 4604                 break;
 4605         case MOD_UNLOAD:
 4606                 return (EBUSY);
 4607         default:
 4608                 return (EOPNOTSUPP);
 4609         }
 4610 
 4611         return (0);
 4612 }
 4613 
 4614 /*********************************************************************
 4615  *
 4616  *  PUBLIC FUNCTION DEFINITIONS
 4617  *     ordered as in iflib.h
 4618  *
 4619  **********************************************************************/
 4620 
 4621 
 4622 static void
 4623 _iflib_assert(if_shared_ctx_t sctx)
 4624 {
 4625         MPASS(sctx->isc_tx_maxsize);
 4626         MPASS(sctx->isc_tx_maxsegsize);
 4627 
 4628         MPASS(sctx->isc_rx_maxsize);
 4629         MPASS(sctx->isc_rx_nsegments);
 4630         MPASS(sctx->isc_rx_maxsegsize);
 4631 
 4632         MPASS(sctx->isc_nrxd_min[0]);
 4633         MPASS(sctx->isc_nrxd_max[0]);
 4634         MPASS(sctx->isc_nrxd_default[0]);
 4635         MPASS(sctx->isc_ntxd_min[0]);
 4636         MPASS(sctx->isc_ntxd_max[0]);
 4637         MPASS(sctx->isc_ntxd_default[0]);
 4638 }
 4639 
 4640 static void
 4641 _iflib_pre_assert(if_softc_ctx_t scctx)
 4642 {
 4643 
 4644         MPASS(scctx->isc_txrx->ift_txd_encap);
 4645         MPASS(scctx->isc_txrx->ift_txd_flush);
 4646         MPASS(scctx->isc_txrx->ift_txd_credits_update);
 4647         MPASS(scctx->isc_txrx->ift_rxd_available);
 4648         MPASS(scctx->isc_txrx->ift_rxd_pkt_get);
 4649         MPASS(scctx->isc_txrx->ift_rxd_refill);
 4650         MPASS(scctx->isc_txrx->ift_rxd_flush);
 4651 }
 4652 
 4653 static int
 4654 iflib_register(if_ctx_t ctx)
 4655 {
 4656         if_shared_ctx_t sctx = ctx->ifc_sctx;
 4657         driver_t *driver = sctx->isc_driver;
 4658         device_t dev = ctx->ifc_dev;
 4659         if_t ifp;
 4660 
 4661         _iflib_assert(sctx);
 4662 
 4663         CTX_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev));
 4664 
 4665         ifp = ctx->ifc_ifp = if_gethandle(IFT_ETHER);
 4666         if (ifp == NULL) {
 4667                 device_printf(dev, "can not allocate ifnet structure\n");
 4668                 return (ENOMEM);
 4669         }
 4670 
 4671         /*
 4672          * Initialize our context's device specific methods
 4673          */
 4674         kobj_init((kobj_t) ctx, (kobj_class_t) driver);
 4675         kobj_class_compile((kobj_class_t) driver);
 4676         driver->refs++;
 4677 
 4678         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 4679         if_setsoftc(ifp, ctx);
 4680         if_setdev(ifp, dev);
 4681         if_setinitfn(ifp, iflib_if_init);
 4682         if_setioctlfn(ifp, iflib_if_ioctl);
 4683         if_settransmitfn(ifp, iflib_if_transmit);
 4684         if_setqflushfn(ifp, iflib_if_qflush);
 4685         if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
 4686 
 4687         ctx->ifc_vlan_attach_event =
 4688                 EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx,
 4689                                                           EVENTHANDLER_PRI_FIRST);
 4690         ctx->ifc_vlan_detach_event =
 4691                 EVENTHANDLER_REGISTER(vlan_unconfig, iflib_vlan_unregister, ctx,
 4692                                                           EVENTHANDLER_PRI_FIRST);
 4693 
 4694         ifmedia_init(&ctx->ifc_media, IFM_IMASK,
 4695                                          iflib_media_change, iflib_media_status);
 4696 
 4697         return (0);
 4698 }
 4699 
 4700 
 4701 static int
 4702 iflib_queues_alloc(if_ctx_t ctx)
 4703 {
 4704         if_shared_ctx_t sctx = ctx->ifc_sctx;
 4705         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 4706         device_t dev = ctx->ifc_dev;
 4707         int nrxqsets = scctx->isc_nrxqsets;
 4708         int ntxqsets = scctx->isc_ntxqsets;
 4709         iflib_txq_t txq;
 4710         iflib_rxq_t rxq;
 4711         iflib_fl_t fl = NULL;
 4712         int i, j, cpu, err, txconf, rxconf;
 4713         iflib_dma_info_t ifdip;
 4714         uint32_t *rxqsizes = scctx->isc_rxqsizes;
 4715         uint32_t *txqsizes = scctx->isc_txqsizes;
 4716         uint8_t nrxqs = sctx->isc_nrxqs;
 4717         uint8_t ntxqs = sctx->isc_ntxqs;
 4718         int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1;
 4719         caddr_t *vaddrs;
 4720         uint64_t *paddrs;
 4721 
 4722         KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1"));
 4723         KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1"));
 4724 
 4725 /* Allocate the TX ring struct memory */
 4726         if (!(ctx->ifc_txqs =
 4727             (iflib_txq_t) malloc(sizeof(struct iflib_txq) *
 4728             ntxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
 4729                 device_printf(dev, "Unable to allocate TX ring memory\n");
 4730                 err = ENOMEM;
 4731                 goto fail;
 4732         }
 4733 
 4734         /* Now allocate the RX */
 4735         if (!(ctx->ifc_rxqs =
 4736             (iflib_rxq_t) malloc(sizeof(struct iflib_rxq) *
 4737             nrxqsets, M_IFLIB, M_NOWAIT | M_ZERO))) {
 4738                 device_printf(dev, "Unable to allocate RX ring memory\n");
 4739                 err = ENOMEM;
 4740                 goto rx_fail;
 4741         }
 4742 
 4743         txq = ctx->ifc_txqs;
 4744         rxq = ctx->ifc_rxqs;
 4745 
 4746         /*
 4747          * XXX handle allocation failure
 4748          */
 4749         for (txconf = i = 0, cpu = CPU_FIRST(); i < ntxqsets; i++, txconf++, txq++, cpu = CPU_NEXT(cpu)) {
 4750                 /* Set up some basics */
 4751 
 4752                 if ((ifdip = malloc(sizeof(struct iflib_dma_info) * ntxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) {
 4753                         device_printf(dev, "failed to allocate iflib_dma_info\n");
 4754                         err = ENOMEM;
 4755                         goto err_tx_desc;
 4756                 }
 4757                 txq->ift_ifdi = ifdip;
 4758                 for (j = 0; j < ntxqs; j++, ifdip++) {
 4759                         if (iflib_dma_alloc(ctx, txqsizes[j], ifdip, BUS_DMA_NOWAIT)) {
 4760                                 device_printf(dev, "Unable to allocate Descriptor memory\n");
 4761                                 err = ENOMEM;
 4762                                 goto err_tx_desc;
 4763                         }
 4764                         txq->ift_txd_size[j] = scctx->isc_txd_size[j];
 4765                         bzero((void *)ifdip->idi_vaddr, txqsizes[j]);
 4766                 }
 4767                 txq->ift_ctx = ctx;
 4768                 txq->ift_id = i;
 4769                 if (sctx->isc_flags & IFLIB_HAS_TXCQ) {
 4770                         txq->ift_br_offset = 1;
 4771                 } else {
 4772                         txq->ift_br_offset = 0;
 4773                 }
 4774                 /* XXX fix this */
 4775                 txq->ift_timer.c_cpu = cpu;
 4776 
 4777                 if (iflib_txsd_alloc(txq)) {
 4778                         device_printf(dev, "Critical Failure setting up TX buffers\n");
 4779                         err = ENOMEM;
 4780                         goto err_tx_desc;
 4781                 }
 4782 
 4783                 /* Initialize the TX lock */
 4784                 snprintf(txq->ift_mtx_name, MTX_NAME_LEN, "%s:tx(%d):callout",
 4785                     device_get_nameunit(dev), txq->ift_id);
 4786                 mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF);
 4787                 callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0);
 4788 
 4789                 snprintf(txq->ift_db_mtx_name, MTX_NAME_LEN, "%s:tx(%d):db",
 4790                          device_get_nameunit(dev), txq->ift_id);
 4791 
 4792                 err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain,
 4793                                       iflib_txq_can_drain, M_IFLIB, M_WAITOK);
 4794                 if (err) {
 4795                         /* XXX free any allocated rings */
 4796                         device_printf(dev, "Unable to allocate buf_ring\n");
 4797                         goto err_tx_desc;
 4798                 }
 4799         }
 4800 
 4801         for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) {
 4802                 /* Set up some basics */
 4803 
 4804                 if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs, M_IFLIB, M_WAITOK|M_ZERO)) == NULL) {
 4805                         device_printf(dev, "failed to allocate iflib_dma_info\n");
 4806                         err = ENOMEM;
 4807                         goto err_tx_desc;
 4808                 }
 4809 
 4810                 rxq->ifr_ifdi = ifdip;
 4811                 /* XXX this needs to be changed if #rx queues != #tx queues */
 4812                 rxq->ifr_ntxqirq = 1;
 4813                 rxq->ifr_txqid[0] = i;
 4814                 for (j = 0; j < nrxqs; j++, ifdip++) {
 4815                         if (iflib_dma_alloc(ctx, rxqsizes[j], ifdip, BUS_DMA_NOWAIT)) {
 4816                                 device_printf(dev, "Unable to allocate Descriptor memory\n");
 4817                                 err = ENOMEM;
 4818                                 goto err_tx_desc;
 4819                         }
 4820                         bzero((void *)ifdip->idi_vaddr, rxqsizes[j]);
 4821                 }
 4822                 rxq->ifr_ctx = ctx;
 4823                 rxq->ifr_id = i;
 4824                 if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
 4825                         rxq->ifr_fl_offset = 1;
 4826                 } else {
 4827                         rxq->ifr_fl_offset = 0;
 4828                 }
 4829                 rxq->ifr_nfl = nfree_lists;
 4830                 if (!(fl =
 4831                           (iflib_fl_t) malloc(sizeof(struct iflib_fl) * nfree_lists, M_IFLIB, M_NOWAIT | M_ZERO))) {
 4832                         device_printf(dev, "Unable to allocate free list memory\n");
 4833                         err = ENOMEM;
 4834                         goto err_tx_desc;
 4835                 }
 4836                 rxq->ifr_fl = fl;
 4837                 for (j = 0; j < nfree_lists; j++) {
 4838                         fl[j].ifl_rxq = rxq;
 4839                         fl[j].ifl_id = j;
 4840                         fl[j].ifl_ifdi = &rxq->ifr_ifdi[j + rxq->ifr_fl_offset];
 4841                         fl[j].ifl_rxd_size = scctx->isc_rxd_size[j];
 4842                 }
 4843         /* Allocate receive buffers for the ring*/
 4844                 if (iflib_rxsd_alloc(rxq)) {
 4845                         device_printf(dev,
 4846                             "Critical Failure setting up receive buffers\n");
 4847                         err = ENOMEM;
 4848                         goto err_rx_desc;
 4849                 }
 4850 
 4851                 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) 
 4852                         fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, M_WAITOK|M_ZERO);
 4853         }
 4854 
 4855         /* TXQs */
 4856         vaddrs = malloc(sizeof(caddr_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
 4857         paddrs = malloc(sizeof(uint64_t)*ntxqsets*ntxqs, M_IFLIB, M_WAITOK);
 4858         for (i = 0; i < ntxqsets; i++) {
 4859                 iflib_dma_info_t di = ctx->ifc_txqs[i].ift_ifdi;
 4860 
 4861                 for (j = 0; j < ntxqs; j++, di++) {
 4862                         vaddrs[i*ntxqs + j] = di->idi_vaddr;
 4863                         paddrs[i*ntxqs + j] = di->idi_paddr;
 4864                 }
 4865         }
 4866         if ((err = IFDI_TX_QUEUES_ALLOC(ctx, vaddrs, paddrs, ntxqs, ntxqsets)) != 0) {
 4867                 device_printf(ctx->ifc_dev, "device queue allocation failed\n");
 4868                 iflib_tx_structures_free(ctx);
 4869                 free(vaddrs, M_IFLIB);
 4870                 free(paddrs, M_IFLIB);
 4871                 goto err_rx_desc;
 4872         }
 4873         free(vaddrs, M_IFLIB);
 4874         free(paddrs, M_IFLIB);
 4875 
 4876         /* RXQs */
 4877         vaddrs = malloc(sizeof(caddr_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
 4878         paddrs = malloc(sizeof(uint64_t)*nrxqsets*nrxqs, M_IFLIB, M_WAITOK);
 4879         for (i = 0; i < nrxqsets; i++) {
 4880                 iflib_dma_info_t di = ctx->ifc_rxqs[i].ifr_ifdi;
 4881 
 4882                 for (j = 0; j < nrxqs; j++, di++) {
 4883                         vaddrs[i*nrxqs + j] = di->idi_vaddr;
 4884                         paddrs[i*nrxqs + j] = di->idi_paddr;
 4885                 }
 4886         }
 4887         if ((err = IFDI_RX_QUEUES_ALLOC(ctx, vaddrs, paddrs, nrxqs, nrxqsets)) != 0) {
 4888                 device_printf(ctx->ifc_dev, "device queue allocation failed\n");
 4889                 iflib_tx_structures_free(ctx);
 4890                 free(vaddrs, M_IFLIB);
 4891                 free(paddrs, M_IFLIB);
 4892                 goto err_rx_desc;
 4893         }
 4894         free(vaddrs, M_IFLIB);
 4895         free(paddrs, M_IFLIB);
 4896 
 4897         return (0);
 4898 
 4899 /* XXX handle allocation failure changes */
 4900 err_rx_desc:
 4901 err_tx_desc:
 4902 rx_fail:
 4903         if (ctx->ifc_rxqs != NULL)
 4904                 free(ctx->ifc_rxqs, M_IFLIB);
 4905         ctx->ifc_rxqs = NULL;
 4906         if (ctx->ifc_txqs != NULL)
 4907                 free(ctx->ifc_txqs, M_IFLIB);
 4908         ctx->ifc_txqs = NULL;
 4909 fail:
 4910         return (err);
 4911 }
 4912 
 4913 static int
 4914 iflib_tx_structures_setup(if_ctx_t ctx)
 4915 {
 4916         iflib_txq_t txq = ctx->ifc_txqs;
 4917         int i;
 4918 
 4919         for (i = 0; i < NTXQSETS(ctx); i++, txq++)
 4920                 iflib_txq_setup(txq);
 4921 
 4922         return (0);
 4923 }
 4924 
 4925 static void
 4926 iflib_tx_structures_free(if_ctx_t ctx)
 4927 {
 4928         iflib_txq_t txq = ctx->ifc_txqs;
 4929         int i, j;
 4930 
 4931         for (i = 0; i < NTXQSETS(ctx); i++, txq++) {
 4932                 iflib_txq_destroy(txq);
 4933                 for (j = 0; j < ctx->ifc_nhwtxqs; j++)
 4934                         iflib_dma_free(&txq->ift_ifdi[j]);
 4935         }
 4936         free(ctx->ifc_txqs, M_IFLIB);
 4937         ctx->ifc_txqs = NULL;
 4938         IFDI_QUEUES_FREE(ctx);
 4939 }
 4940 
 4941 /*********************************************************************
 4942  *
 4943  *  Initialize all receive rings.
 4944  *
 4945  **********************************************************************/
 4946 static int
 4947 iflib_rx_structures_setup(if_ctx_t ctx)
 4948 {
 4949         iflib_rxq_t rxq = ctx->ifc_rxqs;
 4950         int q;
 4951 #if defined(INET6) || defined(INET)
 4952         int i, err;
 4953 #endif
 4954 
 4955         for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) {
 4956 #if defined(INET6) || defined(INET)
 4957                 tcp_lro_free(&rxq->ifr_lc);
 4958                 if ((err = tcp_lro_init_args(&rxq->ifr_lc, ctx->ifc_ifp,
 4959                     TCP_LRO_ENTRIES, min(1024,
 4960                     ctx->ifc_softc_ctx.isc_nrxd[rxq->ifr_fl_offset]))) != 0) {
 4961                         device_printf(ctx->ifc_dev, "LRO Initialization failed!\n");
 4962                         goto fail;
 4963                 }
 4964                 rxq->ifr_lro_enabled = TRUE;
 4965 #endif
 4966                 IFDI_RXQ_SETUP(ctx, rxq->ifr_id);
 4967         }
 4968         return (0);
 4969 #if defined(INET6) || defined(INET)
 4970 fail:
 4971         /*
 4972          * Free RX software descriptors allocated so far, we will only handle
 4973          * the rings that completed, the failing case will have
 4974          * cleaned up for itself. 'q' failed, so its the terminus.
 4975          */
 4976         rxq = ctx->ifc_rxqs;
 4977         for (i = 0; i < q; ++i, rxq++) {
 4978                 iflib_rx_sds_free(rxq);
 4979                 rxq->ifr_cq_gen = rxq->ifr_cq_cidx = rxq->ifr_cq_pidx = 0;
 4980         }
 4981         return (err);
 4982 #endif
 4983 }
 4984 
 4985 /*********************************************************************
 4986  *
 4987  *  Free all receive rings.
 4988  *
 4989  **********************************************************************/
 4990 static void
 4991 iflib_rx_structures_free(if_ctx_t ctx)
 4992 {
 4993         iflib_rxq_t rxq = ctx->ifc_rxqs;
 4994 
 4995         for (int i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) {
 4996                 iflib_rx_sds_free(rxq);
 4997         }
 4998 }
 4999 
 5000 static int
 5001 iflib_qset_structures_setup(if_ctx_t ctx)
 5002 {
 5003         int err;
 5004 
 5005         /*
 5006          * It is expected that the caller takes care of freeing queues if this
 5007          * fails.
 5008          */
 5009         if ((err = iflib_tx_structures_setup(ctx)) != 0) {
 5010                 device_printf(ctx->ifc_dev, "iflib_tx_structures_setup failed: %d\n", err);
 5011                 return (err);
 5012         }
 5013 
 5014         if ((err = iflib_rx_structures_setup(ctx)) != 0)
 5015                 device_printf(ctx->ifc_dev, "iflib_rx_structures_setup failed: %d\n", err);
 5016 
 5017         return (err);
 5018 }
 5019 
 5020 int
 5021 iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid,
 5022                                 driver_filter_t filter, void *filter_arg, driver_intr_t handler, void *arg, char *name)
 5023 {
 5024 
 5025         return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name));
 5026 }
 5027 
 5028 #ifdef SMP
 5029 static int
 5030 find_nth(if_ctx_t ctx, int qid)
 5031 {
 5032         cpuset_t cpus;
 5033         int i, cpuid, eqid, count;
 5034 
 5035         CPU_COPY(&ctx->ifc_cpus, &cpus);
 5036         count = CPU_COUNT(&cpus);
 5037         eqid = qid % count;
 5038         /* clear up to the qid'th bit */
 5039         for (i = 0; i < eqid; i++) {
 5040                 cpuid = CPU_FFS(&cpus);
 5041                 MPASS(cpuid != 0);
 5042                 CPU_CLR(cpuid-1, &cpus);
 5043         }
 5044         cpuid = CPU_FFS(&cpus);
 5045         MPASS(cpuid != 0);
 5046         return (cpuid-1);
 5047 }
 5048 
 5049 #ifdef SCHED_ULE
 5050 extern struct cpu_group *cpu_top;              /* CPU topology */
 5051 
 5052 static int
 5053 find_child_with_core(int cpu, struct cpu_group *grp)
 5054 {
 5055         int i;
 5056 
 5057         if (grp->cg_children == 0)
 5058                 return -1;
 5059 
 5060         MPASS(grp->cg_child);
 5061         for (i = 0; i < grp->cg_children; i++) {
 5062                 if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask))
 5063                         return i;
 5064         }
 5065 
 5066         return -1;
 5067 }
 5068 
 5069 /*
 5070  * Find the nth "close" core to the specified core
 5071  * "close" is defined as the deepest level that shares
 5072  * at least an L2 cache.  With threads, this will be
 5073  * threads on the same core.  If the sahred cache is L3
 5074  * or higher, simply returns the same core.
 5075  */
 5076 static int
 5077 find_close_core(int cpu, int core_offset)
 5078 {
 5079         struct cpu_group *grp;
 5080         int i;
 5081         int fcpu;
 5082         cpuset_t cs;
 5083 
 5084         grp = cpu_top;
 5085         if (grp == NULL)
 5086                 return cpu;
 5087         i = 0;
 5088         while ((i = find_child_with_core(cpu, grp)) != -1) {
 5089                 /* If the child only has one cpu, don't descend */
 5090                 if (grp->cg_child[i].cg_count <= 1)
 5091                         break;
 5092                 grp = &grp->cg_child[i];
 5093         }
 5094 
 5095         /* If they don't share at least an L2 cache, use the same CPU */
 5096         if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE)
 5097                 return cpu;
 5098 
 5099         /* Now pick one */
 5100         CPU_COPY(&grp->cg_mask, &cs);
 5101 
 5102         /* Add the selected CPU offset to core offset. */
 5103         for (i = 0; (fcpu = CPU_FFS(&cs)) != 0; i++) {
 5104                 if (fcpu - 1 == cpu)
 5105                         break;
 5106                 CPU_CLR(fcpu - 1, &cs);
 5107         }
 5108         MPASS(fcpu);
 5109 
 5110         core_offset += i;
 5111 
 5112         CPU_COPY(&grp->cg_mask, &cs);
 5113         for (i = core_offset % grp->cg_count; i > 0; i--) {
 5114                 MPASS(CPU_FFS(&cs));
 5115                 CPU_CLR(CPU_FFS(&cs) - 1, &cs);
 5116         }
 5117         MPASS(CPU_FFS(&cs));
 5118         return CPU_FFS(&cs) - 1;
 5119 }
 5120 #else
 5121 static int
 5122 find_close_core(int cpu, int core_offset __unused)
 5123 {
 5124         return cpu;
 5125 }
 5126 #endif
 5127 
 5128 static int
 5129 get_core_offset(if_ctx_t ctx, iflib_intr_type_t type, int qid)
 5130 {
 5131         switch (type) {
 5132         case IFLIB_INTR_TX:
 5133                 /* TX queues get cores which share at least an L2 cache with the corresponding RX queue */
 5134                 /* XXX handle multiple RX threads per core and more than two core per L2 group */
 5135                 return qid / CPU_COUNT(&ctx->ifc_cpus) + 1;
 5136         case IFLIB_INTR_RX:
 5137         case IFLIB_INTR_RXTX:
 5138                 /* RX queues get the specified core */
 5139                 return qid / CPU_COUNT(&ctx->ifc_cpus);
 5140         default:
 5141                 return -1;
 5142         }
 5143 }
 5144 #else
 5145 #define get_core_offset(ctx, type, qid) CPU_FIRST()
 5146 #define find_close_core(cpuid, tid)     CPU_FIRST()
 5147 #define find_nth(ctx, gid)              CPU_FIRST()
 5148 #endif
 5149 
 5150 /* Just to avoid copy/paste */
 5151 static inline int
 5152 iflib_irq_set_affinity(if_ctx_t ctx, int irq, iflib_intr_type_t type, int qid,
 5153     struct grouptask *gtask, struct taskqgroup *tqg, void *uniq, char *name)
 5154 {
 5155         int cpuid;
 5156         int err, tid;
 5157 
 5158         cpuid = find_nth(ctx, qid);
 5159         tid = get_core_offset(ctx, type, qid);
 5160         MPASS(tid >= 0);
 5161         cpuid = find_close_core(cpuid, tid);
 5162         err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, irq, name);
 5163         if (err) {
 5164                 device_printf(ctx->ifc_dev, "taskqgroup_attach_cpu failed %d\n", err);
 5165                 return (err);
 5166         }
 5167 #ifdef notyet
 5168         if (cpuid > ctx->ifc_cpuid_highest)
 5169                 ctx->ifc_cpuid_highest = cpuid;
 5170 #endif
 5171         return 0;
 5172 }
 5173 
 5174 int
 5175 iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid,
 5176                                                 iflib_intr_type_t type, driver_filter_t *filter,
 5177                                                 void *filter_arg, int qid, char *name)
 5178 {
 5179         struct grouptask *gtask;
 5180         struct taskqgroup *tqg;
 5181         iflib_filter_info_t info;
 5182         gtask_fn_t *fn;
 5183         int tqrid, err;
 5184         driver_filter_t *intr_fast;
 5185         void *q;
 5186 
 5187         info = &ctx->ifc_filter_info;
 5188         tqrid = rid;
 5189 
 5190         switch (type) {
 5191         /* XXX merge tx/rx for netmap? */
 5192         case IFLIB_INTR_TX:
 5193                 q = &ctx->ifc_txqs[qid];
 5194                 info = &ctx->ifc_txqs[qid].ift_filter_info;
 5195                 gtask = &ctx->ifc_txqs[qid].ift_task;
 5196                 tqg = qgroup_if_io_tqg;
 5197                 fn = _task_fn_tx;
 5198                 intr_fast = iflib_fast_intr;
 5199                 GROUPTASK_INIT(gtask, 0, fn, q);
 5200                 break;
 5201         case IFLIB_INTR_RX:
 5202                 q = &ctx->ifc_rxqs[qid];
 5203                 info = &ctx->ifc_rxqs[qid].ifr_filter_info;
 5204                 gtask = &ctx->ifc_rxqs[qid].ifr_task;
 5205                 tqg = qgroup_if_io_tqg;
 5206                 fn = _task_fn_rx;
 5207                 intr_fast = iflib_fast_intr;
 5208                 GROUPTASK_INIT(gtask, 0, fn, q);
 5209                 break;
 5210         case IFLIB_INTR_RXTX:
 5211                 q = &ctx->ifc_rxqs[qid];
 5212                 info = &ctx->ifc_rxqs[qid].ifr_filter_info;
 5213                 gtask = &ctx->ifc_rxqs[qid].ifr_task;
 5214                 tqg = qgroup_if_io_tqg;
 5215                 fn = _task_fn_rx;
 5216                 intr_fast = iflib_fast_intr_rxtx;
 5217                 GROUPTASK_INIT(gtask, 0, fn, q);
 5218                 break;
 5219         case IFLIB_INTR_ADMIN:
 5220                 q = ctx;
 5221                 tqrid = -1;
 5222                 info = &ctx->ifc_filter_info;
 5223                 gtask = &ctx->ifc_admin_task;
 5224                 tqg = qgroup_if_config_tqg;
 5225                 fn = _task_fn_admin;
 5226                 intr_fast = iflib_fast_intr_ctx;
 5227                 break;
 5228         default:
 5229                 panic("unknown net intr type");
 5230         }
 5231 
 5232         info->ifi_filter = filter;
 5233         info->ifi_filter_arg = filter_arg;
 5234         info->ifi_task = gtask;
 5235         info->ifi_ctx = q;
 5236 
 5237         err = _iflib_irq_alloc(ctx, irq, rid, intr_fast, NULL, info,  name);
 5238         if (err != 0) {
 5239                 device_printf(ctx->ifc_dev, "_iflib_irq_alloc failed %d\n", err);
 5240                 return (err);
 5241         }
 5242         if (type == IFLIB_INTR_ADMIN)
 5243                 return (0);
 5244 
 5245         if (tqrid != -1) {
 5246                 err = iflib_irq_set_affinity(ctx, rman_get_start(irq->ii_res), type, qid, gtask, tqg, q, name);
 5247                 if (err)
 5248                         return (err);
 5249         } else {
 5250                 taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name);
 5251         }
 5252 
 5253         return (0);
 5254 }
 5255 
 5256 void
 5257 iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type,  void *arg, int qid, char *name)
 5258 {
 5259         struct grouptask *gtask;
 5260         struct taskqgroup *tqg;
 5261         gtask_fn_t *fn;
 5262         void *q;
 5263         int irq_num = -1;
 5264         int err;
 5265 
 5266         switch (type) {
 5267         case IFLIB_INTR_TX:
 5268                 q = &ctx->ifc_txqs[qid];
 5269                 gtask = &ctx->ifc_txqs[qid].ift_task;
 5270                 tqg = qgroup_if_io_tqg;
 5271                 fn = _task_fn_tx;
 5272                 if (irq != NULL)
 5273                         irq_num = rman_get_start(irq->ii_res);
 5274                 break;
 5275         case IFLIB_INTR_RX:
 5276                 q = &ctx->ifc_rxqs[qid];
 5277                 gtask = &ctx->ifc_rxqs[qid].ifr_task;
 5278                 tqg = qgroup_if_io_tqg;
 5279                 fn = _task_fn_rx;
 5280                 if (irq != NULL)
 5281                         irq_num = rman_get_start(irq->ii_res);
 5282                 break;
 5283         case IFLIB_INTR_IOV:
 5284                 q = ctx;
 5285                 gtask = &ctx->ifc_vflr_task;
 5286                 tqg = qgroup_if_config_tqg;
 5287                 fn = _task_fn_iov;
 5288                 break;
 5289         default:
 5290                 panic("unknown net intr type");
 5291         }
 5292         GROUPTASK_INIT(gtask, 0, fn, q);
 5293         if (irq_num != -1) {
 5294                 err = iflib_irq_set_affinity(ctx, irq_num, type, qid, gtask, tqg, q, name);
 5295                 if (err)
 5296                         taskqgroup_attach(tqg, gtask, q, irq_num, name);
 5297         }
 5298         else {
 5299                 taskqgroup_attach(tqg, gtask, q, irq_num, name);
 5300         }
 5301 }
 5302 
 5303 void
 5304 iflib_irq_free(if_ctx_t ctx, if_irq_t irq)
 5305 {
 5306         if (irq->ii_tag)
 5307                 bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag);
 5308 
 5309         if (irq->ii_res)
 5310                 bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, irq->ii_rid, irq->ii_res);
 5311 }
 5312 
 5313 static int
 5314 iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int *rid, char *name)
 5315 {
 5316         iflib_txq_t txq = ctx->ifc_txqs;
 5317         iflib_rxq_t rxq = ctx->ifc_rxqs;
 5318         if_irq_t irq = &ctx->ifc_legacy_irq;
 5319         iflib_filter_info_t info;
 5320         struct grouptask *gtask;
 5321         struct taskqgroup *tqg;
 5322         gtask_fn_t *fn;
 5323         int tqrid;
 5324         void *q;
 5325         int err;
 5326 
 5327         q = &ctx->ifc_rxqs[0];
 5328         info = &rxq[0].ifr_filter_info;
 5329         gtask = &rxq[0].ifr_task;
 5330         tqg = qgroup_if_io_tqg;
 5331         tqrid = irq->ii_rid = *rid;
 5332         fn = _task_fn_rx;
 5333 
 5334         ctx->ifc_flags |= IFC_LEGACY;
 5335         info->ifi_filter = filter;
 5336         info->ifi_filter_arg = filter_arg;
 5337         info->ifi_task = gtask;
 5338 
 5339         /* We allocate a single interrupt resource */
 5340         if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_ctx, NULL, info, name)) != 0)
 5341                 return (err);
 5342         GROUPTASK_INIT(gtask, 0, fn, q);
 5343         taskqgroup_attach(tqg, gtask, q, rman_get_start(irq->ii_res), name);
 5344 
 5345         GROUPTASK_INIT(&txq->ift_task, 0, _task_fn_tx, txq);
 5346         taskqgroup_attach(qgroup_if_io_tqg, &txq->ift_task, txq, rman_get_start(irq->ii_res), "tx");
 5347         return (0);
 5348 }
 5349 
 5350 void
 5351 iflib_led_create(if_ctx_t ctx)
 5352 {
 5353 
 5354         ctx->ifc_led_dev = led_create(iflib_led_func, ctx,
 5355             device_get_nameunit(ctx->ifc_dev));
 5356 }
 5357 
 5358 void
 5359 iflib_tx_intr_deferred(if_ctx_t ctx, int txqid)
 5360 {
 5361 
 5362         GROUPTASK_ENQUEUE(&ctx->ifc_txqs[txqid].ift_task);
 5363 }
 5364 
 5365 void
 5366 iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid)
 5367 {
 5368 
 5369         GROUPTASK_ENQUEUE(&ctx->ifc_rxqs[rxqid].ifr_task);
 5370 }
 5371 
 5372 void
 5373 iflib_admin_intr_deferred(if_ctx_t ctx)
 5374 {
 5375 #ifdef INVARIANTS
 5376         struct grouptask *gtask;
 5377 
 5378         gtask = &ctx->ifc_admin_task;
 5379         MPASS(gtask != NULL && gtask->gt_taskqueue != NULL);
 5380 #endif
 5381 
 5382         GROUPTASK_ENQUEUE(&ctx->ifc_admin_task);
 5383 }
 5384 
 5385 void
 5386 iflib_iov_intr_deferred(if_ctx_t ctx)
 5387 {
 5388 
 5389         GROUPTASK_ENQUEUE(&ctx->ifc_vflr_task);
 5390 }
 5391 
 5392 void
 5393 iflib_io_tqg_attach(struct grouptask *gt, void *uniq, int cpu, char *name)
 5394 {
 5395 
 5396         taskqgroup_attach_cpu(qgroup_if_io_tqg, gt, uniq, cpu, -1, name);
 5397 }
 5398 
 5399 void
 5400 iflib_config_gtask_init(if_ctx_t ctx, struct grouptask *gtask, gtask_fn_t *fn,
 5401         char *name)
 5402 {
 5403 
 5404         GROUPTASK_INIT(gtask, 0, fn, ctx);
 5405         taskqgroup_attach(qgroup_if_config_tqg, gtask, gtask, -1, name);
 5406 }
 5407 
 5408 void
 5409 iflib_config_gtask_deinit(struct grouptask *gtask)
 5410 {
 5411 
 5412         taskqgroup_detach(qgroup_if_config_tqg, gtask); 
 5413 }
 5414 
 5415 void
 5416 iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate)
 5417 {
 5418         if_t ifp = ctx->ifc_ifp;
 5419         iflib_txq_t txq = ctx->ifc_txqs;
 5420 
 5421         if_setbaudrate(ifp, baudrate);
 5422         if (baudrate >= IF_Gbps(10))
 5423                 ctx->ifc_flags |= IFC_PREFETCH;
 5424 
 5425         /* If link down, disable watchdog */
 5426         if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) {
 5427                 for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++)
 5428                         txq->ift_qstatus = IFLIB_QUEUE_IDLE;
 5429         }
 5430         ctx->ifc_link_state = link_state;
 5431         if_link_state_change(ifp, link_state);
 5432 }
 5433 
 5434 static int
 5435 iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq)
 5436 {
 5437         int credits;
 5438 #ifdef INVARIANTS
 5439         int credits_pre = txq->ift_cidx_processed;
 5440 #endif
 5441 
 5442         if (ctx->isc_txd_credits_update == NULL)
 5443                 return (0);
 5444 
 5445         if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0)
 5446                 return (0);
 5447 
 5448         txq->ift_processed += credits;
 5449         txq->ift_cidx_processed += credits;
 5450 
 5451         MPASS(credits_pre + credits == txq->ift_cidx_processed);
 5452         if (txq->ift_cidx_processed >= txq->ift_size)
 5453                 txq->ift_cidx_processed -= txq->ift_size;
 5454         return (credits);
 5455 }
 5456 
 5457 static int
 5458 iflib_rxd_avail(if_ctx_t ctx, iflib_rxq_t rxq, qidx_t cidx, qidx_t budget)
 5459 {
 5460 
 5461         return (ctx->isc_rxd_available(ctx->ifc_softc, rxq->ifr_id, cidx,
 5462             budget));
 5463 }
 5464 
 5465 void
 5466 iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name,
 5467         const char *description, if_int_delay_info_t info,
 5468         int offset, int value)
 5469 {
 5470         info->iidi_ctx = ctx;
 5471         info->iidi_offset = offset;
 5472         info->iidi_value = value;
 5473         SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev),
 5474             SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)),
 5475             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
 5476             info, 0, iflib_sysctl_int_delay, "I", description);
 5477 }
 5478 
 5479 struct mtx *
 5480 iflib_ctx_lock_get(if_ctx_t ctx)
 5481 {
 5482 
 5483         return (&ctx->ifc_mtx);
 5484 }
 5485 
 5486 static int
 5487 iflib_msix_init(if_ctx_t ctx)
 5488 {
 5489         device_t dev = ctx->ifc_dev;
 5490         if_shared_ctx_t sctx = ctx->ifc_sctx;
 5491         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 5492         int vectors, queues, rx_queues, tx_queues, queuemsgs, msgs;
 5493         int iflib_num_tx_queues, iflib_num_rx_queues;
 5494         int err, admincnt, bar;
 5495 
 5496         iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs;
 5497         iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs;
 5498 
 5499         device_printf(dev, "msix_init qsets capped at %d\n", imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets));
 5500 
 5501         bar = ctx->ifc_softc_ctx.isc_msix_bar;
 5502         admincnt = sctx->isc_admin_intrcnt;
 5503         /* Override by global tuneable */
 5504         {
 5505                 int i;
 5506                 size_t len = sizeof(i);
 5507                 err = kernel_sysctlbyname(curthread, "hw.pci.enable_msix", &i, &len, NULL, 0, NULL, 0);
 5508                 if (err == 0) {
 5509                         if (i == 0)
 5510                                 goto msi;
 5511                 }
 5512                 else {
 5513                         device_printf(dev, "unable to read hw.pci.enable_msix.");
 5514                 }
 5515         }
 5516         /* Override by tuneable */
 5517         if (scctx->isc_disable_msix)
 5518                 goto msi;
 5519 
 5520         /*
 5521         ** When used in a virtualized environment
 5522         ** PCI BUSMASTER capability may not be set
 5523         ** so explicity set it here and rewrite
 5524         ** the ENABLE in the MSIX control register
 5525         ** at this point to cause the host to
 5526         ** successfully initialize us.
 5527         */
 5528         {
 5529                 int msix_ctrl, rid;
 5530 
 5531                 pci_enable_busmaster(dev);
 5532                 rid = 0;
 5533                 if (pci_find_cap(dev, PCIY_MSIX, &rid) == 0 && rid != 0) {
 5534                         rid += PCIR_MSIX_CTRL;
 5535                         msix_ctrl = pci_read_config(dev, rid, 2);
 5536                         msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
 5537                         pci_write_config(dev, rid, msix_ctrl, 2);
 5538                 } else {
 5539                         device_printf(dev, "PCIY_MSIX capability not found; "
 5540                                            "or rid %d == 0.\n", rid);
 5541                         goto msi;
 5542                 }
 5543         }
 5544 
 5545         /*
 5546          * bar == -1 => "trust me I know what I'm doing"
 5547          * Some drivers are for hardware that is so shoddily
 5548          * documented that no one knows which bars are which
 5549          * so the developer has to map all bars. This hack
 5550          * allows shoddy garbage to use msix in this framework.
 5551          */
 5552         if (bar != -1) {
 5553                 ctx->ifc_msix_mem = bus_alloc_resource_any(dev,
 5554                     SYS_RES_MEMORY, &bar, RF_ACTIVE);
 5555                 if (ctx->ifc_msix_mem == NULL) {
 5556                         /* May not be enabled */
 5557                         device_printf(dev, "Unable to map MSIX table \n");
 5558                         goto msi;
 5559                 }
 5560         }
 5561         /* First try MSI/X */
 5562         if ((msgs = pci_msix_count(dev)) == 0) { /* system has msix disabled */
 5563                 device_printf(dev, "System has MSIX disabled \n");
 5564                 bus_release_resource(dev, SYS_RES_MEMORY,
 5565                     bar, ctx->ifc_msix_mem);
 5566                 ctx->ifc_msix_mem = NULL;
 5567                 goto msi;
 5568         }
 5569 #if IFLIB_DEBUG
 5570         /* use only 1 qset in debug mode */
 5571         queuemsgs = min(msgs - admincnt, 1);
 5572 #else
 5573         queuemsgs = msgs - admincnt;
 5574 #endif
 5575 #ifdef RSS
 5576         queues = imin(queuemsgs, rss_getnumbuckets());
 5577 #else
 5578         queues = queuemsgs;
 5579 #endif
 5580         queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues);
 5581         device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n",
 5582                                   CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt);
 5583 #ifdef  RSS
 5584         /* If we're doing RSS, clamp at the number of RSS buckets */
 5585         if (queues > rss_getnumbuckets())
 5586                 queues = rss_getnumbuckets();
 5587 #endif
 5588         if (iflib_num_rx_queues > 0 && iflib_num_rx_queues < queuemsgs - admincnt)
 5589                 rx_queues = iflib_num_rx_queues;
 5590         else
 5591                 rx_queues = queues;
 5592 
 5593         if (rx_queues > scctx->isc_nrxqsets)
 5594                 rx_queues = scctx->isc_nrxqsets;
 5595 
 5596         /*
 5597          * We want this to be all logical CPUs by default
 5598          */
 5599         if (iflib_num_tx_queues > 0 && iflib_num_tx_queues < queues)
 5600                 tx_queues = iflib_num_tx_queues;
 5601         else
 5602                 tx_queues = mp_ncpus;
 5603 
 5604         if (tx_queues > scctx->isc_ntxqsets)
 5605                 tx_queues = scctx->isc_ntxqsets;
 5606 
 5607         if (ctx->ifc_sysctl_qs_eq_override == 0) {
 5608 #ifdef INVARIANTS
 5609                 if (tx_queues != rx_queues)
 5610                         device_printf(dev, "queue equality override not set, capping rx_queues at %d and tx_queues at %d\n",
 5611                                       min(rx_queues, tx_queues), min(rx_queues, tx_queues));
 5612 #endif
 5613                 tx_queues = min(rx_queues, tx_queues);
 5614                 rx_queues = min(rx_queues, tx_queues);
 5615         }
 5616 
 5617         device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues);
 5618 
 5619         vectors = rx_queues + admincnt;
 5620         if ((err = pci_alloc_msix(dev, &vectors)) == 0) {
 5621                 device_printf(dev,
 5622                                           "Using MSIX interrupts with %d vectors\n", vectors);
 5623                 scctx->isc_vectors = vectors;
 5624                 scctx->isc_nrxqsets = rx_queues;
 5625                 scctx->isc_ntxqsets = tx_queues;
 5626                 scctx->isc_intr = IFLIB_INTR_MSIX;
 5627 
 5628                 return (vectors);
 5629         } else {
 5630                 device_printf(dev, "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err);
 5631         }
 5632 msi:
 5633         vectors = pci_msi_count(dev);
 5634         scctx->isc_nrxqsets = 1;
 5635         scctx->isc_ntxqsets = 1;
 5636         scctx->isc_vectors = vectors;
 5637         if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) {
 5638                 device_printf(dev,"Using an MSI interrupt\n");
 5639                 scctx->isc_intr = IFLIB_INTR_MSI;
 5640         } else {
 5641                 device_printf(dev,"Using a Legacy interrupt\n");
 5642                 scctx->isc_intr = IFLIB_INTR_LEGACY;
 5643         }
 5644 
 5645         return (vectors);
 5646 }
 5647 
 5648 char * ring_states[] = { "IDLE", "BUSY", "STALLED", "ABDICATED" };
 5649 
 5650 static int
 5651 mp_ring_state_handler(SYSCTL_HANDLER_ARGS)
 5652 {
 5653         int rc;
 5654         uint16_t *state = ((uint16_t *)oidp->oid_arg1);
 5655         struct sbuf *sb;
 5656         char *ring_state = "UNKNOWN";
 5657 
 5658         /* XXX needed ? */
 5659         rc = sysctl_wire_old_buffer(req, 0);
 5660         MPASS(rc == 0);
 5661         if (rc != 0)
 5662                 return (rc);
 5663         sb = sbuf_new_for_sysctl(NULL, NULL, 80, req);
 5664         MPASS(sb != NULL);
 5665         if (sb == NULL)
 5666                 return (ENOMEM);
 5667         if (state[3] <= 3)
 5668                 ring_state = ring_states[state[3]];
 5669 
 5670         sbuf_printf(sb, "pidx_head: %04hd pidx_tail: %04hd cidx: %04hd state: %s",
 5671                     state[0], state[1], state[2], ring_state);
 5672         rc = sbuf_finish(sb);
 5673         sbuf_delete(sb);
 5674         return(rc);
 5675 }
 5676 
 5677 enum iflib_ndesc_handler {
 5678         IFLIB_NTXD_HANDLER,
 5679         IFLIB_NRXD_HANDLER,
 5680 };
 5681 
 5682 static int
 5683 mp_ndesc_handler(SYSCTL_HANDLER_ARGS)
 5684 {
 5685         if_ctx_t ctx = (void *)arg1;
 5686         enum iflib_ndesc_handler type = arg2;
 5687         char buf[256] = {0};
 5688         qidx_t *ndesc;
 5689         char *p, *next;
 5690         int nqs, rc, i;
 5691 
 5692         MPASS(type == IFLIB_NTXD_HANDLER || type == IFLIB_NRXD_HANDLER);
 5693 
 5694         nqs = 8;
 5695         switch(type) {
 5696         case IFLIB_NTXD_HANDLER:
 5697                 ndesc = ctx->ifc_sysctl_ntxds;
 5698                 if (ctx->ifc_sctx)
 5699                         nqs = ctx->ifc_sctx->isc_ntxqs;
 5700                 break;
 5701         case IFLIB_NRXD_HANDLER:
 5702                 ndesc = ctx->ifc_sysctl_nrxds;
 5703                 if (ctx->ifc_sctx)
 5704                         nqs = ctx->ifc_sctx->isc_nrxqs;
 5705                 break;
 5706         }
 5707         if (nqs == 0)
 5708                 nqs = 8;
 5709 
 5710         for (i=0; i<8; i++) {
 5711                 if (i >= nqs)
 5712                         break;
 5713                 if (i)
 5714                         strcat(buf, ",");
 5715                 sprintf(strchr(buf, 0), "%d", ndesc[i]);
 5716         }
 5717 
 5718         rc = sysctl_handle_string(oidp, buf, sizeof(buf), req);
 5719         if (rc || req->newptr == NULL)
 5720                 return rc;
 5721 
 5722         for (i = 0, next = buf, p = strsep(&next, " ,"); i < 8 && p;
 5723             i++, p = strsep(&next, " ,")) {
 5724                 ndesc[i] = strtoul(p, NULL, 10);
 5725         }
 5726 
 5727         return(rc);
 5728 }
 5729 
 5730 #define NAME_BUFLEN 32
 5731 static void
 5732 iflib_add_device_sysctl_pre(if_ctx_t ctx)
 5733 {
 5734         device_t dev = iflib_get_dev(ctx);
 5735         struct sysctl_oid_list *child, *oid_list;
 5736         struct sysctl_ctx_list *ctx_list;
 5737         struct sysctl_oid *node;
 5738 
 5739         ctx_list = device_get_sysctl_ctx(dev);
 5740         child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
 5741         ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "iflib",
 5742                                                       CTLFLAG_RD, NULL, "IFLIB fields");
 5743         oid_list = SYSCTL_CHILDREN(node);
 5744 
 5745         SYSCTL_ADD_STRING(ctx_list, oid_list, OID_AUTO, "driver_version",
 5746                        CTLFLAG_RD, ctx->ifc_sctx->isc_driver_version, 0,
 5747                        "driver version");
 5748 
 5749         SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_ntxqs",
 5750                        CTLFLAG_RWTUN, &ctx->ifc_sysctl_ntxqs, 0,
 5751                         "# of txqs to use, 0 => use default #");
 5752         SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_nrxqs",
 5753                        CTLFLAG_RWTUN, &ctx->ifc_sysctl_nrxqs, 0,
 5754                         "# of rxqs to use, 0 => use default #");
 5755         SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "override_qs_enable",
 5756                        CTLFLAG_RWTUN, &ctx->ifc_sysctl_qs_eq_override, 0,
 5757                        "permit #txq != #rxq");
 5758         SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix",
 5759                       CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0,
 5760                       "disable MSIX (default 0)");
 5761         SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget",
 5762                        CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0,
 5763                        "set the rx budget");
 5764 
 5765         /* XXX change for per-queue sizes */
 5766         SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds",
 5767                        CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER,
 5768                        mp_ndesc_handler, "A",
 5769                        "list of # of tx descriptors to use, 0 = use default #");
 5770         SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds",
 5771                        CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER,
 5772                        mp_ndesc_handler, "A",
 5773                        "list of # of rx descriptors to use, 0 = use default #");
 5774 }
 5775 
 5776 static void
 5777 iflib_add_device_sysctl_post(if_ctx_t ctx)
 5778 {
 5779         if_shared_ctx_t sctx = ctx->ifc_sctx;
 5780         if_softc_ctx_t scctx = &ctx->ifc_softc_ctx;
 5781         device_t dev = iflib_get_dev(ctx);
 5782         struct sysctl_oid_list *child;
 5783         struct sysctl_ctx_list *ctx_list;
 5784         iflib_fl_t fl;
 5785         iflib_txq_t txq;
 5786         iflib_rxq_t rxq;
 5787         int i, j;
 5788         char namebuf[NAME_BUFLEN];
 5789         char *qfmt;
 5790         struct sysctl_oid *queue_node, *fl_node, *node;
 5791         struct sysctl_oid_list *queue_list, *fl_list;
 5792         ctx_list = device_get_sysctl_ctx(dev);
 5793 
 5794         node = ctx->ifc_sysctl_node;
 5795         child = SYSCTL_CHILDREN(node);
 5796 
 5797         if (scctx->isc_ntxqsets > 100)
 5798                 qfmt = "txq%03d";
 5799         else if (scctx->isc_ntxqsets > 10)
 5800                 qfmt = "txq%02d";
 5801         else
 5802                 qfmt = "txq%d";
 5803         for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) {
 5804                 snprintf(namebuf, NAME_BUFLEN, qfmt, i);
 5805                 queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
 5806                                              CTLFLAG_RD, NULL, "Queue Name");
 5807                 queue_list = SYSCTL_CHILDREN(queue_node);
 5808 #if MEMORY_LOGGING
 5809                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_dequeued",
 5810                                 CTLFLAG_RD,
 5811                                 &txq->ift_dequeued, "total mbufs freed");
 5812                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_enqueued",
 5813                                 CTLFLAG_RD,
 5814                                 &txq->ift_enqueued, "total mbufs enqueued");
 5815 #endif
 5816                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag",
 5817                                    CTLFLAG_RD,
 5818                                    &txq->ift_mbuf_defrag, "# of times m_defrag was called");
 5819                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "m_pullups",
 5820                                    CTLFLAG_RD,
 5821                                    &txq->ift_pullups, "# of times m_pullup was called");
 5822                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "mbuf_defrag_failed",
 5823                                    CTLFLAG_RD,
 5824                                    &txq->ift_mbuf_defrag_failed, "# of times m_defrag failed");
 5825                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_desc_avail",
 5826                                    CTLFLAG_RD,
 5827                                    &txq->ift_no_desc_avail, "# of times no descriptors were available");
 5828                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "tx_map_failed",
 5829                                    CTLFLAG_RD,
 5830                                    &txq->ift_map_failed, "# of times dma map failed");
 5831                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txd_encap_efbig",
 5832                                    CTLFLAG_RD,
 5833                                    &txq->ift_txd_encap_efbig, "# of times txd_encap returned EFBIG");
 5834                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "no_tx_dma_setup",
 5835                                    CTLFLAG_RD,
 5836                                    &txq->ift_no_tx_dma_setup, "# of times map failed for other than EFBIG");
 5837                 SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_pidx",
 5838                                    CTLFLAG_RD,
 5839                                    &txq->ift_pidx, 1, "Producer Index");
 5840                 SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx",
 5841                                    CTLFLAG_RD,
 5842                                    &txq->ift_cidx, 1, "Consumer Index");
 5843                 SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_cidx_processed",
 5844                                    CTLFLAG_RD,
 5845                                    &txq->ift_cidx_processed, 1, "Consumer Index seen by credit update");
 5846                 SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "txq_in_use",
 5847                                    CTLFLAG_RD,
 5848                                    &txq->ift_in_use, 1, "descriptors in use");
 5849                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_processed",
 5850                                    CTLFLAG_RD,
 5851                                    &txq->ift_processed, "descriptors procesed for clean");
 5852                 SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_cleaned",
 5853                                    CTLFLAG_RD,
 5854                                    &txq->ift_cleaned, "total cleaned");
 5855                 SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state",
 5856                                 CTLTYPE_STRING | CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br->state),
 5857                                 0, mp_ring_state_handler, "A", "soft ring state");
 5858                 SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues",
 5859                                        CTLFLAG_RD, &txq->ift_br->enqueues,
 5860                                        "# of enqueues to the mp_ring for this queue");
 5861                 SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_drops",
 5862                                        CTLFLAG_RD, &txq->ift_br->drops,
 5863                                        "# of drops in the mp_ring for this queue");
 5864                 SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_starts",
 5865                                        CTLFLAG_RD, &txq->ift_br->starts,
 5866                                        "# of normal consumer starts in the mp_ring for this queue");
 5867                 SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_stalls",
 5868                                        CTLFLAG_RD, &txq->ift_br->stalls,
 5869                                                "# of consumer stalls in the mp_ring for this queue");
 5870                 SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_restarts",
 5871                                CTLFLAG_RD, &txq->ift_br->restarts,
 5872                                        "# of consumer restarts in the mp_ring for this queue");
 5873                 SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_abdications",
 5874                                        CTLFLAG_RD, &txq->ift_br->abdications,
 5875                                        "# of consumer abdications in the mp_ring for this queue");
 5876         }
 5877 
 5878         if (scctx->isc_nrxqsets > 100)
 5879                 qfmt = "rxq%03d";
 5880         else if (scctx->isc_nrxqsets > 10)
 5881                 qfmt = "rxq%02d";
 5882         else
 5883                 qfmt = "rxq%d";
 5884         for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) {
 5885                 snprintf(namebuf, NAME_BUFLEN, qfmt, i);
 5886                 queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf,
 5887                                              CTLFLAG_RD, NULL, "Queue Name");
 5888                 queue_list = SYSCTL_CHILDREN(queue_node);
 5889                 if (sctx->isc_flags & IFLIB_HAS_RXCQ) {
 5890                         SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_pidx",
 5891                                        CTLFLAG_RD,
 5892                                        &rxq->ifr_cq_pidx, 1, "Producer Index");
 5893                         SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_cidx",
 5894                                        CTLFLAG_RD,
 5895                                        &rxq->ifr_cq_cidx, 1, "Consumer Index");
 5896                 }
 5897 
 5898                 for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) {
 5899                         snprintf(namebuf, NAME_BUFLEN, "rxq_fl%d", j);
 5900                         fl_node = SYSCTL_ADD_NODE(ctx_list, queue_list, OID_AUTO, namebuf,
 5901                                                      CTLFLAG_RD, NULL, "freelist Name");
 5902                         fl_list = SYSCTL_CHILDREN(fl_node);
 5903                         SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "pidx",
 5904                                        CTLFLAG_RD,
 5905                                        &fl->ifl_pidx, 1, "Producer Index");
 5906                         SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "cidx",
 5907                                        CTLFLAG_RD,
 5908                                        &fl->ifl_cidx, 1, "Consumer Index");
 5909                         SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "credits",
 5910                                        CTLFLAG_RD,
 5911                                        &fl->ifl_credits, 1, "credits available");
 5912 #if MEMORY_LOGGING
 5913                         SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_enqueued",
 5914                                         CTLFLAG_RD,
 5915                                         &fl->ifl_m_enqueued, "mbufs allocated");
 5916                         SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_dequeued",
 5917                                         CTLFLAG_RD,
 5918                                         &fl->ifl_m_dequeued, "mbufs freed");
 5919                         SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_enqueued",
 5920                                         CTLFLAG_RD,
 5921                                         &fl->ifl_cl_enqueued, "clusters allocated");
 5922                         SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_cl_dequeued",
 5923                                         CTLFLAG_RD,
 5924                                         &fl->ifl_cl_dequeued, "clusters freed");
 5925 #endif
 5926 
 5927                 }
 5928         }
 5929 
 5930 }
 5931 
 5932 #ifndef __NO_STRICT_ALIGNMENT
 5933 static struct mbuf *
 5934 iflib_fixup_rx(struct mbuf *m)
 5935 {
 5936         struct mbuf *n;
 5937 
 5938         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
 5939                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
 5940                 m->m_data += ETHER_HDR_LEN;
 5941                 n = m;
 5942         } else {
 5943                 MGETHDR(n, M_NOWAIT, MT_DATA);
 5944                 if (n == NULL) {
 5945                         m_freem(m);
 5946                         return (NULL);
 5947                 }
 5948                 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
 5949                 m->m_data += ETHER_HDR_LEN;
 5950                 m->m_len -= ETHER_HDR_LEN;
 5951                 n->m_len = ETHER_HDR_LEN;
 5952                 M_MOVE_PKTHDR(n, m);
 5953                 n->m_next = m;
 5954         }
 5955         return (n);
 5956 }
 5957 #endif

Cache object: 5c610246772b05f1ef929ad073922502


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.