The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/e1000/if_igb.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /******************************************************************************
    2 
    3   Copyright (c) 2001-2015, Intel Corporation 
    4   All rights reserved.
    5   
    6   Redistribution and use in source and binary forms, with or without 
    7   modification, are permitted provided that the following conditions are met:
    8   
    9    1. Redistributions of source code must retain the above copyright notice, 
   10       this list of conditions and the following disclaimer.
   11   
   12    2. Redistributions in binary form must reproduce the above copyright 
   13       notice, this list of conditions and the following disclaimer in the 
   14       documentation and/or other materials provided with the distribution.
   15   
   16    3. Neither the name of the Intel Corporation nor the names of its 
   17       contributors may be used to endorse or promote products derived from 
   18       this software without specific prior written permission.
   19   
   20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
   22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
   23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
   24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
   25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
   26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
   27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
   28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
   29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   30   POSSIBILITY OF SUCH DAMAGE.
   31 
   32 ******************************************************************************/
   33 /*$FreeBSD$*/
   34 
   35 
   36 #include "opt_inet.h"
   37 #include "opt_inet6.h"
   38 
   39 #ifdef HAVE_KERNEL_OPTION_HEADERS
   40 #include "opt_device_polling.h"
   41 #include "opt_altq.h"
   42 #endif
   43 
   44 #include "if_igb.h"
   45 
   46 /*********************************************************************
   47  *  Driver version:
   48  *********************************************************************/
   49 char igb_driver_version[] = "2.5.3-k";
   50 
   51 
   52 /*********************************************************************
   53  *  PCI Device ID Table
   54  *
   55  *  Used by probe to select devices to load on
   56  *  Last field stores an index into e1000_strings
   57  *  Last entry must be all 0s
   58  *
   59  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
   60  *********************************************************************/
   61 
   62 static igb_vendor_info_t igb_vendor_info_array[] =
   63 {
   64         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0},
   65         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0},
   66         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0},
   67         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0},
   68         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0},
   69         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0},
   70         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0},
   71         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0},
   72         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0},
   73         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0},
   74         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0},
   75         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0},
   76         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0},
   77         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0},
   78         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0},
   79         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0},
   80         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0},
   81         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0},
   82         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0},
   83         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0},
   84         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0},
   85         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0},
   86         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0},
   87         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER,  0, 0, 0},
   88         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0},
   89         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII,  0, 0, 0},
   90         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0},
   91         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0},
   92         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0},
   93         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0},
   94         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0},
   95         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0},
   96         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER,  0, 0, 0},
   97         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0},
   98         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII,  0, 0, 0},
   99         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0},
  100         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0},
  101         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0},
  102         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII,  0, 0, 0},
  103         /* required last entry */
  104         {0, 0, 0, 0, 0}
  105 };
  106 
  107 /*********************************************************************
  108  *  Table of branding strings for all supported NICs.
  109  *********************************************************************/
  110 
  111 static char *igb_strings[] = {
  112         "Intel(R) PRO/1000 Network Connection"
  113 };
  114 
  115 /*********************************************************************
  116  *  Function prototypes
  117  *********************************************************************/
  118 static int      igb_probe(device_t);
  119 static int      igb_attach(device_t);
  120 static int      igb_detach(device_t);
  121 static int      igb_shutdown(device_t);
  122 static int      igb_suspend(device_t);
  123 static int      igb_resume(device_t);
  124 #ifndef IGB_LEGACY_TX
  125 static int      igb_mq_start(struct ifnet *, struct mbuf *);
  126 static int      igb_mq_start_locked(struct ifnet *, struct tx_ring *);
  127 static void     igb_qflush(struct ifnet *);
  128 static void     igb_deferred_mq_start(void *, int);
  129 #else
  130 static void     igb_start(struct ifnet *);
  131 static void     igb_start_locked(struct tx_ring *, struct ifnet *ifp);
  132 #endif
  133 static int      igb_ioctl(struct ifnet *, u_long, caddr_t);
  134 static void     igb_init(void *);
  135 static void     igb_init_locked(struct adapter *);
  136 static void     igb_stop(void *);
  137 static void     igb_media_status(struct ifnet *, struct ifmediareq *);
  138 static int      igb_media_change(struct ifnet *);
  139 static void     igb_identify_hardware(struct adapter *);
  140 static int      igb_allocate_pci_resources(struct adapter *);
  141 static int      igb_allocate_msix(struct adapter *);
  142 static int      igb_allocate_legacy(struct adapter *);
  143 static int      igb_setup_msix(struct adapter *);
  144 static void     igb_free_pci_resources(struct adapter *);
  145 static void     igb_local_timer(void *);
  146 static void     igb_reset(struct adapter *);
  147 static int      igb_setup_interface(device_t, struct adapter *);
  148 static int      igb_allocate_queues(struct adapter *);
  149 static void     igb_configure_queues(struct adapter *);
  150 
  151 static int      igb_allocate_transmit_buffers(struct tx_ring *);
  152 static void     igb_setup_transmit_structures(struct adapter *);
  153 static void     igb_setup_transmit_ring(struct tx_ring *);
  154 static void     igb_initialize_transmit_units(struct adapter *);
  155 static void     igb_free_transmit_structures(struct adapter *);
  156 static void     igb_free_transmit_buffers(struct tx_ring *);
  157 
  158 static int      igb_allocate_receive_buffers(struct rx_ring *);
  159 static int      igb_setup_receive_structures(struct adapter *);
  160 static int      igb_setup_receive_ring(struct rx_ring *);
  161 static void     igb_initialize_receive_units(struct adapter *);
  162 static void     igb_free_receive_structures(struct adapter *);
  163 static void     igb_free_receive_buffers(struct rx_ring *);
  164 static void     igb_free_receive_ring(struct rx_ring *);
  165 
  166 static void     igb_enable_intr(struct adapter *);
  167 static void     igb_disable_intr(struct adapter *);
  168 static void     igb_update_stats_counters(struct adapter *);
  169 static bool     igb_txeof(struct tx_ring *);
  170 
  171 static __inline void igb_rx_discard(struct rx_ring *, int);
  172 static __inline void igb_rx_input(struct rx_ring *,
  173                     struct ifnet *, struct mbuf *, u32);
  174 
  175 static bool     igb_rxeof(struct igb_queue *, int, int *);
  176 static void     igb_rx_checksum(u32, struct mbuf *, u32);
  177 static int      igb_tx_ctx_setup(struct tx_ring *,
  178                     struct mbuf *, u32 *, u32 *);
  179 static int      igb_tso_setup(struct tx_ring *,
  180                     struct mbuf *, u32 *, u32 *);
  181 static void     igb_set_promisc(struct adapter *);
  182 static void     igb_disable_promisc(struct adapter *);
  183 static void     igb_set_multi(struct adapter *);
  184 static void     igb_update_link_status(struct adapter *);
  185 static void     igb_refresh_mbufs(struct rx_ring *, int);
  186 
  187 static void     igb_register_vlan(void *, struct ifnet *, u16);
  188 static void     igb_unregister_vlan(void *, struct ifnet *, u16);
  189 static void     igb_setup_vlan_hw_support(struct adapter *);
  190 
  191 static int      igb_xmit(struct tx_ring *, struct mbuf **);
  192 static int      igb_dma_malloc(struct adapter *, bus_size_t,
  193                     struct igb_dma_alloc *, int);
  194 static void     igb_dma_free(struct adapter *, struct igb_dma_alloc *);
  195 static int      igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
  196 static void     igb_print_nvm_info(struct adapter *);
  197 static int      igb_is_valid_ether_addr(u8 *);
  198 static void     igb_add_hw_stats(struct adapter *);
  199 
  200 static void     igb_vf_init_stats(struct adapter *);
  201 static void     igb_update_vf_stats_counters(struct adapter *);
  202 
  203 /* Management and WOL Support */
  204 static void     igb_init_manageability(struct adapter *);
  205 static void     igb_release_manageability(struct adapter *);
  206 static void     igb_get_hw_control(struct adapter *);
  207 static void     igb_release_hw_control(struct adapter *);
  208 static void     igb_enable_wakeup(device_t);
  209 static void     igb_led_func(void *, int);
  210 
  211 static int      igb_irq_fast(void *);
  212 static void     igb_msix_que(void *);
  213 static void     igb_msix_link(void *);
  214 static void     igb_handle_que(void *context, int pending);
  215 static void     igb_handle_link(void *context, int pending);
  216 static void     igb_handle_link_locked(struct adapter *);
  217 
  218 static void     igb_set_sysctl_value(struct adapter *, const char *,
  219                     const char *, int *, int);
  220 static int      igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
  221 static int      igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
  222 static int      igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
  223 
  224 #ifdef DEVICE_POLLING
  225 static poll_handler_t igb_poll;
  226 #endif /* POLLING */
  227 
  228 /*********************************************************************
  229  *  FreeBSD Device Interface Entry Points
  230  *********************************************************************/
  231 
  232 static device_method_t igb_methods[] = {
  233         /* Device interface */
  234         DEVMETHOD(device_probe, igb_probe),
  235         DEVMETHOD(device_attach, igb_attach),
  236         DEVMETHOD(device_detach, igb_detach),
  237         DEVMETHOD(device_shutdown, igb_shutdown),
  238         DEVMETHOD(device_suspend, igb_suspend),
  239         DEVMETHOD(device_resume, igb_resume),
  240         DEVMETHOD_END
  241 };
  242 
  243 static driver_t igb_driver = {
  244         "igb", igb_methods, sizeof(struct adapter),
  245 };
  246 
  247 static devclass_t igb_devclass;
  248 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
  249 MODULE_DEPEND(igb, pci, 1, 1, 1);
  250 MODULE_DEPEND(igb, ether, 1, 1, 1);
  251 
  252 /*********************************************************************
  253  *  Tunable default values.
  254  *********************************************************************/
  255 
  256 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
  257 
  258 /* Descriptor defaults */
  259 static int igb_rxd = IGB_DEFAULT_RXD;
  260 static int igb_txd = IGB_DEFAULT_TXD;
  261 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
  262 TUNABLE_INT("hw.igb.txd", &igb_txd);
  263 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
  264     "Number of receive descriptors per queue");
  265 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
  266     "Number of transmit descriptors per queue");
  267 
  268 /*
  269 ** AIM: Adaptive Interrupt Moderation
  270 ** which means that the interrupt rate
  271 ** is varied over time based on the
  272 ** traffic for that interrupt vector
  273 */
  274 static int igb_enable_aim = TRUE;
  275 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
  276 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0,
  277     "Enable adaptive interrupt moderation");
  278 
  279 /*
  280  * MSIX should be the default for best performance,
  281  * but this allows it to be forced off for testing.
  282  */         
  283 static int igb_enable_msix = 1;
  284 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
  285 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
  286     "Enable MSI-X interrupts");
  287 
  288 /*
  289 ** Tuneable Interrupt rate
  290 */
  291 static int igb_max_interrupt_rate = 8000;
  292 TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate);
  293 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
  294     &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
  295 
  296 #ifndef IGB_LEGACY_TX
  297 /*
  298 ** Tuneable number of buffers in the buf-ring (drbr_xxx)
  299 */
  300 static int igb_buf_ring_size = IGB_BR_SIZE;
  301 TUNABLE_INT("hw.igb.buf_ring_size", &igb_buf_ring_size);
  302 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
  303     &igb_buf_ring_size, 0, "Size of the bufring");
  304 #endif
  305 
  306 /*
  307 ** Header split causes the packet header to
  308 ** be dma'd to a seperate mbuf from the payload.
  309 ** this can have memory alignment benefits. But
  310 ** another plus is that small packets often fit
  311 ** into the header and thus use no cluster. Its
  312 ** a very workload dependent type feature.
  313 */
  314 static int igb_header_split = FALSE;
  315 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
  316 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
  317     "Enable receive mbuf header split");
  318 
  319 /*
  320 ** This will autoconfigure based on the
  321 ** number of CPUs and max supported
  322 ** MSIX messages if left at 0.
  323 */
  324 static int igb_num_queues = 0;
  325 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
  326 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
  327     "Number of queues to configure, 0 indicates autoconfigure");
  328 
  329 /*
  330 ** Global variable to store last used CPU when binding queues
  331 ** to CPUs in igb_allocate_msix.  Starts at CPU_FIRST and increments when a
  332 ** queue is bound to a cpu.
  333 */
  334 static int igb_last_bind_cpu = -1;
  335 
  336 /* How many packets rxeof tries to clean at a time */
  337 static int igb_rx_process_limit = 100;
  338 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
  339 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
  340     &igb_rx_process_limit, 0,
  341     "Maximum number of received packets to process at a time, -1 means unlimited");
  342 
  343 /* How many packets txeof tries to clean at a time */
  344 static int igb_tx_process_limit = -1;
  345 TUNABLE_INT("hw.igb.tx_process_limit", &igb_tx_process_limit);
  346 SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
  347     &igb_tx_process_limit, 0,
  348     "Maximum number of sent packets to process at a time, -1 means unlimited");
  349 
  350 #ifdef DEV_NETMAP       /* see ixgbe.c for details */
  351 #include <dev/netmap/if_igb_netmap.h>
  352 #endif /* DEV_NETMAP */
  353 /*********************************************************************
  354  *  Device identification routine
  355  *
  356  *  igb_probe determines if the driver should be loaded on
  357  *  adapter based on PCI vendor/device id of the adapter.
  358  *
  359  *  return BUS_PROBE_DEFAULT on success, positive on failure
  360  *********************************************************************/
  361 
  362 static int
  363 igb_probe(device_t dev)
  364 {
  365         char            adapter_name[256];
  366         uint16_t        pci_vendor_id = 0;
  367         uint16_t        pci_device_id = 0;
  368         uint16_t        pci_subvendor_id = 0;
  369         uint16_t        pci_subdevice_id = 0;
  370         igb_vendor_info_t *ent;
  371 
  372         INIT_DEBUGOUT("igb_probe: begin");
  373 
  374         pci_vendor_id = pci_get_vendor(dev);
  375         if (pci_vendor_id != IGB_INTEL_VENDOR_ID)
  376                 return (ENXIO);
  377 
  378         pci_device_id = pci_get_device(dev);
  379         pci_subvendor_id = pci_get_subvendor(dev);
  380         pci_subdevice_id = pci_get_subdevice(dev);
  381 
  382         ent = igb_vendor_info_array;
  383         while (ent->vendor_id != 0) {
  384                 if ((pci_vendor_id == ent->vendor_id) &&
  385                     (pci_device_id == ent->device_id) &&
  386 
  387                     ((pci_subvendor_id == ent->subvendor_id) ||
  388                     (ent->subvendor_id == 0)) &&
  389 
  390                     ((pci_subdevice_id == ent->subdevice_id) ||
  391                     (ent->subdevice_id == 0))) {
  392                         sprintf(adapter_name, "%s, Version - %s",
  393                                 igb_strings[ent->index],
  394                                 igb_driver_version);
  395                         device_set_desc_copy(dev, adapter_name);
  396                         return (BUS_PROBE_DEFAULT);
  397                 }
  398                 ent++;
  399         }
  400         return (ENXIO);
  401 }
  402 
  403 /*********************************************************************
  404  *  Device initialization routine
  405  *
  406  *  The attach entry point is called when the driver is being loaded.
  407  *  This routine identifies the type of hardware, allocates all resources
  408  *  and initializes the hardware.
  409  *
  410  *  return 0 on success, positive on failure
  411  *********************************************************************/
  412 
  413 static int
  414 igb_attach(device_t dev)
  415 {
  416         struct adapter  *adapter;
  417         int             error = 0;
  418         u16             eeprom_data;
  419 
  420         INIT_DEBUGOUT("igb_attach: begin");
  421 
  422         if (resource_disabled("igb", device_get_unit(dev))) {
  423                 device_printf(dev, "Disabled by device hint\n");
  424                 return (ENXIO);
  425         }
  426 
  427         adapter = device_get_softc(dev);
  428         adapter->dev = adapter->osdep.dev = dev;
  429         IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
  430 
  431         /* SYSCTLs */
  432         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  433             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  434             OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
  435             igb_sysctl_nvm_info, "I", "NVM Information");
  436 
  437         igb_set_sysctl_value(adapter, "enable_aim",
  438             "Interrupt Moderation", &adapter->enable_aim,
  439             igb_enable_aim);
  440 
  441         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  442             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  443             OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
  444             adapter, 0, igb_set_flowcntl, "I", "Flow Control");
  445 
  446         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
  447 
  448         /* Determine hardware and mac info */
  449         igb_identify_hardware(adapter);
  450 
  451         /* Setup PCI resources */
  452         if (igb_allocate_pci_resources(adapter)) {
  453                 device_printf(dev, "Allocation of PCI resources failed\n");
  454                 error = ENXIO;
  455                 goto err_pci;
  456         }
  457 
  458         /* Do Shared Code initialization */
  459         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
  460                 device_printf(dev, "Setup of Shared code failed\n");
  461                 error = ENXIO;
  462                 goto err_pci;
  463         }
  464 
  465         e1000_get_bus_info(&adapter->hw);
  466 
  467         /* Sysctls for limiting the amount of work done in the taskqueues */
  468         igb_set_sysctl_value(adapter, "rx_processing_limit",
  469             "max number of rx packets to process",
  470             &adapter->rx_process_limit, igb_rx_process_limit);
  471 
  472         igb_set_sysctl_value(adapter, "tx_processing_limit",
  473             "max number of tx packets to process",
  474             &adapter->tx_process_limit, igb_tx_process_limit);
  475 
  476         /*
  477          * Validate number of transmit and receive descriptors. It
  478          * must not exceed hardware maximum, and must be multiple
  479          * of E1000_DBA_ALIGN.
  480          */
  481         if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
  482             (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
  483                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
  484                     IGB_DEFAULT_TXD, igb_txd);
  485                 adapter->num_tx_desc = IGB_DEFAULT_TXD;
  486         } else
  487                 adapter->num_tx_desc = igb_txd;
  488         if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
  489             (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
  490                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
  491                     IGB_DEFAULT_RXD, igb_rxd);
  492                 adapter->num_rx_desc = IGB_DEFAULT_RXD;
  493         } else
  494                 adapter->num_rx_desc = igb_rxd;
  495 
  496         adapter->hw.mac.autoneg = DO_AUTO_NEG;
  497         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
  498         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
  499 
  500         /* Copper options */
  501         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
  502                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
  503                 adapter->hw.phy.disable_polarity_correction = FALSE;
  504                 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
  505         }
  506 
  507         /*
  508          * Set the frame limits assuming
  509          * standard ethernet sized frames.
  510          */
  511         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
  512 
  513         /*
  514         ** Allocate and Setup Queues
  515         */
  516         if (igb_allocate_queues(adapter)) {
  517                 error = ENOMEM;
  518                 goto err_pci;
  519         }
  520 
  521         /* Allocate the appropriate stats memory */
  522         if (adapter->vf_ifp) {
  523                 adapter->stats =
  524                     (struct e1000_vf_stats *)malloc(sizeof \
  525                     (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
  526                 igb_vf_init_stats(adapter);
  527         } else
  528                 adapter->stats =
  529                     (struct e1000_hw_stats *)malloc(sizeof \
  530                     (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
  531         if (adapter->stats == NULL) {
  532                 device_printf(dev, "Can not allocate stats memory\n");
  533                 error = ENOMEM;
  534                 goto err_late;
  535         }
  536 
  537         /* Allocate multicast array memory. */
  538         adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
  539             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
  540         if (adapter->mta == NULL) {
  541                 device_printf(dev, "Can not allocate multicast setup array\n");
  542                 error = ENOMEM;
  543                 goto err_late;
  544         }
  545 
  546         /* Some adapter-specific advanced features */
  547         if (adapter->hw.mac.type >= e1000_i350) {
  548                 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  549                     SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  550                     OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
  551                     adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
  552                 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  553                     SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  554                     OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
  555                     adapter, 0, igb_sysctl_eee, "I",
  556                     "Disable Energy Efficient Ethernet");
  557                 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
  558                         if (adapter->hw.mac.type == e1000_i354)
  559                                 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
  560                         else
  561                                 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
  562                 }
  563         }
  564 
  565         /*
  566         ** Start from a known state, this is
  567         ** important in reading the nvm and
  568         ** mac from that.
  569         */
  570         e1000_reset_hw(&adapter->hw);
  571 
  572         /* Make sure we have a good EEPROM before we read from it */
  573         if (((adapter->hw.mac.type != e1000_i210) &&
  574             (adapter->hw.mac.type != e1000_i211)) &&
  575             (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
  576                 /*
  577                 ** Some PCI-E parts fail the first check due to
  578                 ** the link being in sleep state, call it again,
  579                 ** if it fails a second time its a real issue.
  580                 */
  581                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
  582                         device_printf(dev,
  583                             "The EEPROM Checksum Is Not Valid\n");
  584                         error = EIO;
  585                         goto err_late;
  586                 }
  587         }
  588 
  589         /*
  590         ** Copy the permanent MAC address out of the EEPROM
  591         */
  592         if (e1000_read_mac_addr(&adapter->hw) < 0) {
  593                 device_printf(dev, "EEPROM read error while reading MAC"
  594                     " address\n");
  595                 error = EIO;
  596                 goto err_late;
  597         }
  598         /* Check its sanity */
  599         if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
  600                 device_printf(dev, "Invalid MAC address\n");
  601                 error = EIO;
  602                 goto err_late;
  603         }
  604 
  605         /* Setup OS specific network interface */
  606         if (igb_setup_interface(dev, adapter) != 0)
  607                 goto err_late;
  608 
  609         /* Now get a good starting state */
  610         igb_reset(adapter);
  611 
  612         /* Initialize statistics */
  613         igb_update_stats_counters(adapter);
  614 
  615         adapter->hw.mac.get_link_status = 1;
  616         igb_update_link_status(adapter);
  617 
  618         /* Indicate SOL/IDER usage */
  619         if (e1000_check_reset_block(&adapter->hw))
  620                 device_printf(dev,
  621                     "PHY reset is blocked due to SOL/IDER session.\n");
  622 
  623         /* Determine if we have to control management hardware */
  624         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
  625 
  626         /*
  627          * Setup Wake-on-Lan
  628          */
  629         /* APME bit in EEPROM is mapped to WUC.APME */
  630         eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
  631         if (eeprom_data)
  632                 adapter->wol = E1000_WUFC_MAG;
  633 
  634         /* Register for VLAN events */
  635         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
  636              igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  637         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
  638              igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  639 
  640         igb_add_hw_stats(adapter);
  641 
  642         /* Tell the stack that the interface is not active */
  643         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
  644         adapter->ifp->if_drv_flags |=  IFF_DRV_OACTIVE;
  645 
  646         adapter->led_dev = led_create(igb_led_func, adapter,
  647             device_get_nameunit(dev));
  648 
  649         /* 
  650         ** Configure Interrupts
  651         */
  652         if ((adapter->msix > 1) && (igb_enable_msix))
  653                 error = igb_allocate_msix(adapter);
  654         else /* MSI or Legacy */
  655                 error = igb_allocate_legacy(adapter);
  656         if (error)
  657                 goto err_late;
  658 
  659 #ifdef DEV_NETMAP
  660         igb_netmap_attach(adapter);
  661 #endif /* DEV_NETMAP */
  662         INIT_DEBUGOUT("igb_attach: end");
  663 
  664         return (0);
  665 
  666 err_late:
  667         if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */
  668                 return(error);
  669         igb_free_transmit_structures(adapter);
  670         igb_free_receive_structures(adapter);
  671         igb_release_hw_control(adapter);
  672 err_pci:
  673         igb_free_pci_resources(adapter);
  674         if (adapter->ifp != NULL)
  675                 if_free(adapter->ifp);
  676         free(adapter->mta, M_DEVBUF);
  677         IGB_CORE_LOCK_DESTROY(adapter);
  678 
  679         return (error);
  680 }
  681 
  682 /*********************************************************************
  683  *  Device removal routine
  684  *
  685  *  The detach entry point is called when the driver is being removed.
  686  *  This routine stops the adapter and deallocates all the resources
  687  *  that were allocated for driver operation.
  688  *
  689  *  return 0 on success, positive on failure
  690  *********************************************************************/
  691 
  692 static int
  693 igb_detach(device_t dev)
  694 {
  695         struct adapter  *adapter = device_get_softc(dev);
  696         struct ifnet    *ifp = adapter->ifp;
  697 
  698         INIT_DEBUGOUT("igb_detach: begin");
  699 
  700         /* Make sure VLANS are not using driver */
  701         if (adapter->ifp->if_vlantrunk != NULL) {
  702                 device_printf(dev,"Vlan in use, detach first\n");
  703                 return (EBUSY);
  704         }
  705 
  706         ether_ifdetach(adapter->ifp);
  707 
  708         if (adapter->led_dev != NULL)
  709                 led_destroy(adapter->led_dev);
  710 
  711 #ifdef DEVICE_POLLING
  712         if (ifp->if_capenable & IFCAP_POLLING)
  713                 ether_poll_deregister(ifp);
  714 #endif
  715 
  716         IGB_CORE_LOCK(adapter);
  717         adapter->in_detach = 1;
  718         igb_stop(adapter);
  719         IGB_CORE_UNLOCK(adapter);
  720 
  721         e1000_phy_hw_reset(&adapter->hw);
  722 
  723         /* Give control back to firmware */
  724         igb_release_manageability(adapter);
  725         igb_release_hw_control(adapter);
  726 
  727         /* Unregister VLAN events */
  728         if (adapter->vlan_attach != NULL)
  729                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
  730         if (adapter->vlan_detach != NULL)
  731                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
  732 
  733         callout_drain(&adapter->timer);
  734 
  735 #ifdef DEV_NETMAP
  736         netmap_detach(adapter->ifp);
  737 #endif /* DEV_NETMAP */
  738         igb_free_pci_resources(adapter);
  739         bus_generic_detach(dev);
  740         if_free(ifp);
  741 
  742         igb_free_transmit_structures(adapter);
  743         igb_free_receive_structures(adapter);
  744         if (adapter->mta != NULL)
  745                 free(adapter->mta, M_DEVBUF);
  746 
  747         IGB_CORE_LOCK_DESTROY(adapter);
  748 
  749         return (0);
  750 }
  751 
  752 /*********************************************************************
  753  *
  754  *  Shutdown entry point
  755  *
  756  **********************************************************************/
  757 
  758 static int
  759 igb_shutdown(device_t dev)
  760 {
  761         return igb_suspend(dev);
  762 }
  763 
  764 /*
  765  * Suspend/resume device methods.
  766  */
  767 static int
  768 igb_suspend(device_t dev)
  769 {
  770         struct adapter *adapter = device_get_softc(dev);
  771 
  772         IGB_CORE_LOCK(adapter);
  773 
  774         igb_stop(adapter);
  775 
  776         igb_release_manageability(adapter);
  777         igb_release_hw_control(adapter);
  778         igb_enable_wakeup(dev);
  779 
  780         IGB_CORE_UNLOCK(adapter);
  781 
  782         return bus_generic_suspend(dev);
  783 }
  784 
  785 static int
  786 igb_resume(device_t dev)
  787 {
  788         struct adapter *adapter = device_get_softc(dev);
  789         struct tx_ring  *txr = adapter->tx_rings;
  790         struct ifnet *ifp = adapter->ifp;
  791 
  792         IGB_CORE_LOCK(adapter);
  793         igb_init_locked(adapter);
  794         igb_init_manageability(adapter);
  795 
  796         if ((ifp->if_flags & IFF_UP) &&
  797             (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
  798                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
  799                         IGB_TX_LOCK(txr);
  800 #ifndef IGB_LEGACY_TX
  801                         /* Process the stack queue only if not depleted */
  802                         if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
  803                             !drbr_empty(ifp, txr->br))
  804                                 igb_mq_start_locked(ifp, txr);
  805 #else
  806                         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
  807                                 igb_start_locked(txr, ifp);
  808 #endif
  809                         IGB_TX_UNLOCK(txr);
  810                 }
  811         }
  812         IGB_CORE_UNLOCK(adapter);
  813 
  814         return bus_generic_resume(dev);
  815 }
  816 
  817 
  818 #ifdef IGB_LEGACY_TX
  819 
  820 /*********************************************************************
  821  *  Transmit entry point
  822  *
  823  *  igb_start is called by the stack to initiate a transmit.
  824  *  The driver will remain in this routine as long as there are
  825  *  packets to transmit and transmit resources are available.
  826  *  In case resources are not available stack is notified and
  827  *  the packet is requeued.
  828  **********************************************************************/
  829 
  830 static void
  831 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
  832 {
  833         struct adapter  *adapter = ifp->if_softc;
  834         struct mbuf     *m_head;
  835 
  836         IGB_TX_LOCK_ASSERT(txr);
  837 
  838         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
  839             IFF_DRV_RUNNING)
  840                 return;
  841         if (!adapter->link_active)
  842                 return;
  843 
  844         /* Call cleanup if number of TX descriptors low */
  845         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
  846                 igb_txeof(txr);
  847 
  848         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
  849                 if (txr->tx_avail <= IGB_MAX_SCATTER) {
  850                         txr->queue_status |= IGB_QUEUE_DEPLETED;
  851                         break;
  852                 }
  853                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
  854                 if (m_head == NULL)
  855                         break;
  856                 /*
  857                  *  Encapsulation can modify our pointer, and or make it
  858                  *  NULL on failure.  In that event, we can't requeue.
  859                  */
  860                 if (igb_xmit(txr, &m_head)) {
  861                         if (m_head != NULL)
  862                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
  863                         if (txr->tx_avail <= IGB_MAX_SCATTER)
  864                                 txr->queue_status |= IGB_QUEUE_DEPLETED;
  865                         break;
  866                 }
  867 
  868                 /* Send a copy of the frame to the BPF listener */
  869                 ETHER_BPF_MTAP(ifp, m_head);
  870 
  871                 /* Set watchdog on */
  872                 txr->watchdog_time = ticks;
  873                 txr->queue_status |= IGB_QUEUE_WORKING;
  874         }
  875 }
  876  
  877 /*
  878  * Legacy TX driver routine, called from the
  879  * stack, always uses tx[0], and spins for it.
  880  * Should not be used with multiqueue tx
  881  */
  882 static void
  883 igb_start(struct ifnet *ifp)
  884 {
  885         struct adapter  *adapter = ifp->if_softc;
  886         struct tx_ring  *txr = adapter->tx_rings;
  887 
  888         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  889                 IGB_TX_LOCK(txr);
  890                 igb_start_locked(txr, ifp);
  891                 IGB_TX_UNLOCK(txr);
  892         }
  893         return;
  894 }
  895 
  896 #else /* ~IGB_LEGACY_TX */
  897 
  898 /*
  899 ** Multiqueue Transmit Entry:
  900 **  quick turnaround to the stack
  901 **
  902 */
  903 static int
  904 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
  905 {
  906         struct adapter          *adapter = ifp->if_softc;
  907         struct igb_queue        *que;
  908         struct tx_ring          *txr;
  909         int                     i, err = 0;
  910 
  911         /* Which queue to use */
  912         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
  913                 i = m->m_pkthdr.flowid % adapter->num_queues;
  914         else
  915                 i = curcpu % adapter->num_queues;
  916         txr = &adapter->tx_rings[i];
  917         que = &adapter->queues[i];
  918 
  919         err = drbr_enqueue(ifp, txr->br, m);
  920         if (err)
  921                 return (err);
  922         if (IGB_TX_TRYLOCK(txr)) {
  923                 igb_mq_start_locked(ifp, txr);
  924                 IGB_TX_UNLOCK(txr);
  925         } else
  926                 taskqueue_enqueue(que->tq, &txr->txq_task);
  927 
  928         return (0);
  929 }
  930 
  931 static int
  932 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
  933 {
  934         struct adapter  *adapter = txr->adapter;
  935         struct mbuf     *next;
  936         int             err = 0, enq = 0;
  937 
  938         IGB_TX_LOCK_ASSERT(txr);
  939 
  940         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
  941             adapter->link_active == 0)
  942                 return (ENETDOWN);
  943 
  944         /* Process the queue */
  945         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
  946                 if ((err = igb_xmit(txr, &next)) != 0) {
  947                         if (next == NULL) {
  948                                 /* It was freed, move forward */
  949                                 drbr_advance(ifp, txr->br);
  950                         } else {
  951                                 /* 
  952                                  * Still have one left, it may not be
  953                                  * the same since the transmit function
  954                                  * may have changed it.
  955                                  */
  956                                 drbr_putback(ifp, txr->br, next);
  957                         }
  958                         break;
  959                 }
  960                 drbr_advance(ifp, txr->br);
  961                 enq++;
  962                 ifp->if_obytes += next->m_pkthdr.len;
  963                 if (next->m_flags & M_MCAST)
  964                         ifp->if_omcasts++;
  965                 ETHER_BPF_MTAP(ifp, next);
  966                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
  967                         break;
  968         }
  969         if (enq > 0) {
  970                 /* Set the watchdog */
  971                 txr->queue_status |= IGB_QUEUE_WORKING;
  972                 txr->watchdog_time = ticks;
  973         }
  974         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
  975                 igb_txeof(txr);
  976         if (txr->tx_avail <= IGB_MAX_SCATTER)
  977                 txr->queue_status |= IGB_QUEUE_DEPLETED;
  978         return (err);
  979 }
  980 
  981 /*
  982  * Called from a taskqueue to drain queued transmit packets.
  983  */
  984 static void
  985 igb_deferred_mq_start(void *arg, int pending)
  986 {
  987         struct tx_ring *txr = arg;
  988         struct adapter *adapter = txr->adapter;
  989         struct ifnet *ifp = adapter->ifp;
  990 
  991         IGB_TX_LOCK(txr);
  992         if (!drbr_empty(ifp, txr->br))
  993                 igb_mq_start_locked(ifp, txr);
  994         IGB_TX_UNLOCK(txr);
  995 }
  996 
  997 /*
  998 ** Flush all ring buffers
  999 */
 1000 static void
 1001 igb_qflush(struct ifnet *ifp)
 1002 {
 1003         struct adapter  *adapter = ifp->if_softc;
 1004         struct tx_ring  *txr = adapter->tx_rings;
 1005         struct mbuf     *m;
 1006 
 1007         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1008                 IGB_TX_LOCK(txr);
 1009                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 1010                         m_freem(m);
 1011                 IGB_TX_UNLOCK(txr);
 1012         }
 1013         if_qflush(ifp);
 1014 }
 1015 #endif /* ~IGB_LEGACY_TX */
 1016 
 1017 /*********************************************************************
 1018  *  Ioctl entry point
 1019  *
 1020  *  igb_ioctl is called when the user wants to configure the
 1021  *  interface.
 1022  *
 1023  *  return 0 on success, positive on failure
 1024  **********************************************************************/
 1025 
 1026 static int
 1027 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 1028 {
 1029         struct adapter  *adapter = ifp->if_softc;
 1030         struct ifreq    *ifr = (struct ifreq *)data;
 1031 #if defined(INET) || defined(INET6)
 1032         struct ifaddr   *ifa = (struct ifaddr *)data;
 1033 #endif
 1034         bool            avoid_reset = FALSE;
 1035         int             error = 0;
 1036 
 1037         if (adapter->in_detach)
 1038                 return (error);
 1039 
 1040         switch (command) {
 1041         case SIOCSIFADDR:
 1042 #ifdef INET
 1043                 if (ifa->ifa_addr->sa_family == AF_INET)
 1044                         avoid_reset = TRUE;
 1045 #endif
 1046 #ifdef INET6
 1047                 if (ifa->ifa_addr->sa_family == AF_INET6)
 1048                         avoid_reset = TRUE;
 1049 #endif
 1050                 /*
 1051                 ** Calling init results in link renegotiation,
 1052                 ** so we avoid doing it when possible.
 1053                 */
 1054                 if (avoid_reset) {
 1055                         ifp->if_flags |= IFF_UP;
 1056                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 1057                                 igb_init(adapter);
 1058 #ifdef INET
 1059                         if (!(ifp->if_flags & IFF_NOARP))
 1060                                 arp_ifinit(ifp, ifa);
 1061 #endif
 1062                 } else
 1063                         error = ether_ioctl(ifp, command, data);
 1064                 break;
 1065         case SIOCSIFMTU:
 1066             {
 1067                 int max_frame_size;
 1068 
 1069                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 1070 
 1071                 IGB_CORE_LOCK(adapter);
 1072                 max_frame_size = 9234;
 1073                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
 1074                     ETHER_CRC_LEN) {
 1075                         IGB_CORE_UNLOCK(adapter);
 1076                         error = EINVAL;
 1077                         break;
 1078                 }
 1079 
 1080                 ifp->if_mtu = ifr->ifr_mtu;
 1081                 adapter->max_frame_size =
 1082                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 1083                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1084                         igb_init_locked(adapter);
 1085                 IGB_CORE_UNLOCK(adapter);
 1086                 break;
 1087             }
 1088         case SIOCSIFFLAGS:
 1089                 IOCTL_DEBUGOUT("ioctl rcv'd:\
 1090                     SIOCSIFFLAGS (Set Interface Flags)");
 1091                 IGB_CORE_LOCK(adapter);
 1092                 if (ifp->if_flags & IFF_UP) {
 1093                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 1094                                 if ((ifp->if_flags ^ adapter->if_flags) &
 1095                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 1096                                         igb_disable_promisc(adapter);
 1097                                         igb_set_promisc(adapter);
 1098                                 }
 1099                         } else
 1100                                 igb_init_locked(adapter);
 1101                 } else
 1102                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1103                                 igb_stop(adapter);
 1104                 adapter->if_flags = ifp->if_flags;
 1105                 IGB_CORE_UNLOCK(adapter);
 1106                 break;
 1107         case SIOCADDMULTI:
 1108         case SIOCDELMULTI:
 1109                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 1110                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1111                         IGB_CORE_LOCK(adapter);
 1112                         igb_disable_intr(adapter);
 1113                         igb_set_multi(adapter);
 1114 #ifdef DEVICE_POLLING
 1115                         if (!(ifp->if_capenable & IFCAP_POLLING))
 1116 #endif
 1117                                 igb_enable_intr(adapter);
 1118                         IGB_CORE_UNLOCK(adapter);
 1119                 }
 1120                 break;
 1121         case SIOCSIFMEDIA:
 1122                 /* Check SOL/IDER usage */
 1123                 IGB_CORE_LOCK(adapter);
 1124                 if (e1000_check_reset_block(&adapter->hw)) {
 1125                         IGB_CORE_UNLOCK(adapter);
 1126                         device_printf(adapter->dev, "Media change is"
 1127                             " blocked due to SOL/IDER session.\n");
 1128                         break;
 1129                 }
 1130                 IGB_CORE_UNLOCK(adapter);
 1131         case SIOCGIFMEDIA:
 1132                 IOCTL_DEBUGOUT("ioctl rcv'd: \
 1133                     SIOCxIFMEDIA (Get/Set Interface Media)");
 1134                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 1135                 break;
 1136         case SIOCSIFCAP:
 1137             {
 1138                 int mask, reinit;
 1139 
 1140                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 1141                 reinit = 0;
 1142                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 1143 #ifdef DEVICE_POLLING
 1144                 if (mask & IFCAP_POLLING) {
 1145                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
 1146                                 error = ether_poll_register(igb_poll, ifp);
 1147                                 if (error)
 1148                                         return (error);
 1149                                 IGB_CORE_LOCK(adapter);
 1150                                 igb_disable_intr(adapter);
 1151                                 ifp->if_capenable |= IFCAP_POLLING;
 1152                                 IGB_CORE_UNLOCK(adapter);
 1153                         } else {
 1154                                 error = ether_poll_deregister(ifp);
 1155                                 /* Enable interrupt even in error case */
 1156                                 IGB_CORE_LOCK(adapter);
 1157                                 igb_enable_intr(adapter);
 1158                                 ifp->if_capenable &= ~IFCAP_POLLING;
 1159                                 IGB_CORE_UNLOCK(adapter);
 1160                         }
 1161                 }
 1162 #endif
 1163 #if __FreeBSD_version >= 1000000
 1164                 /* HW cannot turn these on/off separately */
 1165                 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
 1166                         ifp->if_capenable ^= IFCAP_RXCSUM;
 1167                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
 1168                         reinit = 1;
 1169                 }
 1170                 if (mask & IFCAP_TXCSUM) {
 1171                         ifp->if_capenable ^= IFCAP_TXCSUM;
 1172                         reinit = 1;
 1173                 }
 1174                 if (mask & IFCAP_TXCSUM_IPV6) {
 1175                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
 1176                         reinit = 1;
 1177                 }
 1178 #else
 1179                 if (mask & IFCAP_HWCSUM) {
 1180                         ifp->if_capenable ^= IFCAP_HWCSUM;
 1181                         reinit = 1;
 1182                 }
 1183 #endif
 1184                 if (mask & IFCAP_TSO4) {
 1185                         ifp->if_capenable ^= IFCAP_TSO4;
 1186                         reinit = 1;
 1187                 }
 1188                 if (mask & IFCAP_TSO6) {
 1189                         ifp->if_capenable ^= IFCAP_TSO6;
 1190                         reinit = 1;
 1191                 }
 1192                 if (mask & IFCAP_VLAN_HWTAGGING) {
 1193                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 1194                         reinit = 1;
 1195                 }
 1196                 if (mask & IFCAP_VLAN_HWFILTER) {
 1197                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 1198                         reinit = 1;
 1199                 }
 1200                 if (mask & IFCAP_VLAN_HWTSO) {
 1201                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 1202                         reinit = 1;
 1203                 }
 1204                 if (mask & IFCAP_LRO) {
 1205                         ifp->if_capenable ^= IFCAP_LRO;
 1206                         reinit = 1;
 1207                 }
 1208                 if (mask & IFCAP_WOL) {
 1209                         if (mask & IFCAP_WOL_MAGIC)
 1210                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
 1211                         if (mask & IFCAP_WOL_MCAST)
 1212                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
 1213                         if (mask & IFCAP_WOL_UCAST)
 1214                                 ifp->if_capenable ^= IFCAP_WOL_UCAST;
 1215                 }
 1216                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
 1217                         igb_init(adapter);
 1218                 VLAN_CAPABILITIES(ifp);
 1219                 break;
 1220             }
 1221 
 1222         default:
 1223                 error = ether_ioctl(ifp, command, data);
 1224                 break;
 1225         }
 1226 
 1227         return (error);
 1228 }
 1229 
 1230 
 1231 /*********************************************************************
 1232  *  Init entry point
 1233  *
 1234  *  This routine is used in two ways. It is used by the stack as
 1235  *  init entry point in network interface structure. It is also used
 1236  *  by the driver as a hw/sw initialization routine to get to a
 1237  *  consistent state.
 1238  *
 1239  *  return 0 on success, positive on failure
 1240  **********************************************************************/
 1241 
 1242 static void
 1243 igb_init_locked(struct adapter *adapter)
 1244 {
 1245         struct ifnet    *ifp = adapter->ifp;
 1246         device_t        dev = adapter->dev;
 1247 
 1248         INIT_DEBUGOUT("igb_init: begin");
 1249 
 1250         IGB_CORE_LOCK_ASSERT(adapter);
 1251 
 1252         igb_disable_intr(adapter);
 1253         callout_stop(&adapter->timer);
 1254 
 1255         /* Get the latest mac address, User can use a LAA */
 1256         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
 1257               ETHER_ADDR_LEN);
 1258 
 1259         /* Put the address into the Receive Address Array */
 1260         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 1261 
 1262         igb_reset(adapter);
 1263         igb_update_link_status(adapter);
 1264 
 1265         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 1266 
 1267         /* Set hardware offload abilities */
 1268         ifp->if_hwassist = 0;
 1269         if (ifp->if_capenable & IFCAP_TXCSUM) {
 1270 #if __FreeBSD_version >= 1000000
 1271                 ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP);
 1272                 if (adapter->hw.mac.type != e1000_82575)
 1273                         ifp->if_hwassist |= CSUM_IP_SCTP;
 1274 #else
 1275                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 1276 #if __FreeBSD_version >= 800000
 1277                 if (adapter->hw.mac.type != e1000_82575)
 1278                         ifp->if_hwassist |= CSUM_SCTP;
 1279 #endif
 1280 #endif
 1281         }
 1282 
 1283 #if __FreeBSD_version >= 1000000
 1284         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) {
 1285                 ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP);
 1286                 if (adapter->hw.mac.type != e1000_82575)
 1287                         ifp->if_hwassist |= CSUM_IP6_SCTP;
 1288         }
 1289 #endif
 1290         if (ifp->if_capenable & IFCAP_TSO)
 1291                 ifp->if_hwassist |= CSUM_TSO;
 1292 
 1293         /* Clear bad data from Rx FIFOs */
 1294         e1000_rx_fifo_flush_82575(&adapter->hw);
 1295 
 1296         /* Configure for OS presence */
 1297         igb_init_manageability(adapter);
 1298 
 1299         /* Prepare transmit descriptors and buffers */
 1300         igb_setup_transmit_structures(adapter);
 1301         igb_initialize_transmit_units(adapter);
 1302 
 1303         /* Setup Multicast table */
 1304         igb_set_multi(adapter);
 1305 
 1306         /*
 1307         ** Figure out the desired mbuf pool
 1308         ** for doing jumbo/packetsplit
 1309         */
 1310         if (adapter->max_frame_size <= 2048)
 1311                 adapter->rx_mbuf_sz = MCLBYTES;
 1312 #ifndef CONTIGMALLOC_WORKS
 1313        else
 1314                adapter->rx_mbuf_sz = MJUMPAGESIZE;
 1315 #else
 1316         else if (adapter->max_frame_size <= 4096)
 1317                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 1318         else
 1319                 adapter->rx_mbuf_sz = MJUM9BYTES;
 1320 #endif
 1321 
 1322         /* Prepare receive descriptors and buffers */
 1323         if (igb_setup_receive_structures(adapter)) {
 1324                 device_printf(dev, "Could not setup receive structures\n");
 1325                 return;
 1326         }
 1327         igb_initialize_receive_units(adapter);
 1328 
 1329         /* Enable VLAN support */
 1330         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
 1331                 igb_setup_vlan_hw_support(adapter);
 1332                                 
 1333         /* Don't lose promiscuous settings */
 1334         igb_set_promisc(adapter);
 1335 
 1336         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 1337         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 1338 
 1339         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 1340         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
 1341 
 1342         if (adapter->msix > 1) /* Set up queue routing */
 1343                 igb_configure_queues(adapter);
 1344 
 1345         /* this clears any pending interrupts */
 1346         E1000_READ_REG(&adapter->hw, E1000_ICR);
 1347 #ifdef DEVICE_POLLING
 1348         /*
 1349          * Only enable interrupts if we are not polling, make sure
 1350          * they are off otherwise.
 1351          */
 1352         if (ifp->if_capenable & IFCAP_POLLING)
 1353                 igb_disable_intr(adapter);
 1354         else
 1355 #endif /* DEVICE_POLLING */
 1356         {
 1357                 igb_enable_intr(adapter);
 1358                 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
 1359         }
 1360 
 1361         /* Set Energy Efficient Ethernet */
 1362         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 1363                 if (adapter->hw.mac.type == e1000_i354)
 1364                         e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
 1365                 else
 1366                         e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
 1367         }
 1368 }
 1369 
 1370 static void
 1371 igb_init(void *arg)
 1372 {
 1373         struct adapter *adapter = arg;
 1374 
 1375         IGB_CORE_LOCK(adapter);
 1376         igb_init_locked(adapter);
 1377         IGB_CORE_UNLOCK(adapter);
 1378 }
 1379 
 1380 
 1381 static void
 1382 igb_handle_que(void *context, int pending)
 1383 {
 1384         struct igb_queue *que = context;
 1385         struct adapter *adapter = que->adapter;
 1386         struct tx_ring *txr = que->txr;
 1387         struct ifnet    *ifp = adapter->ifp;
 1388 
 1389         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1390                 bool    more;
 1391 
 1392                 more = igb_rxeof(que, adapter->rx_process_limit, NULL);
 1393 
 1394                 IGB_TX_LOCK(txr);
 1395                 igb_txeof(txr);
 1396 #ifndef IGB_LEGACY_TX
 1397                 /* Process the stack queue only if not depleted */
 1398                 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
 1399                     !drbr_empty(ifp, txr->br))
 1400                         igb_mq_start_locked(ifp, txr);
 1401 #else
 1402                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1403                         igb_start_locked(txr, ifp);
 1404 #endif
 1405                 IGB_TX_UNLOCK(txr);
 1406                 /* Do we need another? */
 1407                 if (more) {
 1408                         taskqueue_enqueue(que->tq, &que->que_task);
 1409                         return;
 1410                 }
 1411         }
 1412 
 1413 #ifdef DEVICE_POLLING
 1414         if (ifp->if_capenable & IFCAP_POLLING)
 1415                 return;
 1416 #endif
 1417         /* Reenable this interrupt */
 1418         if (que->eims)
 1419                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
 1420         else
 1421                 igb_enable_intr(adapter);
 1422 }
 1423 
 1424 /* Deal with link in a sleepable context */
 1425 static void
 1426 igb_handle_link(void *context, int pending)
 1427 {
 1428         struct adapter *adapter = context;
 1429 
 1430         IGB_CORE_LOCK(adapter);
 1431         igb_handle_link_locked(adapter);
 1432         IGB_CORE_UNLOCK(adapter);
 1433 }
 1434 
 1435 static void
 1436 igb_handle_link_locked(struct adapter *adapter)
 1437 {
 1438         struct tx_ring  *txr = adapter->tx_rings;
 1439         struct ifnet *ifp = adapter->ifp;
 1440 
 1441         IGB_CORE_LOCK_ASSERT(adapter);
 1442         adapter->hw.mac.get_link_status = 1;
 1443         igb_update_link_status(adapter);
 1444         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
 1445                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1446                         IGB_TX_LOCK(txr);
 1447 #ifndef IGB_LEGACY_TX
 1448                         /* Process the stack queue only if not depleted */
 1449                         if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
 1450                             !drbr_empty(ifp, txr->br))
 1451                                 igb_mq_start_locked(ifp, txr);
 1452 #else
 1453                         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1454                                 igb_start_locked(txr, ifp);
 1455 #endif
 1456                         IGB_TX_UNLOCK(txr);
 1457                 }
 1458         }
 1459 }
 1460 
 1461 /*********************************************************************
 1462  *
 1463  *  MSI/Legacy Deferred
 1464  *  Interrupt Service routine  
 1465  *
 1466  *********************************************************************/
 1467 static int
 1468 igb_irq_fast(void *arg)
 1469 {
 1470         struct adapter          *adapter = arg;
 1471         struct igb_queue        *que = adapter->queues;
 1472         u32                     reg_icr;
 1473 
 1474 
 1475         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1476 
 1477         /* Hot eject?  */
 1478         if (reg_icr == 0xffffffff)
 1479                 return FILTER_STRAY;
 1480 
 1481         /* Definitely not our interrupt.  */
 1482         if (reg_icr == 0x0)
 1483                 return FILTER_STRAY;
 1484 
 1485         if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
 1486                 return FILTER_STRAY;
 1487 
 1488         /*
 1489          * Mask interrupts until the taskqueue is finished running.  This is
 1490          * cheap, just assume that it is needed.  This also works around the
 1491          * MSI message reordering errata on certain systems.
 1492          */
 1493         igb_disable_intr(adapter);
 1494         taskqueue_enqueue(que->tq, &que->que_task);
 1495 
 1496         /* Link status change */
 1497         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
 1498                 taskqueue_enqueue(que->tq, &adapter->link_task);
 1499 
 1500         if (reg_icr & E1000_ICR_RXO)
 1501                 adapter->rx_overruns++;
 1502         return FILTER_HANDLED;
 1503 }
 1504 
 1505 #ifdef DEVICE_POLLING
 1506 #if __FreeBSD_version >= 800000
 1507 #define POLL_RETURN_COUNT(a) (a)
 1508 static int
 1509 #else
 1510 #define POLL_RETURN_COUNT(a)
 1511 static void
 1512 #endif
 1513 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 1514 {
 1515         struct adapter          *adapter = ifp->if_softc;
 1516         struct igb_queue        *que;
 1517         struct tx_ring          *txr;
 1518         u32                     reg_icr, rx_done = 0;
 1519         u32                     loop = IGB_MAX_LOOP;
 1520         bool                    more;
 1521 
 1522         IGB_CORE_LOCK(adapter);
 1523         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 1524                 IGB_CORE_UNLOCK(adapter);
 1525                 return POLL_RETURN_COUNT(rx_done);
 1526         }
 1527 
 1528         if (cmd == POLL_AND_CHECK_STATUS) {
 1529                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1530                 /* Link status change */
 1531                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
 1532                         igb_handle_link_locked(adapter);
 1533 
 1534                 if (reg_icr & E1000_ICR_RXO)
 1535                         adapter->rx_overruns++;
 1536         }
 1537         IGB_CORE_UNLOCK(adapter);
 1538 
 1539         for (int i = 0; i < adapter->num_queues; i++) {
 1540                 que = &adapter->queues[i];
 1541                 txr = que->txr;
 1542 
 1543                 igb_rxeof(que, count, &rx_done);
 1544 
 1545                 IGB_TX_LOCK(txr);
 1546                 do {
 1547                         more = igb_txeof(txr);
 1548                 } while (loop-- && more);
 1549 #ifndef IGB_LEGACY_TX
 1550                 if (!drbr_empty(ifp, txr->br))
 1551                         igb_mq_start_locked(ifp, txr);
 1552 #else
 1553                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1554                         igb_start_locked(txr, ifp);
 1555 #endif
 1556                 IGB_TX_UNLOCK(txr);
 1557         }
 1558 
 1559         return POLL_RETURN_COUNT(rx_done);
 1560 }
 1561 #endif /* DEVICE_POLLING */
 1562 
 1563 /*********************************************************************
 1564  *
 1565  *  MSIX Que Interrupt Service routine
 1566  *
 1567  **********************************************************************/
 1568 static void
 1569 igb_msix_que(void *arg)
 1570 {
 1571         struct igb_queue *que = arg;
 1572         struct adapter *adapter = que->adapter;
 1573         struct ifnet   *ifp = adapter->ifp;
 1574         struct tx_ring *txr = que->txr;
 1575         struct rx_ring *rxr = que->rxr;
 1576         u32             newitr = 0;
 1577         bool            more_rx;
 1578 
 1579         /* Ignore spurious interrupts */
 1580         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 1581                 return;
 1582 
 1583         E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
 1584         ++que->irqs;
 1585 
 1586         IGB_TX_LOCK(txr);
 1587         igb_txeof(txr);
 1588 #ifndef IGB_LEGACY_TX
 1589         /* Process the stack queue only if not depleted */
 1590         if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
 1591             !drbr_empty(ifp, txr->br))
 1592                 igb_mq_start_locked(ifp, txr);
 1593 #else
 1594         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1595                 igb_start_locked(txr, ifp);
 1596 #endif
 1597         IGB_TX_UNLOCK(txr);
 1598 
 1599         more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
 1600 
 1601         if (adapter->enable_aim == FALSE)
 1602                 goto no_calc;
 1603         /*
 1604         ** Do Adaptive Interrupt Moderation:
 1605         **  - Write out last calculated setting
 1606         **  - Calculate based on average size over
 1607         **    the last interval.
 1608         */
 1609         if (que->eitr_setting)
 1610                 E1000_WRITE_REG(&adapter->hw,
 1611                     E1000_EITR(que->msix), que->eitr_setting);
 1612  
 1613         que->eitr_setting = 0;
 1614 
 1615         /* Idle, do nothing */
 1616         if ((txr->bytes == 0) && (rxr->bytes == 0))
 1617                 goto no_calc;
 1618                                 
 1619         /* Used half Default if sub-gig */
 1620         if (adapter->link_speed != 1000)
 1621                 newitr = IGB_DEFAULT_ITR / 2;
 1622         else {
 1623                 if ((txr->bytes) && (txr->packets))
 1624                         newitr = txr->bytes/txr->packets;
 1625                 if ((rxr->bytes) && (rxr->packets))
 1626                         newitr = max(newitr,
 1627                             (rxr->bytes / rxr->packets));
 1628                 newitr += 24; /* account for hardware frame, crc */
 1629                 /* set an upper boundary */
 1630                 newitr = min(newitr, 3000);
 1631                 /* Be nice to the mid range */
 1632                 if ((newitr > 300) && (newitr < 1200))
 1633                         newitr = (newitr / 3);
 1634                 else
 1635                         newitr = (newitr / 2);
 1636         }
 1637         newitr &= 0x7FFC;  /* Mask invalid bits */
 1638         if (adapter->hw.mac.type == e1000_82575)
 1639                 newitr |= newitr << 16;
 1640         else
 1641                 newitr |= E1000_EITR_CNT_IGNR;
 1642                  
 1643         /* save for next interrupt */
 1644         que->eitr_setting = newitr;
 1645 
 1646         /* Reset state */
 1647         txr->bytes = 0;
 1648         txr->packets = 0;
 1649         rxr->bytes = 0;
 1650         rxr->packets = 0;
 1651 
 1652 no_calc:
 1653         /* Schedule a clean task if needed*/
 1654         if (more_rx)
 1655                 taskqueue_enqueue(que->tq, &que->que_task);
 1656         else
 1657                 /* Reenable this interrupt */
 1658                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
 1659         return;
 1660 }
 1661 
 1662 
 1663 /*********************************************************************
 1664  *
 1665  *  MSIX Link Interrupt Service routine
 1666  *
 1667  **********************************************************************/
 1668 
 1669 static void
 1670 igb_msix_link(void *arg)
 1671 {
 1672         struct adapter  *adapter = arg;
 1673         u32             icr;
 1674 
 1675         ++adapter->link_irq;
 1676         icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1677         if (!(icr & E1000_ICR_LSC))
 1678                 goto spurious;
 1679         igb_handle_link(adapter, 0);
 1680 
 1681 spurious:
 1682         /* Rearm */
 1683         E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
 1684         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
 1685         return;
 1686 }
 1687 
 1688 
 1689 /*********************************************************************
 1690  *
 1691  *  Media Ioctl callback
 1692  *
 1693  *  This routine is called whenever the user queries the status of
 1694  *  the interface using ifconfig.
 1695  *
 1696  **********************************************************************/
 1697 static void
 1698 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 1699 {
 1700         struct adapter *adapter = ifp->if_softc;
 1701 
 1702         INIT_DEBUGOUT("igb_media_status: begin");
 1703 
 1704         IGB_CORE_LOCK(adapter);
 1705         igb_update_link_status(adapter);
 1706 
 1707         ifmr->ifm_status = IFM_AVALID;
 1708         ifmr->ifm_active = IFM_ETHER;
 1709 
 1710         if (!adapter->link_active) {
 1711                 IGB_CORE_UNLOCK(adapter);
 1712                 return;
 1713         }
 1714 
 1715         ifmr->ifm_status |= IFM_ACTIVE;
 1716 
 1717         switch (adapter->link_speed) {
 1718         case 10:
 1719                 ifmr->ifm_active |= IFM_10_T;
 1720                 break;
 1721         case 100:
 1722                 /*
 1723                 ** Support for 100Mb SFP - these are Fiber 
 1724                 ** but the media type appears as serdes
 1725                 */
 1726                 if (adapter->hw.phy.media_type ==
 1727                     e1000_media_type_internal_serdes)
 1728                         ifmr->ifm_active |= IFM_100_FX;
 1729                 else
 1730                         ifmr->ifm_active |= IFM_100_TX;
 1731                 break;
 1732         case 1000:
 1733                 ifmr->ifm_active |= IFM_1000_T;
 1734                 break;
 1735         case 2500:
 1736                 ifmr->ifm_active |= IFM_2500_SX;
 1737                 break;
 1738         }
 1739 
 1740         if (adapter->link_duplex == FULL_DUPLEX)
 1741                 ifmr->ifm_active |= IFM_FDX;
 1742         else
 1743                 ifmr->ifm_active |= IFM_HDX;
 1744 
 1745         IGB_CORE_UNLOCK(adapter);
 1746 }
 1747 
 1748 /*********************************************************************
 1749  *
 1750  *  Media Ioctl callback
 1751  *
 1752  *  This routine is called when the user changes speed/duplex using
 1753  *  media/mediopt option with ifconfig.
 1754  *
 1755  **********************************************************************/
 1756 static int
 1757 igb_media_change(struct ifnet *ifp)
 1758 {
 1759         struct adapter *adapter = ifp->if_softc;
 1760         struct ifmedia  *ifm = &adapter->media;
 1761 
 1762         INIT_DEBUGOUT("igb_media_change: begin");
 1763 
 1764         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 1765                 return (EINVAL);
 1766 
 1767         IGB_CORE_LOCK(adapter);
 1768         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 1769         case IFM_AUTO:
 1770                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1771                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 1772                 break;
 1773         case IFM_1000_LX:
 1774         case IFM_1000_SX:
 1775         case IFM_1000_T:
 1776                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1777                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
 1778                 break;
 1779         case IFM_100_TX:
 1780                 adapter->hw.mac.autoneg = FALSE;
 1781                 adapter->hw.phy.autoneg_advertised = 0;
 1782                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1783                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
 1784                 else
 1785                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
 1786                 break;
 1787         case IFM_10_T:
 1788                 adapter->hw.mac.autoneg = FALSE;
 1789                 adapter->hw.phy.autoneg_advertised = 0;
 1790                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1791                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
 1792                 else
 1793                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
 1794                 break;
 1795         default:
 1796                 device_printf(adapter->dev, "Unsupported media type\n");
 1797         }
 1798 
 1799         igb_init_locked(adapter);
 1800         IGB_CORE_UNLOCK(adapter);
 1801 
 1802         return (0);
 1803 }
 1804 
 1805 
 1806 /*********************************************************************
 1807  *
 1808  *  This routine maps the mbufs to Advanced TX descriptors.
 1809  *  
 1810  **********************************************************************/
 1811 static int
 1812 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 1813 {
 1814         struct adapter  *adapter = txr->adapter;
 1815         u32             olinfo_status = 0, cmd_type_len;
 1816         int             i, j, error, nsegs;
 1817         int             first;
 1818         bool            remap = TRUE;
 1819         struct mbuf     *m_head;
 1820         bus_dma_segment_t segs[IGB_MAX_SCATTER];
 1821         bus_dmamap_t    map;
 1822         struct igb_tx_buf *txbuf;
 1823         union e1000_adv_tx_desc *txd = NULL;
 1824 
 1825         m_head = *m_headp;
 1826 
 1827         /* Basic descriptor defines */
 1828         cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
 1829             E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
 1830 
 1831         if (m_head->m_flags & M_VLANTAG)
 1832                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
 1833 
 1834         /*
 1835          * Important to capture the first descriptor
 1836          * used because it will contain the index of
 1837          * the one we tell the hardware to report back
 1838          */
 1839         first = txr->next_avail_desc;
 1840         txbuf = &txr->tx_buffers[first];
 1841         map = txbuf->map;
 1842 
 1843         /*
 1844          * Map the packet for DMA.
 1845          */
 1846 retry:
 1847         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
 1848             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 1849 
 1850         if (__predict_false(error)) {
 1851                 struct mbuf *m;
 1852 
 1853                 switch (error) {
 1854                 case EFBIG:
 1855                         /* Try it again? - one try */
 1856                         if (remap == TRUE) {
 1857                                 remap = FALSE;
 1858                                 m = m_collapse(*m_headp, M_NOWAIT,
 1859                                     IGB_MAX_SCATTER);
 1860                                 if (m == NULL) {
 1861                                         adapter->mbuf_defrag_failed++;
 1862                                         m_freem(*m_headp);
 1863                                         *m_headp = NULL;
 1864                                         return (ENOBUFS);
 1865                                 }
 1866                                 *m_headp = m;
 1867                                 goto retry;
 1868                         } else
 1869                                 return (error);
 1870                 default:
 1871                         txr->no_tx_dma_setup++;
 1872                         m_freem(*m_headp);
 1873                         *m_headp = NULL;
 1874                         return (error);
 1875                 }
 1876         }
 1877 
 1878         /* Make certain there are enough descriptors */
 1879         if (txr->tx_avail < (nsegs + 2)) {
 1880                 txr->no_desc_avail++;
 1881                 bus_dmamap_unload(txr->txtag, map);
 1882                 return (ENOBUFS);
 1883         }
 1884         m_head = *m_headp;
 1885 
 1886         /*
 1887         ** Set up the appropriate offload context
 1888         ** this will consume the first descriptor
 1889         */
 1890         error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
 1891         if (__predict_false(error)) {
 1892                 m_freem(*m_headp);
 1893                 *m_headp = NULL;
 1894                 return (error);
 1895         }
 1896 
 1897         /* 82575 needs the queue index added */
 1898         if (adapter->hw.mac.type == e1000_82575)
 1899                 olinfo_status |= txr->me << 4;
 1900 
 1901         i = txr->next_avail_desc;
 1902         for (j = 0; j < nsegs; j++) {
 1903                 bus_size_t seglen;
 1904                 bus_addr_t segaddr;
 1905 
 1906                 txbuf = &txr->tx_buffers[i];
 1907                 txd = &txr->tx_base[i];
 1908                 seglen = segs[j].ds_len;
 1909                 segaddr = htole64(segs[j].ds_addr);
 1910 
 1911                 txd->read.buffer_addr = segaddr;
 1912                 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
 1913                     cmd_type_len | seglen);
 1914                 txd->read.olinfo_status = htole32(olinfo_status);
 1915 
 1916                 if (++i == txr->num_desc)
 1917                         i = 0;
 1918         }
 1919 
 1920         txd->read.cmd_type_len |=
 1921             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
 1922         txr->tx_avail -= nsegs;
 1923         txr->next_avail_desc = i;
 1924 
 1925         txbuf->m_head = m_head;
 1926         /*
 1927         ** Here we swap the map so the last descriptor,
 1928         ** which gets the completion interrupt has the
 1929         ** real map, and the first descriptor gets the
 1930         ** unused map from this descriptor.
 1931         */
 1932         txr->tx_buffers[first].map = txbuf->map;
 1933         txbuf->map = map;
 1934         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
 1935 
 1936         /* Set the EOP descriptor that will be marked done */
 1937         txbuf = &txr->tx_buffers[first];
 1938         txbuf->eop = txd;
 1939 
 1940         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 1941             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 1942         /*
 1943          * Advance the Transmit Descriptor Tail (Tdt), this tells the
 1944          * hardware that this frame is available to transmit.
 1945          */
 1946         ++txr->total_packets;
 1947         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
 1948 
 1949         return (0);
 1950 }
 1951 static void
 1952 igb_set_promisc(struct adapter *adapter)
 1953 {
 1954         struct ifnet    *ifp = adapter->ifp;
 1955         struct e1000_hw *hw = &adapter->hw;
 1956         u32             reg;
 1957 
 1958         if (adapter->vf_ifp) {
 1959                 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
 1960                 return;
 1961         }
 1962 
 1963         reg = E1000_READ_REG(hw, E1000_RCTL);
 1964         if (ifp->if_flags & IFF_PROMISC) {
 1965                 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
 1966                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 1967         } else if (ifp->if_flags & IFF_ALLMULTI) {
 1968                 reg |= E1000_RCTL_MPE;
 1969                 reg &= ~E1000_RCTL_UPE;
 1970                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 1971         }
 1972 }
 1973 
 1974 static void
 1975 igb_disable_promisc(struct adapter *adapter)
 1976 {
 1977         struct e1000_hw *hw = &adapter->hw;
 1978         struct ifnet    *ifp = adapter->ifp;
 1979         u32             reg;
 1980         int             mcnt = 0;
 1981 
 1982         if (adapter->vf_ifp) {
 1983                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
 1984                 return;
 1985         }
 1986         reg = E1000_READ_REG(hw, E1000_RCTL);
 1987         reg &=  (~E1000_RCTL_UPE);
 1988         if (ifp->if_flags & IFF_ALLMULTI)
 1989                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
 1990         else {
 1991                 struct  ifmultiaddr *ifma;
 1992 #if __FreeBSD_version < 800000
 1993                 IF_ADDR_LOCK(ifp);
 1994 #else   
 1995                 if_maddr_rlock(ifp);
 1996 #endif
 1997                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 1998                         if (ifma->ifma_addr->sa_family != AF_LINK)
 1999                                 continue;
 2000                         if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
 2001                                 break;
 2002                         mcnt++;
 2003                 }
 2004 #if __FreeBSD_version < 800000
 2005                 IF_ADDR_UNLOCK(ifp);
 2006 #else
 2007                 if_maddr_runlock(ifp);
 2008 #endif
 2009         }
 2010         /* Don't disable if in MAX groups */
 2011         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
 2012                 reg &=  (~E1000_RCTL_MPE);
 2013         E1000_WRITE_REG(hw, E1000_RCTL, reg);
 2014 }
 2015 
 2016 
 2017 /*********************************************************************
 2018  *  Multicast Update
 2019  *
 2020  *  This routine is called whenever multicast address list is updated.
 2021  *
 2022  **********************************************************************/
 2023 
 2024 static void
 2025 igb_set_multi(struct adapter *adapter)
 2026 {
 2027         struct ifnet    *ifp = adapter->ifp;
 2028         struct ifmultiaddr *ifma;
 2029         u32 reg_rctl = 0;
 2030         u8  *mta;
 2031 
 2032         int mcnt = 0;
 2033 
 2034         IOCTL_DEBUGOUT("igb_set_multi: begin");
 2035 
 2036         mta = adapter->mta;
 2037         bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
 2038             MAX_NUM_MULTICAST_ADDRESSES);
 2039 
 2040 #if __FreeBSD_version < 800000
 2041         IF_ADDR_LOCK(ifp);
 2042 #else
 2043         if_maddr_rlock(ifp);
 2044 #endif
 2045         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 2046                 if (ifma->ifma_addr->sa_family != AF_LINK)
 2047                         continue;
 2048 
 2049                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
 2050                         break;
 2051 
 2052                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 2053                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
 2054                 mcnt++;
 2055         }
 2056 #if __FreeBSD_version < 800000
 2057         IF_ADDR_UNLOCK(ifp);
 2058 #else
 2059         if_maddr_runlock(ifp);
 2060 #endif
 2061 
 2062         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
 2063                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 2064                 reg_rctl |= E1000_RCTL_MPE;
 2065                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 2066         } else
 2067                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
 2068 }
 2069 
 2070 
 2071 /*********************************************************************
 2072  *  Timer routine:
 2073  *      This routine checks for link status,
 2074  *      updates statistics, and does the watchdog.
 2075  *
 2076  **********************************************************************/
 2077 
 2078 static void
 2079 igb_local_timer(void *arg)
 2080 {
 2081         struct adapter          *adapter = arg;
 2082         device_t                dev = adapter->dev;
 2083         struct ifnet            *ifp = adapter->ifp;
 2084         struct tx_ring          *txr = adapter->tx_rings;
 2085         struct igb_queue        *que = adapter->queues;
 2086         int                     hung = 0, busy = 0;
 2087 
 2088 
 2089         IGB_CORE_LOCK_ASSERT(adapter);
 2090 
 2091         igb_update_link_status(adapter);
 2092         igb_update_stats_counters(adapter);
 2093 
 2094         /*
 2095         ** Check the TX queues status
 2096         **      - central locked handling of OACTIVE
 2097         **      - watchdog only if all queues show hung
 2098         */
 2099         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 2100                 if ((txr->queue_status & IGB_QUEUE_HUNG) &&
 2101                     (adapter->pause_frames == 0))
 2102                         ++hung;
 2103                 if (txr->queue_status & IGB_QUEUE_DEPLETED)
 2104                         ++busy;
 2105                 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
 2106                         taskqueue_enqueue(que->tq, &que->que_task);
 2107         }
 2108         if (hung == adapter->num_queues)
 2109                 goto timeout;
 2110         if (busy == adapter->num_queues)
 2111                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 2112         else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
 2113             (busy < adapter->num_queues))
 2114                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 2115 
 2116         adapter->pause_frames = 0;
 2117         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 2118 #ifndef DEVICE_POLLING
 2119         /* Schedule all queue interrupts - deadlock protection */
 2120         E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
 2121 #endif
 2122         return;
 2123 
 2124 timeout:
 2125         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
 2126         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
 2127             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
 2128             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
 2129         device_printf(dev,"TX(%d) desc avail = %d,"
 2130             "Next TX to Clean = %d\n",
 2131             txr->me, txr->tx_avail, txr->next_to_clean);
 2132         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 2133         adapter->watchdog_events++;
 2134         igb_init_locked(adapter);
 2135 }
 2136 
 2137 static void
 2138 igb_update_link_status(struct adapter *adapter)
 2139 {
 2140         struct e1000_hw         *hw = &adapter->hw;
 2141         struct e1000_fc_info    *fc = &hw->fc;
 2142         struct ifnet            *ifp = adapter->ifp;
 2143         device_t                dev = adapter->dev;
 2144         struct tx_ring          *txr = adapter->tx_rings;
 2145         u32                     link_check, thstat, ctrl;
 2146         char                    *flowctl = NULL;
 2147 
 2148         link_check = thstat = ctrl = 0;
 2149 
 2150         /* Get the cached link value or read for real */
 2151         switch (hw->phy.media_type) {
 2152         case e1000_media_type_copper:
 2153                 if (hw->mac.get_link_status) {
 2154                         /* Do the work to read phy */
 2155                         e1000_check_for_link(hw);
 2156                         link_check = !hw->mac.get_link_status;
 2157                 } else
 2158                         link_check = TRUE;
 2159                 break;
 2160         case e1000_media_type_fiber:
 2161                 e1000_check_for_link(hw);
 2162                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
 2163                                  E1000_STATUS_LU);
 2164                 break;
 2165         case e1000_media_type_internal_serdes:
 2166                 e1000_check_for_link(hw);
 2167                 link_check = adapter->hw.mac.serdes_has_link;
 2168                 break;
 2169         /* VF device is type_unknown */
 2170         case e1000_media_type_unknown:
 2171                 e1000_check_for_link(hw);
 2172                 link_check = !hw->mac.get_link_status;
 2173                 /* Fall thru */
 2174         default:
 2175                 break;
 2176         }
 2177 
 2178         /* Check for thermal downshift or shutdown */
 2179         if (hw->mac.type == e1000_i350) {
 2180                 thstat = E1000_READ_REG(hw, E1000_THSTAT);
 2181                 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
 2182         }
 2183 
 2184         /* Get the flow control for display */
 2185         switch (fc->current_mode) {
 2186         case e1000_fc_rx_pause:
 2187                 flowctl = "RX";
 2188                 break;  
 2189         case e1000_fc_tx_pause:
 2190                 flowctl = "TX";
 2191                 break;  
 2192         case e1000_fc_full:
 2193                 flowctl = "Full";
 2194                 break;  
 2195         case e1000_fc_none:
 2196         default:
 2197                 flowctl = "None";
 2198                 break;  
 2199         }
 2200 
 2201         /* Now we check if a transition has happened */
 2202         if (link_check && (adapter->link_active == 0)) {
 2203                 e1000_get_speed_and_duplex(&adapter->hw, 
 2204                     &adapter->link_speed, &adapter->link_duplex);
 2205                 if (bootverbose)
 2206                         device_printf(dev, "Link is up %d Mbps %s,"
 2207                             " Flow Control: %s\n",
 2208                             adapter->link_speed,
 2209                             ((adapter->link_duplex == FULL_DUPLEX) ?
 2210                             "Full Duplex" : "Half Duplex"), flowctl);
 2211                 adapter->link_active = 1;
 2212                 ifp->if_baudrate = adapter->link_speed * 1000000;
 2213                 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
 2214                     (thstat & E1000_THSTAT_LINK_THROTTLE))
 2215                         device_printf(dev, "Link: thermal downshift\n");
 2216                 /* Delay Link Up for Phy update */
 2217                 if (((hw->mac.type == e1000_i210) ||
 2218                     (hw->mac.type == e1000_i211)) &&
 2219                     (hw->phy.id == I210_I_PHY_ID))
 2220                         msec_delay(I210_LINK_DELAY);
 2221                 /* Reset if the media type changed. */
 2222                 if (hw->dev_spec._82575.media_changed) {
 2223                         hw->dev_spec._82575.media_changed = false;
 2224                         adapter->flags |= IGB_MEDIA_RESET;
 2225                         igb_reset(adapter);
 2226                 }       
 2227                 /* This can sleep */
 2228                 if_link_state_change(ifp, LINK_STATE_UP);
 2229         } else if (!link_check && (adapter->link_active == 1)) {
 2230                 ifp->if_baudrate = adapter->link_speed = 0;
 2231                 adapter->link_duplex = 0;
 2232                 if (bootverbose)
 2233                         device_printf(dev, "Link is Down\n");
 2234                 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
 2235                     (thstat & E1000_THSTAT_PWR_DOWN))
 2236                         device_printf(dev, "Link: thermal shutdown\n");
 2237                 adapter->link_active = 0;
 2238                 /* This can sleep */
 2239                 if_link_state_change(ifp, LINK_STATE_DOWN);
 2240                 /* Reset queue state */
 2241                 for (int i = 0; i < adapter->num_queues; i++, txr++)
 2242                         txr->queue_status = IGB_QUEUE_IDLE;
 2243         }
 2244 }
 2245 
 2246 /*********************************************************************
 2247  *
 2248  *  This routine disables all traffic on the adapter by issuing a
 2249  *  global reset on the MAC and deallocates TX/RX buffers.
 2250  *
 2251  **********************************************************************/
 2252 
 2253 static void
 2254 igb_stop(void *arg)
 2255 {
 2256         struct adapter  *adapter = arg;
 2257         struct ifnet    *ifp = adapter->ifp;
 2258         struct tx_ring *txr = adapter->tx_rings;
 2259 
 2260         IGB_CORE_LOCK_ASSERT(adapter);
 2261 
 2262         INIT_DEBUGOUT("igb_stop: begin");
 2263 
 2264         igb_disable_intr(adapter);
 2265 
 2266         callout_stop(&adapter->timer);
 2267 
 2268         /* Tell the stack that the interface is no longer active */
 2269         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 2270         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 2271 
 2272         /* Disarm watchdog timer. */
 2273         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 2274                 IGB_TX_LOCK(txr);
 2275                 txr->queue_status = IGB_QUEUE_IDLE;
 2276                 IGB_TX_UNLOCK(txr);
 2277         }
 2278 
 2279         e1000_reset_hw(&adapter->hw);
 2280         E1000_WRITE_REG(&adapter->hw, E1000_WUFC, 0);
 2281 
 2282         e1000_led_off(&adapter->hw);
 2283         e1000_cleanup_led(&adapter->hw);
 2284 }
 2285 
 2286 
 2287 /*********************************************************************
 2288  *
 2289  *  Determine hardware revision.
 2290  *
 2291  **********************************************************************/
 2292 static void
 2293 igb_identify_hardware(struct adapter *adapter)
 2294 {
 2295         device_t dev = adapter->dev;
 2296 
 2297         /* Make sure our PCI config space has the necessary stuff set */
 2298         pci_enable_busmaster(dev);
 2299         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
 2300 
 2301         /* Save off the information about this board */
 2302         adapter->hw.vendor_id = pci_get_vendor(dev);
 2303         adapter->hw.device_id = pci_get_device(dev);
 2304         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
 2305         adapter->hw.subsystem_vendor_id =
 2306             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 2307         adapter->hw.subsystem_device_id =
 2308             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 2309 
 2310         /* Set MAC type early for PCI setup */
 2311         e1000_set_mac_type(&adapter->hw);
 2312 
 2313         /* Are we a VF device? */
 2314         if ((adapter->hw.mac.type == e1000_vfadapt) ||
 2315             (adapter->hw.mac.type == e1000_vfadapt_i350))
 2316                 adapter->vf_ifp = 1;
 2317         else
 2318                 adapter->vf_ifp = 0;
 2319 }
 2320 
 2321 static int
 2322 igb_allocate_pci_resources(struct adapter *adapter)
 2323 {
 2324         device_t        dev = adapter->dev;
 2325         int             rid;
 2326 
 2327         rid = PCIR_BAR(0);
 2328         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 2329             &rid, RF_ACTIVE);
 2330         if (adapter->pci_mem == NULL) {
 2331                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 2332                 return (ENXIO);
 2333         }
 2334         adapter->osdep.mem_bus_space_tag =
 2335             rman_get_bustag(adapter->pci_mem);
 2336         adapter->osdep.mem_bus_space_handle =
 2337             rman_get_bushandle(adapter->pci_mem);
 2338         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
 2339 
 2340         adapter->num_queues = 1; /* Defaults for Legacy or MSI */
 2341 
 2342         /* This will setup either MSI/X or MSI */
 2343         adapter->msix = igb_setup_msix(adapter);
 2344         adapter->hw.back = &adapter->osdep;
 2345 
 2346         return (0);
 2347 }
 2348 
 2349 /*********************************************************************
 2350  *
 2351  *  Setup the Legacy or MSI Interrupt handler
 2352  *
 2353  **********************************************************************/
 2354 static int
 2355 igb_allocate_legacy(struct adapter *adapter)
 2356 {
 2357         device_t                dev = adapter->dev;
 2358         struct igb_queue        *que = adapter->queues;
 2359 #ifndef IGB_LEGACY_TX
 2360         struct tx_ring          *txr = adapter->tx_rings;
 2361 #endif
 2362         int                     error, rid = 0;
 2363 
 2364         /* Turn off all interrupts */
 2365         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
 2366 
 2367         /* MSI RID is 1 */
 2368         if (adapter->msix == 1)
 2369                 rid = 1;
 2370 
 2371         /* We allocate a single interrupt resource */
 2372         adapter->res = bus_alloc_resource_any(dev,
 2373             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2374         if (adapter->res == NULL) {
 2375                 device_printf(dev, "Unable to allocate bus resource: "
 2376                     "interrupt\n");
 2377                 return (ENXIO);
 2378         }
 2379 
 2380 #ifndef IGB_LEGACY_TX
 2381         TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
 2382 #endif
 2383 
 2384         /*
 2385          * Try allocating a fast interrupt and the associated deferred
 2386          * processing contexts.
 2387          */
 2388         TASK_INIT(&que->que_task, 0, igb_handle_que, que);
 2389         /* Make tasklet for deferred link handling */
 2390         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
 2391         que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
 2392             taskqueue_thread_enqueue, &que->tq);
 2393         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
 2394             device_get_nameunit(adapter->dev));
 2395         if ((error = bus_setup_intr(dev, adapter->res,
 2396             INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
 2397             adapter, &adapter->tag)) != 0) {
 2398                 device_printf(dev, "Failed to register fast interrupt "
 2399                             "handler: %d\n", error);
 2400                 taskqueue_free(que->tq);
 2401                 que->tq = NULL;
 2402                 return (error);
 2403         }
 2404 
 2405         return (0);
 2406 }
 2407 
 2408 
 2409 /*********************************************************************
 2410  *
 2411  *  Setup the MSIX Queue Interrupt handlers: 
 2412  *
 2413  **********************************************************************/
 2414 static int
 2415 igb_allocate_msix(struct adapter *adapter)
 2416 {
 2417         device_t                dev = adapter->dev;
 2418         struct igb_queue        *que = adapter->queues;
 2419         int                     error, rid, vector = 0;
 2420 
 2421         /* Be sure to start with all interrupts disabled */
 2422         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
 2423         E1000_WRITE_FLUSH(&adapter->hw);
 2424 
 2425         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
 2426                 rid = vector +1;
 2427                 que->res = bus_alloc_resource_any(dev,
 2428                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2429                 if (que->res == NULL) {
 2430                         device_printf(dev,
 2431                             "Unable to allocate bus resource: "
 2432                             "MSIX Queue Interrupt\n");
 2433                         return (ENXIO);
 2434                 }
 2435                 error = bus_setup_intr(dev, que->res,
 2436                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2437                     igb_msix_que, que, &que->tag);
 2438                 if (error) {
 2439                         que->res = NULL;
 2440                         device_printf(dev, "Failed to register Queue handler");
 2441                         return (error);
 2442                 }
 2443 #if __FreeBSD_version >= 800504
 2444                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
 2445 #endif
 2446                 que->msix = vector;
 2447                 if (adapter->hw.mac.type == e1000_82575)
 2448                         que->eims = E1000_EICR_TX_QUEUE0 << i;
 2449                 else
 2450                         que->eims = 1 << vector;
 2451                 /*
 2452                 ** Bind the msix vector, and thus the
 2453                 ** rings to the corresponding cpu.
 2454                 */
 2455                 if (adapter->num_queues > 1) {
 2456                         if (igb_last_bind_cpu < 0)
 2457                                 igb_last_bind_cpu = CPU_FIRST();
 2458                         bus_bind_intr(dev, que->res, igb_last_bind_cpu);
 2459                         device_printf(dev,
 2460                                 "Bound queue %d to cpu %d\n",
 2461                                 i,igb_last_bind_cpu);
 2462                         igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
 2463                 }
 2464 #ifndef IGB_LEGACY_TX
 2465                 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
 2466                     que->txr);
 2467 #endif
 2468                 /* Make tasklet for deferred handling */
 2469                 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
 2470                 que->tq = taskqueue_create("igb_que", M_NOWAIT,
 2471                     taskqueue_thread_enqueue, &que->tq);
 2472                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
 2473                     device_get_nameunit(adapter->dev));
 2474         }
 2475 
 2476         /* And Link */
 2477         rid = vector + 1;
 2478         adapter->res = bus_alloc_resource_any(dev,
 2479             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2480         if (adapter->res == NULL) {
 2481                 device_printf(dev,
 2482                     "Unable to allocate bus resource: "
 2483                     "MSIX Link Interrupt\n");
 2484                 return (ENXIO);
 2485         }
 2486         if ((error = bus_setup_intr(dev, adapter->res,
 2487             INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2488             igb_msix_link, adapter, &adapter->tag)) != 0) {
 2489                 device_printf(dev, "Failed to register Link handler");
 2490                 return (error);
 2491         }
 2492 #if __FreeBSD_version >= 800504
 2493         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
 2494 #endif
 2495         adapter->linkvec = vector;
 2496 
 2497         return (0);
 2498 }
 2499 
 2500 
 2501 static void
 2502 igb_configure_queues(struct adapter *adapter)
 2503 {
 2504         struct  e1000_hw        *hw = &adapter->hw;
 2505         struct  igb_queue       *que;
 2506         u32                     tmp, ivar = 0, newitr = 0;
 2507 
 2508         /* First turn on RSS capability */
 2509         if (adapter->hw.mac.type != e1000_82575)
 2510                 E1000_WRITE_REG(hw, E1000_GPIE,
 2511                     E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
 2512                     E1000_GPIE_PBA | E1000_GPIE_NSICR);
 2513 
 2514         /* Turn on MSIX */
 2515         switch (adapter->hw.mac.type) {
 2516         case e1000_82580:
 2517         case e1000_i350:
 2518         case e1000_i354:
 2519         case e1000_i210:
 2520         case e1000_i211:
 2521         case e1000_vfadapt:
 2522         case e1000_vfadapt_i350:
 2523                 /* RX entries */
 2524                 for (int i = 0; i < adapter->num_queues; i++) {
 2525                         u32 index = i >> 1;
 2526                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2527                         que = &adapter->queues[i];
 2528                         if (i & 1) {
 2529                                 ivar &= 0xFF00FFFF;
 2530                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
 2531                         } else {
 2532                                 ivar &= 0xFFFFFF00;
 2533                                 ivar |= que->msix | E1000_IVAR_VALID;
 2534                         }
 2535                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2536                 }
 2537                 /* TX entries */
 2538                 for (int i = 0; i < adapter->num_queues; i++) {
 2539                         u32 index = i >> 1;
 2540                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2541                         que = &adapter->queues[i];
 2542                         if (i & 1) {
 2543                                 ivar &= 0x00FFFFFF;
 2544                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
 2545                         } else {
 2546                                 ivar &= 0xFFFF00FF;
 2547                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
 2548                         }
 2549                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2550                         adapter->que_mask |= que->eims;
 2551                 }
 2552 
 2553                 /* And for the link interrupt */
 2554                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 2555                 adapter->link_mask = 1 << adapter->linkvec;
 2556                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 2557                 break;
 2558         case e1000_82576:
 2559                 /* RX entries */
 2560                 for (int i = 0; i < adapter->num_queues; i++) {
 2561                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2562                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2563                         que = &adapter->queues[i];
 2564                         if (i < 8) {
 2565                                 ivar &= 0xFFFFFF00;
 2566                                 ivar |= que->msix | E1000_IVAR_VALID;
 2567                         } else {
 2568                                 ivar &= 0xFF00FFFF;
 2569                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
 2570                         }
 2571                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2572                         adapter->que_mask |= que->eims;
 2573                 }
 2574                 /* TX entries */
 2575                 for (int i = 0; i < adapter->num_queues; i++) {
 2576                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2577                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2578                         que = &adapter->queues[i];
 2579                         if (i < 8) {
 2580                                 ivar &= 0xFFFF00FF;
 2581                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
 2582                         } else {
 2583                                 ivar &= 0x00FFFFFF;
 2584                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
 2585                         }
 2586                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2587                         adapter->que_mask |= que->eims;
 2588                 }
 2589 
 2590                 /* And for the link interrupt */
 2591                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 2592                 adapter->link_mask = 1 << adapter->linkvec;
 2593                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 2594                 break;
 2595 
 2596         case e1000_82575:
 2597                 /* enable MSI-X support*/
 2598                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 2599                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 2600                 /* Auto-Mask interrupts upon ICR read. */
 2601                 tmp |= E1000_CTRL_EXT_EIAME;
 2602                 tmp |= E1000_CTRL_EXT_IRCA;
 2603                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 2604 
 2605                 /* Queues */
 2606                 for (int i = 0; i < adapter->num_queues; i++) {
 2607                         que = &adapter->queues[i];
 2608                         tmp = E1000_EICR_RX_QUEUE0 << i;
 2609                         tmp |= E1000_EICR_TX_QUEUE0 << i;
 2610                         que->eims = tmp;
 2611                         E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
 2612                             i, que->eims);
 2613                         adapter->que_mask |= que->eims;
 2614                 }
 2615 
 2616                 /* Link */
 2617                 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
 2618                     E1000_EIMS_OTHER);
 2619                 adapter->link_mask |= E1000_EIMS_OTHER;
 2620         default:
 2621                 break;
 2622         }
 2623 
 2624         /* Set the starting interrupt rate */
 2625         if (igb_max_interrupt_rate > 0)
 2626                 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
 2627 
 2628         if (hw->mac.type == e1000_82575)
 2629                 newitr |= newitr << 16;
 2630         else
 2631                 newitr |= E1000_EITR_CNT_IGNR;
 2632 
 2633         for (int i = 0; i < adapter->num_queues; i++) {
 2634                 que = &adapter->queues[i];
 2635                 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
 2636         }
 2637 
 2638         return;
 2639 }
 2640 
 2641 
 2642 static void
 2643 igb_free_pci_resources(struct adapter *adapter)
 2644 {
 2645         struct          igb_queue *que = adapter->queues;
 2646         device_t        dev = adapter->dev;
 2647         int             rid;
 2648 
 2649         /*
 2650         ** There is a slight possibility of a failure mode
 2651         ** in attach that will result in entering this function
 2652         ** before interrupt resources have been initialized, and
 2653         ** in that case we do not want to execute the loops below
 2654         ** We can detect this reliably by the state of the adapter
 2655         ** res pointer.
 2656         */
 2657         if (adapter->res == NULL)
 2658                 goto mem;
 2659 
 2660         /*
 2661          * First release all the interrupt resources:
 2662          */
 2663         for (int i = 0; i < adapter->num_queues; i++, que++) {
 2664                 rid = que->msix + 1;
 2665                 if (que->tag != NULL) {
 2666                         bus_teardown_intr(dev, que->res, que->tag);
 2667                         que->tag = NULL;
 2668                 }
 2669                 if (que->res != NULL)
 2670                         bus_release_resource(dev,
 2671                             SYS_RES_IRQ, rid, que->res);
 2672         }
 2673 
 2674         /* Clean the Legacy or Link interrupt last */
 2675         if (adapter->linkvec) /* we are doing MSIX */
 2676                 rid = adapter->linkvec + 1;
 2677         else
 2678                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
 2679 
 2680         que = adapter->queues;
 2681         if (adapter->tag != NULL) {
 2682                 taskqueue_drain(que->tq, &adapter->link_task);
 2683                 bus_teardown_intr(dev, adapter->res, adapter->tag);
 2684                 adapter->tag = NULL;
 2685         }
 2686         if (adapter->res != NULL)
 2687                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
 2688 
 2689         for (int i = 0; i < adapter->num_queues; i++, que++) {
 2690                 if (que->tq != NULL) {
 2691 #ifndef IGB_LEGACY_TX
 2692                         taskqueue_drain(que->tq, &que->txr->txq_task);
 2693 #endif
 2694                         taskqueue_drain(que->tq, &que->que_task);
 2695                         taskqueue_free(que->tq);
 2696                 }
 2697         }
 2698 mem:
 2699         if (adapter->msix)
 2700                 pci_release_msi(dev);
 2701 
 2702         if (adapter->msix_mem != NULL)
 2703                 bus_release_resource(dev, SYS_RES_MEMORY,
 2704                     adapter->memrid, adapter->msix_mem);
 2705 
 2706         if (adapter->pci_mem != NULL)
 2707                 bus_release_resource(dev, SYS_RES_MEMORY,
 2708                     PCIR_BAR(0), adapter->pci_mem);
 2709 
 2710 }
 2711 
 2712 /*
 2713  * Setup Either MSI/X or MSI
 2714  */
 2715 static int
 2716 igb_setup_msix(struct adapter *adapter)
 2717 {
 2718         device_t        dev = adapter->dev;
 2719         int             bar, want, queues, msgs, maxqueues;
 2720 
 2721         /* tuneable override */
 2722         if (igb_enable_msix == 0)
 2723                 goto msi;
 2724 
 2725         /* First try MSI/X */
 2726         msgs = pci_msix_count(dev); 
 2727         if (msgs == 0)
 2728                 goto msi;
 2729         /*
 2730         ** Some new devices, as with ixgbe, now may
 2731         ** use a different BAR, so we need to keep
 2732         ** track of which is used.
 2733         */
 2734         adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
 2735         bar = pci_read_config(dev, adapter->memrid, 4);
 2736         if (bar == 0) /* use next bar */
 2737                 adapter->memrid += 4;
 2738         adapter->msix_mem = bus_alloc_resource_any(dev,
 2739             SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
 2740         if (adapter->msix_mem == NULL) {
 2741                 /* May not be enabled */
 2742                 device_printf(adapter->dev,
 2743                     "Unable to map MSIX table \n");
 2744                 goto msi;
 2745         }
 2746 
 2747         /* Figure out a reasonable auto config value */
 2748         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
 2749 
 2750         /* Manual override */
 2751         if (igb_num_queues != 0)
 2752                 queues = igb_num_queues;
 2753 
 2754         /* Sanity check based on HW */
 2755         switch (adapter->hw.mac.type) {
 2756                 case e1000_82575:
 2757                         maxqueues = 4;
 2758                         break;
 2759                 case e1000_82576:
 2760                 case e1000_82580:
 2761                 case e1000_i350:
 2762                 case e1000_i354:
 2763                         maxqueues = 8;
 2764                         break;
 2765                 case e1000_i210:
 2766                         maxqueues = 4;
 2767                         break;
 2768                 case e1000_i211:
 2769                         maxqueues = 2;
 2770                         break;
 2771                 default:  /* VF interfaces */
 2772                         maxqueues = 1;
 2773                         break;
 2774         }
 2775         if (queues > maxqueues)
 2776                 queues = maxqueues;
 2777 
 2778         /* Manual override */
 2779         if (igb_num_queues != 0)
 2780                 queues = igb_num_queues;
 2781 
 2782         /*
 2783         ** One vector (RX/TX pair) per queue
 2784         ** plus an additional for Link interrupt
 2785         */
 2786         want = queues + 1;
 2787         if (msgs >= want)
 2788                 msgs = want;
 2789         else {
 2790                 device_printf(adapter->dev,
 2791                     "MSIX Configuration Problem, "
 2792                     "%d vectors configured, but %d queues wanted!\n",
 2793                     msgs, want);
 2794                 goto msi;
 2795         }
 2796         if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
 2797                 device_printf(adapter->dev,
 2798                     "Using MSIX interrupts with %d vectors\n", msgs);
 2799                 adapter->num_queues = queues;
 2800                 return (msgs);
 2801         }
 2802         /*
 2803         ** If MSIX alloc failed or provided us with
 2804         ** less than needed, free and fall through to MSI
 2805         */
 2806         pci_release_msi(dev);
 2807 
 2808 msi:
 2809         if (adapter->msix_mem != NULL) {
 2810                 bus_release_resource(dev, SYS_RES_MEMORY,
 2811                     adapter->memrid, adapter->msix_mem);
 2812                 adapter->msix_mem = NULL;
 2813         }
 2814         msgs = 1;
 2815         if (pci_alloc_msi(dev, &msgs) == 0) {
 2816                 device_printf(adapter->dev," Using an MSI interrupt\n");
 2817                 return (msgs);
 2818         }
 2819         device_printf(adapter->dev," Using a Legacy interrupt\n");
 2820         return (0);
 2821 }
 2822 
 2823 /*********************************************************************
 2824  *
 2825  *  Initialize the DMA Coalescing feature
 2826  *
 2827  **********************************************************************/
 2828 static void
 2829 igb_init_dmac(struct adapter *adapter, u32 pba)
 2830 {
 2831         device_t        dev = adapter->dev;
 2832         struct e1000_hw *hw = &adapter->hw;
 2833         u32             dmac, reg = ~E1000_DMACR_DMAC_EN;
 2834         u16             hwm;
 2835 
 2836         if (hw->mac.type == e1000_i211)
 2837                 return;
 2838 
 2839         if (hw->mac.type > e1000_82580) {
 2840 
 2841                 if (adapter->dmac == 0) { /* Disabling it */
 2842                         E1000_WRITE_REG(hw, E1000_DMACR, reg);
 2843                         return;
 2844                 } else
 2845                         device_printf(dev, "DMA Coalescing enabled\n");
 2846 
 2847                 /* Set starting threshold */
 2848                 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
 2849 
 2850                 hwm = 64 * pba - adapter->max_frame_size / 16;
 2851                 if (hwm < 64 * (pba - 6))
 2852                         hwm = 64 * (pba - 6);
 2853                 reg = E1000_READ_REG(hw, E1000_FCRTC);
 2854                 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
 2855                 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
 2856                     & E1000_FCRTC_RTH_COAL_MASK);
 2857                 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
 2858 
 2859 
 2860                 dmac = pba - adapter->max_frame_size / 512;
 2861                 if (dmac < pba - 10)
 2862                         dmac = pba - 10;
 2863                 reg = E1000_READ_REG(hw, E1000_DMACR);
 2864                 reg &= ~E1000_DMACR_DMACTHR_MASK;
 2865                 reg |= ((dmac << E1000_DMACR_DMACTHR_SHIFT)
 2866                     & E1000_DMACR_DMACTHR_MASK);
 2867 
 2868                 /* transition to L0x or L1 if available..*/
 2869                 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
 2870 
 2871                 /* Check if status is 2.5Gb backplane connection
 2872                 * before configuration of watchdog timer, which is
 2873                 * in msec values in 12.8usec intervals
 2874                 * watchdog timer= msec values in 32usec intervals
 2875                 * for non 2.5Gb connection
 2876                 */
 2877                 if (hw->mac.type == e1000_i354) {
 2878                         int status = E1000_READ_REG(hw, E1000_STATUS);
 2879                         if ((status & E1000_STATUS_2P5_SKU) &&
 2880                             (!(status & E1000_STATUS_2P5_SKU_OVER)))
 2881                                 reg |= ((adapter->dmac * 5) >> 6);
 2882                         else
 2883                                 reg |= (adapter->dmac >> 5);
 2884                 } else {
 2885                         reg |= (adapter->dmac >> 5);
 2886                 }
 2887 
 2888                 E1000_WRITE_REG(hw, E1000_DMACR, reg);
 2889 
 2890                 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
 2891 
 2892                 /* Set the interval before transition */
 2893                 reg = E1000_READ_REG(hw, E1000_DMCTLX);
 2894                 if (hw->mac.type == e1000_i350)
 2895                         reg |= IGB_DMCTLX_DCFLUSH_DIS;
 2896                 /*
 2897                 ** in 2.5Gb connection, TTLX unit is 0.4 usec
 2898                 ** which is 0x4*2 = 0xA. But delay is still 4 usec
 2899                 */
 2900                 if (hw->mac.type == e1000_i354) {
 2901                         int status = E1000_READ_REG(hw, E1000_STATUS);
 2902                         if ((status & E1000_STATUS_2P5_SKU) &&
 2903                             (!(status & E1000_STATUS_2P5_SKU_OVER)))
 2904                                 reg |= 0xA;
 2905                         else
 2906                                 reg |= 0x4;
 2907                 } else {
 2908                         reg |= 0x4;
 2909                 }
 2910 
 2911                 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
 2912 
 2913                 /* free space in tx packet buffer to wake from DMA coal */
 2914                 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
 2915                     (2 * adapter->max_frame_size)) >> 6);
 2916 
 2917                 /* make low power state decision controlled by DMA coal */
 2918                 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
 2919                 reg &= ~E1000_PCIEMISC_LX_DECISION;
 2920                 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
 2921 
 2922         } else if (hw->mac.type == e1000_82580) {
 2923                 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
 2924                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
 2925                     reg & ~E1000_PCIEMISC_LX_DECISION);
 2926                 E1000_WRITE_REG(hw, E1000_DMACR, 0);
 2927         }
 2928 }
 2929 
 2930 
 2931 /*********************************************************************
 2932  *
 2933  *  Set up an fresh starting state
 2934  *
 2935  **********************************************************************/
 2936 static void
 2937 igb_reset(struct adapter *adapter)
 2938 {
 2939         device_t        dev = adapter->dev;
 2940         struct e1000_hw *hw = &adapter->hw;
 2941         struct e1000_fc_info *fc = &hw->fc;
 2942         struct ifnet    *ifp = adapter->ifp;
 2943         u32             pba = 0;
 2944         u16             hwm;
 2945 
 2946         INIT_DEBUGOUT("igb_reset: begin");
 2947 
 2948         /* Let the firmware know the OS is in control */
 2949         igb_get_hw_control(adapter);
 2950 
 2951         /*
 2952          * Packet Buffer Allocation (PBA)
 2953          * Writing PBA sets the receive portion of the buffer
 2954          * the remainder is used for the transmit buffer.
 2955          */
 2956         switch (hw->mac.type) {
 2957         case e1000_82575:
 2958                 pba = E1000_PBA_32K;
 2959                 break;
 2960         case e1000_82576:
 2961         case e1000_vfadapt:
 2962                 pba = E1000_READ_REG(hw, E1000_RXPBS);
 2963                 pba &= E1000_RXPBS_SIZE_MASK_82576;
 2964                 break;
 2965         case e1000_82580:
 2966         case e1000_i350:
 2967         case e1000_i354:
 2968         case e1000_vfadapt_i350:
 2969                 pba = E1000_READ_REG(hw, E1000_RXPBS);
 2970                 pba = e1000_rxpbs_adjust_82580(pba);
 2971                 break;
 2972         case e1000_i210:
 2973         case e1000_i211:
 2974                 pba = E1000_PBA_34K;
 2975         default:
 2976                 break;
 2977         }
 2978 
 2979         /* Special needs in case of Jumbo frames */
 2980         if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
 2981                 u32 tx_space, min_tx, min_rx;
 2982                 pba = E1000_READ_REG(hw, E1000_PBA);
 2983                 tx_space = pba >> 16;
 2984                 pba &= 0xffff;
 2985                 min_tx = (adapter->max_frame_size +
 2986                     sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
 2987                 min_tx = roundup2(min_tx, 1024);
 2988                 min_tx >>= 10;
 2989                 min_rx = adapter->max_frame_size;
 2990                 min_rx = roundup2(min_rx, 1024);
 2991                 min_rx >>= 10;
 2992                 if (tx_space < min_tx &&
 2993                     ((min_tx - tx_space) < pba)) {
 2994                         pba = pba - (min_tx - tx_space);
 2995                         /*
 2996                          * if short on rx space, rx wins
 2997                          * and must trump tx adjustment
 2998                          */
 2999                         if (pba < min_rx)
 3000                                 pba = min_rx;
 3001                 }
 3002                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 3003         }
 3004 
 3005         INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
 3006 
 3007         /*
 3008          * These parameters control the automatic generation (Tx) and
 3009          * response (Rx) to Ethernet PAUSE frames.
 3010          * - High water mark should allow for at least two frames to be
 3011          *   received after sending an XOFF.
 3012          * - Low water mark works best when it is very near the high water mark.
 3013          *   This allows the receiver to restart by sending XON when it has
 3014          *   drained a bit.
 3015          */
 3016         hwm = min(((pba << 10) * 9 / 10),
 3017             ((pba << 10) - 2 * adapter->max_frame_size));
 3018 
 3019         if (hw->mac.type < e1000_82576) {
 3020                 fc->high_water = hwm & 0xFFF8;  /* 8-byte granularity */
 3021                 fc->low_water = fc->high_water - 8;
 3022         } else {
 3023                 fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
 3024                 fc->low_water = fc->high_water - 16;
 3025         }
 3026 
 3027         fc->pause_time = IGB_FC_PAUSE_TIME;
 3028         fc->send_xon = TRUE;
 3029         if (adapter->fc)
 3030                 fc->requested_mode = adapter->fc;
 3031         else
 3032                 fc->requested_mode = e1000_fc_default;
 3033 
 3034         /* Issue a global reset */
 3035         e1000_reset_hw(hw);
 3036         E1000_WRITE_REG(hw, E1000_WUFC, 0);
 3037 
 3038         /* Reset for AutoMediaDetect */
 3039         if (adapter->flags & IGB_MEDIA_RESET) {
 3040                 e1000_setup_init_funcs(hw, TRUE);
 3041                 e1000_get_bus_info(hw);
 3042                 adapter->flags &= ~IGB_MEDIA_RESET;
 3043         }
 3044 
 3045         if (e1000_init_hw(hw) < 0)
 3046                 device_printf(dev, "Hardware Initialization Failed\n");
 3047 
 3048         /* Setup DMA Coalescing */
 3049         igb_init_dmac(adapter, pba);
 3050 
 3051         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 3052         e1000_get_phy_info(hw);
 3053         e1000_check_for_link(hw);
 3054         return;
 3055 }
 3056 
 3057 /*********************************************************************
 3058  *
 3059  *  Setup networking device structure and register an interface.
 3060  *
 3061  **********************************************************************/
 3062 static int
 3063 igb_setup_interface(device_t dev, struct adapter *adapter)
 3064 {
 3065         struct ifnet   *ifp;
 3066 
 3067         INIT_DEBUGOUT("igb_setup_interface: begin");
 3068 
 3069         ifp = adapter->ifp = if_alloc(IFT_ETHER);
 3070         if (ifp == NULL) {
 3071                 device_printf(dev, "can not allocate ifnet structure\n");
 3072                 return (-1);
 3073         }
 3074         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 3075         ifp->if_init =  igb_init;
 3076         ifp->if_softc = adapter;
 3077         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 3078         ifp->if_ioctl = igb_ioctl;
 3079 
 3080         /* TSO parameters */
 3081         ifp->if_hw_tsomax = IP_MAXPACKET;
 3082         ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER;
 3083         ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE;
 3084 
 3085 #ifndef IGB_LEGACY_TX
 3086         ifp->if_transmit = igb_mq_start;
 3087         ifp->if_qflush = igb_qflush;
 3088 #else
 3089         ifp->if_start = igb_start;
 3090         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
 3091         ifp->if_snd.ifq_drv_maxlen = 0;
 3092         IFQ_SET_READY(&ifp->if_snd);
 3093 #endif
 3094 
 3095         ether_ifattach(ifp, adapter->hw.mac.addr);
 3096 
 3097         ifp->if_capabilities = ifp->if_capenable = 0;
 3098 
 3099         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
 3100 #if __FreeBSD_version >= 1000000
 3101         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
 3102 #endif
 3103         ifp->if_capabilities |= IFCAP_TSO;
 3104         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
 3105         ifp->if_capenable = ifp->if_capabilities;
 3106 
 3107         /* Don't enable LRO by default */
 3108         ifp->if_capabilities |= IFCAP_LRO;
 3109 
 3110 #ifdef DEVICE_POLLING
 3111         ifp->if_capabilities |= IFCAP_POLLING;
 3112 #endif
 3113 
 3114         /*
 3115          * Tell the upper layer(s) we
 3116          * support full VLAN capability.
 3117          */
 3118         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 3119         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
 3120                              |  IFCAP_VLAN_HWTSO
 3121                              |  IFCAP_VLAN_MTU;
 3122         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
 3123                           |  IFCAP_VLAN_HWTSO
 3124                           |  IFCAP_VLAN_MTU;
 3125 
 3126         /*
 3127          * Enable only WOL MAGIC by default if WOL is enabled in EEPROM.
 3128          */
 3129         ifp->if_capabilities |= IFCAP_WOL;
 3130         if (adapter->wol)
 3131                 ifp->if_capenable |= IFCAP_WOL_MAGIC;
 3132 
 3133         /*
 3134         ** Don't turn this on by default, if vlans are
 3135         ** created on another pseudo device (eg. lagg)
 3136         ** then vlan events are not passed thru, breaking
 3137         ** operation, but with HW FILTER off it works. If
 3138         ** using vlans directly on the igb driver you can
 3139         ** enable this and get full hardware tag filtering.
 3140         */
 3141         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 3142 
 3143         /*
 3144          * Specify the media types supported by this adapter and register
 3145          * callbacks to update media and link information
 3146          */
 3147         ifmedia_init(&adapter->media, IFM_IMASK,
 3148             igb_media_change, igb_media_status);
 3149         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 3150             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
 3151                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 
 3152                             0, NULL);
 3153                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 3154         } else {
 3155                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
 3156                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
 3157                             0, NULL);
 3158                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
 3159                             0, NULL);
 3160                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
 3161                             0, NULL);
 3162                 if (adapter->hw.phy.type != e1000_phy_ife) {
 3163                         ifmedia_add(&adapter->media,
 3164                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
 3165                         ifmedia_add(&adapter->media,
 3166                                 IFM_ETHER | IFM_1000_T, 0, NULL);
 3167                 }
 3168         }
 3169         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 3170         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
 3171         return (0);
 3172 }
 3173 
 3174 
 3175 /*
 3176  * Manage DMA'able memory.
 3177  */
 3178 static void
 3179 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 3180 {
 3181         if (error)
 3182                 return;
 3183         *(bus_addr_t *) arg = segs[0].ds_addr;
 3184 }
 3185 
 3186 static int
 3187 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
 3188         struct igb_dma_alloc *dma, int mapflags)
 3189 {
 3190         int error;
 3191 
 3192         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
 3193                                 IGB_DBA_ALIGN, 0,       /* alignment, bounds */
 3194                                 BUS_SPACE_MAXADDR,      /* lowaddr */
 3195                                 BUS_SPACE_MAXADDR,      /* highaddr */
 3196                                 NULL, NULL,             /* filter, filterarg */
 3197                                 size,                   /* maxsize */
 3198                                 1,                      /* nsegments */
 3199                                 size,                   /* maxsegsize */
 3200                                 0,                      /* flags */
 3201                                 NULL,                   /* lockfunc */
 3202                                 NULL,                   /* lockarg */
 3203                                 &dma->dma_tag);
 3204         if (error) {
 3205                 device_printf(adapter->dev,
 3206                     "%s: bus_dma_tag_create failed: %d\n",
 3207                     __func__, error);
 3208                 goto fail_0;
 3209         }
 3210 
 3211         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 3212             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
 3213         if (error) {
 3214                 device_printf(adapter->dev,
 3215                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
 3216                     __func__, (uintmax_t)size, error);
 3217                 goto fail_2;
 3218         }
 3219 
 3220         dma->dma_paddr = 0;
 3221         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 3222             size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
 3223         if (error || dma->dma_paddr == 0) {
 3224                 device_printf(adapter->dev,
 3225                     "%s: bus_dmamap_load failed: %d\n",
 3226                     __func__, error);
 3227                 goto fail_3;
 3228         }
 3229 
 3230         return (0);
 3231 
 3232 fail_3:
 3233         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 3234 fail_2:
 3235         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 3236         bus_dma_tag_destroy(dma->dma_tag);
 3237 fail_0:
 3238         dma->dma_tag = NULL;
 3239 
 3240         return (error);
 3241 }
 3242 
 3243 static void
 3244 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
 3245 {
 3246         if (dma->dma_tag == NULL)
 3247                 return;
 3248         if (dma->dma_paddr != 0) {
 3249                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
 3250                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 3251                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 3252                 dma->dma_paddr = 0;
 3253         }
 3254         if (dma->dma_vaddr != NULL) {
 3255                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 3256                 dma->dma_vaddr = NULL;
 3257         }
 3258         bus_dma_tag_destroy(dma->dma_tag);
 3259         dma->dma_tag = NULL;
 3260 }
 3261 
 3262 
 3263 /*********************************************************************
 3264  *
 3265  *  Allocate memory for the transmit and receive rings, and then
 3266  *  the descriptors associated with each, called only once at attach.
 3267  *
 3268  **********************************************************************/
 3269 static int
 3270 igb_allocate_queues(struct adapter *adapter)
 3271 {
 3272         device_t dev = adapter->dev;
 3273         struct igb_queue        *que = NULL;
 3274         struct tx_ring          *txr = NULL;
 3275         struct rx_ring          *rxr = NULL;
 3276         int rsize, tsize, error = E1000_SUCCESS;
 3277         int txconf = 0, rxconf = 0;
 3278 
 3279         /* First allocate the top level queue structs */
 3280         if (!(adapter->queues =
 3281             (struct igb_queue *) malloc(sizeof(struct igb_queue) *
 3282             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3283                 device_printf(dev, "Unable to allocate queue memory\n");
 3284                 error = ENOMEM;
 3285                 goto fail;
 3286         }
 3287 
 3288         /* Next allocate the TX ring struct memory */
 3289         if (!(adapter->tx_rings =
 3290             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
 3291             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3292                 device_printf(dev, "Unable to allocate TX ring memory\n");
 3293                 error = ENOMEM;
 3294                 goto tx_fail;
 3295         }
 3296 
 3297         /* Now allocate the RX */
 3298         if (!(adapter->rx_rings =
 3299             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
 3300             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3301                 device_printf(dev, "Unable to allocate RX ring memory\n");
 3302                 error = ENOMEM;
 3303                 goto rx_fail;
 3304         }
 3305 
 3306         tsize = roundup2(adapter->num_tx_desc *
 3307             sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
 3308         /*
 3309          * Now set up the TX queues, txconf is needed to handle the
 3310          * possibility that things fail midcourse and we need to
 3311          * undo memory gracefully
 3312          */ 
 3313         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
 3314                 /* Set up some basics */
 3315                 txr = &adapter->tx_rings[i];
 3316                 txr->adapter = adapter;
 3317                 txr->me = i;
 3318                 txr->num_desc = adapter->num_tx_desc;
 3319 
 3320                 /* Initialize the TX lock */
 3321                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
 3322                     device_get_nameunit(dev), txr->me);
 3323                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
 3324 
 3325                 if (igb_dma_malloc(adapter, tsize,
 3326                         &txr->txdma, BUS_DMA_NOWAIT)) {
 3327                         device_printf(dev,
 3328                             "Unable to allocate TX Descriptor memory\n");
 3329                         error = ENOMEM;
 3330                         goto err_tx_desc;
 3331                 }
 3332                 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
 3333                 bzero((void *)txr->tx_base, tsize);
 3334 
 3335                 /* Now allocate transmit buffers for the ring */
 3336                 if (igb_allocate_transmit_buffers(txr)) {
 3337                         device_printf(dev,
 3338                             "Critical Failure setting up transmit buffers\n");
 3339                         error = ENOMEM;
 3340                         goto err_tx_desc;
 3341                 }
 3342 #ifndef IGB_LEGACY_TX
 3343                 /* Allocate a buf ring */
 3344                 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
 3345                     M_WAITOK, &txr->tx_mtx);
 3346 #endif
 3347         }
 3348 
 3349         /*
 3350          * Next the RX queues...
 3351          */ 
 3352         rsize = roundup2(adapter->num_rx_desc *
 3353             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 3354         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
 3355                 rxr = &adapter->rx_rings[i];
 3356                 rxr->adapter = adapter;
 3357                 rxr->me = i;
 3358 
 3359                 /* Initialize the RX lock */
 3360                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
 3361                     device_get_nameunit(dev), txr->me);
 3362                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
 3363 
 3364                 if (igb_dma_malloc(adapter, rsize,
 3365                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
 3366                         device_printf(dev,
 3367                             "Unable to allocate RxDescriptor memory\n");
 3368                         error = ENOMEM;
 3369                         goto err_rx_desc;
 3370                 }
 3371                 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
 3372                 bzero((void *)rxr->rx_base, rsize);
 3373 
 3374                 /* Allocate receive buffers for the ring*/
 3375                 if (igb_allocate_receive_buffers(rxr)) {
 3376                         device_printf(dev,
 3377                             "Critical Failure setting up receive buffers\n");
 3378                         error = ENOMEM;
 3379                         goto err_rx_desc;
 3380                 }
 3381         }
 3382 
 3383         /*
 3384         ** Finally set up the queue holding structs
 3385         */
 3386         for (int i = 0; i < adapter->num_queues; i++) {
 3387                 que = &adapter->queues[i];
 3388                 que->adapter = adapter;
 3389                 que->txr = &adapter->tx_rings[i];
 3390                 que->rxr = &adapter->rx_rings[i];
 3391         }
 3392 
 3393         return (0);
 3394 
 3395 err_rx_desc:
 3396         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
 3397                 igb_dma_free(adapter, &rxr->rxdma);
 3398 err_tx_desc:
 3399         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
 3400                 igb_dma_free(adapter, &txr->txdma);
 3401         free(adapter->rx_rings, M_DEVBUF);
 3402 rx_fail:
 3403 #ifndef IGB_LEGACY_TX
 3404         buf_ring_free(txr->br, M_DEVBUF);
 3405 #endif
 3406         free(adapter->tx_rings, M_DEVBUF);
 3407 tx_fail:
 3408         free(adapter->queues, M_DEVBUF);
 3409 fail:
 3410         return (error);
 3411 }
 3412 
 3413 /*********************************************************************
 3414  *
 3415  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
 3416  *  the information needed to transmit a packet on the wire. This is
 3417  *  called only once at attach, setup is done every reset.
 3418  *
 3419  **********************************************************************/
 3420 static int
 3421 igb_allocate_transmit_buffers(struct tx_ring *txr)
 3422 {
 3423         struct adapter *adapter = txr->adapter;
 3424         device_t dev = adapter->dev;
 3425         struct igb_tx_buf *txbuf;
 3426         int error, i;
 3427 
 3428         /*
 3429          * Setup DMA descriptor areas.
 3430          */
 3431         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 3432                                1, 0,                    /* alignment, bounds */
 3433                                BUS_SPACE_MAXADDR,       /* lowaddr */
 3434                                BUS_SPACE_MAXADDR,       /* highaddr */
 3435                                NULL, NULL,              /* filter, filterarg */
 3436                                IGB_TSO_SIZE,            /* maxsize */
 3437                                IGB_MAX_SCATTER,         /* nsegments */
 3438                                PAGE_SIZE,               /* maxsegsize */
 3439                                0,                       /* flags */
 3440                                NULL,                    /* lockfunc */
 3441                                NULL,                    /* lockfuncarg */
 3442                                &txr->txtag))) {
 3443                 device_printf(dev,"Unable to allocate TX DMA tag\n");
 3444                 goto fail;
 3445         }
 3446 
 3447         if (!(txr->tx_buffers =
 3448             (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
 3449             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3450                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 3451                 error = ENOMEM;
 3452                 goto fail;
 3453         }
 3454 
 3455         /* Create the descriptor buffer dma maps */
 3456         txbuf = txr->tx_buffers;
 3457         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 3458                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
 3459                 if (error != 0) {
 3460                         device_printf(dev, "Unable to create TX DMA map\n");
 3461                         goto fail;
 3462                 }
 3463         }
 3464 
 3465         return 0;
 3466 fail:
 3467         /* We free all, it handles case where we are in the middle */
 3468         igb_free_transmit_structures(adapter);
 3469         return (error);
 3470 }
 3471 
 3472 /*********************************************************************
 3473  *
 3474  *  Initialize a transmit ring.
 3475  *
 3476  **********************************************************************/
 3477 static void
 3478 igb_setup_transmit_ring(struct tx_ring *txr)
 3479 {
 3480         struct adapter *adapter = txr->adapter;
 3481         struct igb_tx_buf *txbuf;
 3482         int i;
 3483 #ifdef DEV_NETMAP
 3484         struct netmap_adapter *na = NA(adapter->ifp);
 3485         struct netmap_slot *slot;
 3486 #endif /* DEV_NETMAP */
 3487 
 3488         /* Clear the old descriptor contents */
 3489         IGB_TX_LOCK(txr);
 3490 #ifdef DEV_NETMAP
 3491         slot = netmap_reset(na, NR_TX, txr->me, 0);
 3492 #endif /* DEV_NETMAP */
 3493         bzero((void *)txr->tx_base,
 3494               (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
 3495         /* Reset indices */
 3496         txr->next_avail_desc = 0;
 3497         txr->next_to_clean = 0;
 3498 
 3499         /* Free any existing tx buffers. */
 3500         txbuf = txr->tx_buffers;
 3501         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 3502                 if (txbuf->m_head != NULL) {
 3503                         bus_dmamap_sync(txr->txtag, txbuf->map,
 3504                             BUS_DMASYNC_POSTWRITE);
 3505                         bus_dmamap_unload(txr->txtag, txbuf->map);
 3506                         m_freem(txbuf->m_head);
 3507                         txbuf->m_head = NULL;
 3508                 }
 3509 #ifdef DEV_NETMAP
 3510                 if (slot) {
 3511                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
 3512                         /* no need to set the address */
 3513                         netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
 3514                 }
 3515 #endif /* DEV_NETMAP */
 3516                 /* clear the watch index */
 3517                 txbuf->eop = NULL;
 3518         }
 3519 
 3520         /* Set number of descriptors available */
 3521         txr->tx_avail = adapter->num_tx_desc;
 3522 
 3523         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3524             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 3525         IGB_TX_UNLOCK(txr);
 3526 }
 3527 
 3528 /*********************************************************************
 3529  *
 3530  *  Initialize all transmit rings.
 3531  *
 3532  **********************************************************************/
 3533 static void
 3534 igb_setup_transmit_structures(struct adapter *adapter)
 3535 {
 3536         struct tx_ring *txr = adapter->tx_rings;
 3537 
 3538         for (int i = 0; i < adapter->num_queues; i++, txr++)
 3539                 igb_setup_transmit_ring(txr);
 3540 
 3541         return;
 3542 }
 3543 
 3544 /*********************************************************************
 3545  *
 3546  *  Enable transmit unit.
 3547  *
 3548  **********************************************************************/
 3549 static void
 3550 igb_initialize_transmit_units(struct adapter *adapter)
 3551 {
 3552         struct tx_ring  *txr = adapter->tx_rings;
 3553         struct e1000_hw *hw = &adapter->hw;
 3554         u32             tctl, txdctl;
 3555 
 3556         INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
 3557         tctl = txdctl = 0;
 3558 
 3559         /* Setup the Tx Descriptor Rings */
 3560         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 3561                 u64 bus_addr = txr->txdma.dma_paddr;
 3562 
 3563                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
 3564                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
 3565                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
 3566                     (uint32_t)(bus_addr >> 32));
 3567                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
 3568                     (uint32_t)bus_addr);
 3569 
 3570                 /* Setup the HW Tx Head and Tail descriptor pointers */
 3571                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
 3572                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
 3573 
 3574                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
 3575                     E1000_READ_REG(hw, E1000_TDBAL(i)),
 3576                     E1000_READ_REG(hw, E1000_TDLEN(i)));
 3577 
 3578                 txr->queue_status = IGB_QUEUE_IDLE;
 3579 
 3580                 txdctl |= IGB_TX_PTHRESH;
 3581                 txdctl |= IGB_TX_HTHRESH << 8;
 3582                 txdctl |= IGB_TX_WTHRESH << 16;
 3583                 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
 3584                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
 3585         }
 3586 
 3587         if (adapter->vf_ifp)
 3588                 return;
 3589 
 3590         e1000_config_collision_dist(hw);
 3591 
 3592         /* Program the Transmit Control Register */
 3593         tctl = E1000_READ_REG(hw, E1000_TCTL);
 3594         tctl &= ~E1000_TCTL_CT;
 3595         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
 3596                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
 3597 
 3598         /* This write will effectively turn on the transmit unit. */
 3599         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
 3600 }
 3601 
 3602 /*********************************************************************
 3603  *
 3604  *  Free all transmit rings.
 3605  *
 3606  **********************************************************************/
 3607 static void
 3608 igb_free_transmit_structures(struct adapter *adapter)
 3609 {
 3610         struct tx_ring *txr = adapter->tx_rings;
 3611 
 3612         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 3613                 IGB_TX_LOCK(txr);
 3614                 igb_free_transmit_buffers(txr);
 3615                 igb_dma_free(adapter, &txr->txdma);
 3616                 IGB_TX_UNLOCK(txr);
 3617                 IGB_TX_LOCK_DESTROY(txr);
 3618         }
 3619         free(adapter->tx_rings, M_DEVBUF);
 3620 }
 3621 
 3622 /*********************************************************************
 3623  *
 3624  *  Free transmit ring related data structures.
 3625  *
 3626  **********************************************************************/
 3627 static void
 3628 igb_free_transmit_buffers(struct tx_ring *txr)
 3629 {
 3630         struct adapter *adapter = txr->adapter;
 3631         struct igb_tx_buf *tx_buffer;
 3632         int             i;
 3633 
 3634         INIT_DEBUGOUT("free_transmit_ring: begin");
 3635 
 3636         if (txr->tx_buffers == NULL)
 3637                 return;
 3638 
 3639         tx_buffer = txr->tx_buffers;
 3640         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
 3641                 if (tx_buffer->m_head != NULL) {
 3642                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
 3643                             BUS_DMASYNC_POSTWRITE);
 3644                         bus_dmamap_unload(txr->txtag,
 3645                             tx_buffer->map);
 3646                         m_freem(tx_buffer->m_head);
 3647                         tx_buffer->m_head = NULL;
 3648                         if (tx_buffer->map != NULL) {
 3649                                 bus_dmamap_destroy(txr->txtag,
 3650                                     tx_buffer->map);
 3651                                 tx_buffer->map = NULL;
 3652                         }
 3653                 } else if (tx_buffer->map != NULL) {
 3654                         bus_dmamap_unload(txr->txtag,
 3655                             tx_buffer->map);
 3656                         bus_dmamap_destroy(txr->txtag,
 3657                             tx_buffer->map);
 3658                         tx_buffer->map = NULL;
 3659                 }
 3660         }
 3661 #ifndef IGB_LEGACY_TX
 3662         if (txr->br != NULL)
 3663                 buf_ring_free(txr->br, M_DEVBUF);
 3664 #endif
 3665         if (txr->tx_buffers != NULL) {
 3666                 free(txr->tx_buffers, M_DEVBUF);
 3667                 txr->tx_buffers = NULL;
 3668         }
 3669         if (txr->txtag != NULL) {
 3670                 bus_dma_tag_destroy(txr->txtag);
 3671                 txr->txtag = NULL;
 3672         }
 3673         return;
 3674 }
 3675 
 3676 /**********************************************************************
 3677  *
 3678  *  Setup work for hardware segmentation offload (TSO) on
 3679  *  adapters using advanced tx descriptors
 3680  *
 3681  **********************************************************************/
 3682 static int
 3683 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
 3684     u32 *cmd_type_len, u32 *olinfo_status)
 3685 {
 3686         struct adapter *adapter = txr->adapter;
 3687         struct e1000_adv_tx_context_desc *TXD;
 3688         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 3689         u32 mss_l4len_idx = 0, paylen;
 3690         u16 vtag = 0, eh_type;
 3691         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
 3692         struct ether_vlan_header *eh;
 3693 #ifdef INET6
 3694         struct ip6_hdr *ip6;
 3695 #endif
 3696 #ifdef INET
 3697         struct ip *ip;
 3698 #endif
 3699         struct tcphdr *th;
 3700 
 3701 
 3702         /*
 3703          * Determine where frame payload starts.
 3704          * Jump over vlan headers if already present
 3705          */
 3706         eh = mtod(mp, struct ether_vlan_header *);
 3707         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 3708                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3709                 eh_type = eh->evl_proto;
 3710         } else {
 3711                 ehdrlen = ETHER_HDR_LEN;
 3712                 eh_type = eh->evl_encap_proto;
 3713         }
 3714 
 3715         switch (ntohs(eh_type)) {
 3716 #ifdef INET6
 3717         case ETHERTYPE_IPV6:
 3718                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3719                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
 3720                 if (ip6->ip6_nxt != IPPROTO_TCP)
 3721                         return (ENXIO);
 3722                 ip_hlen = sizeof(struct ip6_hdr);
 3723                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3724                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
 3725                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
 3726                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
 3727                 break;
 3728 #endif
 3729 #ifdef INET
 3730         case ETHERTYPE_IP:
 3731                 ip = (struct ip *)(mp->m_data + ehdrlen);
 3732                 if (ip->ip_p != IPPROTO_TCP)
 3733                         return (ENXIO);
 3734                 ip->ip_sum = 0;
 3735                 ip_hlen = ip->ip_hl << 2;
 3736                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 3737                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
 3738                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 3739                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3740                 /* Tell transmit desc to also do IPv4 checksum. */
 3741                 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
 3742                 break;
 3743 #endif
 3744         default:
 3745                 device_printf(adapter->dev,
 3746                     "CSUM_TSO but no supported IP version (0x%04x)",
 3747                     ntohs(eh_type));
 3748                 return (ENXIO);
 3749         }
 3750 
 3751         ctxd = txr->next_avail_desc;
 3752         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3753 
 3754         tcp_hlen = th->th_off << 2;
 3755 
 3756         /* This is used in the transmit desc in encap */
 3757         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
 3758 
 3759         /* VLAN MACLEN IPLEN */
 3760         if (mp->m_flags & M_VLANTAG) {
 3761                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3762                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3763         }
 3764 
 3765         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
 3766         vlan_macip_lens |= ip_hlen;
 3767         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
 3768 
 3769         /* ADV DTYPE TUCMD */
 3770         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3771         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3772         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
 3773 
 3774         /* MSS L4LEN IDX */
 3775         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
 3776         mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
 3777         /* 82575 needs the queue index added */
 3778         if (adapter->hw.mac.type == e1000_82575)
 3779                 mss_l4len_idx |= txr->me << 4;
 3780         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 3781 
 3782         TXD->seqnum_seed = htole32(0);
 3783 
 3784         if (++ctxd == txr->num_desc)
 3785                 ctxd = 0;
 3786 
 3787         txr->tx_avail--;
 3788         txr->next_avail_desc = ctxd;
 3789         *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
 3790         *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 3791         *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
 3792         ++txr->tso_tx;
 3793         return (0);
 3794 }
 3795 
 3796 /*********************************************************************
 3797  *
 3798  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
 3799  *
 3800  **********************************************************************/
 3801 
 3802 static int
 3803 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
 3804     u32 *cmd_type_len, u32 *olinfo_status)
 3805 {
 3806         struct e1000_adv_tx_context_desc *TXD;
 3807         struct adapter *adapter = txr->adapter;
 3808         struct ether_vlan_header *eh;
 3809         struct ip *ip;
 3810         struct ip6_hdr *ip6;
 3811         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
 3812         int     ehdrlen, ip_hlen = 0;
 3813         u16     etype;
 3814         u8      ipproto = 0;
 3815         int     ctxd = txr->next_avail_desc;
 3816         u16     vtag = 0;
 3817 
 3818         /* First check if TSO is to be used */
 3819         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
 3820                 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
 3821 
 3822         /* Indicate the whole packet as payload when not doing TSO */
 3823         *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
 3824 
 3825         /* Now ready a context descriptor */
 3826         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3827 
 3828         /*
 3829         ** In advanced descriptors the vlan tag must 
 3830         ** be placed into the context descriptor. Hence
 3831         ** we need to make one even if not doing offloads.
 3832         */
 3833         if (mp->m_flags & M_VLANTAG) {
 3834                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3835                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3836         } else if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) {
 3837                 return (0);
 3838         }
 3839 
 3840         /*
 3841          * Determine where frame payload starts.
 3842          * Jump over vlan headers if already present,
 3843          * helpful for QinQ too.
 3844          */
 3845         eh = mtod(mp, struct ether_vlan_header *);
 3846         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 3847                 etype = ntohs(eh->evl_proto);
 3848                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3849         } else {
 3850                 etype = ntohs(eh->evl_encap_proto);
 3851                 ehdrlen = ETHER_HDR_LEN;
 3852         }
 3853 
 3854         /* Set the ether header length */
 3855         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
 3856 
 3857         switch (etype) {
 3858                 case ETHERTYPE_IP:
 3859                         ip = (struct ip *)(mp->m_data + ehdrlen);
 3860                         ip_hlen = ip->ip_hl << 2;
 3861                         ipproto = ip->ip_p;
 3862                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3863                         break;
 3864                 case ETHERTYPE_IPV6:
 3865                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3866                         ip_hlen = sizeof(struct ip6_hdr);
 3867                         /* XXX-BZ this will go badly in case of ext hdrs. */
 3868                         ipproto = ip6->ip6_nxt;
 3869                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
 3870                         break;
 3871                 default:
 3872                         break;
 3873         }
 3874 
 3875         vlan_macip_lens |= ip_hlen;
 3876         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3877 
 3878         switch (ipproto) {
 3879                 case IPPROTO_TCP:
 3880 #if __FreeBSD_version >= 1000000
 3881                         if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) {
 3882 #else
 3883                         if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
 3884 #endif
 3885                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3886                                 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 3887                         }
 3888                         break;
 3889                 case IPPROTO_UDP:
 3890 #if __FreeBSD_version >= 1000000
 3891                         if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) {
 3892 #else
 3893                         if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
 3894 #endif
 3895                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
 3896                                 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 3897                         }
 3898                         break;
 3899 
 3900 #if __FreeBSD_version >= 800000
 3901                 case IPPROTO_SCTP:
 3902 #if __FreeBSD_version >= 1000000
 3903                         if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) {
 3904 #else
 3905                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP) {
 3906 #endif
 3907                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
 3908                                 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 3909                         }
 3910                         break;
 3911 #endif
 3912                 default:
 3913                         break;
 3914         }
 3915 
 3916         /* 82575 needs the queue index added */
 3917         if (adapter->hw.mac.type == e1000_82575)
 3918                 mss_l4len_idx = txr->me << 4;
 3919 
 3920         /* Now copy bits into descriptor */
 3921         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
 3922         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
 3923         TXD->seqnum_seed = htole32(0);
 3924         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 3925 
 3926         /* We've consumed the first desc, adjust counters */
 3927         if (++ctxd == txr->num_desc)
 3928                 ctxd = 0;
 3929         txr->next_avail_desc = ctxd;
 3930         --txr->tx_avail;
 3931 
 3932         return (0);
 3933 }
 3934 
 3935 /**********************************************************************
 3936  *
 3937  *  Examine each tx_buffer in the used queue. If the hardware is done
 3938  *  processing the packet then free associated resources. The
 3939  *  tx_buffer is put back on the free queue.
 3940  *
 3941  *  TRUE return means there's work in the ring to clean, FALSE its empty.
 3942  **********************************************************************/
 3943 static bool
 3944 igb_txeof(struct tx_ring *txr)
 3945 {
 3946         struct adapter          *adapter = txr->adapter;
 3947         struct ifnet            *ifp = adapter->ifp;
 3948         u32                     work, processed = 0;
 3949         int                     limit = adapter->tx_process_limit;
 3950         struct igb_tx_buf       *buf;
 3951         union e1000_adv_tx_desc *txd;
 3952 
 3953         mtx_assert(&txr->tx_mtx, MA_OWNED);
 3954 
 3955 #ifdef DEV_NETMAP
 3956         if (netmap_tx_irq(ifp, txr->me))
 3957                 return (FALSE);
 3958 #endif /* DEV_NETMAP */
 3959 
 3960         if (txr->tx_avail == txr->num_desc) {
 3961                 txr->queue_status = IGB_QUEUE_IDLE;
 3962                 return FALSE;
 3963         }
 3964 
 3965         /* Get work starting point */
 3966         work = txr->next_to_clean;
 3967         buf = &txr->tx_buffers[work];
 3968         txd = &txr->tx_base[work];
 3969         work -= txr->num_desc; /* The distance to ring end */
 3970         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3971             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 3972         do {
 3973                 union e1000_adv_tx_desc *eop = buf->eop;
 3974                 if (eop == NULL) /* No work */
 3975                         break;
 3976 
 3977                 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
 3978                         break;  /* I/O not complete */
 3979 
 3980                 if (buf->m_head) {
 3981                         txr->bytes +=
 3982                             buf->m_head->m_pkthdr.len;
 3983                         bus_dmamap_sync(txr->txtag,
 3984                             buf->map,
 3985                             BUS_DMASYNC_POSTWRITE);
 3986                         bus_dmamap_unload(txr->txtag,
 3987                             buf->map);
 3988                         m_freem(buf->m_head);
 3989                         buf->m_head = NULL;
 3990                 }
 3991                 buf->eop = NULL;
 3992                 ++txr->tx_avail;
 3993 
 3994                 /* We clean the range if multi segment */
 3995                 while (txd != eop) {
 3996                         ++txd;
 3997                         ++buf;
 3998                         ++work;
 3999                         /* wrap the ring? */
 4000                         if (__predict_false(!work)) {
 4001                                 work -= txr->num_desc;
 4002                                 buf = txr->tx_buffers;
 4003                                 txd = txr->tx_base;
 4004                         }
 4005                         if (buf->m_head) {
 4006                                 txr->bytes +=
 4007                                     buf->m_head->m_pkthdr.len;
 4008                                 bus_dmamap_sync(txr->txtag,
 4009                                     buf->map,
 4010                                     BUS_DMASYNC_POSTWRITE);
 4011                                 bus_dmamap_unload(txr->txtag,
 4012                                     buf->map);
 4013                                 m_freem(buf->m_head);
 4014                                 buf->m_head = NULL;
 4015                         }
 4016                         ++txr->tx_avail;
 4017                         buf->eop = NULL;
 4018 
 4019                 }
 4020                 ++txr->packets;
 4021                 ++processed;
 4022                 ++ifp->if_opackets;
 4023                 txr->watchdog_time = ticks;
 4024 
 4025                 /* Try the next packet */
 4026                 ++txd;
 4027                 ++buf;
 4028                 ++work;
 4029                 /* reset with a wrap */
 4030                 if (__predict_false(!work)) {
 4031                         work -= txr->num_desc;
 4032                         buf = txr->tx_buffers;
 4033                         txd = txr->tx_base;
 4034                 }
 4035                 prefetch(txd);
 4036         } while (__predict_true(--limit));
 4037 
 4038         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 4039             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4040 
 4041         work += txr->num_desc;
 4042         txr->next_to_clean = work;
 4043 
 4044         /*
 4045         ** Watchdog calculation, we know there's
 4046         ** work outstanding or the first return
 4047         ** would have been taken, so none processed
 4048         ** for too long indicates a hang.
 4049         */
 4050         if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
 4051                 txr->queue_status |= IGB_QUEUE_HUNG;
 4052 
 4053         if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
 4054                 txr->queue_status &= ~IGB_QUEUE_DEPLETED;       
 4055 
 4056         if (txr->tx_avail == txr->num_desc) {
 4057                 txr->queue_status = IGB_QUEUE_IDLE;
 4058                 return (FALSE);
 4059         }
 4060 
 4061         return (TRUE);
 4062 }
 4063 
 4064 /*********************************************************************
 4065  *
 4066  *  Refresh mbuf buffers for RX descriptor rings
 4067  *   - now keeps its own state so discards due to resource
 4068  *     exhaustion are unnecessary, if an mbuf cannot be obtained
 4069  *     it just returns, keeping its placeholder, thus it can simply
 4070  *     be recalled to try again.
 4071  *
 4072  **********************************************************************/
 4073 static void
 4074 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
 4075 {
 4076         struct adapter          *adapter = rxr->adapter;
 4077         bus_dma_segment_t       hseg[1];
 4078         bus_dma_segment_t       pseg[1];
 4079         struct igb_rx_buf       *rxbuf;
 4080         struct mbuf             *mh, *mp;
 4081         int                     i, j, nsegs, error;
 4082         bool                    refreshed = FALSE;
 4083 
 4084         i = j = rxr->next_to_refresh;
 4085         /*
 4086         ** Get one descriptor beyond
 4087         ** our work mark to control
 4088         ** the loop.
 4089         */
 4090         if (++j == adapter->num_rx_desc)
 4091                 j = 0;
 4092 
 4093         while (j != limit) {
 4094                 rxbuf = &rxr->rx_buffers[i];
 4095                 /* No hdr mbuf used with header split off */
 4096                 if (rxr->hdr_split == FALSE)
 4097                         goto no_split;
 4098                 if (rxbuf->m_head == NULL) {
 4099                         mh = m_gethdr(M_NOWAIT, MT_DATA);
 4100                         if (mh == NULL)
 4101                                 goto update;
 4102                 } else
 4103                         mh = rxbuf->m_head;
 4104 
 4105                 mh->m_pkthdr.len = mh->m_len = MHLEN;
 4106                 mh->m_len = MHLEN;
 4107                 mh->m_flags |= M_PKTHDR;
 4108                 /* Get the memory mapping */
 4109                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
 4110                     rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
 4111                 if (error != 0) {
 4112                         printf("Refresh mbufs: hdr dmamap load"
 4113                             " failure - %d\n", error);
 4114                         m_free(mh);
 4115                         rxbuf->m_head = NULL;
 4116                         goto update;
 4117                 }
 4118                 rxbuf->m_head = mh;
 4119                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 4120                     BUS_DMASYNC_PREREAD);
 4121                 rxr->rx_base[i].read.hdr_addr =
 4122                     htole64(hseg[0].ds_addr);
 4123 no_split:
 4124                 if (rxbuf->m_pack == NULL) {
 4125                         mp = m_getjcl(M_NOWAIT, MT_DATA,
 4126                             M_PKTHDR, adapter->rx_mbuf_sz);
 4127                         if (mp == NULL)
 4128                                 goto update;
 4129                 } else
 4130                         mp = rxbuf->m_pack;
 4131 
 4132                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
 4133                 /* Get the memory mapping */
 4134                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
 4135                     rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
 4136                 if (error != 0) {
 4137                         printf("Refresh mbufs: payload dmamap load"
 4138                             " failure - %d\n", error);
 4139                         m_free(mp);
 4140                         rxbuf->m_pack = NULL;
 4141                         goto update;
 4142                 }
 4143                 rxbuf->m_pack = mp;
 4144                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 4145                     BUS_DMASYNC_PREREAD);
 4146                 rxr->rx_base[i].read.pkt_addr =
 4147                     htole64(pseg[0].ds_addr);
 4148                 refreshed = TRUE; /* I feel wefreshed :) */
 4149 
 4150                 i = j; /* our next is precalculated */
 4151                 rxr->next_to_refresh = i;
 4152                 if (++j == adapter->num_rx_desc)
 4153                         j = 0;
 4154         }
 4155 update:
 4156         if (refreshed) /* update tail */
 4157                 E1000_WRITE_REG(&adapter->hw,
 4158                     E1000_RDT(rxr->me), rxr->next_to_refresh);
 4159         return;
 4160 }
 4161 
 4162 
 4163 /*********************************************************************
 4164  *
 4165  *  Allocate memory for rx_buffer structures. Since we use one
 4166  *  rx_buffer per received packet, the maximum number of rx_buffer's
 4167  *  that we'll need is equal to the number of receive descriptors
 4168  *  that we've allocated.
 4169  *
 4170  **********************************************************************/
 4171 static int
 4172 igb_allocate_receive_buffers(struct rx_ring *rxr)
 4173 {
 4174         struct  adapter         *adapter = rxr->adapter;
 4175         device_t                dev = adapter->dev;
 4176         struct igb_rx_buf       *rxbuf;
 4177         int                     i, bsize, error;
 4178 
 4179         bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
 4180         if (!(rxr->rx_buffers =
 4181             (struct igb_rx_buf *) malloc(bsize,
 4182             M_DEVBUF, M_NOWAIT | M_ZERO))) {
 4183                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
 4184                 error = ENOMEM;
 4185                 goto fail;
 4186         }
 4187 
 4188         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 4189                                    1, 0,                /* alignment, bounds */
 4190                                    BUS_SPACE_MAXADDR,   /* lowaddr */
 4191                                    BUS_SPACE_MAXADDR,   /* highaddr */
 4192                                    NULL, NULL,          /* filter, filterarg */
 4193                                    MSIZE,               /* maxsize */
 4194                                    1,                   /* nsegments */
 4195                                    MSIZE,               /* maxsegsize */
 4196                                    0,                   /* flags */
 4197                                    NULL,                /* lockfunc */
 4198                                    NULL,                /* lockfuncarg */
 4199                                    &rxr->htag))) {
 4200                 device_printf(dev, "Unable to create RX DMA tag\n");
 4201                 goto fail;
 4202         }
 4203 
 4204         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 4205                                    1, 0,                /* alignment, bounds */
 4206                                    BUS_SPACE_MAXADDR,   /* lowaddr */
 4207                                    BUS_SPACE_MAXADDR,   /* highaddr */
 4208                                    NULL, NULL,          /* filter, filterarg */
 4209                                    MJUM9BYTES,          /* maxsize */
 4210                                    1,                   /* nsegments */
 4211                                    MJUM9BYTES,          /* maxsegsize */
 4212                                    0,                   /* flags */
 4213                                    NULL,                /* lockfunc */
 4214                                    NULL,                /* lockfuncarg */
 4215                                    &rxr->ptag))) {
 4216                 device_printf(dev, "Unable to create RX payload DMA tag\n");
 4217                 goto fail;
 4218         }
 4219 
 4220         for (i = 0; i < adapter->num_rx_desc; i++) {
 4221                 rxbuf = &rxr->rx_buffers[i];
 4222                 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
 4223                 if (error) {
 4224                         device_printf(dev,
 4225                             "Unable to create RX head DMA maps\n");
 4226                         goto fail;
 4227                 }
 4228                 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
 4229                 if (error) {
 4230                         device_printf(dev,
 4231                             "Unable to create RX packet DMA maps\n");
 4232                         goto fail;
 4233                 }
 4234         }
 4235 
 4236         return (0);
 4237 
 4238 fail:
 4239         /* Frees all, but can handle partial completion */
 4240         igb_free_receive_structures(adapter);
 4241         return (error);
 4242 }
 4243 
 4244 
 4245 static void
 4246 igb_free_receive_ring(struct rx_ring *rxr)
 4247 {
 4248         struct  adapter         *adapter = rxr->adapter;
 4249         struct igb_rx_buf       *rxbuf;
 4250 
 4251 
 4252         for (int i = 0; i < adapter->num_rx_desc; i++) {
 4253                 rxbuf = &rxr->rx_buffers[i];
 4254                 if (rxbuf->m_head != NULL) {
 4255                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 4256                             BUS_DMASYNC_POSTREAD);
 4257                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 4258                         rxbuf->m_head->m_flags |= M_PKTHDR;
 4259                         m_freem(rxbuf->m_head);
 4260                 }
 4261                 if (rxbuf->m_pack != NULL) {
 4262                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 4263                             BUS_DMASYNC_POSTREAD);
 4264                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 4265                         rxbuf->m_pack->m_flags |= M_PKTHDR;
 4266                         m_freem(rxbuf->m_pack);
 4267                 }
 4268                 rxbuf->m_head = NULL;
 4269                 rxbuf->m_pack = NULL;
 4270         }
 4271 }
 4272 
 4273 
 4274 /*********************************************************************
 4275  *
 4276  *  Initialize a receive ring and its buffers.
 4277  *
 4278  **********************************************************************/
 4279 static int
 4280 igb_setup_receive_ring(struct rx_ring *rxr)
 4281 {
 4282         struct  adapter         *adapter;
 4283         struct  ifnet           *ifp;
 4284         device_t                dev;
 4285         struct igb_rx_buf       *rxbuf;
 4286         bus_dma_segment_t       pseg[1], hseg[1];
 4287         struct lro_ctrl         *lro = &rxr->lro;
 4288         int                     rsize, nsegs, error = 0;
 4289 #ifdef DEV_NETMAP
 4290         struct netmap_adapter *na = NA(rxr->adapter->ifp);
 4291         struct netmap_slot *slot;
 4292 #endif /* DEV_NETMAP */
 4293 
 4294         adapter = rxr->adapter;
 4295         dev = adapter->dev;
 4296         ifp = adapter->ifp;
 4297 
 4298         /* Clear the ring contents */
 4299         IGB_RX_LOCK(rxr);
 4300 #ifdef DEV_NETMAP
 4301         slot = netmap_reset(na, NR_RX, rxr->me, 0);
 4302 #endif /* DEV_NETMAP */
 4303         rsize = roundup2(adapter->num_rx_desc *
 4304             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 4305         bzero((void *)rxr->rx_base, rsize);
 4306 
 4307         /*
 4308         ** Free current RX buffer structures and their mbufs
 4309         */
 4310         igb_free_receive_ring(rxr);
 4311 
 4312         /* Configure for header split? */
 4313         if (igb_header_split)
 4314                 rxr->hdr_split = TRUE;
 4315 
 4316         /* Now replenish the ring mbufs */
 4317         for (int j = 0; j < adapter->num_rx_desc; ++j) {
 4318                 struct mbuf     *mh, *mp;
 4319 
 4320                 rxbuf = &rxr->rx_buffers[j];
 4321 #ifdef DEV_NETMAP
 4322                 if (slot) {
 4323                         /* slot sj is mapped to the i-th NIC-ring entry */
 4324                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
 4325                         uint64_t paddr;
 4326                         void *addr;
 4327 
 4328                         addr = PNMB(na, slot + sj, &paddr);
 4329                         netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
 4330                         /* Update descriptor */
 4331                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
 4332                         continue;
 4333                 }
 4334 #endif /* DEV_NETMAP */
 4335                 if (rxr->hdr_split == FALSE)
 4336                         goto skip_head;
 4337 
 4338                 /* First the header */
 4339                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
 4340                 if (rxbuf->m_head == NULL) {
 4341                         error = ENOBUFS;
 4342                         goto fail;
 4343                 }
 4344                 m_adj(rxbuf->m_head, ETHER_ALIGN);
 4345                 mh = rxbuf->m_head;
 4346                 mh->m_len = mh->m_pkthdr.len = MHLEN;
 4347                 mh->m_flags |= M_PKTHDR;
 4348                 /* Get the memory mapping */
 4349                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
 4350                     rxbuf->hmap, rxbuf->m_head, hseg,
 4351                     &nsegs, BUS_DMA_NOWAIT);
 4352                 if (error != 0) /* Nothing elegant to do here */
 4353                         goto fail;
 4354                 bus_dmamap_sync(rxr->htag,
 4355                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
 4356                 /* Update descriptor */
 4357                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
 4358 
 4359 skip_head:
 4360                 /* Now the payload cluster */
 4361                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
 4362                     M_PKTHDR, adapter->rx_mbuf_sz);
 4363                 if (rxbuf->m_pack == NULL) {
 4364                         error = ENOBUFS;
 4365                         goto fail;
 4366                 }
 4367                 mp = rxbuf->m_pack;
 4368                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
 4369                 /* Get the memory mapping */
 4370                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
 4371                     rxbuf->pmap, mp, pseg,
 4372                     &nsegs, BUS_DMA_NOWAIT);
 4373                 if (error != 0)
 4374                         goto fail;
 4375                 bus_dmamap_sync(rxr->ptag,
 4376                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
 4377                 /* Update descriptor */
 4378                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
 4379         }
 4380 
 4381         /* Setup our descriptor indices */
 4382         rxr->next_to_check = 0;
 4383         rxr->next_to_refresh = adapter->num_rx_desc - 1;
 4384         rxr->lro_enabled = FALSE;
 4385         rxr->rx_split_packets = 0;
 4386         rxr->rx_bytes = 0;
 4387 
 4388         rxr->fmp = NULL;
 4389         rxr->lmp = NULL;
 4390 
 4391         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4392             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4393 
 4394         /*
 4395         ** Now set up the LRO interface, we
 4396         ** also only do head split when LRO
 4397         ** is enabled, since so often they
 4398         ** are undesireable in similar setups.
 4399         */
 4400         if (ifp->if_capenable & IFCAP_LRO) {
 4401                 error = tcp_lro_init(lro);
 4402                 if (error) {
 4403                         device_printf(dev, "LRO Initialization failed!\n");
 4404                         goto fail;
 4405                 }
 4406                 INIT_DEBUGOUT("RX LRO Initialized\n");
 4407                 rxr->lro_enabled = TRUE;
 4408                 lro->ifp = adapter->ifp;
 4409         }
 4410 
 4411         IGB_RX_UNLOCK(rxr);
 4412         return (0);
 4413 
 4414 fail:
 4415         igb_free_receive_ring(rxr);
 4416         IGB_RX_UNLOCK(rxr);
 4417         return (error);
 4418 }
 4419 
 4420 
 4421 /*********************************************************************
 4422  *
 4423  *  Initialize all receive rings.
 4424  *
 4425  **********************************************************************/
 4426 static int
 4427 igb_setup_receive_structures(struct adapter *adapter)
 4428 {
 4429         struct rx_ring *rxr = adapter->rx_rings;
 4430         int i;
 4431 
 4432         for (i = 0; i < adapter->num_queues; i++, rxr++)
 4433                 if (igb_setup_receive_ring(rxr))
 4434                         goto fail;
 4435 
 4436         return (0);
 4437 fail:
 4438         /*
 4439          * Free RX buffers allocated so far, we will only handle
 4440          * the rings that completed, the failing case will have
 4441          * cleaned up for itself. 'i' is the endpoint.
 4442          */
 4443         for (int j = 0; j < i; ++j) {
 4444                 rxr = &adapter->rx_rings[j];
 4445                 IGB_RX_LOCK(rxr);
 4446                 igb_free_receive_ring(rxr);
 4447                 IGB_RX_UNLOCK(rxr);
 4448         }
 4449 
 4450         return (ENOBUFS);
 4451 }
 4452 
 4453 /*********************************************************************
 4454  *
 4455  *  Enable receive unit.
 4456  *
 4457  **********************************************************************/
 4458 static void
 4459 igb_initialize_receive_units(struct adapter *adapter)
 4460 {
 4461         struct rx_ring  *rxr = adapter->rx_rings;
 4462         struct ifnet    *ifp = adapter->ifp;
 4463         struct e1000_hw *hw = &adapter->hw;
 4464         u32             rctl, rxcsum, psize, srrctl = 0;
 4465 
 4466         INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
 4467 
 4468         /*
 4469          * Make sure receives are disabled while setting
 4470          * up the descriptor ring
 4471          */
 4472         rctl = E1000_READ_REG(hw, E1000_RCTL);
 4473         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
 4474 
 4475         /*
 4476         ** Set up for header split
 4477         */
 4478         if (igb_header_split) {
 4479                 /* Use a standard mbuf for the header */
 4480                 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
 4481                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
 4482         } else
 4483                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
 4484 
 4485         /*
 4486         ** Set up for jumbo frames
 4487         */
 4488         if (ifp->if_mtu > ETHERMTU) {
 4489                 rctl |= E1000_RCTL_LPE;
 4490                 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
 4491                         srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4492                         rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
 4493                 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
 4494                         srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4495                         rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
 4496                 }
 4497                 /* Set maximum packet len */
 4498                 psize = adapter->max_frame_size;
 4499                 /* are we on a vlan? */
 4500                 if (adapter->ifp->if_vlantrunk != NULL)
 4501                         psize += VLAN_TAG_SIZE;
 4502                 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
 4503         } else {
 4504                 rctl &= ~E1000_RCTL_LPE;
 4505                 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4506                 rctl |= E1000_RCTL_SZ_2048;
 4507         }
 4508 
 4509         /*
 4510          * If TX flow control is disabled and there's >1 queue defined,
 4511          * enable DROP.
 4512          *
 4513          * This drops frames rather than hanging the RX MAC for all queues.
 4514          */
 4515         if ((adapter->num_queues > 1) &&
 4516             (adapter->fc == e1000_fc_none ||
 4517              adapter->fc == e1000_fc_rx_pause)) {
 4518                 srrctl |= E1000_SRRCTL_DROP_EN;
 4519         }
 4520 
 4521         /* Setup the Base and Length of the Rx Descriptor Rings */
 4522         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 4523                 u64 bus_addr = rxr->rxdma.dma_paddr;
 4524                 u32 rxdctl;
 4525 
 4526                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
 4527                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
 4528                 E1000_WRITE_REG(hw, E1000_RDBAH(i),
 4529                     (uint32_t)(bus_addr >> 32));
 4530                 E1000_WRITE_REG(hw, E1000_RDBAL(i),
 4531                     (uint32_t)bus_addr);
 4532                 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
 4533                 /* Enable this Queue */
 4534                 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
 4535                 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
 4536                 rxdctl &= 0xFFF00000;
 4537                 rxdctl |= IGB_RX_PTHRESH;
 4538                 rxdctl |= IGB_RX_HTHRESH << 8;
 4539                 rxdctl |= IGB_RX_WTHRESH << 16;
 4540                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
 4541         }
 4542 
 4543         /*
 4544         ** Setup for RX MultiQueue
 4545         */
 4546         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
 4547         if (adapter->num_queues >1) {
 4548                 u32 random[10], mrqc, shift = 0;
 4549                 union igb_reta {
 4550                         u32 dword;
 4551                         u8  bytes[4];
 4552                 } reta;
 4553 
 4554                 arc4rand(&random, sizeof(random), 0);
 4555                 if (adapter->hw.mac.type == e1000_82575)
 4556                         shift = 6;
 4557                 /* Warning FM follows */
 4558                 for (int i = 0; i < 128; i++) {
 4559                         reta.bytes[i & 3] =
 4560                             (i % adapter->num_queues) << shift;
 4561                         if ((i & 3) == 3)
 4562                                 E1000_WRITE_REG(hw,
 4563                                     E1000_RETA(i >> 2), reta.dword);
 4564                 }
 4565                 /* Now fill in hash table */
 4566                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
 4567                 for (int i = 0; i < 10; i++)
 4568                         E1000_WRITE_REG_ARRAY(hw,
 4569                             E1000_RSSRK(0), i, random[i]);
 4570 
 4571                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
 4572                     E1000_MRQC_RSS_FIELD_IPV4_TCP);
 4573                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
 4574                     E1000_MRQC_RSS_FIELD_IPV6_TCP);
 4575                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
 4576                     E1000_MRQC_RSS_FIELD_IPV6_UDP);
 4577                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
 4578                     E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
 4579 
 4580                 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
 4581 
 4582                 /*
 4583                 ** NOTE: Receive Full-Packet Checksum Offload 
 4584                 ** is mutually exclusive with Multiqueue. However
 4585                 ** this is not the same as TCP/IP checksums which
 4586                 ** still work.
 4587                 */
 4588                 rxcsum |= E1000_RXCSUM_PCSD;
 4589 #if __FreeBSD_version >= 800000
 4590                 /* For SCTP Offload */
 4591                 if ((hw->mac.type != e1000_82575) &&
 4592                     (ifp->if_capenable & IFCAP_RXCSUM))
 4593                         rxcsum |= E1000_RXCSUM_CRCOFL;
 4594 #endif
 4595         } else {
 4596                 /* Non RSS setup */
 4597                 if (ifp->if_capenable & IFCAP_RXCSUM) {
 4598                         rxcsum |= E1000_RXCSUM_IPPCSE;
 4599 #if __FreeBSD_version >= 800000
 4600                         if (adapter->hw.mac.type != e1000_82575)
 4601                                 rxcsum |= E1000_RXCSUM_CRCOFL;
 4602 #endif
 4603                 } else
 4604                         rxcsum &= ~E1000_RXCSUM_TUOFL;
 4605         }
 4606         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
 4607 
 4608         /* Setup the Receive Control Register */
 4609         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
 4610         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
 4611                    E1000_RCTL_RDMTS_HALF |
 4612                    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
 4613         /* Strip CRC bytes. */
 4614         rctl |= E1000_RCTL_SECRC;
 4615         /* Make sure VLAN Filters are off */
 4616         rctl &= ~E1000_RCTL_VFE;
 4617         /* Don't store bad packets */
 4618         rctl &= ~E1000_RCTL_SBP;
 4619 
 4620         /* Enable Receives */
 4621         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 4622 
 4623         /*
 4624          * Setup the HW Rx Head and Tail Descriptor Pointers
 4625          *   - needs to be after enable
 4626          */
 4627         for (int i = 0; i < adapter->num_queues; i++) {
 4628                 rxr = &adapter->rx_rings[i];
 4629                 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
 4630 #ifdef DEV_NETMAP
 4631                 /*
 4632                  * an init() while a netmap client is active must
 4633                  * preserve the rx buffers passed to userspace.
 4634                  * In this driver it means we adjust RDT to
 4635                  * something different from next_to_refresh
 4636                  * (which is not used in netmap mode).
 4637                  */
 4638                 if (ifp->if_capenable & IFCAP_NETMAP) {
 4639                         struct netmap_adapter *na = NA(adapter->ifp);
 4640                         struct netmap_kring *kring = &na->rx_rings[i];
 4641                         int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
 4642 
 4643                         if (t >= adapter->num_rx_desc)
 4644                                 t -= adapter->num_rx_desc;
 4645                         else if (t < 0)
 4646                                 t += adapter->num_rx_desc;
 4647                         E1000_WRITE_REG(hw, E1000_RDT(i), t);
 4648                 } else
 4649 #endif /* DEV_NETMAP */
 4650                 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
 4651         }
 4652         return;
 4653 }
 4654 
 4655 /*********************************************************************
 4656  *
 4657  *  Free receive rings.
 4658  *
 4659  **********************************************************************/
 4660 static void
 4661 igb_free_receive_structures(struct adapter *adapter)
 4662 {
 4663         struct rx_ring *rxr = adapter->rx_rings;
 4664 
 4665         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 4666                 struct lro_ctrl *lro = &rxr->lro;
 4667                 igb_free_receive_buffers(rxr);
 4668                 tcp_lro_free(lro);
 4669                 igb_dma_free(adapter, &rxr->rxdma);
 4670         }
 4671 
 4672         free(adapter->rx_rings, M_DEVBUF);
 4673 }
 4674 
 4675 /*********************************************************************
 4676  *
 4677  *  Free receive ring data structures.
 4678  *
 4679  **********************************************************************/
 4680 static void
 4681 igb_free_receive_buffers(struct rx_ring *rxr)
 4682 {
 4683         struct adapter          *adapter = rxr->adapter;
 4684         struct igb_rx_buf       *rxbuf;
 4685         int i;
 4686 
 4687         INIT_DEBUGOUT("free_receive_structures: begin");
 4688 
 4689         /* Cleanup any existing buffers */
 4690         if (rxr->rx_buffers != NULL) {
 4691                 for (i = 0; i < adapter->num_rx_desc; i++) {
 4692                         rxbuf = &rxr->rx_buffers[i];
 4693                         if (rxbuf->m_head != NULL) {
 4694                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 4695                                     BUS_DMASYNC_POSTREAD);
 4696                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 4697                                 rxbuf->m_head->m_flags |= M_PKTHDR;
 4698                                 m_freem(rxbuf->m_head);
 4699                         }
 4700                         if (rxbuf->m_pack != NULL) {
 4701                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 4702                                     BUS_DMASYNC_POSTREAD);
 4703                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 4704                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
 4705                                 m_freem(rxbuf->m_pack);
 4706                         }
 4707                         rxbuf->m_head = NULL;
 4708                         rxbuf->m_pack = NULL;
 4709                         if (rxbuf->hmap != NULL) {
 4710                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
 4711                                 rxbuf->hmap = NULL;
 4712                         }
 4713                         if (rxbuf->pmap != NULL) {
 4714                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
 4715                                 rxbuf->pmap = NULL;
 4716                         }
 4717                 }
 4718                 if (rxr->rx_buffers != NULL) {
 4719                         free(rxr->rx_buffers, M_DEVBUF);
 4720                         rxr->rx_buffers = NULL;
 4721                 }
 4722         }
 4723 
 4724         if (rxr->htag != NULL) {
 4725                 bus_dma_tag_destroy(rxr->htag);
 4726                 rxr->htag = NULL;
 4727         }
 4728         if (rxr->ptag != NULL) {
 4729                 bus_dma_tag_destroy(rxr->ptag);
 4730                 rxr->ptag = NULL;
 4731         }
 4732 }
 4733 
 4734 static __inline void
 4735 igb_rx_discard(struct rx_ring *rxr, int i)
 4736 {
 4737         struct igb_rx_buf       *rbuf;
 4738 
 4739         rbuf = &rxr->rx_buffers[i];
 4740 
 4741         /* Partially received? Free the chain */
 4742         if (rxr->fmp != NULL) {
 4743                 rxr->fmp->m_flags |= M_PKTHDR;
 4744                 m_freem(rxr->fmp);
 4745                 rxr->fmp = NULL;
 4746                 rxr->lmp = NULL;
 4747         }
 4748 
 4749         /*
 4750         ** With advanced descriptors the writeback
 4751         ** clobbers the buffer addrs, so its easier
 4752         ** to just free the existing mbufs and take
 4753         ** the normal refresh path to get new buffers
 4754         ** and mapping.
 4755         */
 4756         if (rbuf->m_head) {
 4757                 m_free(rbuf->m_head);
 4758                 rbuf->m_head = NULL;
 4759                 bus_dmamap_unload(rxr->htag, rbuf->hmap);
 4760         }
 4761 
 4762         if (rbuf->m_pack) {
 4763                 m_free(rbuf->m_pack);
 4764                 rbuf->m_pack = NULL;
 4765                 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
 4766         }
 4767 
 4768         return;
 4769 }
 4770 
 4771 static __inline void
 4772 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
 4773 {
 4774 
 4775         /*
 4776          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
 4777          * should be computed by hardware. Also it should not have VLAN tag in
 4778          * ethernet header.
 4779          */
 4780         if (rxr->lro_enabled &&
 4781             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
 4782             (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
 4783             (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
 4784             (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
 4785             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 
 4786             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
 4787                 /*
 4788                  * Send to the stack if:
 4789                  **  - LRO not enabled, or
 4790                  **  - no LRO resources, or
 4791                  **  - lro enqueue fails
 4792                  */
 4793                 if (rxr->lro.lro_cnt != 0)
 4794                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
 4795                                 return;
 4796         }
 4797         IGB_RX_UNLOCK(rxr);
 4798         (*ifp->if_input)(ifp, m);
 4799         IGB_RX_LOCK(rxr);
 4800 }
 4801 
 4802 /*********************************************************************
 4803  *
 4804  *  This routine executes in interrupt context. It replenishes
 4805  *  the mbufs in the descriptor and sends data which has been
 4806  *  dma'ed into host memory to upper layer.
 4807  *
 4808  *  We loop at most count times if count is > 0, or until done if
 4809  *  count < 0.
 4810  *
 4811  *  Return TRUE if more to clean, FALSE otherwise
 4812  *********************************************************************/
 4813 static bool
 4814 igb_rxeof(struct igb_queue *que, int count, int *done)
 4815 {
 4816         struct adapter          *adapter = que->adapter;
 4817         struct rx_ring          *rxr = que->rxr;
 4818         struct ifnet            *ifp = adapter->ifp;
 4819         struct lro_ctrl         *lro = &rxr->lro;
 4820         struct lro_entry        *queued;
 4821         int                     i, processed = 0, rxdone = 0;
 4822         u32                     ptype, staterr = 0;
 4823         union e1000_adv_rx_desc *cur;
 4824 
 4825         IGB_RX_LOCK(rxr);
 4826         /* Sync the ring. */
 4827         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4828             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 4829 
 4830 #ifdef DEV_NETMAP
 4831         if (netmap_rx_irq(ifp, rxr->me, &processed)) {
 4832                 IGB_RX_UNLOCK(rxr);
 4833                 return (FALSE);
 4834         }
 4835 #endif /* DEV_NETMAP */
 4836 
 4837         /* Main clean loop */
 4838         for (i = rxr->next_to_check; count != 0;) {
 4839                 struct mbuf             *sendmp, *mh, *mp;
 4840                 struct igb_rx_buf       *rxbuf;
 4841                 u16                     hlen, plen, hdr, vtag;
 4842                 bool                    eop = FALSE;
 4843  
 4844                 cur = &rxr->rx_base[i];
 4845                 staterr = le32toh(cur->wb.upper.status_error);
 4846                 if ((staterr & E1000_RXD_STAT_DD) == 0)
 4847                         break;
 4848                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 4849                         break;
 4850                 count--;
 4851                 sendmp = mh = mp = NULL;
 4852                 cur->wb.upper.status_error = 0;
 4853                 rxbuf = &rxr->rx_buffers[i];
 4854                 plen = le16toh(cur->wb.upper.length);
 4855                 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
 4856                 if (((adapter->hw.mac.type == e1000_i350) ||
 4857                     (adapter->hw.mac.type == e1000_i354)) &&
 4858                     (staterr & E1000_RXDEXT_STATERR_LB))
 4859                         vtag = be16toh(cur->wb.upper.vlan);
 4860                 else
 4861                         vtag = le16toh(cur->wb.upper.vlan);
 4862                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
 4863                 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
 4864 
 4865                 /*
 4866                  * Free the frame (all segments) if we're at EOP and
 4867                  * it's an error.
 4868                  *
 4869                  * The datasheet states that EOP + status is only valid for
 4870                  * the final segment in a multi-segment frame.
 4871                  */
 4872                 if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
 4873                         adapter->dropped_pkts++;
 4874                         ++rxr->rx_discarded;
 4875                         igb_rx_discard(rxr, i);
 4876                         goto next_desc;
 4877                 }
 4878 
 4879                 /*
 4880                 ** The way the hardware is configured to
 4881                 ** split, it will ONLY use the header buffer
 4882                 ** when header split is enabled, otherwise we
 4883                 ** get normal behavior, ie, both header and
 4884                 ** payload are DMA'd into the payload buffer.
 4885                 **
 4886                 ** The fmp test is to catch the case where a
 4887                 ** packet spans multiple descriptors, in that
 4888                 ** case only the first header is valid.
 4889                 */
 4890                 if (rxr->hdr_split && rxr->fmp == NULL) {
 4891                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 4892                         hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
 4893                             E1000_RXDADV_HDRBUFLEN_SHIFT;
 4894                         if (hlen > IGB_HDR_BUF)
 4895                                 hlen = IGB_HDR_BUF;
 4896                         mh = rxr->rx_buffers[i].m_head;
 4897                         mh->m_len = hlen;
 4898                         /* clear buf pointer for refresh */
 4899                         rxbuf->m_head = NULL;
 4900                         /*
 4901                         ** Get the payload length, this
 4902                         ** could be zero if its a small
 4903                         ** packet.
 4904                         */
 4905                         if (plen > 0) {
 4906                                 mp = rxr->rx_buffers[i].m_pack;
 4907                                 mp->m_len = plen;
 4908                                 mh->m_next = mp;
 4909                                 /* clear buf pointer */
 4910                                 rxbuf->m_pack = NULL;
 4911                                 rxr->rx_split_packets++;
 4912                         }
 4913                 } else {
 4914                         /*
 4915                         ** Either no header split, or a
 4916                         ** secondary piece of a fragmented
 4917                         ** split packet.
 4918                         */
 4919                         mh = rxr->rx_buffers[i].m_pack;
 4920                         mh->m_len = plen;
 4921                         /* clear buf info for refresh */
 4922                         rxbuf->m_pack = NULL;
 4923                 }
 4924                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 4925 
 4926                 ++processed; /* So we know when to refresh */
 4927 
 4928                 /* Initial frame - setup */
 4929                 if (rxr->fmp == NULL) {
 4930                         mh->m_pkthdr.len = mh->m_len;
 4931                         /* Save the head of the chain */
 4932                         rxr->fmp = mh;
 4933                         rxr->lmp = mh;
 4934                         if (mp != NULL) {
 4935                                 /* Add payload if split */
 4936                                 mh->m_pkthdr.len += mp->m_len;
 4937                                 rxr->lmp = mh->m_next;
 4938                         }
 4939                 } else {
 4940                         /* Chain mbuf's together */
 4941                         rxr->lmp->m_next = mh;
 4942                         rxr->lmp = rxr->lmp->m_next;
 4943                         rxr->fmp->m_pkthdr.len += mh->m_len;
 4944                 }
 4945 
 4946                 if (eop) {
 4947                         rxr->fmp->m_pkthdr.rcvif = ifp;
 4948                         ifp->if_ipackets++;
 4949                         rxr->rx_packets++;
 4950                         /* capture data for AIM */
 4951                         rxr->packets++;
 4952                         rxr->bytes += rxr->fmp->m_pkthdr.len;
 4953                         rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
 4954 
 4955                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
 4956                                 igb_rx_checksum(staterr, rxr->fmp, ptype);
 4957 
 4958                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
 4959                             (staterr & E1000_RXD_STAT_VP) != 0) {
 4960                                 rxr->fmp->m_pkthdr.ether_vtag = vtag;
 4961                                 rxr->fmp->m_flags |= M_VLANTAG;
 4962                         }
 4963 
 4964                         /*
 4965                          * In case of multiqueue, we have RXCSUM.PCSD bit set
 4966                          * and never cleared. This means we have RSS hash
 4967                          * available to be used.
 4968                          */
 4969                         if (adapter->num_queues > 1) {
 4970                                 rxr->fmp->m_pkthdr.flowid = 
 4971                                     le32toh(cur->wb.lower.hi_dword.rss);
 4972                                 /*
 4973                                  * Full RSS support is not avilable in
 4974                                  * FreeBSD 10 so setting the hash type to
 4975                                  * OPAQUE.
 4976                                  */
 4977                                 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
 4978                         } else {
 4979 #ifndef IGB_LEGACY_TX
 4980                                 rxr->fmp->m_pkthdr.flowid = que->msix;
 4981                                 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
 4982 #endif
 4983                         }
 4984                         sendmp = rxr->fmp;
 4985                         /* Make sure to set M_PKTHDR. */
 4986                         sendmp->m_flags |= M_PKTHDR;
 4987                         rxr->fmp = NULL;
 4988                         rxr->lmp = NULL;
 4989                 }
 4990 
 4991 next_desc:
 4992                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4993                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4994 
 4995                 /* Advance our pointers to the next descriptor. */
 4996                 if (++i == adapter->num_rx_desc)
 4997                         i = 0;
 4998                 /*
 4999                 ** Send to the stack or LRO
 5000                 */
 5001                 if (sendmp != NULL) {
 5002                         rxr->next_to_check = i;
 5003                         igb_rx_input(rxr, ifp, sendmp, ptype);
 5004                         i = rxr->next_to_check;
 5005                         rxdone++;
 5006                 }
 5007 
 5008                 /* Every 8 descriptors we go to refresh mbufs */
 5009                 if (processed == 8) {
 5010                         igb_refresh_mbufs(rxr, i);
 5011                         processed = 0;
 5012                 }
 5013         }
 5014 
 5015         /* Catch any remainders */
 5016         if (igb_rx_unrefreshed(rxr))
 5017                 igb_refresh_mbufs(rxr, i);
 5018 
 5019         rxr->next_to_check = i;
 5020 
 5021         /*
 5022          * Flush any outstanding LRO work
 5023          */
 5024         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
 5025                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
 5026                 tcp_lro_flush(lro, queued);
 5027         }
 5028 
 5029         if (done != NULL)
 5030                 *done += rxdone;
 5031 
 5032         IGB_RX_UNLOCK(rxr);
 5033         return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
 5034 }
 5035 
 5036 /*********************************************************************
 5037  *
 5038  *  Verify that the hardware indicated that the checksum is valid.
 5039  *  Inform the stack about the status of checksum so that stack
 5040  *  doesn't spend time verifying the checksum.
 5041  *
 5042  *********************************************************************/
 5043 static void
 5044 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
 5045 {
 5046         u16 status = (u16)staterr;
 5047         u8  errors = (u8) (staterr >> 24);
 5048         int sctp;
 5049 
 5050         /* Ignore Checksum bit is set */
 5051         if (status & E1000_RXD_STAT_IXSM) {
 5052                 mp->m_pkthdr.csum_flags = 0;
 5053                 return;
 5054         }
 5055 
 5056         if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
 5057             (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
 5058                 sctp = 1;
 5059         else
 5060                 sctp = 0;
 5061         if (status & E1000_RXD_STAT_IPCS) {
 5062                 /* Did it pass? */
 5063                 if (!(errors & E1000_RXD_ERR_IPE)) {
 5064                         /* IP Checksum Good */
 5065                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
 5066                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
 5067                 } else
 5068                         mp->m_pkthdr.csum_flags = 0;
 5069         }
 5070 
 5071         if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
 5072                 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
 5073 #if __FreeBSD_version >= 800000
 5074                 if (sctp) /* reassign */
 5075                         type = CSUM_SCTP_VALID;
 5076 #endif
 5077                 /* Did it pass? */
 5078                 if (!(errors & E1000_RXD_ERR_TCPE)) {
 5079                         mp->m_pkthdr.csum_flags |= type;
 5080                         if (sctp == 0)
 5081                                 mp->m_pkthdr.csum_data = htons(0xffff);
 5082                 }
 5083         }
 5084         return;
 5085 }
 5086 
 5087 /*
 5088  * This routine is run via an vlan
 5089  * config EVENT
 5090  */
 5091 static void
 5092 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 5093 {
 5094         struct adapter  *adapter = ifp->if_softc;
 5095         u32             index, bit;
 5096 
 5097         if (ifp->if_softc !=  arg)   /* Not our event */
 5098                 return;
 5099 
 5100         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 5101                 return;
 5102 
 5103         IGB_CORE_LOCK(adapter);
 5104         index = (vtag >> 5) & 0x7F;
 5105         bit = vtag & 0x1F;
 5106         adapter->shadow_vfta[index] |= (1 << bit);
 5107         ++adapter->num_vlans;
 5108         /* Change hw filter setting */
 5109         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
 5110                 igb_setup_vlan_hw_support(adapter);
 5111         IGB_CORE_UNLOCK(adapter);
 5112 }
 5113 
 5114 /*
 5115  * This routine is run via an vlan
 5116  * unconfig EVENT
 5117  */
 5118 static void
 5119 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 5120 {
 5121         struct adapter  *adapter = ifp->if_softc;
 5122         u32             index, bit;
 5123 
 5124         if (ifp->if_softc !=  arg)
 5125                 return;
 5126 
 5127         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 5128                 return;
 5129 
 5130         IGB_CORE_LOCK(adapter);
 5131         index = (vtag >> 5) & 0x7F;
 5132         bit = vtag & 0x1F;
 5133         adapter->shadow_vfta[index] &= ~(1 << bit);
 5134         --adapter->num_vlans;
 5135         /* Change hw filter setting */
 5136         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
 5137                 igb_setup_vlan_hw_support(adapter);
 5138         IGB_CORE_UNLOCK(adapter);
 5139 }
 5140 
 5141 static void
 5142 igb_setup_vlan_hw_support(struct adapter *adapter)
 5143 {
 5144         struct e1000_hw *hw = &adapter->hw;
 5145         struct ifnet    *ifp = adapter->ifp;
 5146         u32             reg;
 5147 
 5148         if (adapter->vf_ifp) {
 5149                 e1000_rlpml_set_vf(hw,
 5150                     adapter->max_frame_size + VLAN_TAG_SIZE);
 5151                 return;
 5152         }
 5153 
 5154         reg = E1000_READ_REG(hw, E1000_CTRL);
 5155         reg |= E1000_CTRL_VME;
 5156         E1000_WRITE_REG(hw, E1000_CTRL, reg);
 5157 
 5158         /* Enable the Filter Table */
 5159         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
 5160                 reg = E1000_READ_REG(hw, E1000_RCTL);
 5161                 reg &= ~E1000_RCTL_CFIEN;
 5162                 reg |= E1000_RCTL_VFE;
 5163                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 5164         }
 5165 
 5166         /* Update the frame size */
 5167         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
 5168             adapter->max_frame_size + VLAN_TAG_SIZE);
 5169 
 5170         /* Don't bother with table if no vlans */
 5171         if ((adapter->num_vlans == 0) ||
 5172             ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
 5173                 return;
 5174         /*
 5175         ** A soft reset zero's out the VFTA, so
 5176         ** we need to repopulate it now.
 5177         */
 5178         for (int i = 0; i < IGB_VFTA_SIZE; i++)
 5179                 if (adapter->shadow_vfta[i] != 0) {
 5180                         if (adapter->vf_ifp)
 5181                                 e1000_vfta_set_vf(hw,
 5182                                     adapter->shadow_vfta[i], TRUE);
 5183                         else
 5184                                 e1000_write_vfta(hw,
 5185                                     i, adapter->shadow_vfta[i]);
 5186                 }
 5187 }
 5188 
 5189 static void
 5190 igb_enable_intr(struct adapter *adapter)
 5191 {
 5192         /* With RSS set up what to auto clear */
 5193         if (adapter->msix_mem) {
 5194                 u32 mask = (adapter->que_mask | adapter->link_mask);
 5195                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
 5196                 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
 5197                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
 5198                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 5199                     E1000_IMS_LSC);
 5200         } else {
 5201                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 5202                     IMS_ENABLE_MASK);
 5203         }
 5204         E1000_WRITE_FLUSH(&adapter->hw);
 5205 
 5206         return;
 5207 }
 5208 
 5209 static void
 5210 igb_disable_intr(struct adapter *adapter)
 5211 {
 5212         if (adapter->msix_mem) {
 5213                 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
 5214                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
 5215         } 
 5216         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
 5217         E1000_WRITE_FLUSH(&adapter->hw);
 5218         return;
 5219 }
 5220 
 5221 /*
 5222  * Bit of a misnomer, what this really means is
 5223  * to enable OS management of the system... aka
 5224  * to disable special hardware management features 
 5225  */
 5226 static void
 5227 igb_init_manageability(struct adapter *adapter)
 5228 {
 5229         if (adapter->has_manage) {
 5230                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
 5231                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 5232 
 5233                 /* disable hardware interception of ARP */
 5234                 manc &= ~(E1000_MANC_ARP_EN);
 5235 
 5236                 /* enable receiving management packets to the host */
 5237                 manc |= E1000_MANC_EN_MNG2HOST;
 5238                 manc2h |= 1 << 5;  /* Mng Port 623 */
 5239                 manc2h |= 1 << 6;  /* Mng Port 664 */
 5240                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
 5241                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 5242         }
 5243 }
 5244 
 5245 /*
 5246  * Give control back to hardware management
 5247  * controller if there is one.
 5248  */
 5249 static void
 5250 igb_release_manageability(struct adapter *adapter)
 5251 {
 5252         if (adapter->has_manage) {
 5253                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 5254 
 5255                 /* re-enable hardware interception of ARP */
 5256                 manc |= E1000_MANC_ARP_EN;
 5257                 manc &= ~E1000_MANC_EN_MNG2HOST;
 5258 
 5259                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 5260         }
 5261 }
 5262 
 5263 /*
 5264  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 5265  * For ASF and Pass Through versions of f/w this means that
 5266  * the driver is loaded. 
 5267  *
 5268  */
 5269 static void
 5270 igb_get_hw_control(struct adapter *adapter)
 5271 {
 5272         u32 ctrl_ext;
 5273 
 5274         if (adapter->vf_ifp)
 5275                 return;
 5276 
 5277         /* Let firmware know the driver has taken over */
 5278         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 5279         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 5280             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
 5281 }
 5282 
 5283 /*
 5284  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 5285  * For ASF and Pass Through versions of f/w this means that the
 5286  * driver is no longer loaded.
 5287  *
 5288  */
 5289 static void
 5290 igb_release_hw_control(struct adapter *adapter)
 5291 {
 5292         u32 ctrl_ext;
 5293 
 5294         if (adapter->vf_ifp)
 5295                 return;
 5296 
 5297         /* Let firmware taken over control of h/w */
 5298         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 5299         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 5300             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
 5301 }
 5302 
 5303 static int
 5304 igb_is_valid_ether_addr(uint8_t *addr)
 5305 {
 5306         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
 5307 
 5308         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
 5309                 return (FALSE);
 5310         }
 5311 
 5312         return (TRUE);
 5313 }
 5314 
 5315 
 5316 /*
 5317  * Enable PCI Wake On Lan capability
 5318  */
 5319 static void
 5320 igb_enable_wakeup(device_t dev)
 5321 {
 5322         struct adapter  *adapter = device_get_softc(dev);
 5323         struct ifnet    *ifp = adapter->ifp;
 5324         u32             pmc, ctrl, ctrl_ext, rctl, wuc;
 5325         u16             status;
 5326 
 5327         if (pci_find_cap(dev, PCIY_PMG, &pmc) != 0)
 5328                 return;
 5329 
 5330         adapter->wol = E1000_READ_REG(&adapter->hw, E1000_WUFC);
 5331         if (ifp->if_capenable & IFCAP_WOL_MAGIC)
 5332                 adapter->wol |=  E1000_WUFC_MAG;
 5333         else
 5334                 adapter->wol &= ~E1000_WUFC_MAG;
 5335 
 5336         if (ifp->if_capenable & IFCAP_WOL_MCAST) {
 5337                 adapter->wol |=  E1000_WUFC_MC;
 5338                 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 5339                 rctl |= E1000_RCTL_MPE;
 5340                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
 5341         } else
 5342                 adapter->wol &= ~E1000_WUFC_MC;
 5343 
 5344         if (ifp->if_capenable & IFCAP_WOL_UCAST)
 5345                 adapter->wol |=  E1000_WUFC_EX;
 5346         else
 5347                 adapter->wol &= ~E1000_WUFC_EX;
 5348 
 5349         if (!(adapter->wol & (E1000_WUFC_EX | E1000_WUFC_MAG | E1000_WUFC_MC)))
 5350                 goto pme;
 5351 
 5352         /* Advertise the wakeup capability */
 5353         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
 5354         ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
 5355         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
 5356 
 5357         /* Keep the laser running on Fiber adapters */
 5358         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
 5359             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
 5360                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 5361                 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
 5362                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
 5363         }
 5364 
 5365         /* Enable wakeup by the MAC */
 5366         wuc = E1000_READ_REG(&adapter->hw, E1000_WUC);
 5367         wuc |= E1000_WUC_PME_EN | E1000_WUC_APME;
 5368         E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc);
 5369         E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 5370 
 5371 pme:
 5372         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
 5373         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
 5374         if (ifp->if_capenable & IFCAP_WOL)
 5375                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
 5376         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
 5377 }
 5378 
 5379 static void
 5380 igb_led_func(void *arg, int onoff)
 5381 {
 5382         struct adapter  *adapter = arg;
 5383 
 5384         IGB_CORE_LOCK(adapter);
 5385         if (onoff) {
 5386                 e1000_setup_led(&adapter->hw);
 5387                 e1000_led_on(&adapter->hw);
 5388         } else {
 5389                 e1000_led_off(&adapter->hw);
 5390                 e1000_cleanup_led(&adapter->hw);
 5391         }
 5392         IGB_CORE_UNLOCK(adapter);
 5393 }
 5394 
 5395 /**********************************************************************
 5396  *
 5397  *  Update the board statistics counters.
 5398  *
 5399  **********************************************************************/
 5400 static void
 5401 igb_update_stats_counters(struct adapter *adapter)
 5402 {
 5403         struct ifnet            *ifp;
 5404         struct e1000_hw         *hw = &adapter->hw;
 5405         struct e1000_hw_stats   *stats;
 5406 
 5407         /* 
 5408         ** The virtual function adapter has only a
 5409         ** small controlled set of stats, do only 
 5410         ** those and return.
 5411         */
 5412         if (adapter->vf_ifp) {
 5413                 igb_update_vf_stats_counters(adapter);
 5414                 return;
 5415         }
 5416 
 5417         stats = (struct e1000_hw_stats  *)adapter->stats;
 5418 
 5419         if (adapter->hw.phy.media_type == e1000_media_type_copper ||
 5420            (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
 5421                 stats->symerrs +=
 5422                     E1000_READ_REG(hw,E1000_SYMERRS);
 5423                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
 5424         }
 5425 
 5426         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
 5427         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
 5428         stats->scc += E1000_READ_REG(hw, E1000_SCC);
 5429         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
 5430 
 5431         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
 5432         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
 5433         stats->colc += E1000_READ_REG(hw, E1000_COLC);
 5434         stats->dc += E1000_READ_REG(hw, E1000_DC);
 5435         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
 5436         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
 5437         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
 5438         /*
 5439         ** For watchdog management we need to know if we have been
 5440         ** paused during the last interval, so capture that here.
 5441         */ 
 5442         adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
 5443         stats->xoffrxc += adapter->pause_frames;
 5444         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
 5445         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
 5446         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
 5447         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
 5448         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
 5449         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
 5450         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
 5451         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
 5452         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
 5453         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
 5454         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
 5455         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
 5456 
 5457         /* For the 64-bit byte counters the low dword must be read first. */
 5458         /* Both registers clear on the read of the high dword */
 5459 
 5460         stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
 5461             ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
 5462         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
 5463             ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
 5464 
 5465         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
 5466         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
 5467         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
 5468         stats->roc += E1000_READ_REG(hw, E1000_ROC);
 5469         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
 5470 
 5471         stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
 5472         stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC);
 5473         stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
 5474 
 5475         stats->tor += E1000_READ_REG(hw, E1000_TORL) +
 5476             ((u64)E1000_READ_REG(hw, E1000_TORH) << 32);
 5477         stats->tot += E1000_READ_REG(hw, E1000_TOTL) +
 5478             ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32);
 5479 
 5480         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
 5481         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
 5482         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
 5483         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
 5484         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
 5485         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
 5486         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
 5487         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
 5488         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
 5489         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
 5490 
 5491         /* Interrupt Counts */
 5492 
 5493         stats->iac += E1000_READ_REG(hw, E1000_IAC);
 5494         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
 5495         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
 5496         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
 5497         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
 5498         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
 5499         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
 5500         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
 5501         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
 5502 
 5503         /* Host to Card Statistics */
 5504 
 5505         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
 5506         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
 5507         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
 5508         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
 5509         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
 5510         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
 5511         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
 5512         stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
 5513             ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
 5514         stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
 5515             ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
 5516         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
 5517         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
 5518         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
 5519 
 5520         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
 5521         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
 5522         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
 5523         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
 5524         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
 5525         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
 5526 
 5527         ifp = adapter->ifp;
 5528         ifp->if_collisions = stats->colc;
 5529 
 5530         /* Rx Errors */
 5531         ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc +
 5532             stats->crcerrs + stats->algnerrc +
 5533             stats->ruc + stats->roc + stats->mpc + stats->cexterr;
 5534 
 5535         /* Tx Errors */
 5536         ifp->if_oerrors = stats->ecol +
 5537             stats->latecol + adapter->watchdog_events;
 5538 
 5539         /* Driver specific counters */
 5540         adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
 5541         adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
 5542         adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
 5543         adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
 5544         adapter->packet_buf_alloc_tx =
 5545             ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
 5546         adapter->packet_buf_alloc_rx =
 5547             (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
 5548 }
 5549 
 5550 
 5551 /**********************************************************************
 5552  *
 5553  *  Initialize the VF board statistics counters.
 5554  *
 5555  **********************************************************************/
 5556 static void
 5557 igb_vf_init_stats(struct adapter *adapter)
 5558 {
 5559         struct e1000_hw *hw = &adapter->hw;
 5560         struct e1000_vf_stats   *stats;
 5561 
 5562         stats = (struct e1000_vf_stats  *)adapter->stats;
 5563         if (stats == NULL)
 5564                 return;
 5565         stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
 5566         stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
 5567         stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
 5568         stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
 5569         stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
 5570 }
 5571  
 5572 /**********************************************************************
 5573  *
 5574  *  Update the VF board statistics counters.
 5575  *
 5576  **********************************************************************/
 5577 static void
 5578 igb_update_vf_stats_counters(struct adapter *adapter)
 5579 {
 5580         struct e1000_hw *hw = &adapter->hw;
 5581         struct e1000_vf_stats   *stats;
 5582 
 5583         if (adapter->link_speed == 0)
 5584                 return;
 5585 
 5586         stats = (struct e1000_vf_stats  *)adapter->stats;
 5587 
 5588         UPDATE_VF_REG(E1000_VFGPRC,
 5589             stats->last_gprc, stats->gprc);
 5590         UPDATE_VF_REG(E1000_VFGORC,
 5591             stats->last_gorc, stats->gorc);
 5592         UPDATE_VF_REG(E1000_VFGPTC,
 5593             stats->last_gptc, stats->gptc);
 5594         UPDATE_VF_REG(E1000_VFGOTC,
 5595             stats->last_gotc, stats->gotc);
 5596         UPDATE_VF_REG(E1000_VFMPRC,
 5597             stats->last_mprc, stats->mprc);
 5598 }
 5599 
 5600 /* Export a single 32-bit register via a read-only sysctl. */
 5601 static int
 5602 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
 5603 {
 5604         struct adapter *adapter;
 5605         u_int val;
 5606 
 5607         adapter = oidp->oid_arg1;
 5608         val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
 5609         return (sysctl_handle_int(oidp, &val, 0, req));
 5610 }
 5611 
 5612 /*
 5613 **  Tuneable interrupt rate handler
 5614 */
 5615 static int
 5616 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
 5617 {
 5618         struct igb_queue        *que = ((struct igb_queue *)oidp->oid_arg1);
 5619         int                     error;
 5620         u32                     reg, usec, rate;
 5621                         
 5622         reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
 5623         usec = ((reg & 0x7FFC) >> 2);
 5624         if (usec > 0)
 5625                 rate = 1000000 / usec;
 5626         else
 5627                 rate = 0;
 5628         error = sysctl_handle_int(oidp, &rate, 0, req);
 5629         if (error || !req->newptr)
 5630                 return error;
 5631         return 0;
 5632 }
 5633 
 5634 /*
 5635  * Add sysctl variables, one per statistic, to the system.
 5636  */
 5637 static void
 5638 igb_add_hw_stats(struct adapter *adapter)
 5639 {
 5640         device_t dev = adapter->dev;
 5641 
 5642         struct tx_ring *txr = adapter->tx_rings;
 5643         struct rx_ring *rxr = adapter->rx_rings;
 5644 
 5645         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 5646         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
 5647         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
 5648         struct e1000_hw_stats *stats = adapter->stats;
 5649 
 5650         struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
 5651         struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
 5652 
 5653 #define QUEUE_NAME_LEN 32
 5654         char namebuf[QUEUE_NAME_LEN];
 5655 
 5656         /* Driver Statistics */
 5657         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 
 5658                         CTLFLAG_RD, &adapter->dropped_pkts,
 5659                         "Driver dropped packets");
 5660         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 
 5661                         CTLFLAG_RD, &adapter->link_irq,
 5662                         "Link MSIX IRQ Handled");
 5663         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
 5664                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
 5665                         "Defragmenting mbuf chain failed");
 5666         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", 
 5667                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
 5668                         "Driver tx dma failure in xmit");
 5669         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
 5670                         CTLFLAG_RD, &adapter->rx_overruns,
 5671                         "RX overruns");
 5672         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
 5673                         CTLFLAG_RD, &adapter->watchdog_events,
 5674                         "Watchdog timeouts");
 5675 
 5676         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", 
 5677                         CTLFLAG_RD, &adapter->device_control,
 5678                         "Device Control Register");
 5679         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", 
 5680                         CTLFLAG_RD, &adapter->rx_control,
 5681                         "Receiver Control Register");
 5682         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", 
 5683                         CTLFLAG_RD, &adapter->int_mask,
 5684                         "Interrupt Mask");
 5685         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", 
 5686                         CTLFLAG_RD, &adapter->eint_mask,
 5687                         "Extended Interrupt Mask");
 5688         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", 
 5689                         CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
 5690                         "Transmit Buffer Packet Allocation");
 5691         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", 
 5692                         CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
 5693                         "Receive Buffer Packet Allocation");
 5694         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
 5695                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
 5696                         "Flow Control High Watermark");
 5697         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 
 5698                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
 5699                         "Flow Control Low Watermark");
 5700 
 5701         for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
 5702                 struct lro_ctrl *lro = &rxr->lro;
 5703 
 5704                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
 5705                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 5706                                             CTLFLAG_RD, NULL, "Queue Name");
 5707                 queue_list = SYSCTL_CHILDREN(queue_node);
 5708 
 5709                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 
 5710                                 CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i],
 5711                                 sizeof(&adapter->queues[i]),
 5712                                 igb_sysctl_interrupt_rate_handler,
 5713                                 "IU", "Interrupt Rate");
 5714 
 5715                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 
 5716                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me),
 5717                                 igb_sysctl_reg_handler, "IU",
 5718                                 "Transmit Descriptor Head");
 5719                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 
 5720                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me),
 5721                                 igb_sysctl_reg_handler, "IU",
 5722                                 "Transmit Descriptor Tail");
 5723                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 
 5724                                 CTLFLAG_RD, &txr->no_desc_avail,
 5725                                 "Queue No Descriptor Available");
 5726                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
 5727                                 CTLFLAG_RD, &txr->total_packets,
 5728                                 "Queue Packets Transmitted");
 5729 
 5730                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 
 5731                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
 5732                                 igb_sysctl_reg_handler, "IU",
 5733                                 "Receive Descriptor Head");
 5734                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 
 5735                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
 5736                                 igb_sysctl_reg_handler, "IU",
 5737                                 "Receive Descriptor Tail");
 5738                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
 5739                                 CTLFLAG_RD, &rxr->rx_packets,
 5740                                 "Queue Packets Received");
 5741                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
 5742                                 CTLFLAG_RD, &rxr->rx_bytes,
 5743                                 "Queue Bytes Received");
 5744                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
 5745                                 CTLFLAG_RD, &lro->lro_queued, 0,
 5746                                 "LRO Queued");
 5747                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
 5748                                 CTLFLAG_RD, &lro->lro_flushed, 0,
 5749                                 "LRO Flushed");
 5750         }
 5751 
 5752         /* MAC stats get their own sub node */
 5753 
 5754         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 
 5755                                     CTLFLAG_RD, NULL, "MAC Statistics");
 5756         stat_list = SYSCTL_CHILDREN(stat_node);
 5757 
 5758         /*
 5759         ** VF adapter has a very limited set of stats
 5760         ** since its not managing the metal, so to speak.
 5761         */
 5762         if (adapter->vf_ifp) {
 5763         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
 5764                         CTLFLAG_RD, &stats->gprc,
 5765                         "Good Packets Received");
 5766         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 5767                         CTLFLAG_RD, &stats->gptc,
 5768                         "Good Packets Transmitted");
 5769         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 
 5770                         CTLFLAG_RD, &stats->gorc, 
 5771                         "Good Octets Received"); 
 5772         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 
 5773                         CTLFLAG_RD, &stats->gotc, 
 5774                         "Good Octets Transmitted"); 
 5775         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
 5776                         CTLFLAG_RD, &stats->mprc,
 5777                         "Multicast Packets Received");
 5778                 return;
 5779         }
 5780 
 5781         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", 
 5782                         CTLFLAG_RD, &stats->ecol,
 5783                         "Excessive collisions");
 5784         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", 
 5785                         CTLFLAG_RD, &stats->scc,
 5786                         "Single collisions");
 5787         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 
 5788                         CTLFLAG_RD, &stats->mcc,
 5789                         "Multiple collisions");
 5790         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", 
 5791                         CTLFLAG_RD, &stats->latecol,
 5792                         "Late collisions");
 5793         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", 
 5794                         CTLFLAG_RD, &stats->colc,
 5795                         "Collision Count");
 5796         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
 5797                         CTLFLAG_RD, &stats->symerrs,
 5798                         "Symbol Errors");
 5799         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
 5800                         CTLFLAG_RD, &stats->sec,
 5801                         "Sequence Errors");
 5802         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
 5803                         CTLFLAG_RD, &stats->dc,
 5804                         "Defer Count");
 5805         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
 5806                         CTLFLAG_RD, &stats->mpc,
 5807                         "Missed Packets");
 5808         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors",
 5809                         CTLFLAG_RD, &stats->rlec,
 5810                         "Receive Length Errors");
 5811         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
 5812                         CTLFLAG_RD, &stats->rnbc,
 5813                         "Receive No Buffers");
 5814         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
 5815                         CTLFLAG_RD, &stats->ruc,
 5816                         "Receive Undersize");
 5817         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
 5818                         CTLFLAG_RD, &stats->rfc,
 5819                         "Fragmented Packets Received");
 5820         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
 5821                         CTLFLAG_RD, &stats->roc,
 5822                         "Oversized Packets Received");
 5823         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
 5824                         CTLFLAG_RD, &stats->rjc,
 5825                         "Recevied Jabber");
 5826         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
 5827                         CTLFLAG_RD, &stats->rxerrc,
 5828                         "Receive Errors");
 5829         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
 5830                         CTLFLAG_RD, &stats->crcerrs,
 5831                         "CRC errors");
 5832         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
 5833                         CTLFLAG_RD, &stats->algnerrc,
 5834                         "Alignment Errors");
 5835         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs",
 5836                         CTLFLAG_RD, &stats->tncrs,
 5837                         "Transmit with No CRS");
 5838         /* On 82575 these are collision counts */
 5839         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
 5840                         CTLFLAG_RD, &stats->cexterr,
 5841                         "Collision/Carrier extension errors");
 5842         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
 5843                         CTLFLAG_RD, &stats->xonrxc,
 5844                         "XON Received");
 5845         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
 5846                         CTLFLAG_RD, &stats->xontxc,
 5847                         "XON Transmitted");
 5848         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
 5849                         CTLFLAG_RD, &stats->xoffrxc,
 5850                         "XOFF Received");
 5851         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
 5852                         CTLFLAG_RD, &stats->xofftxc,
 5853                         "XOFF Transmitted");
 5854         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd",
 5855                         CTLFLAG_RD, &stats->fcruc,
 5856                         "Unsupported Flow Control Received");
 5857         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd",
 5858                         CTLFLAG_RD, &stats->mgprc,
 5859                         "Management Packets Received");
 5860         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop",
 5861                         CTLFLAG_RD, &stats->mgpdc,
 5862                         "Management Packets Dropped");
 5863         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd",
 5864                         CTLFLAG_RD, &stats->mgptc,
 5865                         "Management Packets Transmitted");
 5866         /* Packet Reception Stats */
 5867         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
 5868                         CTLFLAG_RD, &stats->tpr,
 5869                         "Total Packets Received");
 5870         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
 5871                         CTLFLAG_RD, &stats->gprc,
 5872                         "Good Packets Received");
 5873         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
 5874                         CTLFLAG_RD, &stats->bprc,
 5875                         "Broadcast Packets Received");
 5876         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
 5877                         CTLFLAG_RD, &stats->mprc,
 5878                         "Multicast Packets Received");
 5879         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
 5880                         CTLFLAG_RD, &stats->prc64,
 5881                         "64 byte frames received");
 5882         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
 5883                         CTLFLAG_RD, &stats->prc127,
 5884                         "65-127 byte frames received");
 5885         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
 5886                         CTLFLAG_RD, &stats->prc255,
 5887                         "128-255 byte frames received");
 5888         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
 5889                         CTLFLAG_RD, &stats->prc511,
 5890                         "256-511 byte frames received");
 5891         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
 5892                         CTLFLAG_RD, &stats->prc1023,
 5893                         "512-1023 byte frames received");
 5894         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
 5895                         CTLFLAG_RD, &stats->prc1522,
 5896                         "1023-1522 byte frames received");
 5897         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 
 5898                         CTLFLAG_RD, &stats->gorc, 
 5899                         "Good Octets Received");
 5900         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd", 
 5901                         CTLFLAG_RD, &stats->tor, 
 5902                         "Total Octets Received");
 5903 
 5904         /* Packet Transmission Stats */
 5905         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 
 5906                         CTLFLAG_RD, &stats->gotc, 
 5907                         "Good Octets Transmitted"); 
 5908         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd", 
 5909                         CTLFLAG_RD, &stats->tot, 
 5910                         "Total Octets Transmitted");
 5911         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
 5912                         CTLFLAG_RD, &stats->tpt,
 5913                         "Total Packets Transmitted");
 5914         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 5915                         CTLFLAG_RD, &stats->gptc,
 5916                         "Good Packets Transmitted");
 5917         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
 5918                         CTLFLAG_RD, &stats->bptc,
 5919                         "Broadcast Packets Transmitted");
 5920         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
 5921                         CTLFLAG_RD, &stats->mptc,
 5922                         "Multicast Packets Transmitted");
 5923         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
 5924                         CTLFLAG_RD, &stats->ptc64,
 5925                         "64 byte frames transmitted");
 5926         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
 5927                         CTLFLAG_RD, &stats->ptc127,
 5928                         "65-127 byte frames transmitted");
 5929         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
 5930                         CTLFLAG_RD, &stats->ptc255,
 5931                         "128-255 byte frames transmitted");
 5932         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
 5933                         CTLFLAG_RD, &stats->ptc511,
 5934                         "256-511 byte frames transmitted");
 5935         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
 5936                         CTLFLAG_RD, &stats->ptc1023,
 5937                         "512-1023 byte frames transmitted");
 5938         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
 5939                         CTLFLAG_RD, &stats->ptc1522,
 5940                         "1024-1522 byte frames transmitted");
 5941         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
 5942                         CTLFLAG_RD, &stats->tsctc,
 5943                         "TSO Contexts Transmitted");
 5944         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
 5945                         CTLFLAG_RD, &stats->tsctfc,
 5946                         "TSO Contexts Failed");
 5947 
 5948 
 5949         /* Interrupt Stats */
 5950 
 5951         int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 
 5952                                     CTLFLAG_RD, NULL, "Interrupt Statistics");
 5953         int_list = SYSCTL_CHILDREN(int_node);
 5954 
 5955         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
 5956                         CTLFLAG_RD, &stats->iac,
 5957                         "Interrupt Assertion Count");
 5958 
 5959         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
 5960                         CTLFLAG_RD, &stats->icrxptc,
 5961                         "Interrupt Cause Rx Pkt Timer Expire Count");
 5962 
 5963         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
 5964                         CTLFLAG_RD, &stats->icrxatc,
 5965                         "Interrupt Cause Rx Abs Timer Expire Count");
 5966 
 5967         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
 5968                         CTLFLAG_RD, &stats->ictxptc,
 5969                         "Interrupt Cause Tx Pkt Timer Expire Count");
 5970 
 5971         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
 5972                         CTLFLAG_RD, &stats->ictxatc,
 5973                         "Interrupt Cause Tx Abs Timer Expire Count");
 5974 
 5975         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
 5976                         CTLFLAG_RD, &stats->ictxqec,
 5977                         "Interrupt Cause Tx Queue Empty Count");
 5978 
 5979         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
 5980                         CTLFLAG_RD, &stats->ictxqmtc,
 5981                         "Interrupt Cause Tx Queue Min Thresh Count");
 5982 
 5983         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
 5984                         CTLFLAG_RD, &stats->icrxdmtc,
 5985                         "Interrupt Cause Rx Desc Min Thresh Count");
 5986 
 5987         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
 5988                         CTLFLAG_RD, &stats->icrxoc,
 5989                         "Interrupt Cause Receiver Overrun Count");
 5990 
 5991         /* Host to Card Stats */
 5992 
 5993         host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", 
 5994                                     CTLFLAG_RD, NULL, 
 5995                                     "Host to Card Statistics");
 5996 
 5997         host_list = SYSCTL_CHILDREN(host_node);
 5998 
 5999         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
 6000                         CTLFLAG_RD, &stats->cbtmpc,
 6001                         "Circuit Breaker Tx Packet Count");
 6002 
 6003         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
 6004                         CTLFLAG_RD, &stats->htdpmc,
 6005                         "Host Transmit Discarded Packets");
 6006 
 6007         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
 6008                         CTLFLAG_RD, &stats->rpthc,
 6009                         "Rx Packets To Host");
 6010 
 6011         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
 6012                         CTLFLAG_RD, &stats->cbrmpc,
 6013                         "Circuit Breaker Rx Packet Count");
 6014 
 6015         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
 6016                         CTLFLAG_RD, &stats->cbrdpc,
 6017                         "Circuit Breaker Rx Dropped Count");
 6018 
 6019         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
 6020                         CTLFLAG_RD, &stats->hgptc,
 6021                         "Host Good Packets Tx Count");
 6022 
 6023         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
 6024                         CTLFLAG_RD, &stats->htcbdpc,
 6025                         "Host Tx Circuit Breaker Dropped Count");
 6026 
 6027         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
 6028                         CTLFLAG_RD, &stats->hgorc,
 6029                         "Host Good Octets Received Count");
 6030 
 6031         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
 6032                         CTLFLAG_RD, &stats->hgotc,
 6033                         "Host Good Octets Transmit Count");
 6034 
 6035         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
 6036                         CTLFLAG_RD, &stats->lenerrs,
 6037                         "Length Errors");
 6038 
 6039         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
 6040                         CTLFLAG_RD, &stats->scvpc,
 6041                         "SerDes/SGMII Code Violation Pkt Count");
 6042 
 6043         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
 6044                         CTLFLAG_RD, &stats->hrmpc,
 6045                         "Header Redirection Missed Packet Count");
 6046 }
 6047 
 6048 
 6049 /**********************************************************************
 6050  *
 6051  *  This routine provides a way to dump out the adapter eeprom,
 6052  *  often a useful debug/service tool. This only dumps the first
 6053  *  32 words, stuff that matters is in that extent.
 6054  *
 6055  **********************************************************************/
 6056 static int
 6057 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
 6058 {
 6059         struct adapter *adapter;
 6060         int error;
 6061         int result;
 6062 
 6063         result = -1;
 6064         error = sysctl_handle_int(oidp, &result, 0, req);
 6065 
 6066         if (error || !req->newptr)
 6067                 return (error);
 6068 
 6069         /*
 6070          * This value will cause a hex dump of the
 6071          * first 32 16-bit words of the EEPROM to
 6072          * the screen.
 6073          */
 6074         if (result == 1) {
 6075                 adapter = (struct adapter *)arg1;
 6076                 igb_print_nvm_info(adapter);
 6077         }
 6078 
 6079         return (error);
 6080 }
 6081 
 6082 static void
 6083 igb_print_nvm_info(struct adapter *adapter)
 6084 {
 6085         u16     eeprom_data;
 6086         int     i, j, row = 0;
 6087 
 6088         /* Its a bit crude, but it gets the job done */
 6089         printf("\nInterface EEPROM Dump:\n");
 6090         printf("Offset\n0x0000  ");
 6091         for (i = 0, j = 0; i < 32; i++, j++) {
 6092                 if (j == 8) { /* Make the offset block */
 6093                         j = 0; ++row;
 6094                         printf("\n0x00%x0  ",row);
 6095                 }
 6096                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
 6097                 printf("%04x ", eeprom_data);
 6098         }
 6099         printf("\n");
 6100 }
 6101 
 6102 static void
 6103 igb_set_sysctl_value(struct adapter *adapter, const char *name,
 6104         const char *description, int *limit, int value)
 6105 {
 6106         *limit = value;
 6107         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
 6108             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
 6109             OID_AUTO, name, CTLFLAG_RW, limit, value, description);
 6110 }
 6111 
 6112 /*
 6113 ** Set flow control using sysctl:
 6114 ** Flow control values:
 6115 **      0 - off
 6116 **      1 - rx pause
 6117 **      2 - tx pause
 6118 **      3 - full
 6119 */
 6120 static int
 6121 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
 6122 {
 6123         int             error;
 6124         static int      input = 3; /* default is full */
 6125         struct adapter  *adapter = (struct adapter *) arg1;
 6126 
 6127         error = sysctl_handle_int(oidp, &input, 0, req);
 6128 
 6129         if ((error) || (req->newptr == NULL))
 6130                 return (error);
 6131 
 6132         switch (input) {
 6133                 case e1000_fc_rx_pause:
 6134                 case e1000_fc_tx_pause:
 6135                 case e1000_fc_full:
 6136                 case e1000_fc_none:
 6137                         adapter->hw.fc.requested_mode = input;
 6138                         adapter->fc = input;
 6139                         break;
 6140                 default:
 6141                         /* Do nothing */
 6142                         return (error);
 6143         }
 6144 
 6145         adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
 6146         e1000_force_mac_fc(&adapter->hw);
 6147         /* XXX TODO: update DROP_EN on each RX queue if appropriate */
 6148         return (error);
 6149 }
 6150 
 6151 /*
 6152 ** Manage DMA Coalesce:
 6153 ** Control values:
 6154 **      0/1 - off/on
 6155 **      Legal timer values are:
 6156 **      250,500,1000-10000 in thousands
 6157 */
 6158 static int
 6159 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
 6160 {
 6161         struct adapter *adapter = (struct adapter *) arg1;
 6162         int             error;
 6163 
 6164         error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
 6165 
 6166         if ((error) || (req->newptr == NULL))
 6167                 return (error);
 6168 
 6169         switch (adapter->dmac) {
 6170                 case 0:
 6171                         /* Disabling */
 6172                         break;
 6173                 case 1: /* Just enable and use default */
 6174                         adapter->dmac = 1000;
 6175                         break;
 6176                 case 250:
 6177                 case 500:
 6178                 case 1000:
 6179                 case 2000:
 6180                 case 3000:
 6181                 case 4000:
 6182                 case 5000:
 6183                 case 6000:
 6184                 case 7000:
 6185                 case 8000:
 6186                 case 9000:
 6187                 case 10000:
 6188                         /* Legal values - allow */
 6189                         break;
 6190                 default:
 6191                         /* Do nothing, illegal value */
 6192                         adapter->dmac = 0;
 6193                         return (EINVAL);
 6194         }
 6195         /* Reinit the interface */
 6196         igb_init(adapter);
 6197         return (error);
 6198 }
 6199 
 6200 /*
 6201 ** Manage Energy Efficient Ethernet:
 6202 ** Control values:
 6203 **     0/1 - enabled/disabled
 6204 */
 6205 static int
 6206 igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
 6207 {
 6208         struct adapter  *adapter = (struct adapter *) arg1;
 6209         int             error, value;
 6210 
 6211         value = adapter->hw.dev_spec._82575.eee_disable;
 6212         error = sysctl_handle_int(oidp, &value, 0, req);
 6213         if (error || req->newptr == NULL)
 6214                 return (error);
 6215         IGB_CORE_LOCK(adapter);
 6216         adapter->hw.dev_spec._82575.eee_disable = (value != 0);
 6217         igb_init_locked(adapter);
 6218         IGB_CORE_UNLOCK(adapter);
 6219         return (0);
 6220 }

Cache object: 9ab94dda0f577c7700bd755278ca7ce0


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.