The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/e1000/if_igb.c

Version: -  FREEBSD  -  FREEBSD11  -  FREEBSD10  -  FREEBSD9  -  FREEBSD92  -  FREEBSD91  -  FREEBSD90  -  FREEBSD8  -  FREEBSD82  -  FREEBSD81  -  FREEBSD80  -  FREEBSD7  -  FREEBSD74  -  FREEBSD73  -  FREEBSD72  -  FREEBSD71  -  FREEBSD70  -  FREEBSD6  -  FREEBSD64  -  FREEBSD63  -  FREEBSD62  -  FREEBSD61  -  FREEBSD60  -  FREEBSD5  -  FREEBSD55  -  FREEBSD54  -  FREEBSD53  -  FREEBSD52  -  FREEBSD51  -  FREEBSD50  -  FREEBSD4  -  FREEBSD3  -  FREEBSD22  -  linux-2.6  -  linux-2.4.22  -  MK83  -  MK84  -  PLAN9  -  DFBSD  -  NETBSD  -  NETBSD5  -  NETBSD4  -  NETBSD3  -  NETBSD20  -  OPENBSD  -  xnu-517  -  xnu-792  -  xnu-792.6.70  -  xnu-1228  -  xnu-1456.1.26  -  xnu-1699.24.8  -  xnu-2050.18.24  -  OPENSOLARIS  -  minix-3-1-1 
SearchContext: -  none  -  3  -  10 

    1 /******************************************************************************
    2 
    3   Copyright (c) 2001-2015, Intel Corporation 
    4   All rights reserved.
    5   
    6   Redistribution and use in source and binary forms, with or without 
    7   modification, are permitted provided that the following conditions are met:
    8   
    9    1. Redistributions of source code must retain the above copyright notice, 
   10       this list of conditions and the following disclaimer.
   11   
   12    2. Redistributions in binary form must reproduce the above copyright 
   13       notice, this list of conditions and the following disclaimer in the 
   14       documentation and/or other materials provided with the distribution.
   15   
   16    3. Neither the name of the Intel Corporation nor the names of its 
   17       contributors may be used to endorse or promote products derived from 
   18       this software without specific prior written permission.
   19   
   20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
   22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
   23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
   24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
   25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
   26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
   27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
   28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
   29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   30   POSSIBILITY OF SUCH DAMAGE.
   31 
   32 ******************************************************************************/
   33 /*$FreeBSD: head/sys/dev/e1000/if_igb.c 302384 2016-07-07 03:39:18Z sbruno $*/
   34 
   35 
   36 #include "opt_inet.h"
   37 #include "opt_inet6.h"
   38 #include "opt_rss.h"
   39 
   40 #ifdef HAVE_KERNEL_OPTION_HEADERS
   41 #include "opt_device_polling.h"
   42 #include "opt_altq.h"
   43 #endif
   44 
   45 #include "if_igb.h"
   46 
   47 /*********************************************************************
   48  *  Driver version:
   49  *********************************************************************/
   50 char igb_driver_version[] = "2.5.3-k";
   51 
   52 
   53 /*********************************************************************
   54  *  PCI Device ID Table
   55  *
   56  *  Used by probe to select devices to load on
   57  *  Last field stores an index into e1000_strings
   58  *  Last entry must be all 0s
   59  *
   60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
   61  *********************************************************************/
   62 
   63 static igb_vendor_info_t igb_vendor_info_array[] =
   64 {
   65         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0},
   66         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0},
   67         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0},
   68         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0},
   69         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0},
   70         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0},
   71         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0},
   72         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0},
   73         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0},
   74         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0},
   75         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0},
   76         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0},
   77         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0},
   78         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0},
   79         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0},
   80         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0},
   81         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0},
   82         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0},
   83         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0},
   84         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0},
   85         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0},
   86         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0},
   87         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0},
   88         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER,  0, 0, 0},
   89         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0},
   90         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII,  0, 0, 0},
   91         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0},
   92         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0},
   93         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0},
   94         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0},
   95         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0},
   96         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0},
   97         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER,  0, 0, 0},
   98         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0},
   99         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII,  0, 0, 0},
  100         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0},
  101         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0},
  102         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0},
  103         {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII,  0, 0, 0},
  104         /* required last entry */
  105         {0, 0, 0, 0, 0}
  106 };
  107 
  108 /*********************************************************************
  109  *  Table of branding strings for all supported NICs.
  110  *********************************************************************/
  111 
  112 static char *igb_strings[] = {
  113         "Intel(R) PRO/1000 Network Connection"
  114 };
  115 
  116 /*********************************************************************
  117  *  Function prototypes
  118  *********************************************************************/
  119 static int      igb_probe(device_t);
  120 static int      igb_attach(device_t);
  121 static int      igb_detach(device_t);
  122 static int      igb_shutdown(device_t);
  123 static int      igb_suspend(device_t);
  124 static int      igb_resume(device_t);
  125 #ifndef IGB_LEGACY_TX
  126 static int      igb_mq_start(struct ifnet *, struct mbuf *);
  127 static int      igb_mq_start_locked(struct ifnet *, struct tx_ring *);
  128 static void     igb_qflush(struct ifnet *);
  129 static void     igb_deferred_mq_start(void *, int);
  130 #else
  131 static void     igb_start(struct ifnet *);
  132 static void     igb_start_locked(struct tx_ring *, struct ifnet *ifp);
  133 #endif
  134 static int      igb_ioctl(struct ifnet *, u_long, caddr_t);
  135 static uint64_t igb_get_counter(if_t, ift_counter);
  136 static void     igb_init(void *);
  137 static void     igb_init_locked(struct adapter *);
  138 static void     igb_stop(void *);
  139 static void     igb_media_status(struct ifnet *, struct ifmediareq *);
  140 static int      igb_media_change(struct ifnet *);
  141 static void     igb_identify_hardware(struct adapter *);
  142 static int      igb_allocate_pci_resources(struct adapter *);
  143 static int      igb_allocate_msix(struct adapter *);
  144 static int      igb_allocate_legacy(struct adapter *);
  145 static int      igb_setup_msix(struct adapter *);
  146 static void     igb_free_pci_resources(struct adapter *);
  147 static void     igb_local_timer(void *);
  148 static void     igb_reset(struct adapter *);
  149 static int      igb_setup_interface(device_t, struct adapter *);
  150 static int      igb_allocate_queues(struct adapter *);
  151 static void     igb_configure_queues(struct adapter *);
  152 
  153 static int      igb_allocate_transmit_buffers(struct tx_ring *);
  154 static void     igb_setup_transmit_structures(struct adapter *);
  155 static void     igb_setup_transmit_ring(struct tx_ring *);
  156 static void     igb_initialize_transmit_units(struct adapter *);
  157 static void     igb_free_transmit_structures(struct adapter *);
  158 static void     igb_free_transmit_buffers(struct tx_ring *);
  159 
  160 static int      igb_allocate_receive_buffers(struct rx_ring *);
  161 static int      igb_setup_receive_structures(struct adapter *);
  162 static int      igb_setup_receive_ring(struct rx_ring *);
  163 static void     igb_initialize_receive_units(struct adapter *);
  164 static void     igb_free_receive_structures(struct adapter *);
  165 static void     igb_free_receive_buffers(struct rx_ring *);
  166 static void     igb_free_receive_ring(struct rx_ring *);
  167 
  168 static void     igb_enable_intr(struct adapter *);
  169 static void     igb_disable_intr(struct adapter *);
  170 static void     igb_update_stats_counters(struct adapter *);
  171 static bool     igb_txeof(struct tx_ring *);
  172 
  173 static __inline void igb_rx_discard(struct rx_ring *, int);
  174 static __inline void igb_rx_input(struct rx_ring *,
  175                     struct ifnet *, struct mbuf *, u32);
  176 
  177 static bool     igb_rxeof(struct igb_queue *, int, int *);
  178 static void     igb_rx_checksum(u32, struct mbuf *, u32);
  179 static int      igb_tx_ctx_setup(struct tx_ring *,
  180                     struct mbuf *, u32 *, u32 *);
  181 static int      igb_tso_setup(struct tx_ring *,
  182                     struct mbuf *, u32 *, u32 *);
  183 static void     igb_set_promisc(struct adapter *);
  184 static void     igb_disable_promisc(struct adapter *);
  185 static void     igb_set_multi(struct adapter *);
  186 static void     igb_update_link_status(struct adapter *);
  187 static void     igb_refresh_mbufs(struct rx_ring *, int);
  188 
  189 static void     igb_register_vlan(void *, struct ifnet *, u16);
  190 static void     igb_unregister_vlan(void *, struct ifnet *, u16);
  191 static void     igb_setup_vlan_hw_support(struct adapter *);
  192 
  193 static int      igb_xmit(struct tx_ring *, struct mbuf **);
  194 static int      igb_dma_malloc(struct adapter *, bus_size_t,
  195                     struct igb_dma_alloc *, int);
  196 static void     igb_dma_free(struct adapter *, struct igb_dma_alloc *);
  197 static int      igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
  198 static void     igb_print_nvm_info(struct adapter *);
  199 static int      igb_is_valid_ether_addr(u8 *);
  200 static void     igb_add_hw_stats(struct adapter *);
  201 
  202 static void     igb_vf_init_stats(struct adapter *);
  203 static void     igb_update_vf_stats_counters(struct adapter *);
  204 
  205 /* Management and WOL Support */
  206 static void     igb_init_manageability(struct adapter *);
  207 static void     igb_release_manageability(struct adapter *);
  208 static void     igb_get_hw_control(struct adapter *);
  209 static void     igb_release_hw_control(struct adapter *);
  210 static void     igb_enable_wakeup(device_t);
  211 static void     igb_led_func(void *, int);
  212 
  213 static int      igb_irq_fast(void *);
  214 static void     igb_msix_que(void *);
  215 static void     igb_msix_link(void *);
  216 static void     igb_handle_que(void *context, int pending);
  217 static void     igb_handle_link(void *context, int pending);
  218 static void     igb_handle_link_locked(struct adapter *);
  219 
  220 static void     igb_set_sysctl_value(struct adapter *, const char *,
  221                     const char *, int *, int);
  222 static int      igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
  223 static int      igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
  224 static int      igb_sysctl_eee(SYSCTL_HANDLER_ARGS);
  225 
  226 #ifdef DEVICE_POLLING
  227 static poll_handler_t igb_poll;
  228 #endif /* POLLING */
  229 
  230 /*********************************************************************
  231  *  FreeBSD Device Interface Entry Points
  232  *********************************************************************/
  233 
  234 static device_method_t igb_methods[] = {
  235         /* Device interface */
  236         DEVMETHOD(device_probe, igb_probe),
  237         DEVMETHOD(device_attach, igb_attach),
  238         DEVMETHOD(device_detach, igb_detach),
  239         DEVMETHOD(device_shutdown, igb_shutdown),
  240         DEVMETHOD(device_suspend, igb_suspend),
  241         DEVMETHOD(device_resume, igb_resume),
  242         DEVMETHOD_END
  243 };
  244 
  245 static driver_t igb_driver = {
  246         "igb", igb_methods, sizeof(struct adapter),
  247 };
  248 
  249 static devclass_t igb_devclass;
  250 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
  251 MODULE_DEPEND(igb, pci, 1, 1, 1);
  252 MODULE_DEPEND(igb, ether, 1, 1, 1);
  253 #ifdef DEV_NETMAP
  254 MODULE_DEPEND(igb, netmap, 1, 1, 1);
  255 #endif /* DEV_NETMAP */
  256 
  257 /*********************************************************************
  258  *  Tunable default values.
  259  *********************************************************************/
  260 
  261 static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
  262 
  263 /* Descriptor defaults */
  264 static int igb_rxd = IGB_DEFAULT_RXD;
  265 static int igb_txd = IGB_DEFAULT_TXD;
  266 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
  267     "Number of receive descriptors per queue");
  268 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
  269     "Number of transmit descriptors per queue");
  270 
  271 /*
  272 ** AIM: Adaptive Interrupt Moderation
  273 ** which means that the interrupt rate
  274 ** is varied over time based on the
  275 ** traffic for that interrupt vector
  276 */
  277 static int igb_enable_aim = TRUE;
  278 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0,
  279     "Enable adaptive interrupt moderation");
  280 
  281 /*
  282  * MSIX should be the default for best performance,
  283  * but this allows it to be forced off for testing.
  284  */         
  285 static int igb_enable_msix = 1;
  286 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
  287     "Enable MSI-X interrupts");
  288 
  289 /*
  290 ** Tuneable Interrupt rate
  291 */
  292 static int igb_max_interrupt_rate = 8000;
  293 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
  294     &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
  295 
  296 #ifndef IGB_LEGACY_TX
  297 /*
  298 ** Tuneable number of buffers in the buf-ring (drbr_xxx)
  299 */
  300 static int igb_buf_ring_size = IGB_BR_SIZE;
  301 SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN,
  302     &igb_buf_ring_size, 0, "Size of the bufring");
  303 #endif
  304 
  305 /*
  306 ** Header split causes the packet header to
  307 ** be dma'd to a separate mbuf from the payload.
  308 ** this can have memory alignment benefits. But
  309 ** another plus is that small packets often fit
  310 ** into the header and thus use no cluster. Its
  311 ** a very workload dependent type feature.
  312 */
  313 static int igb_header_split = FALSE;
  314 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
  315     "Enable receive mbuf header split");
  316 
  317 /*
  318 ** This will autoconfigure based on the
  319 ** number of CPUs and max supported
  320 ** MSIX messages if left at 0.
  321 */
  322 static int igb_num_queues = 0;
  323 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
  324     "Number of queues to configure, 0 indicates autoconfigure");
  325 
  326 /*
  327 ** Global variable to store last used CPU when binding queues
  328 ** to CPUs in igb_allocate_msix.  Starts at CPU_FIRST and increments when a
  329 ** queue is bound to a cpu.
  330 */
  331 static int igb_last_bind_cpu = -1;
  332 
  333 /* How many packets rxeof tries to clean at a time */
  334 static int igb_rx_process_limit = 100;
  335 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
  336     &igb_rx_process_limit, 0,
  337     "Maximum number of received packets to process at a time, -1 means unlimited");
  338 
  339 /* How many packets txeof tries to clean at a time */
  340 static int igb_tx_process_limit = -1;
  341 SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
  342     &igb_tx_process_limit, 0,
  343     "Maximum number of sent packets to process at a time, -1 means unlimited");
  344 
  345 #ifdef DEV_NETMAP       /* see ixgbe.c for details */
  346 #include <dev/netmap/if_igb_netmap.h>
  347 #endif /* DEV_NETMAP */
  348 /*********************************************************************
  349  *  Device identification routine
  350  *
  351  *  igb_probe determines if the driver should be loaded on
  352  *  adapter based on PCI vendor/device id of the adapter.
  353  *
  354  *  return BUS_PROBE_DEFAULT on success, positive on failure
  355  *********************************************************************/
  356 
  357 static int
  358 igb_probe(device_t dev)
  359 {
  360         char            adapter_name[256];
  361         uint16_t        pci_vendor_id = 0;
  362         uint16_t        pci_device_id = 0;
  363         uint16_t        pci_subvendor_id = 0;
  364         uint16_t        pci_subdevice_id = 0;
  365         igb_vendor_info_t *ent;
  366 
  367         INIT_DEBUGOUT("igb_probe: begin");
  368 
  369         pci_vendor_id = pci_get_vendor(dev);
  370         if (pci_vendor_id != IGB_INTEL_VENDOR_ID)
  371                 return (ENXIO);
  372 
  373         pci_device_id = pci_get_device(dev);
  374         pci_subvendor_id = pci_get_subvendor(dev);
  375         pci_subdevice_id = pci_get_subdevice(dev);
  376 
  377         ent = igb_vendor_info_array;
  378         while (ent->vendor_id != 0) {
  379                 if ((pci_vendor_id == ent->vendor_id) &&
  380                     (pci_device_id == ent->device_id) &&
  381 
  382                     ((pci_subvendor_id == ent->subvendor_id) ||
  383                     (ent->subvendor_id == 0)) &&
  384 
  385                     ((pci_subdevice_id == ent->subdevice_id) ||
  386                     (ent->subdevice_id == 0))) {
  387                         sprintf(adapter_name, "%s, Version - %s",
  388                                 igb_strings[ent->index],
  389                                 igb_driver_version);
  390                         device_set_desc_copy(dev, adapter_name);
  391                         return (BUS_PROBE_DEFAULT);
  392                 }
  393                 ent++;
  394         }
  395         return (ENXIO);
  396 }
  397 
  398 /*********************************************************************
  399  *  Device initialization routine
  400  *
  401  *  The attach entry point is called when the driver is being loaded.
  402  *  This routine identifies the type of hardware, allocates all resources
  403  *  and initializes the hardware.
  404  *
  405  *  return 0 on success, positive on failure
  406  *********************************************************************/
  407 
  408 static int
  409 igb_attach(device_t dev)
  410 {
  411         struct adapter  *adapter;
  412         int             error = 0;
  413         u16             eeprom_data;
  414 
  415         INIT_DEBUGOUT("igb_attach: begin");
  416 
  417         if (resource_disabled("igb", device_get_unit(dev))) {
  418                 device_printf(dev, "Disabled by device hint\n");
  419                 return (ENXIO);
  420         }
  421 
  422         adapter = device_get_softc(dev);
  423         adapter->dev = adapter->osdep.dev = dev;
  424         IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
  425 
  426         /* SYSCTLs */
  427         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  428             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  429             OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
  430             igb_sysctl_nvm_info, "I", "NVM Information");
  431 
  432         igb_set_sysctl_value(adapter, "enable_aim",
  433             "Interrupt Moderation", &adapter->enable_aim,
  434             igb_enable_aim);
  435 
  436         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  437             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  438             OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
  439             adapter, 0, igb_set_flowcntl, "I", "Flow Control");
  440 
  441         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
  442 
  443         /* Determine hardware and mac info */
  444         igb_identify_hardware(adapter);
  445 
  446         /* Setup PCI resources */
  447         if (igb_allocate_pci_resources(adapter)) {
  448                 device_printf(dev, "Allocation of PCI resources failed\n");
  449                 error = ENXIO;
  450                 goto err_pci;
  451         }
  452 
  453         /* Do Shared Code initialization */
  454         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
  455                 device_printf(dev, "Setup of Shared code failed\n");
  456                 error = ENXIO;
  457                 goto err_pci;
  458         }
  459 
  460         e1000_get_bus_info(&adapter->hw);
  461 
  462         /* Sysctls for limiting the amount of work done in the taskqueues */
  463         igb_set_sysctl_value(adapter, "rx_processing_limit",
  464             "max number of rx packets to process",
  465             &adapter->rx_process_limit, igb_rx_process_limit);
  466 
  467         igb_set_sysctl_value(adapter, "tx_processing_limit",
  468             "max number of tx packets to process",
  469             &adapter->tx_process_limit, igb_tx_process_limit);
  470 
  471         /*
  472          * Validate number of transmit and receive descriptors. It
  473          * must not exceed hardware maximum, and must be multiple
  474          * of E1000_DBA_ALIGN.
  475          */
  476         if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
  477             (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
  478                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
  479                     IGB_DEFAULT_TXD, igb_txd);
  480                 adapter->num_tx_desc = IGB_DEFAULT_TXD;
  481         } else
  482                 adapter->num_tx_desc = igb_txd;
  483         if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
  484             (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
  485                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
  486                     IGB_DEFAULT_RXD, igb_rxd);
  487                 adapter->num_rx_desc = IGB_DEFAULT_RXD;
  488         } else
  489                 adapter->num_rx_desc = igb_rxd;
  490 
  491         adapter->hw.mac.autoneg = DO_AUTO_NEG;
  492         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
  493         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
  494 
  495         /* Copper options */
  496         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
  497                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
  498                 adapter->hw.phy.disable_polarity_correction = FALSE;
  499                 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
  500         }
  501 
  502         /*
  503          * Set the frame limits assuming
  504          * standard ethernet sized frames.
  505          */
  506         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
  507 
  508         /*
  509         ** Allocate and Setup Queues
  510         */
  511         if (igb_allocate_queues(adapter)) {
  512                 error = ENOMEM;
  513                 goto err_pci;
  514         }
  515 
  516         /* Allocate the appropriate stats memory */
  517         if (adapter->vf_ifp) {
  518                 adapter->stats =
  519                     (struct e1000_vf_stats *)malloc(sizeof \
  520                     (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
  521                 igb_vf_init_stats(adapter);
  522         } else
  523                 adapter->stats =
  524                     (struct e1000_hw_stats *)malloc(sizeof \
  525                     (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
  526         if (adapter->stats == NULL) {
  527                 device_printf(dev, "Can not allocate stats memory\n");
  528                 error = ENOMEM;
  529                 goto err_late;
  530         }
  531 
  532         /* Allocate multicast array memory. */
  533         adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
  534             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
  535         if (adapter->mta == NULL) {
  536                 device_printf(dev, "Can not allocate multicast setup array\n");
  537                 error = ENOMEM;
  538                 goto err_late;
  539         }
  540 
  541         /* Some adapter-specific advanced features */
  542         if (adapter->hw.mac.type >= e1000_i350) {
  543                 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  544                     SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  545                     OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
  546                     adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
  547                 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  548                     SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  549                     OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW,
  550                     adapter, 0, igb_sysctl_eee, "I",
  551                     "Disable Energy Efficient Ethernet");
  552                 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
  553                         if (adapter->hw.mac.type == e1000_i354)
  554                                 e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
  555                         else
  556                                 e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
  557                 }
  558         }
  559 
  560         /*
  561         ** Start from a known state, this is
  562         ** important in reading the nvm and
  563         ** mac from that.
  564         */
  565         e1000_reset_hw(&adapter->hw);
  566 
  567         /* Make sure we have a good EEPROM before we read from it */
  568         if (((adapter->hw.mac.type != e1000_i210) &&
  569             (adapter->hw.mac.type != e1000_i211)) &&
  570             (e1000_validate_nvm_checksum(&adapter->hw) < 0)) {
  571                 /*
  572                 ** Some PCI-E parts fail the first check due to
  573                 ** the link being in sleep state, call it again,
  574                 ** if it fails a second time its a real issue.
  575                 */
  576                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
  577                         device_printf(dev,
  578                             "The EEPROM Checksum Is Not Valid\n");
  579                         error = EIO;
  580                         goto err_late;
  581                 }
  582         }
  583 
  584         /*
  585         ** Copy the permanent MAC address out of the EEPROM
  586         */
  587         if (e1000_read_mac_addr(&adapter->hw) < 0) {
  588                 device_printf(dev, "EEPROM read error while reading MAC"
  589                     " address\n");
  590                 error = EIO;
  591                 goto err_late;
  592         }
  593         /* Check its sanity */
  594         if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
  595                 device_printf(dev, "Invalid MAC address\n");
  596                 error = EIO;
  597                 goto err_late;
  598         }
  599 
  600         /* Setup OS specific network interface */
  601         if (igb_setup_interface(dev, adapter) != 0)
  602                 goto err_late;
  603 
  604         /* Now get a good starting state */
  605         igb_reset(adapter);
  606 
  607         /* Initialize statistics */
  608         igb_update_stats_counters(adapter);
  609 
  610         adapter->hw.mac.get_link_status = 1;
  611         igb_update_link_status(adapter);
  612 
  613         /* Indicate SOL/IDER usage */
  614         if (e1000_check_reset_block(&adapter->hw))
  615                 device_printf(dev,
  616                     "PHY reset is blocked due to SOL/IDER session.\n");
  617 
  618         /* Determine if we have to control management hardware */
  619         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
  620 
  621         /*
  622          * Setup Wake-on-Lan
  623          */
  624         /* APME bit in EEPROM is mapped to WUC.APME */
  625         eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
  626         if (eeprom_data)
  627                 adapter->wol = E1000_WUFC_MAG;
  628 
  629         /* Register for VLAN events */
  630         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
  631              igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  632         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
  633              igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  634 
  635         igb_add_hw_stats(adapter);
  636 
  637         /* Tell the stack that the interface is not active */
  638         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
  639         adapter->ifp->if_drv_flags |=  IFF_DRV_OACTIVE;
  640 
  641         adapter->led_dev = led_create(igb_led_func, adapter,
  642             device_get_nameunit(dev));
  643 
  644         /* 
  645         ** Configure Interrupts
  646         */
  647         if ((adapter->msix > 1) && (igb_enable_msix))
  648                 error = igb_allocate_msix(adapter);
  649         else /* MSI or Legacy */
  650                 error = igb_allocate_legacy(adapter);
  651         if (error)
  652                 goto err_late;
  653 
  654 #ifdef DEV_NETMAP
  655         igb_netmap_attach(adapter);
  656 #endif /* DEV_NETMAP */
  657         INIT_DEBUGOUT("igb_attach: end");
  658 
  659         return (0);
  660 
  661 err_late:
  662         if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */
  663                 return(error);
  664         igb_free_transmit_structures(adapter);
  665         igb_free_receive_structures(adapter);
  666         igb_release_hw_control(adapter);
  667 err_pci:
  668         igb_free_pci_resources(adapter);
  669         if (adapter->ifp != NULL)
  670                 if_free(adapter->ifp);
  671         free(adapter->mta, M_DEVBUF);
  672         IGB_CORE_LOCK_DESTROY(adapter);
  673 
  674         return (error);
  675 }
  676 
  677 /*********************************************************************
  678  *  Device removal routine
  679  *
  680  *  The detach entry point is called when the driver is being removed.
  681  *  This routine stops the adapter and deallocates all the resources
  682  *  that were allocated for driver operation.
  683  *
  684  *  return 0 on success, positive on failure
  685  *********************************************************************/
  686 
  687 static int
  688 igb_detach(device_t dev)
  689 {
  690         struct adapter  *adapter = device_get_softc(dev);
  691         struct ifnet    *ifp = adapter->ifp;
  692 
  693         INIT_DEBUGOUT("igb_detach: begin");
  694 
  695         /* Make sure VLANS are not using driver */
  696         if (adapter->ifp->if_vlantrunk != NULL) {
  697                 device_printf(dev,"Vlan in use, detach first\n");
  698                 return (EBUSY);
  699         }
  700 
  701         ether_ifdetach(adapter->ifp);
  702 
  703         if (adapter->led_dev != NULL)
  704                 led_destroy(adapter->led_dev);
  705 
  706 #ifdef DEVICE_POLLING
  707         if (ifp->if_capenable & IFCAP_POLLING)
  708                 ether_poll_deregister(ifp);
  709 #endif
  710 
  711         IGB_CORE_LOCK(adapter);
  712         adapter->in_detach = 1;
  713         igb_stop(adapter);
  714         IGB_CORE_UNLOCK(adapter);
  715 
  716         e1000_phy_hw_reset(&adapter->hw);
  717 
  718         /* Give control back to firmware */
  719         igb_release_manageability(adapter);
  720         igb_release_hw_control(adapter);
  721 
  722         if (adapter->wol) {
  723                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
  724                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
  725                 igb_enable_wakeup(dev);
  726         }
  727 
  728         /* Unregister VLAN events */
  729         if (adapter->vlan_attach != NULL)
  730                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
  731         if (adapter->vlan_detach != NULL)
  732                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
  733 
  734         callout_drain(&adapter->timer);
  735 
  736 #ifdef DEV_NETMAP
  737         netmap_detach(adapter->ifp);
  738 #endif /* DEV_NETMAP */
  739         igb_free_pci_resources(adapter);
  740         bus_generic_detach(dev);
  741         if_free(ifp);
  742 
  743         igb_free_transmit_structures(adapter);
  744         igb_free_receive_structures(adapter);
  745         if (adapter->mta != NULL)
  746                 free(adapter->mta, M_DEVBUF);
  747 
  748         IGB_CORE_LOCK_DESTROY(adapter);
  749 
  750         return (0);
  751 }
  752 
  753 /*********************************************************************
  754  *
  755  *  Shutdown entry point
  756  *
  757  **********************************************************************/
  758 
  759 static int
  760 igb_shutdown(device_t dev)
  761 {
  762         return igb_suspend(dev);
  763 }
  764 
  765 /*
  766  * Suspend/resume device methods.
  767  */
  768 static int
  769 igb_suspend(device_t dev)
  770 {
  771         struct adapter *adapter = device_get_softc(dev);
  772 
  773         IGB_CORE_LOCK(adapter);
  774 
  775         igb_stop(adapter);
  776 
  777         igb_release_manageability(adapter);
  778         igb_release_hw_control(adapter);
  779 
  780         if (adapter->wol) {
  781                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
  782                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
  783                 igb_enable_wakeup(dev);
  784         }
  785 
  786         IGB_CORE_UNLOCK(adapter);
  787 
  788         return bus_generic_suspend(dev);
  789 }
  790 
  791 static int
  792 igb_resume(device_t dev)
  793 {
  794         struct adapter *adapter = device_get_softc(dev);
  795         struct tx_ring  *txr = adapter->tx_rings;
  796         struct ifnet *ifp = adapter->ifp;
  797 
  798         IGB_CORE_LOCK(adapter);
  799         igb_init_locked(adapter);
  800         igb_init_manageability(adapter);
  801 
  802         if ((ifp->if_flags & IFF_UP) &&
  803             (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
  804                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
  805                         IGB_TX_LOCK(txr);
  806 #ifndef IGB_LEGACY_TX
  807                         /* Process the stack queue only if not depleted */
  808                         if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
  809                             !drbr_empty(ifp, txr->br))
  810                                 igb_mq_start_locked(ifp, txr);
  811 #else
  812                         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
  813                                 igb_start_locked(txr, ifp);
  814 #endif
  815                         IGB_TX_UNLOCK(txr);
  816                 }
  817         }
  818         IGB_CORE_UNLOCK(adapter);
  819 
  820         return bus_generic_resume(dev);
  821 }
  822 
  823 
  824 #ifdef IGB_LEGACY_TX
  825 
  826 /*********************************************************************
  827  *  Transmit entry point
  828  *
  829  *  igb_start is called by the stack to initiate a transmit.
  830  *  The driver will remain in this routine as long as there are
  831  *  packets to transmit and transmit resources are available.
  832  *  In case resources are not available stack is notified and
  833  *  the packet is requeued.
  834  **********************************************************************/
  835 
  836 static void
  837 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
  838 {
  839         struct adapter  *adapter = ifp->if_softc;
  840         struct mbuf     *m_head;
  841 
  842         IGB_TX_LOCK_ASSERT(txr);
  843 
  844         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
  845             IFF_DRV_RUNNING)
  846                 return;
  847         if (!adapter->link_active)
  848                 return;
  849 
  850         /* Call cleanup if number of TX descriptors low */
  851         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
  852                 igb_txeof(txr);
  853 
  854         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
  855                 if (txr->tx_avail <= IGB_MAX_SCATTER) {
  856                         txr->queue_status |= IGB_QUEUE_DEPLETED;
  857                         break;
  858                 }
  859                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
  860                 if (m_head == NULL)
  861                         break;
  862                 /*
  863                  *  Encapsulation can modify our pointer, and or make it
  864                  *  NULL on failure.  In that event, we can't requeue.
  865                  */
  866                 if (igb_xmit(txr, &m_head)) {
  867                         if (m_head != NULL)
  868                                 IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
  869                         if (txr->tx_avail <= IGB_MAX_SCATTER)
  870                                 txr->queue_status |= IGB_QUEUE_DEPLETED;
  871                         break;
  872                 }
  873 
  874                 /* Send a copy of the frame to the BPF listener */
  875                 ETHER_BPF_MTAP(ifp, m_head);
  876 
  877                 /* Set watchdog on */
  878                 txr->watchdog_time = ticks;
  879                 txr->queue_status |= IGB_QUEUE_WORKING;
  880         }
  881 }
  882  
  883 /*
  884  * Legacy TX driver routine, called from the
  885  * stack, always uses tx[0], and spins for it.
  886  * Should not be used with multiqueue tx
  887  */
  888 static void
  889 igb_start(struct ifnet *ifp)
  890 {
  891         struct adapter  *adapter = ifp->if_softc;
  892         struct tx_ring  *txr = adapter->tx_rings;
  893 
  894         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  895                 IGB_TX_LOCK(txr);
  896                 igb_start_locked(txr, ifp);
  897                 IGB_TX_UNLOCK(txr);
  898         }
  899         return;
  900 }
  901 
  902 #else /* ~IGB_LEGACY_TX */
  903 
  904 /*
  905 ** Multiqueue Transmit Entry:
  906 **  quick turnaround to the stack
  907 **
  908 */
  909 static int
  910 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
  911 {
  912         struct adapter          *adapter = ifp->if_softc;
  913         struct igb_queue        *que;
  914         struct tx_ring          *txr;
  915         int                     i, err = 0;
  916 #ifdef  RSS
  917         uint32_t                bucket_id;
  918 #endif
  919 
  920         /* Which queue to use */
  921         /*
  922          * When doing RSS, map it to the same outbound queue
  923          * as the incoming flow would be mapped to.
  924          *
  925          * If everything is setup correctly, it should be the
  926          * same bucket that the current CPU we're on is.
  927          */
  928         if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
  929 #ifdef  RSS
  930                 if (rss_hash2bucket(m->m_pkthdr.flowid,
  931                     M_HASHTYPE_GET(m), &bucket_id) == 0) {
  932                         /* XXX TODO: spit out something if bucket_id > num_queues? */
  933                         i = bucket_id % adapter->num_queues;
  934                 } else {
  935 #endif
  936                         i = m->m_pkthdr.flowid % adapter->num_queues;
  937 #ifdef  RSS
  938                 }
  939 #endif
  940         } else {
  941                 i = curcpu % adapter->num_queues;
  942         }
  943         txr = &adapter->tx_rings[i];
  944         que = &adapter->queues[i];
  945 
  946         err = drbr_enqueue(ifp, txr->br, m);
  947         if (err)
  948                 return (err);
  949         if (IGB_TX_TRYLOCK(txr)) {
  950                 igb_mq_start_locked(ifp, txr);
  951                 IGB_TX_UNLOCK(txr);
  952         } else
  953                 taskqueue_enqueue(que->tq, &txr->txq_task);
  954 
  955         return (0);
  956 }
  957 
  958 static int
  959 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr)
  960 {
  961         struct adapter  *adapter = txr->adapter;
  962         struct mbuf     *next;
  963         int             err = 0, enq = 0;
  964 
  965         IGB_TX_LOCK_ASSERT(txr);
  966 
  967         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
  968             adapter->link_active == 0)
  969                 return (ENETDOWN);
  970 
  971         /* Process the queue */
  972         while ((next = drbr_peek(ifp, txr->br)) != NULL) {
  973                 if ((err = igb_xmit(txr, &next)) != 0) {
  974                         if (next == NULL) {
  975                                 /* It was freed, move forward */
  976                                 drbr_advance(ifp, txr->br);
  977                         } else {
  978                                 /* 
  979                                  * Still have one left, it may not be
  980                                  * the same since the transmit function
  981                                  * may have changed it.
  982                                  */
  983                                 drbr_putback(ifp, txr->br, next);
  984                         }
  985                         break;
  986                 }
  987                 drbr_advance(ifp, txr->br);
  988                 enq++;
  989                 if (next->m_flags & M_MCAST && adapter->vf_ifp)
  990                         if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
  991                 ETHER_BPF_MTAP(ifp, next);
  992                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
  993                         break;
  994         }
  995         if (enq > 0) {
  996                 /* Set the watchdog */
  997                 txr->queue_status |= IGB_QUEUE_WORKING;
  998                 txr->watchdog_time = ticks;
  999         }
 1000         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
 1001                 igb_txeof(txr);
 1002         if (txr->tx_avail <= IGB_MAX_SCATTER)
 1003                 txr->queue_status |= IGB_QUEUE_DEPLETED;
 1004         return (err);
 1005 }
 1006 
 1007 /*
 1008  * Called from a taskqueue to drain queued transmit packets.
 1009  */
 1010 static void
 1011 igb_deferred_mq_start(void *arg, int pending)
 1012 {
 1013         struct tx_ring *txr = arg;
 1014         struct adapter *adapter = txr->adapter;
 1015         struct ifnet *ifp = adapter->ifp;
 1016 
 1017         IGB_TX_LOCK(txr);
 1018         if (!drbr_empty(ifp, txr->br))
 1019                 igb_mq_start_locked(ifp, txr);
 1020         IGB_TX_UNLOCK(txr);
 1021 }
 1022 
 1023 /*
 1024 ** Flush all ring buffers
 1025 */
 1026 static void
 1027 igb_qflush(struct ifnet *ifp)
 1028 {
 1029         struct adapter  *adapter = ifp->if_softc;
 1030         struct tx_ring  *txr = adapter->tx_rings;
 1031         struct mbuf     *m;
 1032 
 1033         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1034                 IGB_TX_LOCK(txr);
 1035                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 1036                         m_freem(m);
 1037                 IGB_TX_UNLOCK(txr);
 1038         }
 1039         if_qflush(ifp);
 1040 }
 1041 #endif /* ~IGB_LEGACY_TX */
 1042 
 1043 /*********************************************************************
 1044  *  Ioctl entry point
 1045  *
 1046  *  igb_ioctl is called when the user wants to configure the
 1047  *  interface.
 1048  *
 1049  *  return 0 on success, positive on failure
 1050  **********************************************************************/
 1051 
 1052 static int
 1053 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 1054 {
 1055         struct adapter  *adapter = ifp->if_softc;
 1056         struct ifreq    *ifr = (struct ifreq *)data;
 1057 #if defined(INET) || defined(INET6)
 1058         struct ifaddr   *ifa = (struct ifaddr *)data;
 1059 #endif
 1060         bool            avoid_reset = FALSE;
 1061         int             error = 0;
 1062 
 1063         if (adapter->in_detach)
 1064                 return (error);
 1065 
 1066         switch (command) {
 1067         case SIOCSIFADDR:
 1068 #ifdef INET
 1069                 if (ifa->ifa_addr->sa_family == AF_INET)
 1070                         avoid_reset = TRUE;
 1071 #endif
 1072 #ifdef INET6
 1073                 if (ifa->ifa_addr->sa_family == AF_INET6)
 1074                         avoid_reset = TRUE;
 1075 #endif
 1076                 /*
 1077                 ** Calling init results in link renegotiation,
 1078                 ** so we avoid doing it when possible.
 1079                 */
 1080                 if (avoid_reset) {
 1081                         ifp->if_flags |= IFF_UP;
 1082                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 1083                                 igb_init(adapter);
 1084 #ifdef INET
 1085                         if (!(ifp->if_flags & IFF_NOARP))
 1086                                 arp_ifinit(ifp, ifa);
 1087 #endif
 1088                 } else
 1089                         error = ether_ioctl(ifp, command, data);
 1090                 break;
 1091         case SIOCSIFMTU:
 1092             {
 1093                 int max_frame_size;
 1094 
 1095                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 1096 
 1097                 IGB_CORE_LOCK(adapter);
 1098                 max_frame_size = 9234;
 1099                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
 1100                     ETHER_CRC_LEN) {
 1101                         IGB_CORE_UNLOCK(adapter);
 1102                         error = EINVAL;
 1103                         break;
 1104                 }
 1105 
 1106                 ifp->if_mtu = ifr->ifr_mtu;
 1107                 adapter->max_frame_size =
 1108                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 1109                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING))
 1110                         igb_init_locked(adapter);
 1111                 IGB_CORE_UNLOCK(adapter);
 1112                 break;
 1113             }
 1114         case SIOCSIFFLAGS:
 1115                 IOCTL_DEBUGOUT("ioctl rcv'd:\
 1116                     SIOCSIFFLAGS (Set Interface Flags)");
 1117                 IGB_CORE_LOCK(adapter);
 1118                 if (ifp->if_flags & IFF_UP) {
 1119                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 1120                                 if ((ifp->if_flags ^ adapter->if_flags) &
 1121                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 1122                                         igb_disable_promisc(adapter);
 1123                                         igb_set_promisc(adapter);
 1124                                 }
 1125                         } else
 1126                                 igb_init_locked(adapter);
 1127                 } else
 1128                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1129                                 igb_stop(adapter);
 1130                 adapter->if_flags = ifp->if_flags;
 1131                 IGB_CORE_UNLOCK(adapter);
 1132                 break;
 1133         case SIOCADDMULTI:
 1134         case SIOCDELMULTI:
 1135                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 1136                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1137                         IGB_CORE_LOCK(adapter);
 1138                         igb_disable_intr(adapter);
 1139                         igb_set_multi(adapter);
 1140 #ifdef DEVICE_POLLING
 1141                         if (!(ifp->if_capenable & IFCAP_POLLING))
 1142 #endif
 1143                                 igb_enable_intr(adapter);
 1144                         IGB_CORE_UNLOCK(adapter);
 1145                 }
 1146                 break;
 1147         case SIOCSIFMEDIA:
 1148                 /* Check SOL/IDER usage */
 1149                 IGB_CORE_LOCK(adapter);
 1150                 if (e1000_check_reset_block(&adapter->hw)) {
 1151                         IGB_CORE_UNLOCK(adapter);
 1152                         device_printf(adapter->dev, "Media change is"
 1153                             " blocked due to SOL/IDER session.\n");
 1154                         break;
 1155                 }
 1156                 IGB_CORE_UNLOCK(adapter);
 1157         case SIOCGIFMEDIA:
 1158                 IOCTL_DEBUGOUT("ioctl rcv'd: \
 1159                     SIOCxIFMEDIA (Get/Set Interface Media)");
 1160                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 1161                 break;
 1162         case SIOCSIFCAP:
 1163             {
 1164                 int mask, reinit;
 1165 
 1166                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 1167                 reinit = 0;
 1168                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 1169 #ifdef DEVICE_POLLING
 1170                 if (mask & IFCAP_POLLING) {
 1171                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
 1172                                 error = ether_poll_register(igb_poll, ifp);
 1173                                 if (error)
 1174                                         return (error);
 1175                                 IGB_CORE_LOCK(adapter);
 1176                                 igb_disable_intr(adapter);
 1177                                 ifp->if_capenable |= IFCAP_POLLING;
 1178                                 IGB_CORE_UNLOCK(adapter);
 1179                         } else {
 1180                                 error = ether_poll_deregister(ifp);
 1181                                 /* Enable interrupt even in error case */
 1182                                 IGB_CORE_LOCK(adapter);
 1183                                 igb_enable_intr(adapter);
 1184                                 ifp->if_capenable &= ~IFCAP_POLLING;
 1185                                 IGB_CORE_UNLOCK(adapter);
 1186                         }
 1187                 }
 1188 #endif
 1189 #if __FreeBSD_version >= 1000000
 1190                 /* HW cannot turn these on/off separately */
 1191                 if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
 1192                         ifp->if_capenable ^= IFCAP_RXCSUM;
 1193                         ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
 1194                         reinit = 1;
 1195                 }
 1196                 if (mask & IFCAP_TXCSUM) {
 1197                         ifp->if_capenable ^= IFCAP_TXCSUM;
 1198                         reinit = 1;
 1199                 }
 1200                 if (mask & IFCAP_TXCSUM_IPV6) {
 1201                         ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
 1202                         reinit = 1;
 1203                 }
 1204 #else
 1205                 if (mask & IFCAP_HWCSUM) {
 1206                         ifp->if_capenable ^= IFCAP_HWCSUM;
 1207                         reinit = 1;
 1208                 }
 1209 #endif
 1210                 if (mask & IFCAP_TSO4) {
 1211                         ifp->if_capenable ^= IFCAP_TSO4;
 1212                         reinit = 1;
 1213                 }
 1214                 if (mask & IFCAP_TSO6) {
 1215                         ifp->if_capenable ^= IFCAP_TSO6;
 1216                         reinit = 1;
 1217                 }
 1218                 if (mask & IFCAP_VLAN_HWTAGGING) {
 1219                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 1220                         reinit = 1;
 1221                 }
 1222                 if (mask & IFCAP_VLAN_HWFILTER) {
 1223                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 1224                         reinit = 1;
 1225                 }
 1226                 if (mask & IFCAP_VLAN_HWTSO) {
 1227                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 1228                         reinit = 1;
 1229                 }
 1230                 if (mask & IFCAP_LRO) {
 1231                         ifp->if_capenable ^= IFCAP_LRO;
 1232                         reinit = 1;
 1233                 }
 1234                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
 1235                         igb_init(adapter);
 1236                 VLAN_CAPABILITIES(ifp);
 1237                 break;
 1238             }
 1239 
 1240         default:
 1241                 error = ether_ioctl(ifp, command, data);
 1242                 break;
 1243         }
 1244 
 1245         return (error);
 1246 }
 1247 
 1248 
 1249 /*********************************************************************
 1250  *  Init entry point
 1251  *
 1252  *  This routine is used in two ways. It is used by the stack as
 1253  *  init entry point in network interface structure. It is also used
 1254  *  by the driver as a hw/sw initialization routine to get to a
 1255  *  consistent state.
 1256  *
 1257  *  return 0 on success, positive on failure
 1258  **********************************************************************/
 1259 
 1260 static void
 1261 igb_init_locked(struct adapter *adapter)
 1262 {
 1263         struct ifnet    *ifp = adapter->ifp;
 1264         device_t        dev = adapter->dev;
 1265 
 1266         INIT_DEBUGOUT("igb_init: begin");
 1267 
 1268         IGB_CORE_LOCK_ASSERT(adapter);
 1269 
 1270         igb_disable_intr(adapter);
 1271         callout_stop(&adapter->timer);
 1272 
 1273         /* Get the latest mac address, User can use a LAA */
 1274         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
 1275               ETHER_ADDR_LEN);
 1276 
 1277         /* Put the address into the Receive Address Array */
 1278         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 1279 
 1280         igb_reset(adapter);
 1281         igb_update_link_status(adapter);
 1282 
 1283         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 1284 
 1285         /* Set hardware offload abilities */
 1286         ifp->if_hwassist = 0;
 1287         if (ifp->if_capenable & IFCAP_TXCSUM) {
 1288 #if __FreeBSD_version >= 1000000
 1289                 ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP);
 1290                 if (adapter->hw.mac.type != e1000_82575)
 1291                         ifp->if_hwassist |= CSUM_IP_SCTP;
 1292 #else
 1293                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 1294 #if __FreeBSD_version >= 800000
 1295                 if (adapter->hw.mac.type != e1000_82575)
 1296                         ifp->if_hwassist |= CSUM_SCTP;
 1297 #endif
 1298 #endif
 1299         }
 1300 
 1301 #if __FreeBSD_version >= 1000000
 1302         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) {
 1303                 ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP);
 1304                 if (adapter->hw.mac.type != e1000_82575)
 1305                         ifp->if_hwassist |= CSUM_IP6_SCTP;
 1306         }
 1307 #endif
 1308         if (ifp->if_capenable & IFCAP_TSO)
 1309                 ifp->if_hwassist |= CSUM_TSO;
 1310 
 1311         /* Clear bad data from Rx FIFOs */
 1312         e1000_rx_fifo_flush_82575(&adapter->hw);
 1313 
 1314         /* Configure for OS presence */
 1315         igb_init_manageability(adapter);
 1316 
 1317         /* Prepare transmit descriptors and buffers */
 1318         igb_setup_transmit_structures(adapter);
 1319         igb_initialize_transmit_units(adapter);
 1320 
 1321         /* Setup Multicast table */
 1322         igb_set_multi(adapter);
 1323 
 1324         /*
 1325         ** Figure out the desired mbuf pool
 1326         ** for doing jumbo/packetsplit
 1327         */
 1328         if (adapter->max_frame_size <= 2048)
 1329                 adapter->rx_mbuf_sz = MCLBYTES;
 1330         else if (adapter->max_frame_size <= 4096)
 1331                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 1332         else
 1333                 adapter->rx_mbuf_sz = MJUM9BYTES;
 1334 
 1335         /* Prepare receive descriptors and buffers */
 1336         if (igb_setup_receive_structures(adapter)) {
 1337                 device_printf(dev, "Could not setup receive structures\n");
 1338                 return;
 1339         }
 1340         igb_initialize_receive_units(adapter);
 1341 
 1342         /* Enable VLAN support */
 1343         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
 1344                 igb_setup_vlan_hw_support(adapter);
 1345                                 
 1346         /* Don't lose promiscuous settings */
 1347         igb_set_promisc(adapter);
 1348 
 1349         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 1350         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 1351 
 1352         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 1353         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
 1354 
 1355         if (adapter->msix > 1) /* Set up queue routing */
 1356                 igb_configure_queues(adapter);
 1357 
 1358         /* this clears any pending interrupts */
 1359         E1000_READ_REG(&adapter->hw, E1000_ICR);
 1360 #ifdef DEVICE_POLLING
 1361         /*
 1362          * Only enable interrupts if we are not polling, make sure
 1363          * they are off otherwise.
 1364          */
 1365         if (ifp->if_capenable & IFCAP_POLLING)
 1366                 igb_disable_intr(adapter);
 1367         else
 1368 #endif /* DEVICE_POLLING */
 1369         {
 1370                 igb_enable_intr(adapter);
 1371                 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
 1372         }
 1373 
 1374         /* Set Energy Efficient Ethernet */
 1375         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 1376                 if (adapter->hw.mac.type == e1000_i354)
 1377                         e1000_set_eee_i354(&adapter->hw, TRUE, TRUE);
 1378                 else
 1379                         e1000_set_eee_i350(&adapter->hw, TRUE, TRUE);
 1380         }
 1381 }
 1382 
 1383 static void
 1384 igb_init(void *arg)
 1385 {
 1386         struct adapter *adapter = arg;
 1387 
 1388         IGB_CORE_LOCK(adapter);
 1389         igb_init_locked(adapter);
 1390         IGB_CORE_UNLOCK(adapter);
 1391 }
 1392 
 1393 
 1394 static void
 1395 igb_handle_que(void *context, int pending)
 1396 {
 1397         struct igb_queue *que = context;
 1398         struct adapter *adapter = que->adapter;
 1399         struct tx_ring *txr = que->txr;
 1400         struct ifnet    *ifp = adapter->ifp;
 1401 
 1402         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1403                 bool    more;
 1404 
 1405                 more = igb_rxeof(que, adapter->rx_process_limit, NULL);
 1406 
 1407                 IGB_TX_LOCK(txr);
 1408                 igb_txeof(txr);
 1409 #ifndef IGB_LEGACY_TX
 1410                 /* Process the stack queue only if not depleted */
 1411                 if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
 1412                     !drbr_empty(ifp, txr->br))
 1413                         igb_mq_start_locked(ifp, txr);
 1414 #else
 1415                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1416                         igb_start_locked(txr, ifp);
 1417 #endif
 1418                 IGB_TX_UNLOCK(txr);
 1419                 /* Do we need another? */
 1420                 if (more) {
 1421                         taskqueue_enqueue(que->tq, &que->que_task);
 1422                         return;
 1423                 }
 1424         }
 1425 
 1426 #ifdef DEVICE_POLLING
 1427         if (ifp->if_capenable & IFCAP_POLLING)
 1428                 return;
 1429 #endif
 1430         /* Reenable this interrupt */
 1431         if (que->eims)
 1432                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
 1433         else
 1434                 igb_enable_intr(adapter);
 1435 }
 1436 
 1437 /* Deal with link in a sleepable context */
 1438 static void
 1439 igb_handle_link(void *context, int pending)
 1440 {
 1441         struct adapter *adapter = context;
 1442 
 1443         IGB_CORE_LOCK(adapter);
 1444         igb_handle_link_locked(adapter);
 1445         IGB_CORE_UNLOCK(adapter);
 1446 }
 1447 
 1448 static void
 1449 igb_handle_link_locked(struct adapter *adapter)
 1450 {
 1451         struct tx_ring  *txr = adapter->tx_rings;
 1452         struct ifnet *ifp = adapter->ifp;
 1453 
 1454         IGB_CORE_LOCK_ASSERT(adapter);
 1455         adapter->hw.mac.get_link_status = 1;
 1456         igb_update_link_status(adapter);
 1457         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) {
 1458                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1459                         IGB_TX_LOCK(txr);
 1460 #ifndef IGB_LEGACY_TX
 1461                         /* Process the stack queue only if not depleted */
 1462                         if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
 1463                             !drbr_empty(ifp, txr->br))
 1464                                 igb_mq_start_locked(ifp, txr);
 1465 #else
 1466                         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1467                                 igb_start_locked(txr, ifp);
 1468 #endif
 1469                         IGB_TX_UNLOCK(txr);
 1470                 }
 1471         }
 1472 }
 1473 
 1474 /*********************************************************************
 1475  *
 1476  *  MSI/Legacy Deferred
 1477  *  Interrupt Service routine  
 1478  *
 1479  *********************************************************************/
 1480 static int
 1481 igb_irq_fast(void *arg)
 1482 {
 1483         struct adapter          *adapter = arg;
 1484         struct igb_queue        *que = adapter->queues;
 1485         u32                     reg_icr;
 1486 
 1487 
 1488         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1489 
 1490         /* Hot eject?  */
 1491         if (reg_icr == 0xffffffff)
 1492                 return FILTER_STRAY;
 1493 
 1494         /* Definitely not our interrupt.  */
 1495         if (reg_icr == 0x0)
 1496                 return FILTER_STRAY;
 1497 
 1498         if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
 1499                 return FILTER_STRAY;
 1500 
 1501         /*
 1502          * Mask interrupts until the taskqueue is finished running.  This is
 1503          * cheap, just assume that it is needed.  This also works around the
 1504          * MSI message reordering errata on certain systems.
 1505          */
 1506         igb_disable_intr(adapter);
 1507         taskqueue_enqueue(que->tq, &que->que_task);
 1508 
 1509         /* Link status change */
 1510         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
 1511                 taskqueue_enqueue(que->tq, &adapter->link_task);
 1512 
 1513         if (reg_icr & E1000_ICR_RXO)
 1514                 adapter->rx_overruns++;
 1515         return FILTER_HANDLED;
 1516 }
 1517 
 1518 #ifdef DEVICE_POLLING
 1519 #if __FreeBSD_version >= 800000
 1520 #define POLL_RETURN_COUNT(a) (a)
 1521 static int
 1522 #else
 1523 #define POLL_RETURN_COUNT(a)
 1524 static void
 1525 #endif
 1526 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 1527 {
 1528         struct adapter          *adapter = ifp->if_softc;
 1529         struct igb_queue        *que;
 1530         struct tx_ring          *txr;
 1531         u32                     reg_icr, rx_done = 0;
 1532         u32                     loop = IGB_MAX_LOOP;
 1533         bool                    more;
 1534 
 1535         IGB_CORE_LOCK(adapter);
 1536         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 1537                 IGB_CORE_UNLOCK(adapter);
 1538                 return POLL_RETURN_COUNT(rx_done);
 1539         }
 1540 
 1541         if (cmd == POLL_AND_CHECK_STATUS) {
 1542                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1543                 /* Link status change */
 1544                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
 1545                         igb_handle_link_locked(adapter);
 1546 
 1547                 if (reg_icr & E1000_ICR_RXO)
 1548                         adapter->rx_overruns++;
 1549         }
 1550         IGB_CORE_UNLOCK(adapter);
 1551 
 1552         for (int i = 0; i < adapter->num_queues; i++) {
 1553                 que = &adapter->queues[i];
 1554                 txr = que->txr;
 1555 
 1556                 igb_rxeof(que, count, &rx_done);
 1557 
 1558                 IGB_TX_LOCK(txr);
 1559                 do {
 1560                         more = igb_txeof(txr);
 1561                 } while (loop-- && more);
 1562 #ifndef IGB_LEGACY_TX
 1563                 if (!drbr_empty(ifp, txr->br))
 1564                         igb_mq_start_locked(ifp, txr);
 1565 #else
 1566                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1567                         igb_start_locked(txr, ifp);
 1568 #endif
 1569                 IGB_TX_UNLOCK(txr);
 1570         }
 1571 
 1572         return POLL_RETURN_COUNT(rx_done);
 1573 }
 1574 #endif /* DEVICE_POLLING */
 1575 
 1576 /*********************************************************************
 1577  *
 1578  *  MSIX Que Interrupt Service routine
 1579  *
 1580  **********************************************************************/
 1581 static void
 1582 igb_msix_que(void *arg)
 1583 {
 1584         struct igb_queue *que = arg;
 1585         struct adapter *adapter = que->adapter;
 1586         struct ifnet   *ifp = adapter->ifp;
 1587         struct tx_ring *txr = que->txr;
 1588         struct rx_ring *rxr = que->rxr;
 1589         u32             newitr = 0;
 1590         bool            more_rx;
 1591 
 1592         /* Ignore spurious interrupts */
 1593         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 1594                 return;
 1595 
 1596         E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
 1597         ++que->irqs;
 1598 
 1599         IGB_TX_LOCK(txr);
 1600         igb_txeof(txr);
 1601 #ifndef IGB_LEGACY_TX
 1602         /* Process the stack queue only if not depleted */
 1603         if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) &&
 1604             !drbr_empty(ifp, txr->br))
 1605                 igb_mq_start_locked(ifp, txr);
 1606 #else
 1607         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1608                 igb_start_locked(txr, ifp);
 1609 #endif
 1610         IGB_TX_UNLOCK(txr);
 1611 
 1612         more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
 1613 
 1614         if (adapter->enable_aim == FALSE)
 1615                 goto no_calc;
 1616         /*
 1617         ** Do Adaptive Interrupt Moderation:
 1618         **  - Write out last calculated setting
 1619         **  - Calculate based on average size over
 1620         **    the last interval.
 1621         */
 1622         if (que->eitr_setting)
 1623                 E1000_WRITE_REG(&adapter->hw,
 1624                     E1000_EITR(que->msix), que->eitr_setting);
 1625  
 1626         que->eitr_setting = 0;
 1627 
 1628         /* Idle, do nothing */
 1629         if ((txr->bytes == 0) && (rxr->bytes == 0))
 1630                 goto no_calc;
 1631                                 
 1632         /* Used half Default if sub-gig */
 1633         if (adapter->link_speed != 1000)
 1634                 newitr = IGB_DEFAULT_ITR / 2;
 1635         else {
 1636                 if ((txr->bytes) && (txr->packets))
 1637                         newitr = txr->bytes/txr->packets;
 1638                 if ((rxr->bytes) && (rxr->packets))
 1639                         newitr = max(newitr,
 1640                             (rxr->bytes / rxr->packets));
 1641                 newitr += 24; /* account for hardware frame, crc */
 1642                 /* set an upper boundary */
 1643                 newitr = min(newitr, 3000);
 1644                 /* Be nice to the mid range */
 1645                 if ((newitr > 300) && (newitr < 1200))
 1646                         newitr = (newitr / 3);
 1647                 else
 1648                         newitr = (newitr / 2);
 1649         }
 1650         newitr &= 0x7FFC;  /* Mask invalid bits */
 1651         if (adapter->hw.mac.type == e1000_82575)
 1652                 newitr |= newitr << 16;
 1653         else
 1654                 newitr |= E1000_EITR_CNT_IGNR;
 1655                  
 1656         /* save for next interrupt */
 1657         que->eitr_setting = newitr;
 1658 
 1659         /* Reset state */
 1660         txr->bytes = 0;
 1661         txr->packets = 0;
 1662         rxr->bytes = 0;
 1663         rxr->packets = 0;
 1664 
 1665 no_calc:
 1666         /* Schedule a clean task if needed*/
 1667         if (more_rx)
 1668                 taskqueue_enqueue(que->tq, &que->que_task);
 1669         else
 1670                 /* Reenable this interrupt */
 1671                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
 1672         return;
 1673 }
 1674 
 1675 
 1676 /*********************************************************************
 1677  *
 1678  *  MSIX Link Interrupt Service routine
 1679  *
 1680  **********************************************************************/
 1681 
 1682 static void
 1683 igb_msix_link(void *arg)
 1684 {
 1685         struct adapter  *adapter = arg;
 1686         u32             icr;
 1687 
 1688         ++adapter->link_irq;
 1689         icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1690         if (!(icr & E1000_ICR_LSC))
 1691                 goto spurious;
 1692         igb_handle_link(adapter, 0);
 1693 
 1694 spurious:
 1695         /* Rearm */
 1696         E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
 1697         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
 1698         return;
 1699 }
 1700 
 1701 
 1702 /*********************************************************************
 1703  *
 1704  *  Media Ioctl callback
 1705  *
 1706  *  This routine is called whenever the user queries the status of
 1707  *  the interface using ifconfig.
 1708  *
 1709  **********************************************************************/
 1710 static void
 1711 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 1712 {
 1713         struct adapter *adapter = ifp->if_softc;
 1714 
 1715         INIT_DEBUGOUT("igb_media_status: begin");
 1716 
 1717         IGB_CORE_LOCK(adapter);
 1718         igb_update_link_status(adapter);
 1719 
 1720         ifmr->ifm_status = IFM_AVALID;
 1721         ifmr->ifm_active = IFM_ETHER;
 1722 
 1723         if (!adapter->link_active) {
 1724                 IGB_CORE_UNLOCK(adapter);
 1725                 return;
 1726         }
 1727 
 1728         ifmr->ifm_status |= IFM_ACTIVE;
 1729 
 1730         switch (adapter->link_speed) {
 1731         case 10:
 1732                 ifmr->ifm_active |= IFM_10_T;
 1733                 break;
 1734         case 100:
 1735                 /*
 1736                 ** Support for 100Mb SFP - these are Fiber 
 1737                 ** but the media type appears as serdes
 1738                 */
 1739                 if (adapter->hw.phy.media_type ==
 1740                     e1000_media_type_internal_serdes)
 1741                         ifmr->ifm_active |= IFM_100_FX;
 1742                 else
 1743                         ifmr->ifm_active |= IFM_100_TX;
 1744                 break;
 1745         case 1000:
 1746                 ifmr->ifm_active |= IFM_1000_T;
 1747                 break;
 1748         case 2500:
 1749                 ifmr->ifm_active |= IFM_2500_SX;
 1750                 break;
 1751         }
 1752 
 1753         if (adapter->link_duplex == FULL_DUPLEX)
 1754                 ifmr->ifm_active |= IFM_FDX;
 1755         else
 1756                 ifmr->ifm_active |= IFM_HDX;
 1757 
 1758         IGB_CORE_UNLOCK(adapter);
 1759 }
 1760 
 1761 /*********************************************************************
 1762  *
 1763  *  Media Ioctl callback
 1764  *
 1765  *  This routine is called when the user changes speed/duplex using
 1766  *  media/mediopt option with ifconfig.
 1767  *
 1768  **********************************************************************/
 1769 static int
 1770 igb_media_change(struct ifnet *ifp)
 1771 {
 1772         struct adapter *adapter = ifp->if_softc;
 1773         struct ifmedia  *ifm = &adapter->media;
 1774 
 1775         INIT_DEBUGOUT("igb_media_change: begin");
 1776 
 1777         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 1778                 return (EINVAL);
 1779 
 1780         IGB_CORE_LOCK(adapter);
 1781         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 1782         case IFM_AUTO:
 1783                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1784                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 1785                 break;
 1786         case IFM_1000_LX:
 1787         case IFM_1000_SX:
 1788         case IFM_1000_T:
 1789                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1790                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
 1791                 break;
 1792         case IFM_100_TX:
 1793                 adapter->hw.mac.autoneg = FALSE;
 1794                 adapter->hw.phy.autoneg_advertised = 0;
 1795                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1796                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
 1797                 else
 1798                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
 1799                 break;
 1800         case IFM_10_T:
 1801                 adapter->hw.mac.autoneg = FALSE;
 1802                 adapter->hw.phy.autoneg_advertised = 0;
 1803                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1804                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
 1805                 else
 1806                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
 1807                 break;
 1808         default:
 1809                 device_printf(adapter->dev, "Unsupported media type\n");
 1810         }
 1811 
 1812         igb_init_locked(adapter);
 1813         IGB_CORE_UNLOCK(adapter);
 1814 
 1815         return (0);
 1816 }
 1817 
 1818 
 1819 /*********************************************************************
 1820  *
 1821  *  This routine maps the mbufs to Advanced TX descriptors.
 1822  *  
 1823  **********************************************************************/
 1824 static int
 1825 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 1826 {
 1827         struct adapter  *adapter = txr->adapter;
 1828         u32             olinfo_status = 0, cmd_type_len;
 1829         int             i, j, error, nsegs;
 1830         int             first;
 1831         bool            remap = TRUE;
 1832         struct mbuf     *m_head;
 1833         bus_dma_segment_t segs[IGB_MAX_SCATTER];
 1834         bus_dmamap_t    map;
 1835         struct igb_tx_buf *txbuf;
 1836         union e1000_adv_tx_desc *txd = NULL;
 1837 
 1838         m_head = *m_headp;
 1839 
 1840         /* Basic descriptor defines */
 1841         cmd_type_len = (E1000_ADVTXD_DTYP_DATA |
 1842             E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT);
 1843 
 1844         if (m_head->m_flags & M_VLANTAG)
 1845                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
 1846 
 1847         /*
 1848          * Important to capture the first descriptor
 1849          * used because it will contain the index of
 1850          * the one we tell the hardware to report back
 1851          */
 1852         first = txr->next_avail_desc;
 1853         txbuf = &txr->tx_buffers[first];
 1854         map = txbuf->map;
 1855 
 1856         /*
 1857          * Map the packet for DMA.
 1858          */
 1859 retry:
 1860         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
 1861             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 1862 
 1863         if (__predict_false(error)) {
 1864                 struct mbuf *m;
 1865 
 1866                 switch (error) {
 1867                 case EFBIG:
 1868                         /* Try it again? - one try */
 1869                         if (remap == TRUE) {
 1870                                 remap = FALSE;
 1871                                 m = m_collapse(*m_headp, M_NOWAIT,
 1872                                     IGB_MAX_SCATTER);
 1873                                 if (m == NULL) {
 1874                                         adapter->mbuf_defrag_failed++;
 1875                                         m_freem(*m_headp);
 1876                                         *m_headp = NULL;
 1877                                         return (ENOBUFS);
 1878                                 }
 1879                                 *m_headp = m;
 1880                                 goto retry;
 1881                         } else
 1882                                 return (error);
 1883                 default:
 1884                         txr->no_tx_dma_setup++;
 1885                         m_freem(*m_headp);
 1886                         *m_headp = NULL;
 1887                         return (error);
 1888                 }
 1889         }
 1890 
 1891         /* Make certain there are enough descriptors */
 1892         if (txr->tx_avail < (nsegs + 2)) {
 1893                 txr->no_desc_avail++;
 1894                 bus_dmamap_unload(txr->txtag, map);
 1895                 return (ENOBUFS);
 1896         }
 1897         m_head = *m_headp;
 1898 
 1899         /*
 1900         ** Set up the appropriate offload context
 1901         ** this will consume the first descriptor
 1902         */
 1903         error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
 1904         if (__predict_false(error)) {
 1905                 m_freem(*m_headp);
 1906                 *m_headp = NULL;
 1907                 return (error);
 1908         }
 1909 
 1910         /* 82575 needs the queue index added */
 1911         if (adapter->hw.mac.type == e1000_82575)
 1912                 olinfo_status |= txr->me << 4;
 1913 
 1914         i = txr->next_avail_desc;
 1915         for (j = 0; j < nsegs; j++) {
 1916                 bus_size_t seglen;
 1917                 bus_addr_t segaddr;
 1918 
 1919                 txbuf = &txr->tx_buffers[i];
 1920                 txd = &txr->tx_base[i];
 1921                 seglen = segs[j].ds_len;
 1922                 segaddr = htole64(segs[j].ds_addr);
 1923 
 1924                 txd->read.buffer_addr = segaddr;
 1925                 txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS |
 1926                     cmd_type_len | seglen);
 1927                 txd->read.olinfo_status = htole32(olinfo_status);
 1928 
 1929                 if (++i == txr->num_desc)
 1930                         i = 0;
 1931         }
 1932 
 1933         txd->read.cmd_type_len |=
 1934             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
 1935         txr->tx_avail -= nsegs;
 1936         txr->next_avail_desc = i;
 1937 
 1938         txbuf->m_head = m_head;
 1939         /*
 1940         ** Here we swap the map so the last descriptor,
 1941         ** which gets the completion interrupt has the
 1942         ** real map, and the first descriptor gets the
 1943         ** unused map from this descriptor.
 1944         */
 1945         txr->tx_buffers[first].map = txbuf->map;
 1946         txbuf->map = map;
 1947         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
 1948 
 1949         /* Set the EOP descriptor that will be marked done */
 1950         txbuf = &txr->tx_buffers[first];
 1951         txbuf->eop = txd;
 1952 
 1953         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 1954             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 1955         /*
 1956          * Advance the Transmit Descriptor Tail (Tdt), this tells the
 1957          * hardware that this frame is available to transmit.
 1958          */
 1959         ++txr->total_packets;
 1960         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
 1961 
 1962         return (0);
 1963 }
 1964 static void
 1965 igb_set_promisc(struct adapter *adapter)
 1966 {
 1967         struct ifnet    *ifp = adapter->ifp;
 1968         struct e1000_hw *hw = &adapter->hw;
 1969         u32             reg;
 1970 
 1971         if (adapter->vf_ifp) {
 1972                 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
 1973                 return;
 1974         }
 1975 
 1976         reg = E1000_READ_REG(hw, E1000_RCTL);
 1977         if (ifp->if_flags & IFF_PROMISC) {
 1978                 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
 1979                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 1980         } else if (ifp->if_flags & IFF_ALLMULTI) {
 1981                 reg |= E1000_RCTL_MPE;
 1982                 reg &= ~E1000_RCTL_UPE;
 1983                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 1984         }
 1985 }
 1986 
 1987 static void
 1988 igb_disable_promisc(struct adapter *adapter)
 1989 {
 1990         struct e1000_hw *hw = &adapter->hw;
 1991         struct ifnet    *ifp = adapter->ifp;
 1992         u32             reg;
 1993         int             mcnt = 0;
 1994 
 1995         if (adapter->vf_ifp) {
 1996                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
 1997                 return;
 1998         }
 1999         reg = E1000_READ_REG(hw, E1000_RCTL);
 2000         reg &=  (~E1000_RCTL_UPE);
 2001         if (ifp->if_flags & IFF_ALLMULTI)
 2002                 mcnt = MAX_NUM_MULTICAST_ADDRESSES;
 2003         else {
 2004                 struct  ifmultiaddr *ifma;
 2005 #if __FreeBSD_version < 800000
 2006                 IF_ADDR_LOCK(ifp);
 2007 #else   
 2008                 if_maddr_rlock(ifp);
 2009 #endif
 2010                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 2011                         if (ifma->ifma_addr->sa_family != AF_LINK)
 2012                                 continue;
 2013                         if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
 2014                                 break;
 2015                         mcnt++;
 2016                 }
 2017 #if __FreeBSD_version < 800000
 2018                 IF_ADDR_UNLOCK(ifp);
 2019 #else
 2020                 if_maddr_runlock(ifp);
 2021 #endif
 2022         }
 2023         /* Don't disable if in MAX groups */
 2024         if (mcnt < MAX_NUM_MULTICAST_ADDRESSES)
 2025                 reg &=  (~E1000_RCTL_MPE);
 2026         E1000_WRITE_REG(hw, E1000_RCTL, reg);
 2027 }
 2028 
 2029 
 2030 /*********************************************************************
 2031  *  Multicast Update
 2032  *
 2033  *  This routine is called whenever multicast address list is updated.
 2034  *
 2035  **********************************************************************/
 2036 
 2037 static void
 2038 igb_set_multi(struct adapter *adapter)
 2039 {
 2040         struct ifnet    *ifp = adapter->ifp;
 2041         struct ifmultiaddr *ifma;
 2042         u32 reg_rctl = 0;
 2043         u8  *mta;
 2044 
 2045         int mcnt = 0;
 2046 
 2047         IOCTL_DEBUGOUT("igb_set_multi: begin");
 2048 
 2049         mta = adapter->mta;
 2050         bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
 2051             MAX_NUM_MULTICAST_ADDRESSES);
 2052 
 2053 #if __FreeBSD_version < 800000
 2054         IF_ADDR_LOCK(ifp);
 2055 #else
 2056         if_maddr_rlock(ifp);
 2057 #endif
 2058         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 2059                 if (ifma->ifma_addr->sa_family != AF_LINK)
 2060                         continue;
 2061 
 2062                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
 2063                         break;
 2064 
 2065                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 2066                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
 2067                 mcnt++;
 2068         }
 2069 #if __FreeBSD_version < 800000
 2070         IF_ADDR_UNLOCK(ifp);
 2071 #else
 2072         if_maddr_runlock(ifp);
 2073 #endif
 2074 
 2075         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
 2076                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 2077                 reg_rctl |= E1000_RCTL_MPE;
 2078                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 2079         } else
 2080                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
 2081 }
 2082 
 2083 
 2084 /*********************************************************************
 2085  *  Timer routine:
 2086  *      This routine checks for link status,
 2087  *      updates statistics, and does the watchdog.
 2088  *
 2089  **********************************************************************/
 2090 
 2091 static void
 2092 igb_local_timer(void *arg)
 2093 {
 2094         struct adapter          *adapter = arg;
 2095         device_t                dev = adapter->dev;
 2096         struct ifnet            *ifp = adapter->ifp;
 2097         struct tx_ring          *txr = adapter->tx_rings;
 2098         struct igb_queue        *que = adapter->queues;
 2099         int                     hung = 0, busy = 0;
 2100 
 2101 
 2102         IGB_CORE_LOCK_ASSERT(adapter);
 2103 
 2104         igb_update_link_status(adapter);
 2105         igb_update_stats_counters(adapter);
 2106 
 2107         /*
 2108         ** Check the TX queues status
 2109         **      - central locked handling of OACTIVE
 2110         **      - watchdog only if all queues show hung
 2111         */
 2112         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
 2113                 if ((txr->queue_status & IGB_QUEUE_HUNG) &&
 2114                     (adapter->pause_frames == 0))
 2115                         ++hung;
 2116                 if (txr->queue_status & IGB_QUEUE_DEPLETED)
 2117                         ++busy;
 2118                 if ((txr->queue_status & IGB_QUEUE_IDLE) == 0)
 2119                         taskqueue_enqueue(que->tq, &que->que_task);
 2120         }
 2121         if (hung == adapter->num_queues)
 2122                 goto timeout;
 2123         if (busy == adapter->num_queues)
 2124                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 2125         else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) &&
 2126             (busy < adapter->num_queues))
 2127                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 2128 
 2129         adapter->pause_frames = 0;
 2130         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 2131 #ifndef DEVICE_POLLING
 2132         /* Schedule all queue interrupts - deadlock protection */
 2133         E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
 2134 #endif
 2135         return;
 2136 
 2137 timeout:
 2138         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
 2139         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
 2140             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
 2141             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
 2142         device_printf(dev,"TX(%d) desc avail = %d,"
 2143             "Next TX to Clean = %d\n",
 2144             txr->me, txr->tx_avail, txr->next_to_clean);
 2145         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 2146         adapter->watchdog_events++;
 2147         igb_init_locked(adapter);
 2148 }
 2149 
 2150 static void
 2151 igb_update_link_status(struct adapter *adapter)
 2152 {
 2153         struct e1000_hw         *hw = &adapter->hw;
 2154         struct e1000_fc_info    *fc = &hw->fc;
 2155         struct ifnet            *ifp = adapter->ifp;
 2156         device_t                dev = adapter->dev;
 2157         struct tx_ring          *txr = adapter->tx_rings;
 2158         u32                     link_check, thstat, ctrl;
 2159         char                    *flowctl = NULL;
 2160 
 2161         link_check = thstat = ctrl = 0;
 2162 
 2163         /* Get the cached link value or read for real */
 2164         switch (hw->phy.media_type) {
 2165         case e1000_media_type_copper:
 2166                 if (hw->mac.get_link_status) {
 2167                         /* Do the work to read phy */
 2168                         e1000_check_for_link(hw);
 2169                         link_check = !hw->mac.get_link_status;
 2170                 } else
 2171                         link_check = TRUE;
 2172                 break;
 2173         case e1000_media_type_fiber:
 2174                 e1000_check_for_link(hw);
 2175                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
 2176                                  E1000_STATUS_LU);
 2177                 break;
 2178         case e1000_media_type_internal_serdes:
 2179                 e1000_check_for_link(hw);
 2180                 link_check = adapter->hw.mac.serdes_has_link;
 2181                 break;
 2182         /* VF device is type_unknown */
 2183         case e1000_media_type_unknown:
 2184                 e1000_check_for_link(hw);
 2185                 link_check = !hw->mac.get_link_status;
 2186                 /* Fall thru */
 2187         default:
 2188                 break;
 2189         }
 2190 
 2191         /* Check for thermal downshift or shutdown */
 2192         if (hw->mac.type == e1000_i350) {
 2193                 thstat = E1000_READ_REG(hw, E1000_THSTAT);
 2194                 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
 2195         }
 2196 
 2197         /* Get the flow control for display */
 2198         switch (fc->current_mode) {
 2199         case e1000_fc_rx_pause:
 2200                 flowctl = "RX";
 2201                 break;  
 2202         case e1000_fc_tx_pause:
 2203                 flowctl = "TX";
 2204                 break;  
 2205         case e1000_fc_full:
 2206                 flowctl = "Full";
 2207                 break;  
 2208         case e1000_fc_none:
 2209         default:
 2210                 flowctl = "None";
 2211                 break;  
 2212         }
 2213 
 2214         /* Now we check if a transition has happened */
 2215         if (link_check && (adapter->link_active == 0)) {
 2216                 e1000_get_speed_and_duplex(&adapter->hw, 
 2217                     &adapter->link_speed, &adapter->link_duplex);
 2218                 if (bootverbose)
 2219                         device_printf(dev, "Link is up %d Mbps %s,"
 2220                             " Flow Control: %s\n",
 2221                             adapter->link_speed,
 2222                             ((adapter->link_duplex == FULL_DUPLEX) ?
 2223                             "Full Duplex" : "Half Duplex"), flowctl);
 2224                 adapter->link_active = 1;
 2225                 ifp->if_baudrate = adapter->link_speed * 1000000;
 2226                 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
 2227                     (thstat & E1000_THSTAT_LINK_THROTTLE))
 2228                         device_printf(dev, "Link: thermal downshift\n");
 2229                 /* Delay Link Up for Phy update */
 2230                 if (((hw->mac.type == e1000_i210) ||
 2231                     (hw->mac.type == e1000_i211)) &&
 2232                     (hw->phy.id == I210_I_PHY_ID))
 2233                         msec_delay(I210_LINK_DELAY);
 2234                 /* Reset if the media type changed. */
 2235                 if (hw->dev_spec._82575.media_changed) {
 2236                         hw->dev_spec._82575.media_changed = false;
 2237                         adapter->flags |= IGB_MEDIA_RESET;
 2238                         igb_reset(adapter);
 2239                 }       
 2240                 /* This can sleep */
 2241                 if_link_state_change(ifp, LINK_STATE_UP);
 2242         } else if (!link_check && (adapter->link_active == 1)) {
 2243                 ifp->if_baudrate = adapter->link_speed = 0;
 2244                 adapter->link_duplex = 0;
 2245                 if (bootverbose)
 2246                         device_printf(dev, "Link is Down\n");
 2247                 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
 2248                     (thstat & E1000_THSTAT_PWR_DOWN))
 2249                         device_printf(dev, "Link: thermal shutdown\n");
 2250                 adapter->link_active = 0;
 2251                 /* This can sleep */
 2252                 if_link_state_change(ifp, LINK_STATE_DOWN);
 2253                 /* Reset queue state */
 2254                 for (int i = 0; i < adapter->num_queues; i++, txr++)
 2255                         txr->queue_status = IGB_QUEUE_IDLE;
 2256         }
 2257 }
 2258 
 2259 /*********************************************************************
 2260  *
 2261  *  This routine disables all traffic on the adapter by issuing a
 2262  *  global reset on the MAC and deallocates TX/RX buffers.
 2263  *
 2264  **********************************************************************/
 2265 
 2266 static void
 2267 igb_stop(void *arg)
 2268 {
 2269         struct adapter  *adapter = arg;
 2270         struct ifnet    *ifp = adapter->ifp;
 2271         struct tx_ring *txr = adapter->tx_rings;
 2272 
 2273         IGB_CORE_LOCK_ASSERT(adapter);
 2274 
 2275         INIT_DEBUGOUT("igb_stop: begin");
 2276 
 2277         igb_disable_intr(adapter);
 2278 
 2279         callout_stop(&adapter->timer);
 2280 
 2281         /* Tell the stack that the interface is no longer active */
 2282         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 2283         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 2284 
 2285         /* Disarm watchdog timer. */
 2286         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 2287                 IGB_TX_LOCK(txr);
 2288                 txr->queue_status = IGB_QUEUE_IDLE;
 2289                 IGB_TX_UNLOCK(txr);
 2290         }
 2291 
 2292         e1000_reset_hw(&adapter->hw);
 2293         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
 2294 
 2295         e1000_led_off(&adapter->hw);
 2296         e1000_cleanup_led(&adapter->hw);
 2297 }
 2298 
 2299 
 2300 /*********************************************************************
 2301  *
 2302  *  Determine hardware revision.
 2303  *
 2304  **********************************************************************/
 2305 static void
 2306 igb_identify_hardware(struct adapter *adapter)
 2307 {
 2308         device_t dev = adapter->dev;
 2309 
 2310         /* Make sure our PCI config space has the necessary stuff set */
 2311         pci_enable_busmaster(dev);
 2312         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
 2313 
 2314         /* Save off the information about this board */
 2315         adapter->hw.vendor_id = pci_get_vendor(dev);
 2316         adapter->hw.device_id = pci_get_device(dev);
 2317         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
 2318         adapter->hw.subsystem_vendor_id =
 2319             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 2320         adapter->hw.subsystem_device_id =
 2321             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 2322 
 2323         /* Set MAC type early for PCI setup */
 2324         e1000_set_mac_type(&adapter->hw);
 2325 
 2326         /* Are we a VF device? */
 2327         if ((adapter->hw.mac.type == e1000_vfadapt) ||
 2328             (adapter->hw.mac.type == e1000_vfadapt_i350))
 2329                 adapter->vf_ifp = 1;
 2330         else
 2331                 adapter->vf_ifp = 0;
 2332 }
 2333 
 2334 static int
 2335 igb_allocate_pci_resources(struct adapter *adapter)
 2336 {
 2337         device_t        dev = adapter->dev;
 2338         int             rid;
 2339 
 2340         rid = PCIR_BAR(0);
 2341         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 2342             &rid, RF_ACTIVE);
 2343         if (adapter->pci_mem == NULL) {
 2344                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 2345                 return (ENXIO);
 2346         }
 2347         adapter->osdep.mem_bus_space_tag =
 2348             rman_get_bustag(adapter->pci_mem);
 2349         adapter->osdep.mem_bus_space_handle =
 2350             rman_get_bushandle(adapter->pci_mem);
 2351         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
 2352 
 2353         adapter->num_queues = 1; /* Defaults for Legacy or MSI */
 2354 
 2355         /* This will setup either MSI/X or MSI */
 2356         adapter->msix = igb_setup_msix(adapter);
 2357         adapter->hw.back = &adapter->osdep;
 2358 
 2359         return (0);
 2360 }
 2361 
 2362 /*********************************************************************
 2363  *
 2364  *  Setup the Legacy or MSI Interrupt handler
 2365  *
 2366  **********************************************************************/
 2367 static int
 2368 igb_allocate_legacy(struct adapter *adapter)
 2369 {
 2370         device_t                dev = adapter->dev;
 2371         struct igb_queue        *que = adapter->queues;
 2372 #ifndef IGB_LEGACY_TX
 2373         struct tx_ring          *txr = adapter->tx_rings;
 2374 #endif
 2375         int                     error, rid = 0;
 2376 
 2377         /* Turn off all interrupts */
 2378         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
 2379 
 2380         /* MSI RID is 1 */
 2381         if (adapter->msix == 1)
 2382                 rid = 1;
 2383 
 2384         /* We allocate a single interrupt resource */
 2385         adapter->res = bus_alloc_resource_any(dev,
 2386             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2387         if (adapter->res == NULL) {
 2388                 device_printf(dev, "Unable to allocate bus resource: "
 2389                     "interrupt\n");
 2390                 return (ENXIO);
 2391         }
 2392 
 2393 #ifndef IGB_LEGACY_TX
 2394         TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
 2395 #endif
 2396 
 2397         /*
 2398          * Try allocating a fast interrupt and the associated deferred
 2399          * processing contexts.
 2400          */
 2401         TASK_INIT(&que->que_task, 0, igb_handle_que, que);
 2402         /* Make tasklet for deferred link handling */
 2403         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
 2404         que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
 2405             taskqueue_thread_enqueue, &que->tq);
 2406         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
 2407             device_get_nameunit(adapter->dev));
 2408         if ((error = bus_setup_intr(dev, adapter->res,
 2409             INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
 2410             adapter, &adapter->tag)) != 0) {
 2411                 device_printf(dev, "Failed to register fast interrupt "
 2412                             "handler: %d\n", error);
 2413                 taskqueue_free(que->tq);
 2414                 que->tq = NULL;
 2415                 return (error);
 2416         }
 2417 
 2418         return (0);
 2419 }
 2420 
 2421 
 2422 /*********************************************************************
 2423  *
 2424  *  Setup the MSIX Queue Interrupt handlers: 
 2425  *
 2426  **********************************************************************/
 2427 static int
 2428 igb_allocate_msix(struct adapter *adapter)
 2429 {
 2430         device_t                dev = adapter->dev;
 2431         struct igb_queue        *que = adapter->queues;
 2432         int                     error, rid, vector = 0;
 2433         int                     cpu_id = 0;
 2434 #ifdef  RSS
 2435         cpuset_t cpu_mask;
 2436 #endif
 2437 
 2438         /* Be sure to start with all interrupts disabled */
 2439         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
 2440         E1000_WRITE_FLUSH(&adapter->hw);
 2441 
 2442 #ifdef  RSS
 2443         /*
 2444          * If we're doing RSS, the number of queues needs to
 2445          * match the number of RSS buckets that are configured.
 2446          *
 2447          * + If there's more queues than RSS buckets, we'll end
 2448          *   up with queues that get no traffic.
 2449          *
 2450          * + If there's more RSS buckets than queues, we'll end
 2451          *   up having multiple RSS buckets map to the same queue,
 2452          *   so there'll be some contention.
 2453          */
 2454         if (adapter->num_queues != rss_getnumbuckets()) {
 2455                 device_printf(dev,
 2456                     "%s: number of queues (%d) != number of RSS buckets (%d)"
 2457                     "; performance will be impacted.\n",
 2458                     __func__,
 2459                     adapter->num_queues,
 2460                     rss_getnumbuckets());
 2461         }
 2462 #endif
 2463 
 2464         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
 2465                 rid = vector +1;
 2466                 que->res = bus_alloc_resource_any(dev,
 2467                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2468                 if (que->res == NULL) {
 2469                         device_printf(dev,
 2470                             "Unable to allocate bus resource: "
 2471                             "MSIX Queue Interrupt\n");
 2472                         return (ENXIO);
 2473                 }
 2474                 error = bus_setup_intr(dev, que->res,
 2475                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2476                     igb_msix_que, que, &que->tag);
 2477                 if (error) {
 2478                         que->res = NULL;
 2479                         device_printf(dev, "Failed to register Queue handler");
 2480                         return (error);
 2481                 }
 2482 #if __FreeBSD_version >= 800504
 2483                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
 2484 #endif
 2485                 que->msix = vector;
 2486                 if (adapter->hw.mac.type == e1000_82575)
 2487                         que->eims = E1000_EICR_TX_QUEUE0 << i;
 2488                 else
 2489                         que->eims = 1 << vector;
 2490 
 2491 #ifdef  RSS
 2492                 /*
 2493                  * The queue ID is used as the RSS layer bucket ID.
 2494                  * We look up the queue ID -> RSS CPU ID and select
 2495                  * that.
 2496                  */
 2497                 cpu_id = rss_getcpu(i % rss_getnumbuckets());
 2498 #else
 2499                 /*
 2500                  * Bind the msix vector, and thus the
 2501                  * rings to the corresponding cpu.
 2502                  *
 2503                  * This just happens to match the default RSS round-robin
 2504                  * bucket -> queue -> CPU allocation.
 2505                  */
 2506                 if (adapter->num_queues > 1) {
 2507                         if (igb_last_bind_cpu < 0)
 2508                                 igb_last_bind_cpu = CPU_FIRST();
 2509                         cpu_id = igb_last_bind_cpu;
 2510                 }
 2511 #endif
 2512 
 2513                 if (adapter->num_queues > 1) {
 2514                         bus_bind_intr(dev, que->res, cpu_id);
 2515 #ifdef  RSS
 2516                         device_printf(dev,
 2517                                 "Bound queue %d to RSS bucket %d\n",
 2518                                 i, cpu_id);
 2519 #else
 2520                         device_printf(dev,
 2521                                 "Bound queue %d to cpu %d\n",
 2522                                 i, cpu_id);
 2523 #endif
 2524                 }
 2525 
 2526 #ifndef IGB_LEGACY_TX
 2527                 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
 2528                     que->txr);
 2529 #endif
 2530                 /* Make tasklet for deferred handling */
 2531                 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
 2532                 que->tq = taskqueue_create("igb_que", M_NOWAIT,
 2533                     taskqueue_thread_enqueue, &que->tq);
 2534                 if (adapter->num_queues > 1) {
 2535                         /*
 2536                          * Only pin the taskqueue thread to a CPU if
 2537                          * RSS is in use.
 2538                          *
 2539                          * This again just happens to match the default RSS
 2540                          * round-robin bucket -> queue -> CPU allocation.
 2541                          */
 2542 #ifdef  RSS
 2543                         CPU_SETOF(cpu_id, &cpu_mask);
 2544                         taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
 2545                             &cpu_mask,
 2546                             "%s que (bucket %d)",
 2547                             device_get_nameunit(adapter->dev),
 2548                             cpu_id);
 2549 #else
 2550                         taskqueue_start_threads(&que->tq, 1, PI_NET,
 2551                             "%s que (qid %d)",
 2552                             device_get_nameunit(adapter->dev),
 2553                             cpu_id);
 2554 #endif
 2555                 } else {
 2556                         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
 2557                             device_get_nameunit(adapter->dev));
 2558                 }
 2559 
 2560                 /* Finally update the last bound CPU id */
 2561                 if (adapter->num_queues > 1)
 2562                         igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu);
 2563         }
 2564 
 2565         /* And Link */
 2566         rid = vector + 1;
 2567         adapter->res = bus_alloc_resource_any(dev,
 2568             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2569         if (adapter->res == NULL) {
 2570                 device_printf(dev,
 2571                     "Unable to allocate bus resource: "
 2572                     "MSIX Link Interrupt\n");
 2573                 return (ENXIO);
 2574         }
 2575         if ((error = bus_setup_intr(dev, adapter->res,
 2576             INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2577             igb_msix_link, adapter, &adapter->tag)) != 0) {
 2578                 device_printf(dev, "Failed to register Link handler");
 2579                 return (error);
 2580         }
 2581 #if __FreeBSD_version >= 800504
 2582         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
 2583 #endif
 2584         adapter->linkvec = vector;
 2585 
 2586         return (0);
 2587 }
 2588 
 2589 
 2590 static void
 2591 igb_configure_queues(struct adapter *adapter)
 2592 {
 2593         struct  e1000_hw        *hw = &adapter->hw;
 2594         struct  igb_queue       *que;
 2595         u32                     tmp, ivar = 0, newitr = 0;
 2596 
 2597         /* First turn on RSS capability */
 2598         if (adapter->hw.mac.type != e1000_82575)
 2599                 E1000_WRITE_REG(hw, E1000_GPIE,
 2600                     E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
 2601                     E1000_GPIE_PBA | E1000_GPIE_NSICR);
 2602 
 2603         /* Turn on MSIX */
 2604         switch (adapter->hw.mac.type) {
 2605         case e1000_82580:
 2606         case e1000_i350:
 2607         case e1000_i354:
 2608         case e1000_i210:
 2609         case e1000_i211:
 2610         case e1000_vfadapt:
 2611         case e1000_vfadapt_i350:
 2612                 /* RX entries */
 2613                 for (int i = 0; i < adapter->num_queues; i++) {
 2614                         u32 index = i >> 1;
 2615                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2616                         que = &adapter->queues[i];
 2617                         if (i & 1) {
 2618                                 ivar &= 0xFF00FFFF;
 2619                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
 2620                         } else {
 2621                                 ivar &= 0xFFFFFF00;
 2622                                 ivar |= que->msix | E1000_IVAR_VALID;
 2623                         }
 2624                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2625                 }
 2626                 /* TX entries */
 2627                 for (int i = 0; i < adapter->num_queues; i++) {
 2628                         u32 index = i >> 1;
 2629                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2630                         que = &adapter->queues[i];
 2631                         if (i & 1) {
 2632                                 ivar &= 0x00FFFFFF;
 2633                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
 2634                         } else {
 2635                                 ivar &= 0xFFFF00FF;
 2636                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
 2637                         }
 2638                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2639                         adapter->que_mask |= que->eims;
 2640                 }
 2641 
 2642                 /* And for the link interrupt */
 2643                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 2644                 adapter->link_mask = 1 << adapter->linkvec;
 2645                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 2646                 break;
 2647         case e1000_82576:
 2648                 /* RX entries */
 2649                 for (int i = 0; i < adapter->num_queues; i++) {
 2650                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2651                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2652                         que = &adapter->queues[i];
 2653                         if (i < 8) {
 2654                                 ivar &= 0xFFFFFF00;
 2655                                 ivar |= que->msix | E1000_IVAR_VALID;
 2656                         } else {
 2657                                 ivar &= 0xFF00FFFF;
 2658                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
 2659                         }
 2660                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2661                         adapter->que_mask |= que->eims;
 2662                 }
 2663                 /* TX entries */
 2664                 for (int i = 0; i < adapter->num_queues; i++) {
 2665                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2666                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2667                         que = &adapter->queues[i];
 2668                         if (i < 8) {
 2669                                 ivar &= 0xFFFF00FF;
 2670                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
 2671                         } else {
 2672                                 ivar &= 0x00FFFFFF;
 2673                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
 2674                         }
 2675                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2676                         adapter->que_mask |= que->eims;
 2677                 }
 2678 
 2679                 /* And for the link interrupt */
 2680                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 2681                 adapter->link_mask = 1 << adapter->linkvec;
 2682                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 2683                 break;
 2684 
 2685         case e1000_82575:
 2686                 /* enable MSI-X support*/
 2687                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 2688                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 2689                 /* Auto-Mask interrupts upon ICR read. */
 2690                 tmp |= E1000_CTRL_EXT_EIAME;
 2691                 tmp |= E1000_CTRL_EXT_IRCA;
 2692                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 2693 
 2694                 /* Queues */
 2695                 for (int i = 0; i < adapter->num_queues; i++) {
 2696                         que = &adapter->queues[i];
 2697                         tmp = E1000_EICR_RX_QUEUE0 << i;
 2698                         tmp |= E1000_EICR_TX_QUEUE0 << i;
 2699                         que->eims = tmp;
 2700                         E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
 2701                             i, que->eims);
 2702                         adapter->que_mask |= que->eims;
 2703                 }
 2704 
 2705                 /* Link */
 2706                 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
 2707                     E1000_EIMS_OTHER);
 2708                 adapter->link_mask |= E1000_EIMS_OTHER;
 2709         default:
 2710                 break;
 2711         }
 2712 
 2713         /* Set the starting interrupt rate */
 2714         if (igb_max_interrupt_rate > 0)
 2715                 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
 2716 
 2717         if (hw->mac.type == e1000_82575)
 2718                 newitr |= newitr << 16;
 2719         else
 2720                 newitr |= E1000_EITR_CNT_IGNR;
 2721 
 2722         for (int i = 0; i < adapter->num_queues; i++) {
 2723                 que = &adapter->queues[i];
 2724                 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
 2725         }
 2726 
 2727         return;
 2728 }
 2729 
 2730 
 2731 static void
 2732 igb_free_pci_resources(struct adapter *adapter)
 2733 {
 2734         struct          igb_queue *que = adapter->queues;
 2735         device_t        dev = adapter->dev;
 2736         int             rid;
 2737 
 2738         /*
 2739         ** There is a slight possibility of a failure mode
 2740         ** in attach that will result in entering this function
 2741         ** before interrupt resources have been initialized, and
 2742         ** in that case we do not want to execute the loops below
 2743         ** We can detect this reliably by the state of the adapter
 2744         ** res pointer.
 2745         */
 2746         if (adapter->res == NULL)
 2747                 goto mem;
 2748 
 2749         /*
 2750          * First release all the interrupt resources:
 2751          */
 2752         for (int i = 0; i < adapter->num_queues; i++, que++) {
 2753                 rid = que->msix + 1;
 2754                 if (que->tag != NULL) {
 2755                         bus_teardown_intr(dev, que->res, que->tag);
 2756                         que->tag = NULL;
 2757                 }
 2758                 if (que->res != NULL)
 2759                         bus_release_resource(dev,
 2760                             SYS_RES_IRQ, rid, que->res);
 2761         }
 2762 
 2763         /* Clean the Legacy or Link interrupt last */
 2764         if (adapter->linkvec) /* we are doing MSIX */
 2765                 rid = adapter->linkvec + 1;
 2766         else
 2767                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
 2768 
 2769         que = adapter->queues;
 2770         if (adapter->tag != NULL) {
 2771                 taskqueue_drain(que->tq, &adapter->link_task);
 2772                 bus_teardown_intr(dev, adapter->res, adapter->tag);
 2773                 adapter->tag = NULL;
 2774         }
 2775         if (adapter->res != NULL)
 2776                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
 2777 
 2778         for (int i = 0; i < adapter->num_queues; i++, que++) {
 2779                 if (que->tq != NULL) {
 2780 #ifndef IGB_LEGACY_TX
 2781                         taskqueue_drain(que->tq, &que->txr->txq_task);
 2782 #endif
 2783                         taskqueue_drain(que->tq, &que->que_task);
 2784                         taskqueue_free(que->tq);
 2785                 }
 2786         }
 2787 mem:
 2788         if (adapter->msix)
 2789                 pci_release_msi(dev);
 2790 
 2791         if (adapter->msix_mem != NULL)
 2792                 bus_release_resource(dev, SYS_RES_MEMORY,
 2793                     adapter->memrid, adapter->msix_mem);
 2794 
 2795         if (adapter->pci_mem != NULL)
 2796                 bus_release_resource(dev, SYS_RES_MEMORY,
 2797                     PCIR_BAR(0), adapter->pci_mem);
 2798 
 2799 }
 2800 
 2801 /*
 2802  * Setup Either MSI/X or MSI
 2803  */
 2804 static int
 2805 igb_setup_msix(struct adapter *adapter)
 2806 {
 2807         device_t        dev = adapter->dev;
 2808         int             bar, want, queues, msgs, maxqueues;
 2809 
 2810         /* tuneable override */
 2811         if (igb_enable_msix == 0)
 2812                 goto msi;
 2813 
 2814         /* First try MSI/X */
 2815         msgs = pci_msix_count(dev); 
 2816         if (msgs == 0)
 2817                 goto msi;
 2818         /*
 2819         ** Some new devices, as with ixgbe, now may
 2820         ** use a different BAR, so we need to keep
 2821         ** track of which is used.
 2822         */
 2823         adapter->memrid = PCIR_BAR(IGB_MSIX_BAR);
 2824         bar = pci_read_config(dev, adapter->memrid, 4);
 2825         if (bar == 0) /* use next bar */
 2826                 adapter->memrid += 4;
 2827         adapter->msix_mem = bus_alloc_resource_any(dev,
 2828             SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE);
 2829         if (adapter->msix_mem == NULL) {
 2830                 /* May not be enabled */
 2831                 device_printf(adapter->dev,
 2832                     "Unable to map MSIX table \n");
 2833                 goto msi;
 2834         }
 2835 
 2836         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
 2837 
 2838         /* Override via tuneable */
 2839         if (igb_num_queues != 0)
 2840                 queues = igb_num_queues;
 2841 
 2842 #ifdef  RSS
 2843         /* If we're doing RSS, clamp at the number of RSS buckets */
 2844         if (queues > rss_getnumbuckets())
 2845                 queues = rss_getnumbuckets();
 2846 #endif
 2847 
 2848 
 2849         /* Sanity check based on HW */
 2850         switch (adapter->hw.mac.type) {
 2851                 case e1000_82575:
 2852                         maxqueues = 4;
 2853                         break;
 2854                 case e1000_82576:
 2855                 case e1000_82580:
 2856                 case e1000_i350:
 2857                 case e1000_i354:
 2858                         maxqueues = 8;
 2859                         break;
 2860                 case e1000_i210:
 2861                         maxqueues = 4;
 2862                         break;
 2863                 case e1000_i211:
 2864                         maxqueues = 2;
 2865                         break;
 2866                 default:  /* VF interfaces */
 2867                         maxqueues = 1;
 2868                         break;
 2869         }
 2870 
 2871         /* Final clamp on the actual hardware capability */
 2872         if (queues > maxqueues)
 2873                 queues = maxqueues;
 2874 
 2875         /*
 2876         ** One vector (RX/TX pair) per queue
 2877         ** plus an additional for Link interrupt
 2878         */
 2879         want = queues + 1;
 2880         if (msgs >= want)
 2881                 msgs = want;
 2882         else {
 2883                 device_printf(adapter->dev,
 2884                     "MSIX Configuration Problem, "
 2885                     "%d vectors configured, but %d queues wanted!\n",
 2886                     msgs, want);
 2887                 goto msi;
 2888         }
 2889         if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) {
 2890                 device_printf(adapter->dev,
 2891                     "Using MSIX interrupts with %d vectors\n", msgs);
 2892                 adapter->num_queues = queues;
 2893                 return (msgs);
 2894         }
 2895         /*
 2896         ** If MSIX alloc failed or provided us with
 2897         ** less than needed, free and fall through to MSI
 2898         */
 2899         pci_release_msi(dev);
 2900 
 2901 msi:
 2902         if (adapter->msix_mem != NULL) {
 2903                 bus_release_resource(dev, SYS_RES_MEMORY,
 2904                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
 2905                 adapter->msix_mem = NULL;
 2906         }
 2907         msgs = 1;
 2908         if (pci_alloc_msi(dev, &msgs) == 0) {
 2909                 device_printf(adapter->dev," Using an MSI interrupt\n");
 2910                 return (msgs);
 2911         }
 2912         device_printf(adapter->dev," Using a Legacy interrupt\n");
 2913         return (0);
 2914 }
 2915 
 2916 /*********************************************************************
 2917  *
 2918  *  Initialize the DMA Coalescing feature
 2919  *
 2920  **********************************************************************/
 2921 static void
 2922 igb_init_dmac(struct adapter *adapter, u32 pba)
 2923 {
 2924         device_t        dev = adapter->dev;
 2925         struct e1000_hw *hw = &adapter->hw;
 2926         u32             dmac, reg = ~E1000_DMACR_DMAC_EN;
 2927         u16             hwm;
 2928 
 2929         if (hw->mac.type == e1000_i211)
 2930                 return;
 2931 
 2932         if (hw->mac.type > e1000_82580) {
 2933 
 2934                 if (adapter->dmac == 0) { /* Disabling it */
 2935                         E1000_WRITE_REG(hw, E1000_DMACR, reg);
 2936                         return;
 2937                 } else
 2938                         device_printf(dev, "DMA Coalescing enabled\n");
 2939 
 2940                 /* Set starting threshold */
 2941                 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
 2942 
 2943                 hwm = 64 * pba - adapter->max_frame_size / 16;
 2944                 if (hwm < 64 * (pba - 6))
 2945                         hwm = 64 * (pba - 6);
 2946                 reg = E1000_READ_REG(hw, E1000_FCRTC);
 2947                 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
 2948                 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
 2949                     & E1000_FCRTC_RTH_COAL_MASK);
 2950                 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
 2951 
 2952 
 2953                 dmac = pba - adapter->max_frame_size / 512;
 2954                 if (dmac < pba - 10)
 2955                         dmac = pba - 10;
 2956                 reg = E1000_READ_REG(hw, E1000_DMACR);
 2957                 reg &= ~E1000_DMACR_DMACTHR_MASK;
 2958                 reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT)
 2959                     & E1000_DMACR_DMACTHR_MASK);
 2960 
 2961                 /* transition to L0x or L1 if available..*/
 2962                 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
 2963 
 2964                 /* Check if status is 2.5Gb backplane connection
 2965                 * before configuration of watchdog timer, which is
 2966                 * in msec values in 12.8usec intervals
 2967                 * watchdog timer= msec values in 32usec intervals
 2968                 * for non 2.5Gb connection
 2969                 */
 2970                 if (hw->mac.type == e1000_i354) {
 2971                         int status = E1000_READ_REG(hw, E1000_STATUS);
 2972                         if ((status & E1000_STATUS_2P5_SKU) &&
 2973                             (!(status & E1000_STATUS_2P5_SKU_OVER)))
 2974                                 reg |= ((adapter->dmac * 5) >> 6);
 2975                         else
 2976                                 reg |= (adapter->dmac >> 5);
 2977                 } else {
 2978                         reg |= (adapter->dmac >> 5);
 2979                 }
 2980 
 2981                 E1000_WRITE_REG(hw, E1000_DMACR, reg);
 2982 
 2983                 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
 2984 
 2985                 /* Set the interval before transition */
 2986                 reg = E1000_READ_REG(hw, E1000_DMCTLX);
 2987                 if (hw->mac.type == e1000_i350)
 2988                         reg |= IGB_DMCTLX_DCFLUSH_DIS;
 2989                 /*
 2990                 ** in 2.5Gb connection, TTLX unit is 0.4 usec
 2991                 ** which is 0x4*2 = 0xA. But delay is still 4 usec
 2992                 */
 2993                 if (hw->mac.type == e1000_i354) {
 2994                         int status = E1000_READ_REG(hw, E1000_STATUS);
 2995                         if ((status & E1000_STATUS_2P5_SKU) &&
 2996                             (!(status & E1000_STATUS_2P5_SKU_OVER)))
 2997                                 reg |= 0xA;
 2998                         else
 2999                                 reg |= 0x4;
 3000                 } else {
 3001                         reg |= 0x4;
 3002                 }
 3003 
 3004                 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
 3005 
 3006                 /* free space in tx packet buffer to wake from DMA coal */
 3007                 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE -
 3008                     (2 * adapter->max_frame_size)) >> 6);
 3009 
 3010                 /* make low power state decision controlled by DMA coal */
 3011                 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
 3012                 reg &= ~E1000_PCIEMISC_LX_DECISION;
 3013                 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
 3014 
 3015         } else if (hw->mac.type == e1000_82580) {
 3016                 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
 3017                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
 3018                     reg & ~E1000_PCIEMISC_LX_DECISION);
 3019                 E1000_WRITE_REG(hw, E1000_DMACR, 0);
 3020         }
 3021 }
 3022 
 3023 
 3024 /*********************************************************************
 3025  *
 3026  *  Set up an fresh starting state
 3027  *
 3028  **********************************************************************/
 3029 static void
 3030 igb_reset(struct adapter *adapter)
 3031 {
 3032         device_t        dev = adapter->dev;
 3033         struct e1000_hw *hw = &adapter->hw;
 3034         struct e1000_fc_info *fc = &hw->fc;
 3035         struct ifnet    *ifp = adapter->ifp;
 3036         u32             pba = 0;
 3037         u16             hwm;
 3038 
 3039         INIT_DEBUGOUT("igb_reset: begin");
 3040 
 3041         /* Let the firmware know the OS is in control */
 3042         igb_get_hw_control(adapter);
 3043 
 3044         /*
 3045          * Packet Buffer Allocation (PBA)
 3046          * Writing PBA sets the receive portion of the buffer
 3047          * the remainder is used for the transmit buffer.
 3048          */
 3049         switch (hw->mac.type) {
 3050         case e1000_82575:
 3051                 pba = E1000_PBA_32K;
 3052                 break;
 3053         case e1000_82576:
 3054         case e1000_vfadapt:
 3055                 pba = E1000_READ_REG(hw, E1000_RXPBS);
 3056                 pba &= E1000_RXPBS_SIZE_MASK_82576;
 3057                 break;
 3058         case e1000_82580:
 3059         case e1000_i350:
 3060         case e1000_i354:
 3061         case e1000_vfadapt_i350:
 3062                 pba = E1000_READ_REG(hw, E1000_RXPBS);
 3063                 pba = e1000_rxpbs_adjust_82580(pba);
 3064                 break;
 3065         case e1000_i210:
 3066         case e1000_i211:
 3067                 pba = E1000_PBA_34K;
 3068         default:
 3069                 break;
 3070         }
 3071 
 3072         /* Special needs in case of Jumbo frames */
 3073         if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
 3074                 u32 tx_space, min_tx, min_rx;
 3075                 pba = E1000_READ_REG(hw, E1000_PBA);
 3076                 tx_space = pba >> 16;
 3077                 pba &= 0xffff;
 3078                 min_tx = (adapter->max_frame_size +
 3079                     sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
 3080                 min_tx = roundup2(min_tx, 1024);
 3081                 min_tx >>= 10;
 3082                 min_rx = adapter->max_frame_size;
 3083                 min_rx = roundup2(min_rx, 1024);
 3084                 min_rx >>= 10;
 3085                 if (tx_space < min_tx &&
 3086                     ((min_tx - tx_space) < pba)) {
 3087                         pba = pba - (min_tx - tx_space);
 3088                         /*
 3089                          * if short on rx space, rx wins
 3090                          * and must trump tx adjustment
 3091                          */
 3092                         if (pba < min_rx)
 3093                                 pba = min_rx;
 3094                 }
 3095                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 3096         }
 3097 
 3098         INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
 3099 
 3100         /*
 3101          * These parameters control the automatic generation (Tx) and
 3102          * response (Rx) to Ethernet PAUSE frames.
 3103          * - High water mark should allow for at least two frames to be
 3104          *   received after sending an XOFF.
 3105          * - Low water mark works best when it is very near the high water mark.
 3106          *   This allows the receiver to restart by sending XON when it has
 3107          *   drained a bit.
 3108          */
 3109         hwm = min(((pba << 10) * 9 / 10),
 3110             ((pba << 10) - 2 * adapter->max_frame_size));
 3111 
 3112         if (hw->mac.type < e1000_82576) {
 3113                 fc->high_water = hwm & 0xFFF8;  /* 8-byte granularity */
 3114                 fc->low_water = fc->high_water - 8;
 3115         } else {
 3116                 fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
 3117                 fc->low_water = fc->high_water - 16;
 3118         }
 3119 
 3120         fc->pause_time = IGB_FC_PAUSE_TIME;
 3121         fc->send_xon = TRUE;
 3122         if (adapter->fc)
 3123                 fc->requested_mode = adapter->fc;
 3124         else
 3125                 fc->requested_mode = e1000_fc_default;
 3126 
 3127         /* Issue a global reset */
 3128         e1000_reset_hw(hw);
 3129         E1000_WRITE_REG(hw, E1000_WUC, 0);
 3130 
 3131         /* Reset for AutoMediaDetect */
 3132         if (adapter->flags & IGB_MEDIA_RESET) {
 3133                 e1000_setup_init_funcs(hw, TRUE);
 3134                 e1000_get_bus_info(hw);
 3135                 adapter->flags &= ~IGB_MEDIA_RESET;
 3136         }
 3137 
 3138         if (e1000_init_hw(hw) < 0)
 3139                 device_printf(dev, "Hardware Initialization Failed\n");
 3140 
 3141         /* Setup DMA Coalescing */
 3142         igb_init_dmac(adapter, pba);
 3143 
 3144         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 3145         e1000_get_phy_info(hw);
 3146         e1000_check_for_link(hw);
 3147         return;
 3148 }
 3149 
 3150 /*********************************************************************
 3151  *
 3152  *  Setup networking device structure and register an interface.
 3153  *
 3154  **********************************************************************/
 3155 static int
 3156 igb_setup_interface(device_t dev, struct adapter *adapter)
 3157 {
 3158         struct ifnet   *ifp;
 3159 
 3160         INIT_DEBUGOUT("igb_setup_interface: begin");
 3161 
 3162         ifp = adapter->ifp = if_alloc(IFT_ETHER);
 3163         if (ifp == NULL) {
 3164                 device_printf(dev, "can not allocate ifnet structure\n");
 3165                 return (-1);
 3166         }
 3167         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 3168         ifp->if_init =  igb_init;
 3169         ifp->if_softc = adapter;
 3170         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 3171         ifp->if_ioctl = igb_ioctl;
 3172         ifp->if_get_counter = igb_get_counter;
 3173 
 3174         /* TSO parameters */
 3175         ifp->if_hw_tsomax = IP_MAXPACKET;
 3176         ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER;
 3177         ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE;
 3178 
 3179 #ifndef IGB_LEGACY_TX
 3180         ifp->if_transmit = igb_mq_start;
 3181         ifp->if_qflush = igb_qflush;
 3182 #else
 3183         ifp->if_start = igb_start;
 3184         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
 3185         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
 3186         IFQ_SET_READY(&ifp->if_snd);
 3187 #endif
 3188 
 3189         ether_ifattach(ifp, adapter->hw.mac.addr);
 3190 
 3191         ifp->if_capabilities = ifp->if_capenable = 0;
 3192 
 3193         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
 3194 #if __FreeBSD_version >= 1000000
 3195         ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
 3196 #endif
 3197         ifp->if_capabilities |= IFCAP_TSO;
 3198         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
 3199         ifp->if_capenable = ifp->if_capabilities;
 3200 
 3201         /* Don't enable LRO by default */
 3202         ifp->if_capabilities |= IFCAP_LRO;
 3203 
 3204 #ifdef DEVICE_POLLING
 3205         ifp->if_capabilities |= IFCAP_POLLING;
 3206 #endif
 3207 
 3208         /*
 3209          * Tell the upper layer(s) we
 3210          * support full VLAN capability.
 3211          */
 3212         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
 3213         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
 3214                              |  IFCAP_VLAN_HWTSO
 3215                              |  IFCAP_VLAN_MTU;
 3216         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
 3217                           |  IFCAP_VLAN_HWTSO
 3218                           |  IFCAP_VLAN_MTU;
 3219 
 3220         /*
 3221         ** Don't turn this on by default, if vlans are
 3222         ** created on another pseudo device (eg. lagg)
 3223         ** then vlan events are not passed thru, breaking
 3224         ** operation, but with HW FILTER off it works. If
 3225         ** using vlans directly on the igb driver you can
 3226         ** enable this and get full hardware tag filtering.
 3227         */
 3228         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 3229 
 3230         /*
 3231          * Specify the media types supported by this adapter and register
 3232          * callbacks to update media and link information
 3233          */
 3234         ifmedia_init(&adapter->media, IFM_IMASK,
 3235             igb_media_change, igb_media_status);
 3236         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 3237             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
 3238                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 
 3239                             0, NULL);
 3240                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 3241         } else {
 3242                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
 3243                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
 3244                             0, NULL);
 3245                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
 3246                             0, NULL);
 3247                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
 3248                             0, NULL);
 3249                 if (adapter->hw.phy.type != e1000_phy_ife) {
 3250                         ifmedia_add(&adapter->media,
 3251                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
 3252                         ifmedia_add(&adapter->media,
 3253                                 IFM_ETHER | IFM_1000_T, 0, NULL);
 3254                 }
 3255         }
 3256         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 3257         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
 3258         return (0);
 3259 }
 3260 
 3261 
 3262 /*
 3263  * Manage DMA'able memory.
 3264  */
 3265 static void
 3266 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 3267 {
 3268         if (error)
 3269                 return;
 3270         *(bus_addr_t *) arg = segs[0].ds_addr;
 3271 }
 3272 
 3273 static int
 3274 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
 3275         struct igb_dma_alloc *dma, int mapflags)
 3276 {
 3277         int error;
 3278 
 3279         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
 3280                                 IGB_DBA_ALIGN, 0,       /* alignment, bounds */
 3281                                 BUS_SPACE_MAXADDR,      /* lowaddr */
 3282                                 BUS_SPACE_MAXADDR,      /* highaddr */
 3283                                 NULL, NULL,             /* filter, filterarg */
 3284                                 size,                   /* maxsize */
 3285                                 1,                      /* nsegments */
 3286                                 size,                   /* maxsegsize */
 3287                                 0,                      /* flags */
 3288                                 NULL,                   /* lockfunc */
 3289                                 NULL,                   /* lockarg */
 3290                                 &dma->dma_tag);
 3291         if (error) {
 3292                 device_printf(adapter->dev,
 3293                     "%s: bus_dma_tag_create failed: %d\n",
 3294                     __func__, error);
 3295                 goto fail_0;
 3296         }
 3297 
 3298         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 3299             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
 3300         if (error) {
 3301                 device_printf(adapter->dev,
 3302                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
 3303                     __func__, (uintmax_t)size, error);
 3304                 goto fail_2;
 3305         }
 3306 
 3307         dma->dma_paddr = 0;
 3308         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 3309             size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
 3310         if (error || dma->dma_paddr == 0) {
 3311                 device_printf(adapter->dev,
 3312                     "%s: bus_dmamap_load failed: %d\n",
 3313                     __func__, error);
 3314                 goto fail_3;
 3315         }
 3316 
 3317         return (0);
 3318 
 3319 fail_3:
 3320         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 3321 fail_2:
 3322         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 3323         bus_dma_tag_destroy(dma->dma_tag);
 3324 fail_0:
 3325         dma->dma_tag = NULL;
 3326 
 3327         return (error);
 3328 }
 3329 
 3330 static void
 3331 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
 3332 {
 3333         if (dma->dma_tag == NULL)
 3334                 return;
 3335         if (dma->dma_paddr != 0) {
 3336                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
 3337                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 3338                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 3339                 dma->dma_paddr = 0;
 3340         }
 3341         if (dma->dma_vaddr != NULL) {
 3342                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 3343                 dma->dma_vaddr = NULL;
 3344         }
 3345         bus_dma_tag_destroy(dma->dma_tag);
 3346         dma->dma_tag = NULL;
 3347 }
 3348 
 3349 
 3350 /*********************************************************************
 3351  *
 3352  *  Allocate memory for the transmit and receive rings, and then
 3353  *  the descriptors associated with each, called only once at attach.
 3354  *
 3355  **********************************************************************/
 3356 static int
 3357 igb_allocate_queues(struct adapter *adapter)
 3358 {
 3359         device_t dev = adapter->dev;
 3360         struct igb_queue        *que = NULL;
 3361         struct tx_ring          *txr = NULL;
 3362         struct rx_ring          *rxr = NULL;
 3363         int rsize, tsize, error = E1000_SUCCESS;
 3364         int txconf = 0, rxconf = 0;
 3365 
 3366         /* First allocate the top level queue structs */
 3367         if (!(adapter->queues =
 3368             (struct igb_queue *) malloc(sizeof(struct igb_queue) *
 3369             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3370                 device_printf(dev, "Unable to allocate queue memory\n");
 3371                 error = ENOMEM;
 3372                 goto fail;
 3373         }
 3374 
 3375         /* Next allocate the TX ring struct memory */
 3376         if (!(adapter->tx_rings =
 3377             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
 3378             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3379                 device_printf(dev, "Unable to allocate TX ring memory\n");
 3380                 error = ENOMEM;
 3381                 goto tx_fail;
 3382         }
 3383 
 3384         /* Now allocate the RX */
 3385         if (!(adapter->rx_rings =
 3386             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
 3387             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3388                 device_printf(dev, "Unable to allocate RX ring memory\n");
 3389                 error = ENOMEM;
 3390                 goto rx_fail;
 3391         }
 3392 
 3393         tsize = roundup2(adapter->num_tx_desc *
 3394             sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
 3395         /*
 3396          * Now set up the TX queues, txconf is needed to handle the
 3397          * possibility that things fail midcourse and we need to
 3398          * undo memory gracefully
 3399          */ 
 3400         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
 3401                 /* Set up some basics */
 3402                 txr = &adapter->tx_rings[i];
 3403                 txr->adapter = adapter;
 3404                 txr->me = i;
 3405                 txr->num_desc = adapter->num_tx_desc;
 3406 
 3407                 /* Initialize the TX lock */
 3408                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
 3409                     device_get_nameunit(dev), txr->me);
 3410                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
 3411 
 3412                 if (igb_dma_malloc(adapter, tsize,
 3413                         &txr->txdma, BUS_DMA_NOWAIT)) {
 3414                         device_printf(dev,
 3415                             "Unable to allocate TX Descriptor memory\n");
 3416                         error = ENOMEM;
 3417                         goto err_tx_desc;
 3418                 }
 3419                 txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr;
 3420                 bzero((void *)txr->tx_base, tsize);
 3421 
 3422                 /* Now allocate transmit buffers for the ring */
 3423                 if (igb_allocate_transmit_buffers(txr)) {
 3424                         device_printf(dev,
 3425                             "Critical Failure setting up transmit buffers\n");
 3426                         error = ENOMEM;
 3427                         goto err_tx_desc;
 3428                 }
 3429 #ifndef IGB_LEGACY_TX
 3430                 /* Allocate a buf ring */
 3431                 txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF,
 3432                     M_WAITOK, &txr->tx_mtx);
 3433 #endif
 3434         }
 3435 
 3436         /*
 3437          * Next the RX queues...
 3438          */ 
 3439         rsize = roundup2(adapter->num_rx_desc *
 3440             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 3441         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
 3442                 rxr = &adapter->rx_rings[i];
 3443                 rxr->adapter = adapter;
 3444                 rxr->me = i;
 3445 
 3446                 /* Initialize the RX lock */
 3447                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
 3448                     device_get_nameunit(dev), txr->me);
 3449                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
 3450 
 3451                 if (igb_dma_malloc(adapter, rsize,
 3452                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
 3453                         device_printf(dev,
 3454                             "Unable to allocate RxDescriptor memory\n");
 3455                         error = ENOMEM;
 3456                         goto err_rx_desc;
 3457                 }
 3458                 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
 3459                 bzero((void *)rxr->rx_base, rsize);
 3460 
 3461                 /* Allocate receive buffers for the ring*/
 3462                 if (igb_allocate_receive_buffers(rxr)) {
 3463                         device_printf(dev,
 3464                             "Critical Failure setting up receive buffers\n");
 3465                         error = ENOMEM;
 3466                         goto err_rx_desc;
 3467                 }
 3468         }
 3469 
 3470         /*
 3471         ** Finally set up the queue holding structs
 3472         */
 3473         for (int i = 0; i < adapter->num_queues; i++) {
 3474                 que = &adapter->queues[i];
 3475                 que->adapter = adapter;
 3476                 que->txr = &adapter->tx_rings[i];
 3477                 que->rxr = &adapter->rx_rings[i];
 3478         }
 3479 
 3480         return (0);
 3481 
 3482 err_rx_desc:
 3483         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
 3484                 igb_dma_free(adapter, &rxr->rxdma);
 3485 err_tx_desc:
 3486         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
 3487                 igb_dma_free(adapter, &txr->txdma);
 3488         free(adapter->rx_rings, M_DEVBUF);
 3489 rx_fail:
 3490 #ifndef IGB_LEGACY_TX
 3491         buf_ring_free(txr->br, M_DEVBUF);
 3492 #endif
 3493         free(adapter->tx_rings, M_DEVBUF);
 3494 tx_fail:
 3495         free(adapter->queues, M_DEVBUF);
 3496 fail:
 3497         return (error);
 3498 }
 3499 
 3500 /*********************************************************************
 3501  *
 3502  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
 3503  *  the information needed to transmit a packet on the wire. This is
 3504  *  called only once at attach, setup is done every reset.
 3505  *
 3506  **********************************************************************/
 3507 static int
 3508 igb_allocate_transmit_buffers(struct tx_ring *txr)
 3509 {
 3510         struct adapter *adapter = txr->adapter;
 3511         device_t dev = adapter->dev;
 3512         struct igb_tx_buf *txbuf;
 3513         int error, i;
 3514 
 3515         /*
 3516          * Setup DMA descriptor areas.
 3517          */
 3518         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 3519                                1, 0,                    /* alignment, bounds */
 3520                                BUS_SPACE_MAXADDR,       /* lowaddr */
 3521                                BUS_SPACE_MAXADDR,       /* highaddr */
 3522                                NULL, NULL,              /* filter, filterarg */
 3523                                IGB_TSO_SIZE,            /* maxsize */
 3524                                IGB_MAX_SCATTER,         /* nsegments */
 3525                                PAGE_SIZE,               /* maxsegsize */
 3526                                0,                       /* flags */
 3527                                NULL,                    /* lockfunc */
 3528                                NULL,                    /* lockfuncarg */
 3529                                &txr->txtag))) {
 3530                 device_printf(dev,"Unable to allocate TX DMA tag\n");
 3531                 goto fail;
 3532         }
 3533 
 3534         if (!(txr->tx_buffers =
 3535             (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) *
 3536             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3537                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 3538                 error = ENOMEM;
 3539                 goto fail;
 3540         }
 3541 
 3542         /* Create the descriptor buffer dma maps */
 3543         txbuf = txr->tx_buffers;
 3544         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 3545                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
 3546                 if (error != 0) {
 3547                         device_printf(dev, "Unable to create TX DMA map\n");
 3548                         goto fail;
 3549                 }
 3550         }
 3551 
 3552         return 0;
 3553 fail:
 3554         /* We free all, it handles case where we are in the middle */
 3555         igb_free_transmit_structures(adapter);
 3556         return (error);
 3557 }
 3558 
 3559 /*********************************************************************
 3560  *
 3561  *  Initialize a transmit ring.
 3562  *
 3563  **********************************************************************/
 3564 static void
 3565 igb_setup_transmit_ring(struct tx_ring *txr)
 3566 {
 3567         struct adapter *adapter = txr->adapter;
 3568         struct igb_tx_buf *txbuf;
 3569         int i;
 3570 #ifdef DEV_NETMAP
 3571         struct netmap_adapter *na = NA(adapter->ifp);
 3572         struct netmap_slot *slot;
 3573 #endif /* DEV_NETMAP */
 3574 
 3575         /* Clear the old descriptor contents */
 3576         IGB_TX_LOCK(txr);
 3577 #ifdef DEV_NETMAP
 3578         slot = netmap_reset(na, NR_TX, txr->me, 0);
 3579 #endif /* DEV_NETMAP */
 3580         bzero((void *)txr->tx_base,
 3581               (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
 3582         /* Reset indices */
 3583         txr->next_avail_desc = 0;
 3584         txr->next_to_clean = 0;
 3585 
 3586         /* Free any existing tx buffers. */
 3587         txbuf = txr->tx_buffers;
 3588         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 3589                 if (txbuf->m_head != NULL) {
 3590                         bus_dmamap_sync(txr->txtag, txbuf->map,
 3591                             BUS_DMASYNC_POSTWRITE);
 3592                         bus_dmamap_unload(txr->txtag, txbuf->map);
 3593                         m_freem(txbuf->m_head);
 3594                         txbuf->m_head = NULL;
 3595                 }
 3596 #ifdef DEV_NETMAP
 3597                 if (slot) {
 3598                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
 3599                         /* no need to set the address */
 3600                         netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si));
 3601                 }
 3602 #endif /* DEV_NETMAP */
 3603                 /* clear the watch index */
 3604                 txbuf->eop = NULL;
 3605         }
 3606 
 3607         /* Set number of descriptors available */
 3608         txr->tx_avail = adapter->num_tx_desc;
 3609 
 3610         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3611             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 3612         IGB_TX_UNLOCK(txr);
 3613 }
 3614 
 3615 /*********************************************************************
 3616  *
 3617  *  Initialize all transmit rings.
 3618  *
 3619  **********************************************************************/
 3620 static void
 3621 igb_setup_transmit_structures(struct adapter *adapter)
 3622 {
 3623         struct tx_ring *txr = adapter->tx_rings;
 3624 
 3625         for (int i = 0; i < adapter->num_queues; i++, txr++)
 3626                 igb_setup_transmit_ring(txr);
 3627 
 3628         return;
 3629 }
 3630 
 3631 /*********************************************************************
 3632  *
 3633  *  Enable transmit unit.
 3634  *
 3635  **********************************************************************/
 3636 static void
 3637 igb_initialize_transmit_units(struct adapter *adapter)
 3638 {
 3639         struct tx_ring  *txr = adapter->tx_rings;
 3640         struct e1000_hw *hw = &adapter->hw;
 3641         u32             tctl, txdctl;
 3642 
 3643         INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
 3644         tctl = txdctl = 0;
 3645 
 3646         /* Setup the Tx Descriptor Rings */
 3647         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 3648                 u64 bus_addr = txr->txdma.dma_paddr;
 3649 
 3650                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
 3651                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
 3652                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
 3653                     (uint32_t)(bus_addr >> 32));
 3654                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
 3655                     (uint32_t)bus_addr);
 3656 
 3657                 /* Setup the HW Tx Head and Tail descriptor pointers */
 3658                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
 3659                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
 3660 
 3661                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
 3662                     E1000_READ_REG(hw, E1000_TDBAL(i)),
 3663                     E1000_READ_REG(hw, E1000_TDLEN(i)));
 3664 
 3665                 txr->queue_status = IGB_QUEUE_IDLE;
 3666 
 3667                 txdctl |= IGB_TX_PTHRESH;
 3668                 txdctl |= IGB_TX_HTHRESH << 8;
 3669                 txdctl |= IGB_TX_WTHRESH << 16;
 3670                 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
 3671                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
 3672         }
 3673 
 3674         if (adapter->vf_ifp)
 3675                 return;
 3676 
 3677         e1000_config_collision_dist(hw);
 3678 
 3679         /* Program the Transmit Control Register */
 3680         tctl = E1000_READ_REG(hw, E1000_TCTL);
 3681         tctl &= ~E1000_TCTL_CT;
 3682         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
 3683                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
 3684 
 3685         /* This write will effectively turn on the transmit unit. */
 3686         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
 3687 }
 3688 
 3689 /*********************************************************************
 3690  *
 3691  *  Free all transmit rings.
 3692  *
 3693  **********************************************************************/
 3694 static void
 3695 igb_free_transmit_structures(struct adapter *adapter)
 3696 {
 3697         struct tx_ring *txr = adapter->tx_rings;
 3698 
 3699         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 3700                 IGB_TX_LOCK(txr);
 3701                 igb_free_transmit_buffers(txr);
 3702                 igb_dma_free(adapter, &txr->txdma);
 3703                 IGB_TX_UNLOCK(txr);
 3704                 IGB_TX_LOCK_DESTROY(txr);
 3705         }
 3706         free(adapter->tx_rings, M_DEVBUF);
 3707 }
 3708 
 3709 /*********************************************************************
 3710  *
 3711  *  Free transmit ring related data structures.
 3712  *
 3713  **********************************************************************/
 3714 static void
 3715 igb_free_transmit_buffers(struct tx_ring *txr)
 3716 {
 3717         struct adapter *adapter = txr->adapter;
 3718         struct igb_tx_buf *tx_buffer;
 3719         int             i;
 3720 
 3721         INIT_DEBUGOUT("free_transmit_ring: begin");
 3722 
 3723         if (txr->tx_buffers == NULL)
 3724                 return;
 3725 
 3726         tx_buffer = txr->tx_buffers;
 3727         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
 3728                 if (tx_buffer->m_head != NULL) {
 3729                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
 3730                             BUS_DMASYNC_POSTWRITE);
 3731                         bus_dmamap_unload(txr->txtag,
 3732                             tx_buffer->map);
 3733                         m_freem(tx_buffer->m_head);
 3734                         tx_buffer->m_head = NULL;
 3735                         if (tx_buffer->map != NULL) {
 3736                                 bus_dmamap_destroy(txr->txtag,
 3737                                     tx_buffer->map);
 3738                                 tx_buffer->map = NULL;
 3739                         }
 3740                 } else if (tx_buffer->map != NULL) {
 3741                         bus_dmamap_unload(txr->txtag,
 3742                             tx_buffer->map);
 3743                         bus_dmamap_destroy(txr->txtag,
 3744                             tx_buffer->map);
 3745                         tx_buffer->map = NULL;
 3746                 }
 3747         }
 3748 #ifndef IGB_LEGACY_TX
 3749         if (txr->br != NULL)
 3750                 buf_ring_free(txr->br, M_DEVBUF);
 3751 #endif
 3752         if (txr->tx_buffers != NULL) {
 3753                 free(txr->tx_buffers, M_DEVBUF);
 3754                 txr->tx_buffers = NULL;
 3755         }
 3756         if (txr->txtag != NULL) {
 3757                 bus_dma_tag_destroy(txr->txtag);
 3758                 txr->txtag = NULL;
 3759         }
 3760         return;
 3761 }
 3762 
 3763 /**********************************************************************
 3764  *
 3765  *  Setup work for hardware segmentation offload (TSO) on
 3766  *  adapters using advanced tx descriptors
 3767  *
 3768  **********************************************************************/
 3769 static int
 3770 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp,
 3771     u32 *cmd_type_len, u32 *olinfo_status)
 3772 {
 3773         struct adapter *adapter = txr->adapter;
 3774         struct e1000_adv_tx_context_desc *TXD;
 3775         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 3776         u32 mss_l4len_idx = 0, paylen;
 3777         u16 vtag = 0, eh_type;
 3778         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
 3779         struct ether_vlan_header *eh;
 3780 #ifdef INET6
 3781         struct ip6_hdr *ip6;
 3782 #endif
 3783 #ifdef INET
 3784         struct ip *ip;
 3785 #endif
 3786         struct tcphdr *th;
 3787 
 3788 
 3789         /*
 3790          * Determine where frame payload starts.
 3791          * Jump over vlan headers if already present
 3792          */
 3793         eh = mtod(mp, struct ether_vlan_header *);
 3794         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 3795                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3796                 eh_type = eh->evl_proto;
 3797         } else {
 3798                 ehdrlen = ETHER_HDR_LEN;
 3799                 eh_type = eh->evl_encap_proto;
 3800         }
 3801 
 3802         switch (ntohs(eh_type)) {
 3803 #ifdef INET6
 3804         case ETHERTYPE_IPV6:
 3805                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3806                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
 3807                 if (ip6->ip6_nxt != IPPROTO_TCP)
 3808                         return (ENXIO);
 3809                 ip_hlen = sizeof(struct ip6_hdr);
 3810                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3811                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
 3812                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
 3813                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
 3814                 break;
 3815 #endif
 3816 #ifdef INET
 3817         case ETHERTYPE_IP:
 3818                 ip = (struct ip *)(mp->m_data + ehdrlen);
 3819                 if (ip->ip_p != IPPROTO_TCP)
 3820                         return (ENXIO);
 3821                 ip->ip_sum = 0;
 3822                 ip_hlen = ip->ip_hl << 2;
 3823                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 3824                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
 3825                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 3826                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3827                 /* Tell transmit desc to also do IPv4 checksum. */
 3828                 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
 3829                 break;
 3830 #endif
 3831         default:
 3832                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
 3833                     __func__, ntohs(eh_type));
 3834                 break;
 3835         }
 3836 
 3837         ctxd = txr->next_avail_desc;
 3838         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3839 
 3840         tcp_hlen = th->th_off << 2;
 3841 
 3842         /* This is used in the transmit desc in encap */
 3843         paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
 3844 
 3845         /* VLAN MACLEN IPLEN */
 3846         if (mp->m_flags & M_VLANTAG) {
 3847                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3848                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3849         }
 3850 
 3851         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
 3852         vlan_macip_lens |= ip_hlen;
 3853         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
 3854 
 3855         /* ADV DTYPE TUCMD */
 3856         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3857         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3858         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
 3859 
 3860         /* MSS L4LEN IDX */
 3861         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
 3862         mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
 3863         /* 82575 needs the queue index added */
 3864         if (adapter->hw.mac.type == e1000_82575)
 3865                 mss_l4len_idx |= txr->me << 4;
 3866         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 3867 
 3868         TXD->seqnum_seed = htole32(0);
 3869 
 3870         if (++ctxd == txr->num_desc)
 3871                 ctxd = 0;
 3872 
 3873         txr->tx_avail--;
 3874         txr->next_avail_desc = ctxd;
 3875         *cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
 3876         *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 3877         *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT;
 3878         ++txr->tso_tx;
 3879         return (0);
 3880 }
 3881 
 3882 /*********************************************************************
 3883  *
 3884  *  Advanced Context Descriptor setup for VLAN, CSUM or TSO
 3885  *
 3886  **********************************************************************/
 3887 
 3888 static int
 3889 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
 3890     u32 *cmd_type_len, u32 *olinfo_status)
 3891 {
 3892         struct e1000_adv_tx_context_desc *TXD;
 3893         struct adapter *adapter = txr->adapter;
 3894         struct ether_vlan_header *eh;
 3895         struct ip *ip;
 3896         struct ip6_hdr *ip6;
 3897         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
 3898         int     ehdrlen, ip_hlen = 0;
 3899         u16     etype;
 3900         u8      ipproto = 0;
 3901         int     offload = TRUE;
 3902         int     ctxd = txr->next_avail_desc;
 3903         u16     vtag = 0;
 3904 
 3905         /* First check if TSO is to be used */
 3906         if (mp->m_pkthdr.csum_flags & CSUM_TSO)
 3907                 return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status));
 3908 
 3909         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
 3910                 offload = FALSE;
 3911 
 3912         /* Indicate the whole packet as payload when not doing TSO */
 3913         *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
 3914 
 3915         /* Now ready a context descriptor */
 3916         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3917 
 3918         /*
 3919         ** In advanced descriptors the vlan tag must 
 3920         ** be placed into the context descriptor. Hence
 3921         ** we need to make one even if not doing offloads.
 3922         */
 3923         if (mp->m_flags & M_VLANTAG) {
 3924                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3925                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3926         } else if (offload == FALSE) /* ... no offload to do */
 3927                 return (0);
 3928 
 3929         /*
 3930          * Determine where frame payload starts.
 3931          * Jump over vlan headers if already present,
 3932          * helpful for QinQ too.
 3933          */
 3934         eh = mtod(mp, struct ether_vlan_header *);
 3935         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 3936                 etype = ntohs(eh->evl_proto);
 3937                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3938         } else {
 3939                 etype = ntohs(eh->evl_encap_proto);
 3940                 ehdrlen = ETHER_HDR_LEN;
 3941         }
 3942 
 3943         /* Set the ether header length */
 3944         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
 3945 
 3946         switch (etype) {
 3947                 case ETHERTYPE_IP:
 3948                         ip = (struct ip *)(mp->m_data + ehdrlen);
 3949                         ip_hlen = ip->ip_hl << 2;
 3950                         ipproto = ip->ip_p;
 3951                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3952                         break;
 3953                 case ETHERTYPE_IPV6:
 3954                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3955                         ip_hlen = sizeof(struct ip6_hdr);
 3956                         /* XXX-BZ this will go badly in case of ext hdrs. */
 3957                         ipproto = ip6->ip6_nxt;
 3958                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
 3959                         break;
 3960                 default:
 3961                         offload = FALSE;
 3962                         break;
 3963         }
 3964 
 3965         vlan_macip_lens |= ip_hlen;
 3966         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3967 
 3968         switch (ipproto) {
 3969                 case IPPROTO_TCP:
 3970 #if __FreeBSD_version >= 1000000
 3971                         if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))
 3972 #else
 3973                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
 3974 #endif
 3975                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3976                         break;
 3977                 case IPPROTO_UDP:
 3978 #if __FreeBSD_version >= 1000000
 3979                         if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP))
 3980 #else
 3981                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
 3982 #endif
 3983                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
 3984                         break;
 3985 
 3986 #if __FreeBSD_version >= 800000
 3987                 case IPPROTO_SCTP:
 3988 #if __FreeBSD_version >= 1000000
 3989                         if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP))
 3990 #else
 3991                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
 3992 #endif
 3993                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
 3994                         break;
 3995 #endif
 3996                 default:
 3997                         offload = FALSE;
 3998                         break;
 3999         }
 4000 
 4001         if (offload) /* For the TX descriptor setup */
 4002                 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 4003 
 4004         /* 82575 needs the queue index added */
 4005         if (adapter->hw.mac.type == e1000_82575)
 4006                 mss_l4len_idx = txr->me << 4;
 4007 
 4008         /* Now copy bits into descriptor */
 4009         TXD->vlan_macip_lens = htole32(vlan_macip_lens);
 4010         TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
 4011         TXD->seqnum_seed = htole32(0);
 4012         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 4013 
 4014         /* We've consumed the first desc, adjust counters */
 4015         if (++ctxd == txr->num_desc)
 4016                 ctxd = 0;
 4017         txr->next_avail_desc = ctxd;
 4018         --txr->tx_avail;
 4019 
 4020         return (0);
 4021 }
 4022 
 4023 /**********************************************************************
 4024  *
 4025  *  Examine each tx_buffer in the used queue. If the hardware is done
 4026  *  processing the packet then free associated resources. The
 4027  *  tx_buffer is put back on the free queue.
 4028  *
 4029  *  TRUE return means there's work in the ring to clean, FALSE its empty.
 4030  **********************************************************************/
 4031 static bool
 4032 igb_txeof(struct tx_ring *txr)
 4033 {
 4034         struct adapter          *adapter = txr->adapter;
 4035 #ifdef DEV_NETMAP
 4036         struct ifnet            *ifp = adapter->ifp;
 4037 #endif /* DEV_NETMAP */
 4038         u32                     work, processed = 0;
 4039         int                     limit = adapter->tx_process_limit;
 4040         struct igb_tx_buf       *buf;
 4041         union e1000_adv_tx_desc *txd;
 4042 
 4043         mtx_assert(&txr->tx_mtx, MA_OWNED);
 4044 
 4045 #ifdef DEV_NETMAP
 4046         if (netmap_tx_irq(ifp, txr->me))
 4047                 return (FALSE);
 4048 #endif /* DEV_NETMAP */
 4049 
 4050         if (txr->tx_avail == txr->num_desc) {
 4051                 txr->queue_status = IGB_QUEUE_IDLE;
 4052                 return FALSE;
 4053         }
 4054 
 4055         /* Get work starting point */
 4056         work = txr->next_to_clean;
 4057         buf = &txr->tx_buffers[work];
 4058         txd = &txr->tx_base[work];
 4059         work -= txr->num_desc; /* The distance to ring end */
 4060         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 4061             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 4062         do {
 4063                 union e1000_adv_tx_desc *eop = buf->eop;
 4064                 if (eop == NULL) /* No work */
 4065                         break;
 4066 
 4067                 if ((eop->wb.status & E1000_TXD_STAT_DD) == 0)
 4068                         break;  /* I/O not complete */
 4069 
 4070                 if (buf->m_head) {
 4071                         txr->bytes +=
 4072                             buf->m_head->m_pkthdr.len;
 4073                         bus_dmamap_sync(txr->txtag,
 4074                             buf->map,
 4075                             BUS_DMASYNC_POSTWRITE);
 4076                         bus_dmamap_unload(txr->txtag,
 4077                             buf->map);
 4078                         m_freem(buf->m_head);
 4079                         buf->m_head = NULL;
 4080                 }
 4081                 buf->eop = NULL;
 4082                 ++txr->tx_avail;
 4083 
 4084                 /* We clean the range if multi segment */
 4085                 while (txd != eop) {
 4086                         ++txd;
 4087                         ++buf;
 4088                         ++work;
 4089                         /* wrap the ring? */
 4090                         if (__predict_false(!work)) {
 4091                                 work -= txr->num_desc;
 4092                                 buf = txr->tx_buffers;
 4093                                 txd = txr->tx_base;
 4094                         }
 4095                         if (buf->m_head) {
 4096                                 txr->bytes +=
 4097                                     buf->m_head->m_pkthdr.len;
 4098                                 bus_dmamap_sync(txr->txtag,
 4099                                     buf->map,
 4100                                     BUS_DMASYNC_POSTWRITE);
 4101                                 bus_dmamap_unload(txr->txtag,
 4102                                     buf->map);
 4103                                 m_freem(buf->m_head);
 4104                                 buf->m_head = NULL;
 4105                         }
 4106                         ++txr->tx_avail;
 4107                         buf->eop = NULL;
 4108 
 4109                 }
 4110                 ++txr->packets;
 4111                 ++processed;
 4112                 txr->watchdog_time = ticks;
 4113 
 4114                 /* Try the next packet */
 4115                 ++txd;
 4116                 ++buf;
 4117                 ++work;
 4118                 /* reset with a wrap */
 4119                 if (__predict_false(!work)) {
 4120                         work -= txr->num_desc;
 4121                         buf = txr->tx_buffers;
 4122                         txd = txr->tx_base;
 4123                 }
 4124                 prefetch(txd);
 4125         } while (__predict_true(--limit));
 4126 
 4127         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 4128             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4129 
 4130         work += txr->num_desc;
 4131         txr->next_to_clean = work;
 4132 
 4133         /*
 4134         ** Watchdog calculation, we know there's
 4135         ** work outstanding or the first return
 4136         ** would have been taken, so none processed
 4137         ** for too long indicates a hang.
 4138         */
 4139         if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
 4140                 txr->queue_status |= IGB_QUEUE_HUNG;
 4141 
 4142         if (txr->tx_avail >= IGB_QUEUE_THRESHOLD)
 4143                 txr->queue_status &= ~IGB_QUEUE_DEPLETED;       
 4144 
 4145         if (txr->tx_avail == txr->num_desc) {
 4146                 txr->queue_status = IGB_QUEUE_IDLE;
 4147                 return (FALSE);
 4148         }
 4149 
 4150         return (TRUE);
 4151 }
 4152 
 4153 /*********************************************************************
 4154  *
 4155  *  Refresh mbuf buffers for RX descriptor rings
 4156  *   - now keeps its own state so discards due to resource
 4157  *     exhaustion are unnecessary, if an mbuf cannot be obtained
 4158  *     it just returns, keeping its placeholder, thus it can simply
 4159  *     be recalled to try again.
 4160  *
 4161  **********************************************************************/
 4162 static void
 4163 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
 4164 {
 4165         struct adapter          *adapter = rxr->adapter;
 4166         bus_dma_segment_t       hseg[1];
 4167         bus_dma_segment_t       pseg[1];
 4168         struct igb_rx_buf       *rxbuf;
 4169         struct mbuf             *mh, *mp;
 4170         int                     i, j, nsegs, error;
 4171         bool                    refreshed = FALSE;
 4172 
 4173         i = j = rxr->next_to_refresh;
 4174         /*
 4175         ** Get one descriptor beyond
 4176         ** our work mark to control
 4177         ** the loop.
 4178         */
 4179         if (++j == adapter->num_rx_desc)
 4180                 j = 0;
 4181 
 4182         while (j != limit) {
 4183                 rxbuf = &rxr->rx_buffers[i];
 4184                 /* No hdr mbuf used with header split off */
 4185                 if (rxr->hdr_split == FALSE)
 4186                         goto no_split;
 4187                 if (rxbuf->m_head == NULL) {
 4188                         mh = m_gethdr(M_NOWAIT, MT_DATA);
 4189                         if (mh == NULL)
 4190                                 goto update;
 4191                 } else
 4192                         mh = rxbuf->m_head;
 4193 
 4194                 mh->m_pkthdr.len = mh->m_len = MHLEN;
 4195                 mh->m_len = MHLEN;
 4196                 mh->m_flags |= M_PKTHDR;
 4197                 /* Get the memory mapping */
 4198                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
 4199                     rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
 4200                 if (error != 0) {
 4201                         printf("Refresh mbufs: hdr dmamap load"
 4202                             " failure - %d\n", error);
 4203                         m_free(mh);
 4204                         rxbuf->m_head = NULL;
 4205                         goto update;
 4206                 }
 4207                 rxbuf->m_head = mh;
 4208                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 4209                     BUS_DMASYNC_PREREAD);
 4210                 rxr->rx_base[i].read.hdr_addr =
 4211                     htole64(hseg[0].ds_addr);
 4212 no_split:
 4213                 if (rxbuf->m_pack == NULL) {
 4214                         mp = m_getjcl(M_NOWAIT, MT_DATA,
 4215                             M_PKTHDR, adapter->rx_mbuf_sz);
 4216                         if (mp == NULL)
 4217                                 goto update;
 4218                 } else
 4219                         mp = rxbuf->m_pack;
 4220 
 4221                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
 4222                 /* Get the memory mapping */
 4223                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
 4224                     rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
 4225                 if (error != 0) {
 4226                         printf("Refresh mbufs: payload dmamap load"
 4227                             " failure - %d\n", error);
 4228                         m_free(mp);
 4229                         rxbuf->m_pack = NULL;
 4230                         goto update;
 4231                 }
 4232                 rxbuf->m_pack = mp;
 4233                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 4234                     BUS_DMASYNC_PREREAD);
 4235                 rxr->rx_base[i].read.pkt_addr =
 4236                     htole64(pseg[0].ds_addr);
 4237                 refreshed = TRUE; /* I feel wefreshed :) */
 4238 
 4239                 i = j; /* our next is precalculated */
 4240                 rxr->next_to_refresh = i;
 4241                 if (++j == adapter->num_rx_desc)
 4242                         j = 0;
 4243         }
 4244 update:
 4245         if (refreshed) /* update tail */
 4246                 E1000_WRITE_REG(&adapter->hw,
 4247                     E1000_RDT(rxr->me), rxr->next_to_refresh);
 4248         return;
 4249 }
 4250 
 4251 
 4252 /*********************************************************************
 4253  *
 4254  *  Allocate memory for rx_buffer structures. Since we use one
 4255  *  rx_buffer per received packet, the maximum number of rx_buffer's
 4256  *  that we'll need is equal to the number of receive descriptors
 4257  *  that we've allocated.
 4258  *
 4259  **********************************************************************/
 4260 static int
 4261 igb_allocate_receive_buffers(struct rx_ring *rxr)
 4262 {
 4263         struct  adapter         *adapter = rxr->adapter;
 4264         device_t                dev = adapter->dev;
 4265         struct igb_rx_buf       *rxbuf;
 4266         int                     i, bsize, error;
 4267 
 4268         bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
 4269         if (!(rxr->rx_buffers =
 4270             (struct igb_rx_buf *) malloc(bsize,
 4271             M_DEVBUF, M_NOWAIT | M_ZERO))) {
 4272                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
 4273                 error = ENOMEM;
 4274                 goto fail;
 4275         }
 4276 
 4277         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 4278                                    1, 0,                /* alignment, bounds */
 4279                                    BUS_SPACE_MAXADDR,   /* lowaddr */
 4280                                    BUS_SPACE_MAXADDR,   /* highaddr */
 4281                                    NULL, NULL,          /* filter, filterarg */
 4282                                    MSIZE,               /* maxsize */
 4283                                    1,                   /* nsegments */
 4284                                    MSIZE,               /* maxsegsize */
 4285                                    0,                   /* flags */
 4286                                    NULL,                /* lockfunc */
 4287                                    NULL,                /* lockfuncarg */
 4288                                    &rxr->htag))) {
 4289                 device_printf(dev, "Unable to create RX DMA tag\n");
 4290                 goto fail;
 4291         }
 4292 
 4293         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 4294                                    1, 0,                /* alignment, bounds */
 4295                                    BUS_SPACE_MAXADDR,   /* lowaddr */
 4296                                    BUS_SPACE_MAXADDR,   /* highaddr */
 4297                                    NULL, NULL,          /* filter, filterarg */
 4298                                    MJUM9BYTES,          /* maxsize */
 4299                                    1,                   /* nsegments */
 4300                                    MJUM9BYTES,          /* maxsegsize */
 4301                                    0,                   /* flags */
 4302                                    NULL,                /* lockfunc */
 4303                                    NULL,                /* lockfuncarg */
 4304                                    &rxr->ptag))) {
 4305                 device_printf(dev, "Unable to create RX payload DMA tag\n");
 4306                 goto fail;
 4307         }
 4308 
 4309         for (i = 0; i < adapter->num_rx_desc; i++) {
 4310                 rxbuf = &rxr->rx_buffers[i];
 4311                 error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap);
 4312                 if (error) {
 4313                         device_printf(dev,
 4314                             "Unable to create RX head DMA maps\n");
 4315                         goto fail;
 4316                 }
 4317                 error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap);
 4318                 if (error) {
 4319                         device_printf(dev,
 4320                             "Unable to create RX packet DMA maps\n");
 4321                         goto fail;
 4322                 }
 4323         }
 4324 
 4325         return (0);
 4326 
 4327 fail:
 4328         /* Frees all, but can handle partial completion */
 4329         igb_free_receive_structures(adapter);
 4330         return (error);
 4331 }
 4332 
 4333 
 4334 static void
 4335 igb_free_receive_ring(struct rx_ring *rxr)
 4336 {
 4337         struct  adapter         *adapter = rxr->adapter;
 4338         struct igb_rx_buf       *rxbuf;
 4339 
 4340 
 4341         for (int i = 0; i < adapter->num_rx_desc; i++) {
 4342                 rxbuf = &rxr->rx_buffers[i];
 4343                 if (rxbuf->m_head != NULL) {
 4344                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 4345                             BUS_DMASYNC_POSTREAD);
 4346                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 4347                         rxbuf->m_head->m_flags |= M_PKTHDR;
 4348                         m_freem(rxbuf->m_head);
 4349                 }
 4350                 if (rxbuf->m_pack != NULL) {
 4351                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 4352                             BUS_DMASYNC_POSTREAD);
 4353                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 4354                         rxbuf->m_pack->m_flags |= M_PKTHDR;
 4355                         m_freem(rxbuf->m_pack);
 4356                 }
 4357                 rxbuf->m_head = NULL;
 4358                 rxbuf->m_pack = NULL;
 4359         }
 4360 }
 4361 
 4362 
 4363 /*********************************************************************
 4364  *
 4365  *  Initialize a receive ring and its buffers.
 4366  *
 4367  **********************************************************************/
 4368 static int
 4369 igb_setup_receive_ring(struct rx_ring *rxr)
 4370 {
 4371         struct  adapter         *adapter;
 4372         struct  ifnet           *ifp;
 4373         device_t                dev;
 4374         struct igb_rx_buf       *rxbuf;
 4375         bus_dma_segment_t       pseg[1], hseg[1];
 4376         struct lro_ctrl         *lro = &rxr->lro;
 4377         int                     rsize, nsegs, error = 0;
 4378 #ifdef DEV_NETMAP
 4379         struct netmap_adapter *na = NA(rxr->adapter->ifp);
 4380         struct netmap_slot *slot;
 4381 #endif /* DEV_NETMAP */
 4382 
 4383         adapter = rxr->adapter;
 4384         dev = adapter->dev;
 4385         ifp = adapter->ifp;
 4386 
 4387         /* Clear the ring contents */
 4388         IGB_RX_LOCK(rxr);
 4389 #ifdef DEV_NETMAP
 4390         slot = netmap_reset(na, NR_RX, rxr->me, 0);
 4391 #endif /* DEV_NETMAP */
 4392         rsize = roundup2(adapter->num_rx_desc *
 4393             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 4394         bzero((void *)rxr->rx_base, rsize);
 4395 
 4396         /*
 4397         ** Free current RX buffer structures and their mbufs
 4398         */
 4399         igb_free_receive_ring(rxr);
 4400 
 4401         /* Configure for header split? */
 4402         if (igb_header_split)
 4403                 rxr->hdr_split = TRUE;
 4404 
 4405         /* Now replenish the ring mbufs */
 4406         for (int j = 0; j < adapter->num_rx_desc; ++j) {
 4407                 struct mbuf     *mh, *mp;
 4408 
 4409                 rxbuf = &rxr->rx_buffers[j];
 4410 #ifdef DEV_NETMAP
 4411                 if (slot) {
 4412                         /* slot sj is mapped to the j-th NIC-ring entry */
 4413                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
 4414                         uint64_t paddr;
 4415                         void *addr;
 4416 
 4417                         addr = PNMB(na, slot + sj, &paddr);
 4418                         netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr);
 4419                         /* Update descriptor */
 4420                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
 4421                         continue;
 4422                 }
 4423 #endif /* DEV_NETMAP */
 4424                 if (rxr->hdr_split == FALSE)
 4425                         goto skip_head;
 4426 
 4427                 /* First the header */
 4428                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
 4429                 if (rxbuf->m_head == NULL) {
 4430                         error = ENOBUFS;
 4431                         goto fail;
 4432                 }
 4433                 m_adj(rxbuf->m_head, ETHER_ALIGN);
 4434                 mh = rxbuf->m_head;
 4435                 mh->m_len = mh->m_pkthdr.len = MHLEN;
 4436                 mh->m_flags |= M_PKTHDR;
 4437                 /* Get the memory mapping */
 4438                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
 4439                     rxbuf->hmap, rxbuf->m_head, hseg,
 4440                     &nsegs, BUS_DMA_NOWAIT);
 4441                 if (error != 0) /* Nothing elegant to do here */
 4442                         goto fail;
 4443                 bus_dmamap_sync(rxr->htag,
 4444                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
 4445                 /* Update descriptor */
 4446                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
 4447 
 4448 skip_head:
 4449                 /* Now the payload cluster */
 4450                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
 4451                     M_PKTHDR, adapter->rx_mbuf_sz);
 4452                 if (rxbuf->m_pack == NULL) {
 4453                         error = ENOBUFS;
 4454                         goto fail;
 4455                 }
 4456                 mp = rxbuf->m_pack;
 4457                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
 4458                 /* Get the memory mapping */
 4459                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
 4460                     rxbuf->pmap, mp, pseg,
 4461                     &nsegs, BUS_DMA_NOWAIT);
 4462                 if (error != 0)
 4463                         goto fail;
 4464                 bus_dmamap_sync(rxr->ptag,
 4465                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
 4466                 /* Update descriptor */
 4467                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
 4468         }
 4469 
 4470         /* Setup our descriptor indices */
 4471         rxr->next_to_check = 0;
 4472         rxr->next_to_refresh = adapter->num_rx_desc - 1;
 4473         rxr->lro_enabled = FALSE;
 4474         rxr->rx_split_packets = 0;
 4475         rxr->rx_bytes = 0;
 4476 
 4477         rxr->fmp = NULL;
 4478         rxr->lmp = NULL;
 4479 
 4480         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4481             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4482 
 4483         /*
 4484         ** Now set up the LRO interface, we
 4485         ** also only do head split when LRO
 4486         ** is enabled, since so often they
 4487         ** are undesirable in similar setups.
 4488         */
 4489         if (ifp->if_capenable & IFCAP_LRO) {
 4490                 error = tcp_lro_init(lro);
 4491                 if (error) {
 4492                         device_printf(dev, "LRO Initialization failed!\n");
 4493                         goto fail;
 4494                 }
 4495                 INIT_DEBUGOUT("RX LRO Initialized\n");
 4496                 rxr->lro_enabled = TRUE;
 4497                 lro->ifp = adapter->ifp;
 4498         }
 4499 
 4500         IGB_RX_UNLOCK(rxr);
 4501         return (0);
 4502 
 4503 fail:
 4504         igb_free_receive_ring(rxr);
 4505         IGB_RX_UNLOCK(rxr);
 4506         return (error);
 4507 }
 4508 
 4509 
 4510 /*********************************************************************
 4511  *
 4512  *  Initialize all receive rings.
 4513  *
 4514  **********************************************************************/
 4515 static int
 4516 igb_setup_receive_structures(struct adapter *adapter)
 4517 {
 4518         struct rx_ring *rxr = adapter->rx_rings;
 4519         int i;
 4520 
 4521         for (i = 0; i < adapter->num_queues; i++, rxr++)
 4522                 if (igb_setup_receive_ring(rxr))
 4523                         goto fail;
 4524 
 4525         return (0);
 4526 fail:
 4527         /*
 4528          * Free RX buffers allocated so far, we will only handle
 4529          * the rings that completed, the failing case will have
 4530          * cleaned up for itself. 'i' is the endpoint.
 4531          */
 4532         for (int j = 0; j < i; ++j) {
 4533                 rxr = &adapter->rx_rings[j];
 4534                 IGB_RX_LOCK(rxr);
 4535                 igb_free_receive_ring(rxr);
 4536                 IGB_RX_UNLOCK(rxr);
 4537         }
 4538 
 4539         return (ENOBUFS);
 4540 }
 4541 
 4542 /*
 4543  * Initialise the RSS mapping for NICs that support multiple transmit/
 4544  * receive rings.
 4545  */
 4546 static void
 4547 igb_initialise_rss_mapping(struct adapter *adapter)
 4548 {
 4549         struct e1000_hw *hw = &adapter->hw;
 4550         int i;
 4551         int queue_id;
 4552         u32 reta;
 4553         u32 rss_key[10], mrqc, shift = 0;
 4554 
 4555         /* XXX? */
 4556         if (adapter->hw.mac.type == e1000_82575)
 4557                 shift = 6;
 4558 
 4559         /*
 4560          * The redirection table controls which destination
 4561          * queue each bucket redirects traffic to.
 4562          * Each DWORD represents four queues, with the LSB
 4563          * being the first queue in the DWORD.
 4564          *
 4565          * This just allocates buckets to queues using round-robin
 4566          * allocation.
 4567          *
 4568          * NOTE: It Just Happens to line up with the default
 4569          * RSS allocation method.
 4570          */
 4571 
 4572         /* Warning FM follows */
 4573         reta = 0;
 4574         for (i = 0; i < 128; i++) {
 4575 #ifdef  RSS
 4576                 queue_id = rss_get_indirection_to_bucket(i);
 4577                 /*
 4578                  * If we have more queues than buckets, we'll
 4579                  * end up mapping buckets to a subset of the
 4580                  * queues.
 4581                  *
 4582                  * If we have more buckets than queues, we'll
 4583                  * end up instead assigning multiple buckets
 4584                  * to queues.
 4585                  *
 4586                  * Both are suboptimal, but we need to handle
 4587                  * the case so we don't go out of bounds
 4588                  * indexing arrays and such.
 4589                  */
 4590                 queue_id = queue_id % adapter->num_queues;
 4591 #else
 4592                 queue_id = (i % adapter->num_queues);
 4593 #endif
 4594                 /* Adjust if required */
 4595                 queue_id = queue_id << shift;
 4596 
 4597                 /*
 4598                  * The low 8 bits are for hash value (n+0);
 4599                  * The next 8 bits are for hash value (n+1), etc.
 4600                  */
 4601                 reta = reta >> 8;
 4602                 reta = reta | ( ((uint32_t) queue_id) << 24);
 4603                 if ((i & 3) == 3) {
 4604                         E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
 4605                         reta = 0;
 4606                 }
 4607         }
 4608 
 4609         /* Now fill in hash table */
 4610 
 4611         /*
 4612          * MRQC: Multiple Receive Queues Command
 4613          * Set queuing to RSS control, number depends on the device.
 4614          */
 4615         mrqc = E1000_MRQC_ENABLE_RSS_8Q;
 4616 
 4617 #ifdef  RSS
 4618         /* XXX ew typecasting */
 4619         rss_getkey((uint8_t *) &rss_key);
 4620 #else
 4621         arc4rand(&rss_key, sizeof(rss_key), 0);
 4622 #endif
 4623         for (i = 0; i < 10; i++)
 4624                 E1000_WRITE_REG_ARRAY(hw,
 4625                     E1000_RSSRK(0), i, rss_key[i]);
 4626 
 4627         /*
 4628          * Configure the RSS fields to hash upon.
 4629          */
 4630         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
 4631             E1000_MRQC_RSS_FIELD_IPV4_TCP);
 4632         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
 4633             E1000_MRQC_RSS_FIELD_IPV6_TCP);
 4634         mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
 4635             E1000_MRQC_RSS_FIELD_IPV6_UDP);
 4636         mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
 4637             E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
 4638 
 4639         E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
 4640 }
 4641 
 4642 /*********************************************************************
 4643  *
 4644  *  Enable receive unit.
 4645  *
 4646  **********************************************************************/
 4647 static void
 4648 igb_initialize_receive_units(struct adapter *adapter)
 4649 {
 4650         struct rx_ring  *rxr = adapter->rx_rings;
 4651         struct ifnet    *ifp = adapter->ifp;
 4652         struct e1000_hw *hw = &adapter->hw;
 4653         u32             rctl, rxcsum, psize, srrctl = 0;
 4654 
 4655         INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
 4656 
 4657         /*
 4658          * Make sure receives are disabled while setting
 4659          * up the descriptor ring
 4660          */
 4661         rctl = E1000_READ_REG(hw, E1000_RCTL);
 4662         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
 4663 
 4664         /*
 4665         ** Set up for header split
 4666         */
 4667         if (igb_header_split) {
 4668                 /* Use a standard mbuf for the header */
 4669                 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
 4670                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
 4671         } else
 4672                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
 4673 
 4674         /*
 4675         ** Set up for jumbo frames
 4676         */
 4677         if (ifp->if_mtu > ETHERMTU) {
 4678                 rctl |= E1000_RCTL_LPE;
 4679                 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
 4680                         srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4681                         rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
 4682                 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
 4683                         srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4684                         rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
 4685                 }
 4686                 /* Set maximum packet len */
 4687                 psize = adapter->max_frame_size;
 4688                 /* are we on a vlan? */
 4689                 if (adapter->ifp->if_vlantrunk != NULL)
 4690                         psize += VLAN_TAG_SIZE;
 4691                 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
 4692         } else {
 4693                 rctl &= ~E1000_RCTL_LPE;
 4694                 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4695                 rctl |= E1000_RCTL_SZ_2048;
 4696         }
 4697 
 4698         /*
 4699          * If TX flow control is disabled and there's >1 queue defined,
 4700          * enable DROP.
 4701          *
 4702          * This drops frames rather than hanging the RX MAC for all queues.
 4703          */
 4704         if ((adapter->num_queues > 1) &&
 4705             (adapter->fc == e1000_fc_none ||
 4706              adapter->fc == e1000_fc_rx_pause)) {
 4707                 srrctl |= E1000_SRRCTL_DROP_EN;
 4708         }
 4709 
 4710         /* Setup the Base and Length of the Rx Descriptor Rings */
 4711         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 4712                 u64 bus_addr = rxr->rxdma.dma_paddr;
 4713                 u32 rxdctl;
 4714 
 4715                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
 4716                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
 4717                 E1000_WRITE_REG(hw, E1000_RDBAH(i),
 4718                     (uint32_t)(bus_addr >> 32));
 4719                 E1000_WRITE_REG(hw, E1000_RDBAL(i),
 4720                     (uint32_t)bus_addr);
 4721                 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
 4722                 /* Enable this Queue */
 4723                 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
 4724                 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
 4725                 rxdctl &= 0xFFF00000;
 4726                 rxdctl |= IGB_RX_PTHRESH;
 4727                 rxdctl |= IGB_RX_HTHRESH << 8;
 4728                 rxdctl |= IGB_RX_WTHRESH << 16;
 4729                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
 4730         }
 4731 
 4732         /*
 4733         ** Setup for RX MultiQueue
 4734         */
 4735         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
 4736         if (adapter->num_queues >1) {
 4737 
 4738                 /* rss setup */
 4739                 igb_initialise_rss_mapping(adapter);
 4740 
 4741                 /*
 4742                 ** NOTE: Receive Full-Packet Checksum Offload 
 4743                 ** is mutually exclusive with Multiqueue. However
 4744                 ** this is not the same as TCP/IP checksums which
 4745                 ** still work.
 4746                 */
 4747                 rxcsum |= E1000_RXCSUM_PCSD;
 4748 #if __FreeBSD_version >= 800000
 4749                 /* For SCTP Offload */
 4750                 if ((hw->mac.type != e1000_82575) &&
 4751                     (ifp->if_capenable & IFCAP_RXCSUM))
 4752                         rxcsum |= E1000_RXCSUM_CRCOFL;
 4753 #endif
 4754         } else {
 4755                 /* Non RSS setup */
 4756                 if (ifp->if_capenable & IFCAP_RXCSUM) {
 4757                         rxcsum |= E1000_RXCSUM_IPPCSE;
 4758 #if __FreeBSD_version >= 800000
 4759                         if (adapter->hw.mac.type != e1000_82575)
 4760                                 rxcsum |= E1000_RXCSUM_CRCOFL;
 4761 #endif
 4762                 } else
 4763                         rxcsum &= ~E1000_RXCSUM_TUOFL;
 4764         }
 4765         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
 4766 
 4767         /* Setup the Receive Control Register */
 4768         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
 4769         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
 4770                    E1000_RCTL_RDMTS_HALF |
 4771                    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
 4772         /* Strip CRC bytes. */
 4773         rctl |= E1000_RCTL_SECRC;
 4774         /* Make sure VLAN Filters are off */
 4775         rctl &= ~E1000_RCTL_VFE;
 4776         /* Don't store bad packets */
 4777         rctl &= ~E1000_RCTL_SBP;
 4778 
 4779         /* Enable Receives */
 4780         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 4781 
 4782         /*
 4783          * Setup the HW Rx Head and Tail Descriptor Pointers
 4784          *   - needs to be after enable
 4785          */
 4786         for (int i = 0; i < adapter->num_queues; i++) {
 4787                 rxr = &adapter->rx_rings[i];
 4788                 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
 4789 #ifdef DEV_NETMAP
 4790                 /*
 4791                  * an init() while a netmap client is active must
 4792                  * preserve the rx buffers passed to userspace.
 4793                  * In this driver it means we adjust RDT to
 4794                  * something different from next_to_refresh
 4795                  * (which is not used in netmap mode).
 4796                  */
 4797                 if (ifp->if_capenable & IFCAP_NETMAP) {
 4798                         struct netmap_adapter *na = NA(adapter->ifp);
 4799                         struct netmap_kring *kring = &na->rx_rings[i];
 4800                         int t = rxr->next_to_refresh - nm_kr_rxspace(kring);
 4801 
 4802                         if (t >= adapter->num_rx_desc)
 4803                                 t -= adapter->num_rx_desc;
 4804                         else if (t < 0)
 4805                                 t += adapter->num_rx_desc;
 4806                         E1000_WRITE_REG(hw, E1000_RDT(i), t);
 4807                 } else
 4808 #endif /* DEV_NETMAP */
 4809                 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
 4810         }
 4811         return;
 4812 }
 4813 
 4814 /*********************************************************************
 4815  *
 4816  *  Free receive rings.
 4817  *
 4818  **********************************************************************/
 4819 static void
 4820 igb_free_receive_structures(struct adapter *adapter)
 4821 {
 4822         struct rx_ring *rxr = adapter->rx_rings;
 4823 
 4824         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 4825                 struct lro_ctrl *lro = &rxr->lro;
 4826                 igb_free_receive_buffers(rxr);
 4827                 tcp_lro_free(lro);
 4828                 igb_dma_free(adapter, &rxr->rxdma);
 4829         }
 4830 
 4831         free(adapter->rx_rings, M_DEVBUF);
 4832 }
 4833 
 4834 /*********************************************************************
 4835  *
 4836  *  Free receive ring data structures.
 4837  *
 4838  **********************************************************************/
 4839 static void
 4840 igb_free_receive_buffers(struct rx_ring *rxr)
 4841 {
 4842         struct adapter          *adapter = rxr->adapter;
 4843         struct igb_rx_buf       *rxbuf;
 4844         int i;
 4845 
 4846         INIT_DEBUGOUT("free_receive_structures: begin");
 4847 
 4848         /* Cleanup any existing buffers */
 4849         if (rxr->rx_buffers != NULL) {
 4850                 for (i = 0; i < adapter->num_rx_desc; i++) {
 4851                         rxbuf = &rxr->rx_buffers[i];
 4852                         if (rxbuf->m_head != NULL) {
 4853                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 4854                                     BUS_DMASYNC_POSTREAD);
 4855                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 4856                                 rxbuf->m_head->m_flags |= M_PKTHDR;
 4857                                 m_freem(rxbuf->m_head);
 4858                         }
 4859                         if (rxbuf->m_pack != NULL) {
 4860                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 4861                                     BUS_DMASYNC_POSTREAD);
 4862                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 4863                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
 4864                                 m_freem(rxbuf->m_pack);
 4865                         }
 4866                         rxbuf->m_head = NULL;
 4867                         rxbuf->m_pack = NULL;
 4868                         if (rxbuf->hmap != NULL) {
 4869                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
 4870                                 rxbuf->hmap = NULL;
 4871                         }
 4872                         if (rxbuf->pmap != NULL) {
 4873                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
 4874                                 rxbuf->pmap = NULL;
 4875                         }
 4876                 }
 4877                 if (rxr->rx_buffers != NULL) {
 4878                         free(rxr->rx_buffers, M_DEVBUF);
 4879                         rxr->rx_buffers = NULL;
 4880                 }
 4881         }
 4882 
 4883         if (rxr->htag != NULL) {
 4884                 bus_dma_tag_destroy(rxr->htag);
 4885                 rxr->htag = NULL;
 4886         }
 4887         if (rxr->ptag != NULL) {
 4888                 bus_dma_tag_destroy(rxr->ptag);
 4889                 rxr->ptag = NULL;
 4890         }
 4891 }
 4892 
 4893 static __inline void
 4894 igb_rx_discard(struct rx_ring *rxr, int i)
 4895 {
 4896         struct igb_rx_buf       *rbuf;
 4897 
 4898         rbuf = &rxr->rx_buffers[i];
 4899 
 4900         /* Partially received? Free the chain */
 4901         if (rxr->fmp != NULL) {
 4902                 rxr->fmp->m_flags |= M_PKTHDR;
 4903                 m_freem(rxr->fmp);
 4904                 rxr->fmp = NULL;
 4905                 rxr->lmp = NULL;
 4906         }
 4907 
 4908         /*
 4909         ** With advanced descriptors the writeback
 4910         ** clobbers the buffer addrs, so its easier
 4911         ** to just free the existing mbufs and take
 4912         ** the normal refresh path to get new buffers
 4913         ** and mapping.
 4914         */
 4915         if (rbuf->m_head) {
 4916                 m_free(rbuf->m_head);
 4917                 rbuf->m_head = NULL;
 4918                 bus_dmamap_unload(rxr->htag, rbuf->hmap);
 4919         }
 4920 
 4921         if (rbuf->m_pack) {
 4922                 m_free(rbuf->m_pack);
 4923                 rbuf->m_pack = NULL;
 4924                 bus_dmamap_unload(rxr->ptag, rbuf->pmap);
 4925         }
 4926 
 4927         return;
 4928 }
 4929 
 4930 static __inline void
 4931 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
 4932 {
 4933 
 4934         /*
 4935          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
 4936          * should be computed by hardware. Also it should not have VLAN tag in
 4937          * ethernet header.
 4938          */
 4939         if (rxr->lro_enabled &&
 4940             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
 4941             (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
 4942             (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
 4943             (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
 4944             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 
 4945             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
 4946                 /*
 4947                  * Send to the stack if:
 4948                  **  - LRO not enabled, or
 4949                  **  - no LRO resources, or
 4950                  **  - lro enqueue fails
 4951                  */
 4952                 if (rxr->lro.lro_cnt != 0)
 4953                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
 4954                                 return;
 4955         }
 4956         IGB_RX_UNLOCK(rxr);
 4957         (*ifp->if_input)(ifp, m);
 4958         IGB_RX_LOCK(rxr);
 4959 }
 4960 
 4961 /*********************************************************************
 4962  *
 4963  *  This routine executes in interrupt context. It replenishes
 4964  *  the mbufs in the descriptor and sends data which has been
 4965  *  dma'ed into host memory to upper layer.
 4966  *
 4967  *  We loop at most count times if count is > 0, or until done if
 4968  *  count < 0.
 4969  *
 4970  *  Return TRUE if more to clean, FALSE otherwise
 4971  *********************************************************************/
 4972 static bool
 4973 igb_rxeof(struct igb_queue *que, int count, int *done)
 4974 {
 4975         struct adapter          *adapter = que->adapter;
 4976         struct rx_ring          *rxr = que->rxr;
 4977         struct ifnet            *ifp = adapter->ifp;
 4978         struct lro_ctrl         *lro = &rxr->lro;
 4979         int                     i, processed = 0, rxdone = 0;
 4980         u32                     ptype, staterr = 0;
 4981         union e1000_adv_rx_desc *cur;
 4982 
 4983         IGB_RX_LOCK(rxr);
 4984         /* Sync the ring. */
 4985         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4986             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 4987 
 4988 #ifdef DEV_NETMAP
 4989         if (netmap_rx_irq(ifp, rxr->me, &processed)) {
 4990                 IGB_RX_UNLOCK(rxr);
 4991                 return (FALSE);
 4992         }
 4993 #endif /* DEV_NETMAP */
 4994 
 4995         /* Main clean loop */
 4996         for (i = rxr->next_to_check; count != 0;) {
 4997                 struct mbuf             *sendmp, *mh, *mp;
 4998                 struct igb_rx_buf       *rxbuf;
 4999                 u16                     hlen, plen, hdr, vtag, pkt_info;
 5000                 bool                    eop = FALSE;
 5001  
 5002                 cur = &rxr->rx_base[i];
 5003                 staterr = le32toh(cur->wb.upper.status_error);
 5004                 if ((staterr & E1000_RXD_STAT_DD) == 0)
 5005                         break;
 5006                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 5007                         break;
 5008                 count--;
 5009                 sendmp = mh = mp = NULL;
 5010                 cur->wb.upper.status_error = 0;
 5011                 rxbuf = &rxr->rx_buffers[i];
 5012                 plen = le16toh(cur->wb.upper.length);
 5013                 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
 5014                 if (((adapter->hw.mac.type == e1000_i350) ||
 5015                     (adapter->hw.mac.type == e1000_i354)) &&
 5016                     (staterr & E1000_RXDEXT_STATERR_LB))
 5017                         vtag = be16toh(cur->wb.upper.vlan);
 5018                 else
 5019                         vtag = le16toh(cur->wb.upper.vlan);
 5020                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
 5021                 pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info);
 5022                 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
 5023 
 5024                 /*
 5025                  * Free the frame (all segments) if we're at EOP and
 5026                  * it's an error.
 5027                  *
 5028                  * The datasheet states that EOP + status is only valid for
 5029                  * the final segment in a multi-segment frame.
 5030                  */
 5031                 if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) {
 5032                         adapter->dropped_pkts++;
 5033                         ++rxr->rx_discarded;
 5034                         igb_rx_discard(rxr, i);
 5035                         goto next_desc;
 5036                 }
 5037 
 5038                 /*
 5039                 ** The way the hardware is configured to
 5040                 ** split, it will ONLY use the header buffer
 5041                 ** when header split is enabled, otherwise we
 5042                 ** get normal behavior, ie, both header and
 5043                 ** payload are DMA'd into the payload buffer.
 5044                 **
 5045                 ** The fmp test is to catch the case where a
 5046                 ** packet spans multiple descriptors, in that
 5047                 ** case only the first header is valid.
 5048                 */
 5049                 if (rxr->hdr_split && rxr->fmp == NULL) {
 5050                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 5051                         hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
 5052                             E1000_RXDADV_HDRBUFLEN_SHIFT;
 5053                         if (hlen > IGB_HDR_BUF)
 5054                                 hlen = IGB_HDR_BUF;
 5055                         mh = rxr->rx_buffers[i].m_head;
 5056                         mh->m_len = hlen;
 5057                         /* clear buf pointer for refresh */
 5058                         rxbuf->m_head = NULL;
 5059                         /*
 5060                         ** Get the payload length, this
 5061                         ** could be zero if its a small
 5062                         ** packet.
 5063                         */
 5064                         if (plen > 0) {
 5065                                 mp = rxr->rx_buffers[i].m_pack;
 5066                                 mp->m_len = plen;
 5067                                 mh->m_next = mp;
 5068                                 /* clear buf pointer */
 5069                                 rxbuf->m_pack = NULL;
 5070                                 rxr->rx_split_packets++;
 5071                         }
 5072                 } else {
 5073                         /*
 5074                         ** Either no header split, or a
 5075                         ** secondary piece of a fragmented
 5076                         ** split packet.
 5077                         */
 5078                         mh = rxr->rx_buffers[i].m_pack;
 5079                         mh->m_len = plen;
 5080                         /* clear buf info for refresh */
 5081                         rxbuf->m_pack = NULL;
 5082                 }
 5083                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 5084 
 5085                 ++processed; /* So we know when to refresh */
 5086 
 5087                 /* Initial frame - setup */
 5088                 if (rxr->fmp == NULL) {
 5089                         mh->m_pkthdr.len = mh->m_len;
 5090                         /* Save the head of the chain */
 5091                         rxr->fmp = mh;
 5092                         rxr->lmp = mh;
 5093                         if (mp != NULL) {
 5094                                 /* Add payload if split */
 5095                                 mh->m_pkthdr.len += mp->m_len;
 5096                                 rxr->lmp = mh->m_next;
 5097                         }
 5098                 } else {
 5099                         /* Chain mbuf's together */
 5100                         rxr->lmp->m_next = mh;
 5101                         rxr->lmp = rxr->lmp->m_next;
 5102                         rxr->fmp->m_pkthdr.len += mh->m_len;
 5103                 }
 5104 
 5105                 if (eop) {
 5106                         rxr->fmp->m_pkthdr.rcvif = ifp;
 5107                         rxr->rx_packets++;
 5108                         /* capture data for AIM */
 5109                         rxr->packets++;
 5110                         rxr->bytes += rxr->fmp->m_pkthdr.len;
 5111                         rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
 5112 
 5113                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
 5114                                 igb_rx_checksum(staterr, rxr->fmp, ptype);
 5115 
 5116                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
 5117                             (staterr & E1000_RXD_STAT_VP) != 0) {
 5118                                 rxr->fmp->m_pkthdr.ether_vtag = vtag;
 5119                                 rxr->fmp->m_flags |= M_VLANTAG;
 5120                         }
 5121 
 5122                         /*
 5123                          * In case of multiqueue, we have RXCSUM.PCSD bit set
 5124                          * and never cleared. This means we have RSS hash
 5125                          * available to be used.
 5126                          */
 5127                         if (adapter->num_queues > 1) {
 5128                                 rxr->fmp->m_pkthdr.flowid = 
 5129                                     le32toh(cur->wb.lower.hi_dword.rss);
 5130                                 switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
 5131                                         case E1000_RXDADV_RSSTYPE_IPV4_TCP:
 5132                                                 M_HASHTYPE_SET(rxr->fmp,
 5133                                                     M_HASHTYPE_RSS_TCP_IPV4);
 5134                                         break;
 5135                                         case E1000_RXDADV_RSSTYPE_IPV4:
 5136                                                 M_HASHTYPE_SET(rxr->fmp,
 5137                                                     M_HASHTYPE_RSS_IPV4);
 5138                                         break;
 5139                                         case E1000_RXDADV_RSSTYPE_IPV6_TCP:
 5140                                                 M_HASHTYPE_SET(rxr->fmp,
 5141                                                     M_HASHTYPE_RSS_TCP_IPV6);
 5142                                         break;
 5143                                         case E1000_RXDADV_RSSTYPE_IPV6_EX:
 5144                                                 M_HASHTYPE_SET(rxr->fmp,
 5145                                                     M_HASHTYPE_RSS_IPV6_EX);
 5146                                         break;
 5147                                         case E1000_RXDADV_RSSTYPE_IPV6:
 5148                                                 M_HASHTYPE_SET(rxr->fmp,
 5149                                                     M_HASHTYPE_RSS_IPV6);
 5150                                         break;
 5151                                         case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
 5152                                                 M_HASHTYPE_SET(rxr->fmp,
 5153                                                     M_HASHTYPE_RSS_TCP_IPV6_EX);
 5154                                         break;
 5155                                         default:
 5156                                                 /* XXX fallthrough */
 5157                                                 M_HASHTYPE_SET(rxr->fmp,
 5158                                                     M_HASHTYPE_OPAQUE_HASH);
 5159                                 }
 5160                         } else {
 5161 #ifndef IGB_LEGACY_TX
 5162                                 rxr->fmp->m_pkthdr.flowid = que->msix;
 5163                                 M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE);
 5164 #endif
 5165                         }
 5166                         sendmp = rxr->fmp;
 5167                         /* Make sure to set M_PKTHDR. */
 5168                         sendmp->m_flags |= M_PKTHDR;
 5169                         rxr->fmp = NULL;
 5170                         rxr->lmp = NULL;
 5171                 }
 5172 
 5173 next_desc:
 5174                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 5175                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 5176 
 5177                 /* Advance our pointers to the next descriptor. */
 5178                 if (++i == adapter->num_rx_desc)
 5179                         i = 0;
 5180                 /*
 5181                 ** Send to the stack or LRO
 5182                 */
 5183                 if (sendmp != NULL) {
 5184                         rxr->next_to_check = i;
 5185                         igb_rx_input(rxr, ifp, sendmp, ptype);
 5186                         i = rxr->next_to_check;
 5187                         rxdone++;
 5188                 }
 5189 
 5190                 /* Every 8 descriptors we go to refresh mbufs */
 5191                 if (processed == 8) {
 5192                         igb_refresh_mbufs(rxr, i);
 5193                         processed = 0;
 5194                 }
 5195         }
 5196 
 5197         /* Catch any remainders */
 5198         if (igb_rx_unrefreshed(rxr))
 5199                 igb_refresh_mbufs(rxr, i);
 5200 
 5201         rxr->next_to_check = i;
 5202 
 5203         /*
 5204          * Flush any outstanding LRO work
 5205          */
 5206         tcp_lro_flush_all(lro);
 5207 
 5208         if (done != NULL)
 5209                 *done += rxdone;
 5210 
 5211         IGB_RX_UNLOCK(rxr);
 5212         return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
 5213 }
 5214 
 5215 /*********************************************************************
 5216  *
 5217  *  Verify that the hardware indicated that the checksum is valid.
 5218  *  Inform the stack about the status of checksum so that stack
 5219  *  doesn't spend time verifying the checksum.
 5220  *
 5221  *********************************************************************/
 5222 static void
 5223 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
 5224 {
 5225         u16 status = (u16)staterr;
 5226         u8  errors = (u8) (staterr >> 24);
 5227         int sctp;
 5228 
 5229         /* Ignore Checksum bit is set */
 5230         if (status & E1000_RXD_STAT_IXSM) {
 5231                 mp->m_pkthdr.csum_flags = 0;
 5232                 return;
 5233         }
 5234 
 5235         if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
 5236             (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
 5237                 sctp = 1;
 5238         else
 5239                 sctp = 0;
 5240         if (status & E1000_RXD_STAT_IPCS) {
 5241                 /* Did it pass? */
 5242                 if (!(errors & E1000_RXD_ERR_IPE)) {
 5243                         /* IP Checksum Good */
 5244                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
 5245                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
 5246                 } else
 5247                         mp->m_pkthdr.csum_flags = 0;
 5248         }
 5249 
 5250         if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
 5251                 u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
 5252 #if __FreeBSD_version >= 800000
 5253                 if (sctp) /* reassign */
 5254                         type = CSUM_SCTP_VALID;
 5255 #endif
 5256                 /* Did it pass? */
 5257                 if (!(errors & E1000_RXD_ERR_TCPE)) {
 5258                         mp->m_pkthdr.csum_flags |= type;
 5259                         if (sctp == 0)
 5260                                 mp->m_pkthdr.csum_data = htons(0xffff);
 5261                 }
 5262         }
 5263         return;
 5264 }
 5265 
 5266 /*
 5267  * This routine is run via an vlan
 5268  * config EVENT
 5269  */
 5270 static void
 5271 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 5272 {
 5273         struct adapter  *adapter = ifp->if_softc;
 5274         u32             index, bit;
 5275 
 5276         if (ifp->if_softc !=  arg)   /* Not our event */
 5277                 return;
 5278 
 5279         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 5280                 return;
 5281 
 5282         IGB_CORE_LOCK(adapter);
 5283         index = (vtag >> 5) & 0x7F;
 5284         bit = vtag & 0x1F;
 5285         adapter->shadow_vfta[index] |= (1 << bit);
 5286         ++adapter->num_vlans;
 5287         /* Change hw filter setting */
 5288         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
 5289                 igb_setup_vlan_hw_support(adapter);
 5290         IGB_CORE_UNLOCK(adapter);
 5291 }
 5292 
 5293 /*
 5294  * This routine is run via an vlan
 5295  * unconfig EVENT
 5296  */
 5297 static void
 5298 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 5299 {
 5300         struct adapter  *adapter = ifp->if_softc;
 5301         u32             index, bit;
 5302 
 5303         if (ifp->if_softc !=  arg)
 5304                 return;
 5305 
 5306         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 5307                 return;
 5308 
 5309         IGB_CORE_LOCK(adapter);
 5310         index = (vtag >> 5) & 0x7F;
 5311         bit = vtag & 0x1F;
 5312         adapter->shadow_vfta[index] &= ~(1 << bit);
 5313         --adapter->num_vlans;
 5314         /* Change hw filter setting */
 5315         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
 5316                 igb_setup_vlan_hw_support(adapter);
 5317         IGB_CORE_UNLOCK(adapter);
 5318 }
 5319 
 5320 static void
 5321 igb_setup_vlan_hw_support(struct adapter *adapter)
 5322 {
 5323         struct e1000_hw *hw = &adapter->hw;
 5324         struct ifnet    *ifp = adapter->ifp;
 5325         u32             reg;
 5326 
 5327         if (adapter->vf_ifp) {
 5328                 e1000_rlpml_set_vf(hw,
 5329                     adapter->max_frame_size + VLAN_TAG_SIZE);
 5330                 return;
 5331         }
 5332 
 5333         reg = E1000_READ_REG(hw, E1000_CTRL);
 5334         reg |= E1000_CTRL_VME;
 5335         E1000_WRITE_REG(hw, E1000_CTRL, reg);
 5336 
 5337         /* Enable the Filter Table */
 5338         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
 5339                 reg = E1000_READ_REG(hw, E1000_RCTL);
 5340                 reg &= ~E1000_RCTL_CFIEN;
 5341                 reg |= E1000_RCTL_VFE;
 5342                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 5343         }
 5344 
 5345         /* Update the frame size */
 5346         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
 5347             adapter->max_frame_size + VLAN_TAG_SIZE);
 5348 
 5349         /* Don't bother with table if no vlans */
 5350         if ((adapter->num_vlans == 0) ||
 5351             ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
 5352                 return;
 5353         /*
 5354         ** A soft reset zero's out the VFTA, so
 5355         ** we need to repopulate it now.
 5356         */
 5357         for (int i = 0; i < IGB_VFTA_SIZE; i++)
 5358                 if (adapter->shadow_vfta[i] != 0) {
 5359                         if (adapter->vf_ifp)
 5360                                 e1000_vfta_set_vf(hw,
 5361                                     adapter->shadow_vfta[i], TRUE);
 5362                         else
 5363                                 e1000_write_vfta(hw,
 5364                                     i, adapter->shadow_vfta[i]);
 5365                 }
 5366 }
 5367 
 5368 static void
 5369 igb_enable_intr(struct adapter *adapter)
 5370 {
 5371         /* With RSS set up what to auto clear */
 5372         if (adapter->msix_mem) {
 5373                 u32 mask = (adapter->que_mask | adapter->link_mask);
 5374                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
 5375                 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
 5376                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
 5377                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 5378                     E1000_IMS_LSC);
 5379         } else {
 5380                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 5381                     IMS_ENABLE_MASK);
 5382         }
 5383         E1000_WRITE_FLUSH(&adapter->hw);
 5384 
 5385         return;
 5386 }
 5387 
 5388 static void
 5389 igb_disable_intr(struct adapter *adapter)
 5390 {
 5391         if (adapter->msix_mem) {
 5392                 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
 5393                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
 5394         } 
 5395         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
 5396         E1000_WRITE_FLUSH(&adapter->hw);
 5397         return;
 5398 }
 5399 
 5400 /*
 5401  * Bit of a misnomer, what this really means is
 5402  * to enable OS management of the system... aka
 5403  * to disable special hardware management features 
 5404  */
 5405 static void
 5406 igb_init_manageability(struct adapter *adapter)
 5407 {
 5408         if (adapter->has_manage) {
 5409                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
 5410                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 5411 
 5412                 /* disable hardware interception of ARP */
 5413                 manc &= ~(E1000_MANC_ARP_EN);
 5414 
 5415                 /* enable receiving management packets to the host */
 5416                 manc |= E1000_MANC_EN_MNG2HOST;
 5417                 manc2h |= 1 << 5;  /* Mng Port 623 */
 5418                 manc2h |= 1 << 6;  /* Mng Port 664 */
 5419                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
 5420                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 5421         }
 5422 }
 5423 
 5424 /*
 5425  * Give control back to hardware management
 5426  * controller if there is one.
 5427  */
 5428 static void
 5429 igb_release_manageability(struct adapter *adapter)
 5430 {
 5431         if (adapter->has_manage) {
 5432                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 5433 
 5434                 /* re-enable hardware interception of ARP */
 5435                 manc |= E1000_MANC_ARP_EN;
 5436                 manc &= ~E1000_MANC_EN_MNG2HOST;
 5437 
 5438                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 5439         }
 5440 }
 5441 
 5442 /*
 5443  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 5444  * For ASF and Pass Through versions of f/w this means that
 5445  * the driver is loaded. 
 5446  *
 5447  */
 5448 static void
 5449 igb_get_hw_control(struct adapter *adapter)
 5450 {
 5451         u32 ctrl_ext;
 5452 
 5453         if (adapter->vf_ifp)
 5454                 return;
 5455 
 5456         /* Let firmware know the driver has taken over */
 5457         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 5458         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 5459             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
 5460 }
 5461 
 5462 /*
 5463  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 5464  * For ASF and Pass Through versions of f/w this means that the
 5465  * driver is no longer loaded.
 5466  *
 5467  */
 5468 static void
 5469 igb_release_hw_control(struct adapter *adapter)
 5470 {
 5471         u32 ctrl_ext;
 5472 
 5473         if (adapter->vf_ifp)
 5474                 return;
 5475 
 5476         /* Let firmware taken over control of h/w */
 5477         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 5478         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 5479             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
 5480 }
 5481 
 5482 static int
 5483 igb_is_valid_ether_addr(uint8_t *addr)
 5484 {
 5485         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
 5486 
 5487         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
 5488                 return (FALSE);
 5489         }
 5490 
 5491         return (TRUE);
 5492 }
 5493 
 5494 
 5495 /*
 5496  * Enable PCI Wake On Lan capability
 5497  */
 5498 static void
 5499 igb_enable_wakeup(device_t dev)
 5500 {
 5501         u16     cap, status;
 5502         u8      id;
 5503 
 5504         /* First find the capabilities pointer*/
 5505         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
 5506         /* Read the PM Capabilities */
 5507         id = pci_read_config(dev, cap, 1);
 5508         if (id != PCIY_PMG)     /* Something wrong */
 5509                 return;
 5510         /* OK, we have the power capabilities, so
 5511            now get the status register */
 5512         cap += PCIR_POWER_STATUS;
 5513         status = pci_read_config(dev, cap, 2);
 5514         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
 5515         pci_write_config(dev, cap, status, 2);
 5516         return;
 5517 }
 5518 
 5519 static void
 5520 igb_led_func(void *arg, int onoff)
 5521 {
 5522         struct adapter  *adapter = arg;
 5523 
 5524         IGB_CORE_LOCK(adapter);
 5525         if (onoff) {
 5526                 e1000_setup_led(&adapter->hw);
 5527                 e1000_led_on(&adapter->hw);
 5528         } else {
 5529                 e1000_led_off(&adapter->hw);
 5530                 e1000_cleanup_led(&adapter->hw);
 5531         }
 5532         IGB_CORE_UNLOCK(adapter);
 5533 }
 5534 
 5535 static uint64_t
 5536 igb_get_vf_counter(if_t ifp, ift_counter cnt)
 5537 {
 5538         struct adapter *adapter;
 5539         struct e1000_vf_stats *stats;
 5540 #ifndef IGB_LEGACY_TX
 5541         struct tx_ring *txr;
 5542         uint64_t rv;
 5543 #endif
 5544 
 5545         adapter = if_getsoftc(ifp);
 5546         stats = (struct e1000_vf_stats *)adapter->stats;
 5547 
 5548         switch (cnt) {
 5549         case IFCOUNTER_IPACKETS:
 5550                 return (stats->gprc);
 5551         case IFCOUNTER_OPACKETS:
 5552                 return (stats->gptc);
 5553         case IFCOUNTER_IBYTES:
 5554                 return (stats->gorc);
 5555         case IFCOUNTER_OBYTES:
 5556                 return (stats->gotc);
 5557         case IFCOUNTER_IMCASTS:
 5558                 return (stats->mprc);
 5559         case IFCOUNTER_IERRORS:
 5560                 return (adapter->dropped_pkts);
 5561         case IFCOUNTER_OERRORS:
 5562                 return (adapter->watchdog_events);
 5563 #ifndef IGB_LEGACY_TX
 5564         case IFCOUNTER_OQDROPS:
 5565                 rv = 0;
 5566                 txr = adapter->tx_rings;
 5567                 for (int i = 0; i < adapter->num_queues; i++, txr++)
 5568                         rv += txr->br->br_drops;
 5569                 return (rv);
 5570 #endif
 5571         default:
 5572                 return (if_get_counter_default(ifp, cnt));
 5573         }
 5574 }
 5575 
 5576 static uint64_t
 5577 igb_get_counter(if_t ifp, ift_counter cnt)
 5578 {
 5579         struct adapter *adapter;
 5580         struct e1000_hw_stats *stats;
 5581 #ifndef IGB_LEGACY_TX
 5582         struct tx_ring *txr;
 5583         uint64_t rv;
 5584 #endif
 5585 
 5586         adapter = if_getsoftc(ifp);
 5587         if (adapter->vf_ifp)
 5588                 return (igb_get_vf_counter(ifp, cnt));
 5589 
 5590         stats = (struct e1000_hw_stats *)adapter->stats;
 5591 
 5592         switch (cnt) {
 5593         case IFCOUNTER_IPACKETS:
 5594                 return (stats->gprc);
 5595         case IFCOUNTER_OPACKETS:
 5596                 return (stats->gptc);
 5597         case IFCOUNTER_IBYTES:
 5598                 return (stats->gorc);
 5599         case IFCOUNTER_OBYTES:
 5600                 return (stats->gotc);
 5601         case IFCOUNTER_IMCASTS:
 5602                 return (stats->mprc);
 5603         case IFCOUNTER_OMCASTS:
 5604                 return (stats->mptc);
 5605         case IFCOUNTER_IERRORS:
 5606                 return (adapter->dropped_pkts + stats->rxerrc +
 5607                     stats->crcerrs + stats->algnerrc +
 5608                     stats->ruc + stats->roc + stats->cexterr);
 5609         case IFCOUNTER_OERRORS:
 5610                 return (stats->ecol + stats->latecol +
 5611                     adapter->watchdog_events);
 5612         case IFCOUNTER_COLLISIONS:
 5613                 return (stats->colc);
 5614         case IFCOUNTER_IQDROPS:
 5615                 return (stats->mpc);
 5616 #ifndef IGB_LEGACY_TX
 5617         case IFCOUNTER_OQDROPS:
 5618                 rv = 0;
 5619                 txr = adapter->tx_rings;
 5620                 for (int i = 0; i < adapter->num_queues; i++, txr++)
 5621                         rv += txr->br->br_drops;
 5622                 return (rv);
 5623 #endif
 5624         default:
 5625                 return (if_get_counter_default(ifp, cnt));
 5626         }
 5627 }
 5628 
 5629 /**********************************************************************
 5630  *
 5631  *  Update the board statistics counters.
 5632  *
 5633  **********************************************************************/
 5634 static void
 5635 igb_update_stats_counters(struct adapter *adapter)
 5636 {
 5637         struct e1000_hw         *hw = &adapter->hw;
 5638         struct e1000_hw_stats   *stats;
 5639 
 5640         /* 
 5641         ** The virtual function adapter has only a
 5642         ** small controlled set of stats, do only 
 5643         ** those and return.
 5644         */
 5645         if (adapter->vf_ifp) {
 5646                 igb_update_vf_stats_counters(adapter);
 5647                 return;
 5648         }
 5649 
 5650         stats = (struct e1000_hw_stats  *)adapter->stats;
 5651 
 5652         if (adapter->hw.phy.media_type == e1000_media_type_copper ||
 5653            (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
 5654                 stats->symerrs +=
 5655                     E1000_READ_REG(hw,E1000_SYMERRS);
 5656                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
 5657         }
 5658 
 5659         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
 5660         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
 5661         stats->scc += E1000_READ_REG(hw, E1000_SCC);
 5662         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
 5663 
 5664         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
 5665         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
 5666         stats->colc += E1000_READ_REG(hw, E1000_COLC);
 5667         stats->dc += E1000_READ_REG(hw, E1000_DC);
 5668         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
 5669         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
 5670         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
 5671         /*
 5672         ** For watchdog management we need to know if we have been
 5673         ** paused during the last interval, so capture that here.
 5674         */ 
 5675         adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
 5676         stats->xoffrxc += adapter->pause_frames;
 5677         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
 5678         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
 5679         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
 5680         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
 5681         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
 5682         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
 5683         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
 5684         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
 5685         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
 5686         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
 5687         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
 5688         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
 5689 
 5690         /* For the 64-bit byte counters the low dword must be read first. */
 5691         /* Both registers clear on the read of the high dword */
 5692 
 5693         stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
 5694             ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
 5695         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
 5696             ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
 5697 
 5698         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
 5699         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
 5700         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
 5701         stats->roc += E1000_READ_REG(hw, E1000_ROC);
 5702         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
 5703 
 5704         stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
 5705         stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC);
 5706         stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
 5707 
 5708         stats->tor += E1000_READ_REG(hw, E1000_TORL) +
 5709             ((u64)E1000_READ_REG(hw, E1000_TORH) << 32);
 5710         stats->tot += E1000_READ_REG(hw, E1000_TOTL) +
 5711             ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32);
 5712 
 5713         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
 5714         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
 5715         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
 5716         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
 5717         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
 5718         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
 5719         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
 5720         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
 5721         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
 5722         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
 5723 
 5724         /* Interrupt Counts */
 5725 
 5726         stats->iac += E1000_READ_REG(hw, E1000_IAC);
 5727         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
 5728         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
 5729         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
 5730         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
 5731         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
 5732         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
 5733         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
 5734         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
 5735 
 5736         /* Host to Card Statistics */
 5737 
 5738         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
 5739         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
 5740         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
 5741         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
 5742         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
 5743         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
 5744         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
 5745         stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
 5746             ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
 5747         stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
 5748             ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
 5749         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
 5750         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
 5751         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
 5752 
 5753         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
 5754         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
 5755         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
 5756         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
 5757         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
 5758         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
 5759 
 5760         /* Driver specific counters */
 5761         adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
 5762         adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
 5763         adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
 5764         adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
 5765         adapter->packet_buf_alloc_tx =
 5766             ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
 5767         adapter->packet_buf_alloc_rx =
 5768             (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
 5769 }
 5770 
 5771 
 5772 /**********************************************************************
 5773  *
 5774  *  Initialize the VF board statistics counters.
 5775  *
 5776  **********************************************************************/
 5777 static void
 5778 igb_vf_init_stats(struct adapter *adapter)
 5779 {
 5780         struct e1000_hw *hw = &adapter->hw;
 5781         struct e1000_vf_stats   *stats;
 5782 
 5783         stats = (struct e1000_vf_stats  *)adapter->stats;
 5784         if (stats == NULL)
 5785                 return;
 5786         stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
 5787         stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
 5788         stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
 5789         stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
 5790         stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
 5791 }
 5792  
 5793 /**********************************************************************
 5794  *
 5795  *  Update the VF board statistics counters.
 5796  *
 5797  **********************************************************************/
 5798 static void
 5799 igb_update_vf_stats_counters(struct adapter *adapter)
 5800 {
 5801         struct e1000_hw *hw = &adapter->hw;
 5802         struct e1000_vf_stats   *stats;
 5803 
 5804         if (adapter->link_speed == 0)
 5805                 return;
 5806 
 5807         stats = (struct e1000_vf_stats  *)adapter->stats;
 5808 
 5809         UPDATE_VF_REG(E1000_VFGPRC,
 5810             stats->last_gprc, stats->gprc);
 5811         UPDATE_VF_REG(E1000_VFGORC,
 5812             stats->last_gorc, stats->gorc);
 5813         UPDATE_VF_REG(E1000_VFGPTC,
 5814             stats->last_gptc, stats->gptc);
 5815         UPDATE_VF_REG(E1000_VFGOTC,
 5816             stats->last_gotc, stats->gotc);
 5817         UPDATE_VF_REG(E1000_VFMPRC,
 5818             stats->last_mprc, stats->mprc);
 5819 }
 5820 
 5821 /* Export a single 32-bit register via a read-only sysctl. */
 5822 static int
 5823 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
 5824 {
 5825         struct adapter *adapter;
 5826         u_int val;
 5827 
 5828         adapter = oidp->oid_arg1;
 5829         val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
 5830         return (sysctl_handle_int(oidp, &val, 0, req));
 5831 }
 5832 
 5833 /*
 5834 **  Tuneable interrupt rate handler
 5835 */
 5836 static int
 5837 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
 5838 {
 5839         struct igb_queue        *que = ((struct igb_queue *)oidp->oid_arg1);
 5840         int                     error;
 5841         u32                     reg, usec, rate;
 5842                         
 5843         reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
 5844         usec = ((reg & 0x7FFC) >> 2);
 5845         if (usec > 0)
 5846                 rate = 1000000 / usec;
 5847         else
 5848                 rate = 0;
 5849         error = sysctl_handle_int(oidp, &rate, 0, req);
 5850         if (error || !req->newptr)
 5851                 return error;
 5852         return 0;
 5853 }
 5854 
 5855 /*
 5856  * Add sysctl variables, one per statistic, to the system.
 5857  */
 5858 static void
 5859 igb_add_hw_stats(struct adapter *adapter)
 5860 {
 5861         device_t dev = adapter->dev;
 5862 
 5863         struct tx_ring *txr = adapter->tx_rings;
 5864         struct rx_ring *rxr = adapter->rx_rings;
 5865 
 5866         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 5867         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
 5868         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
 5869         struct e1000_hw_stats *stats = adapter->stats;
 5870 
 5871         struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
 5872         struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
 5873 
 5874 #define QUEUE_NAME_LEN 32
 5875         char namebuf[QUEUE_NAME_LEN];
 5876 
 5877         /* Driver Statistics */
 5878         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 
 5879                         CTLFLAG_RD, &adapter->dropped_pkts,
 5880                         "Driver dropped packets");
 5881         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", 
 5882                         CTLFLAG_RD, &adapter->link_irq,
 5883                         "Link MSIX IRQ Handled");
 5884         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail",
 5885                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
 5886                         "Defragmenting mbuf chain failed");
 5887         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", 
 5888                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
 5889                         "Driver tx dma failure in xmit");
 5890         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
 5891                         CTLFLAG_RD, &adapter->rx_overruns,
 5892                         "RX overruns");
 5893         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
 5894                         CTLFLAG_RD, &adapter->watchdog_events,
 5895                         "Watchdog timeouts");
 5896 
 5897         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", 
 5898                         CTLFLAG_RD, &adapter->device_control,
 5899                         "Device Control Register");
 5900         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", 
 5901                         CTLFLAG_RD, &adapter->rx_control,
 5902                         "Receiver Control Register");
 5903         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", 
 5904                         CTLFLAG_RD, &adapter->int_mask,
 5905                         "Interrupt Mask");
 5906         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", 
 5907                         CTLFLAG_RD, &adapter->eint_mask,
 5908                         "Extended Interrupt Mask");
 5909         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", 
 5910                         CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
 5911                         "Transmit Buffer Packet Allocation");
 5912         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", 
 5913                         CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
 5914                         "Receive Buffer Packet Allocation");
 5915         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
 5916                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
 5917                         "Flow Control High Watermark");
 5918         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 
 5919                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
 5920                         "Flow Control Low Watermark");
 5921 
 5922         for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
 5923                 struct lro_ctrl *lro = &rxr->lro;
 5924 
 5925                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
 5926                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 5927                                             CTLFLAG_RD, NULL, "Queue Name");
 5928                 queue_list = SYSCTL_CHILDREN(queue_node);
 5929 
 5930                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 
 5931                                 CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i],
 5932                                 sizeof(&adapter->queues[i]),
 5933                                 igb_sysctl_interrupt_rate_handler,
 5934                                 "IU", "Interrupt Rate");
 5935 
 5936                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 
 5937                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me),
 5938                                 igb_sysctl_reg_handler, "IU",
 5939                                 "Transmit Descriptor Head");
 5940                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 
 5941                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me),
 5942                                 igb_sysctl_reg_handler, "IU",
 5943                                 "Transmit Descriptor Tail");
 5944                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 
 5945                                 CTLFLAG_RD, &txr->no_desc_avail,
 5946                                 "Queue Descriptors Unavailable");
 5947                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
 5948                                 CTLFLAG_RD, &txr->total_packets,
 5949                                 "Queue Packets Transmitted");
 5950 
 5951                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 
 5952                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
 5953                                 igb_sysctl_reg_handler, "IU",
 5954                                 "Receive Descriptor Head");
 5955                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 
 5956                                 CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
 5957                                 igb_sysctl_reg_handler, "IU",
 5958                                 "Receive Descriptor Tail");
 5959                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
 5960                                 CTLFLAG_RD, &rxr->rx_packets,
 5961                                 "Queue Packets Received");
 5962                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
 5963                                 CTLFLAG_RD, &rxr->rx_bytes,
 5964                                 "Queue Bytes Received");
 5965                 SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_queued",
 5966                                 CTLFLAG_RD, &lro->lro_queued, 0,
 5967                                 "LRO Queued");
 5968                 SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_flushed",
 5969                                 CTLFLAG_RD, &lro->lro_flushed, 0,
 5970                                 "LRO Flushed");
 5971         }
 5972 
 5973         /* MAC stats get their own sub node */
 5974 
 5975         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 
 5976                                     CTLFLAG_RD, NULL, "MAC Statistics");
 5977         stat_list = SYSCTL_CHILDREN(stat_node);
 5978 
 5979         /*
 5980         ** VF adapter has a very limited set of stats
 5981         ** since its not managing the metal, so to speak.
 5982         */
 5983         if (adapter->vf_ifp) {
 5984         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
 5985                         CTLFLAG_RD, &stats->gprc,
 5986                         "Good Packets Received");
 5987         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 5988                         CTLFLAG_RD, &stats->gptc,
 5989                         "Good Packets Transmitted");
 5990         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 
 5991                         CTLFLAG_RD, &stats->gorc, 
 5992                         "Good Octets Received"); 
 5993         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 
 5994                         CTLFLAG_RD, &stats->gotc, 
 5995                         "Good Octets Transmitted"); 
 5996         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
 5997                         CTLFLAG_RD, &stats->mprc,
 5998                         "Multicast Packets Received");
 5999                 return;
 6000         }
 6001 
 6002         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", 
 6003                         CTLFLAG_RD, &stats->ecol,
 6004                         "Excessive collisions");
 6005         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", 
 6006                         CTLFLAG_RD, &stats->scc,
 6007                         "Single collisions");
 6008         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 
 6009                         CTLFLAG_RD, &stats->mcc,
 6010                         "Multiple collisions");
 6011         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", 
 6012                         CTLFLAG_RD, &stats->latecol,
 6013                         "Late collisions");
 6014         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", 
 6015                         CTLFLAG_RD, &stats->colc,
 6016                         "Collision Count");
 6017         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
 6018                         CTLFLAG_RD, &stats->symerrs,
 6019                         "Symbol Errors");
 6020         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
 6021                         CTLFLAG_RD, &stats->sec,
 6022                         "Sequence Errors");
 6023         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
 6024                         CTLFLAG_RD, &stats->dc,
 6025                         "Defer Count");
 6026         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
 6027                         CTLFLAG_RD, &stats->mpc,
 6028                         "Missed Packets");
 6029         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors",
 6030                         CTLFLAG_RD, &stats->rlec,
 6031                         "Receive Length Errors");
 6032         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
 6033                         CTLFLAG_RD, &stats->rnbc,
 6034                         "Receive No Buffers");
 6035         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
 6036                         CTLFLAG_RD, &stats->ruc,
 6037                         "Receive Undersize");
 6038         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
 6039                         CTLFLAG_RD, &stats->rfc,
 6040                         "Fragmented Packets Received");
 6041         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
 6042                         CTLFLAG_RD, &stats->roc,
 6043                         "Oversized Packets Received");
 6044         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
 6045                         CTLFLAG_RD, &stats->rjc,
 6046                         "Recevied Jabber");
 6047         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
 6048                         CTLFLAG_RD, &stats->rxerrc,
 6049                         "Receive Errors");
 6050         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
 6051                         CTLFLAG_RD, &stats->crcerrs,
 6052                         "CRC errors");
 6053         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
 6054                         CTLFLAG_RD, &stats->algnerrc,
 6055                         "Alignment Errors");
 6056         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs",
 6057                         CTLFLAG_RD, &stats->tncrs,
 6058                         "Transmit with No CRS");
 6059         /* On 82575 these are collision counts */
 6060         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
 6061                         CTLFLAG_RD, &stats->cexterr,
 6062                         "Collision/Carrier extension errors");
 6063         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
 6064                         CTLFLAG_RD, &stats->xonrxc,
 6065                         "XON Received");
 6066         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
 6067                         CTLFLAG_RD, &stats->xontxc,
 6068                         "XON Transmitted");
 6069         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
 6070                         CTLFLAG_RD, &stats->xoffrxc,
 6071                         "XOFF Received");
 6072         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
 6073                         CTLFLAG_RD, &stats->xofftxc,
 6074                         "XOFF Transmitted");
 6075         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd",
 6076                         CTLFLAG_RD, &stats->fcruc,
 6077                         "Unsupported Flow Control Received");
 6078         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd",
 6079                         CTLFLAG_RD, &stats->mgprc,
 6080                         "Management Packets Received");
 6081         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop",
 6082                         CTLFLAG_RD, &stats->mgpdc,
 6083                         "Management Packets Dropped");
 6084         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd",
 6085                         CTLFLAG_RD, &stats->mgptc,
 6086                         "Management Packets Transmitted");
 6087         /* Packet Reception Stats */
 6088         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
 6089                         CTLFLAG_RD, &stats->tpr,
 6090                         "Total Packets Received");
 6091         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
 6092                         CTLFLAG_RD, &stats->gprc,
 6093                         "Good Packets Received");
 6094         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
 6095                         CTLFLAG_RD, &stats->bprc,
 6096                         "Broadcast Packets Received");
 6097         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
 6098                         CTLFLAG_RD, &stats->mprc,
 6099                         "Multicast Packets Received");
 6100         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
 6101                         CTLFLAG_RD, &stats->prc64,
 6102                         "64 byte frames received");
 6103         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
 6104                         CTLFLAG_RD, &stats->prc127,
 6105                         "65-127 byte frames received");
 6106         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
 6107                         CTLFLAG_RD, &stats->prc255,
 6108                         "128-255 byte frames received");
 6109         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
 6110                         CTLFLAG_RD, &stats->prc511,
 6111                         "256-511 byte frames received");
 6112         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
 6113                         CTLFLAG_RD, &stats->prc1023,
 6114                         "512-1023 byte frames received");
 6115         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
 6116                         CTLFLAG_RD, &stats->prc1522,
 6117                         "1023-1522 byte frames received");
 6118         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 
 6119                         CTLFLAG_RD, &stats->gorc, 
 6120                         "Good Octets Received");
 6121         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd", 
 6122                         CTLFLAG_RD, &stats->tor, 
 6123                         "Total Octets Received");
 6124 
 6125         /* Packet Transmission Stats */
 6126         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 
 6127                         CTLFLAG_RD, &stats->gotc, 
 6128                         "Good Octets Transmitted"); 
 6129         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd", 
 6130                         CTLFLAG_RD, &stats->tot, 
 6131                         "Total Octets Transmitted");
 6132         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
 6133                         CTLFLAG_RD, &stats->tpt,
 6134                         "Total Packets Transmitted");
 6135         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 6136                         CTLFLAG_RD, &stats->gptc,
 6137                         "Good Packets Transmitted");
 6138         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
 6139                         CTLFLAG_RD, &stats->bptc,
 6140                         "Broadcast Packets Transmitted");
 6141         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
 6142                         CTLFLAG_RD, &stats->mptc,
 6143                         "Multicast Packets Transmitted");
 6144         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
 6145                         CTLFLAG_RD, &stats->ptc64,
 6146                         "64 byte frames transmitted");
 6147         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
 6148                         CTLFLAG_RD, &stats->ptc127,
 6149                         "65-127 byte frames transmitted");
 6150         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
 6151                         CTLFLAG_RD, &stats->ptc255,
 6152                         "128-255 byte frames transmitted");
 6153         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
 6154                         CTLFLAG_RD, &stats->ptc511,
 6155                         "256-511 byte frames transmitted");
 6156         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
 6157                         CTLFLAG_RD, &stats->ptc1023,
 6158                         "512-1023 byte frames transmitted");
 6159         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
 6160                         CTLFLAG_RD, &stats->ptc1522,
 6161                         "1024-1522 byte frames transmitted");
 6162         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
 6163                         CTLFLAG_RD, &stats->tsctc,
 6164                         "TSO Contexts Transmitted");
 6165         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
 6166                         CTLFLAG_RD, &stats->tsctfc,
 6167                         "TSO Contexts Failed");
 6168 
 6169 
 6170         /* Interrupt Stats */
 6171 
 6172         int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 
 6173                                     CTLFLAG_RD, NULL, "Interrupt Statistics");
 6174         int_list = SYSCTL_CHILDREN(int_node);
 6175 
 6176         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
 6177                         CTLFLAG_RD, &stats->iac,
 6178                         "Interrupt Assertion Count");
 6179 
 6180         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
 6181                         CTLFLAG_RD, &stats->icrxptc,
 6182                         "Interrupt Cause Rx Pkt Timer Expire Count");
 6183 
 6184         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
 6185                         CTLFLAG_RD, &stats->icrxatc,
 6186                         "Interrupt Cause Rx Abs Timer Expire Count");
 6187 
 6188         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
 6189                         CTLFLAG_RD, &stats->ictxptc,
 6190                         "Interrupt Cause Tx Pkt Timer Expire Count");
 6191 
 6192         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
 6193                         CTLFLAG_RD, &stats->ictxatc,
 6194                         "Interrupt Cause Tx Abs Timer Expire Count");
 6195 
 6196         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
 6197                         CTLFLAG_RD, &stats->ictxqec,
 6198                         "Interrupt Cause Tx Queue Empty Count");
 6199 
 6200         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
 6201                         CTLFLAG_RD, &stats->ictxqmtc,
 6202                         "Interrupt Cause Tx Queue Min Thresh Count");
 6203 
 6204         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
 6205                         CTLFLAG_RD, &stats->icrxdmtc,
 6206                         "Interrupt Cause Rx Desc Min Thresh Count");
 6207 
 6208         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
 6209                         CTLFLAG_RD, &stats->icrxoc,
 6210                         "Interrupt Cause Receiver Overrun Count");
 6211 
 6212         /* Host to Card Stats */
 6213 
 6214         host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", 
 6215                                     CTLFLAG_RD, NULL, 
 6216                                     "Host to Card Statistics");
 6217 
 6218         host_list = SYSCTL_CHILDREN(host_node);
 6219 
 6220         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
 6221                         CTLFLAG_RD, &stats->cbtmpc,
 6222                         "Circuit Breaker Tx Packet Count");
 6223 
 6224         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
 6225                         CTLFLAG_RD, &stats->htdpmc,
 6226                         "Host Transmit Discarded Packets");
 6227 
 6228         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
 6229                         CTLFLAG_RD, &stats->rpthc,
 6230                         "Rx Packets To Host");
 6231 
 6232         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
 6233                         CTLFLAG_RD, &stats->cbrmpc,
 6234                         "Circuit Breaker Rx Packet Count");
 6235 
 6236         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
 6237                         CTLFLAG_RD, &stats->cbrdpc,
 6238                         "Circuit Breaker Rx Dropped Count");
 6239 
 6240         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
 6241                         CTLFLAG_RD, &stats->hgptc,
 6242                         "Host Good Packets Tx Count");
 6243 
 6244         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
 6245                         CTLFLAG_RD, &stats->htcbdpc,
 6246                         "Host Tx Circuit Breaker Dropped Count");
 6247 
 6248         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
 6249                         CTLFLAG_RD, &stats->hgorc,
 6250                         "Host Good Octets Received Count");
 6251 
 6252         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
 6253                         CTLFLAG_RD, &stats->hgotc,
 6254                         "Host Good Octets Transmit Count");
 6255 
 6256         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
 6257                         CTLFLAG_RD, &stats->lenerrs,
 6258                         "Length Errors");
 6259 
 6260         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
 6261                         CTLFLAG_RD, &stats->scvpc,
 6262                         "SerDes/SGMII Code Violation Pkt Count");
 6263 
 6264         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
 6265                         CTLFLAG_RD, &stats->hrmpc,
 6266                         "Header Redirection Missed Packet Count");
 6267 }
 6268 
 6269 
 6270 /**********************************************************************
 6271  *
 6272  *  This routine provides a way to dump out the adapter eeprom,
 6273  *  often a useful debug/service tool. This only dumps the first
 6274  *  32 words, stuff that matters is in that extent.
 6275  *
 6276  **********************************************************************/
 6277 static int
 6278 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
 6279 {
 6280         struct adapter *adapter;
 6281         int error;
 6282         int result;
 6283 
 6284         result = -1;
 6285         error = sysctl_handle_int(oidp, &result, 0, req);
 6286 
 6287         if (error || !req->newptr)
 6288                 return (error);
 6289 
 6290         /*
 6291          * This value will cause a hex dump of the
 6292          * first 32 16-bit words of the EEPROM to
 6293          * the screen.
 6294          */
 6295         if (result == 1) {
 6296                 adapter = (struct adapter *)arg1;
 6297                 igb_print_nvm_info(adapter);
 6298         }
 6299 
 6300         return (error);
 6301 }
 6302 
 6303 static void
 6304 igb_print_nvm_info(struct adapter *adapter)
 6305 {
 6306         u16     eeprom_data;
 6307         int     i, j, row = 0;
 6308 
 6309         /* Its a bit crude, but it gets the job done */
 6310         printf("\nInterface EEPROM Dump:\n");
 6311         printf("Offset\n0x0000  ");
 6312         for (i = 0, j = 0; i < 32; i++, j++) {
 6313                 if (j == 8) { /* Make the offset block */
 6314                         j = 0; ++row;
 6315                         printf("\n0x00%x0  ",row);
 6316                 }
 6317                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
 6318                 printf("%04x ", eeprom_data);
 6319         }
 6320         printf("\n");
 6321 }
 6322 
 6323 static void
 6324 igb_set_sysctl_value(struct adapter *adapter, const char *name,
 6325         const char *description, int *limit, int value)
 6326 {
 6327         *limit = value;
 6328         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
 6329             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
 6330             OID_AUTO, name, CTLFLAG_RW, limit, value, description);
 6331 }
 6332 
 6333 /*
 6334 ** Set flow control using sysctl:
 6335 ** Flow control values:
 6336 **      0 - off
 6337 **      1 - rx pause
 6338 **      2 - tx pause
 6339 **      3 - full
 6340 */
 6341 static int
 6342 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
 6343 {
 6344         int             error;
 6345         static int      input = 3; /* default is full */
 6346         struct adapter  *adapter = (struct adapter *) arg1;
 6347 
 6348         error = sysctl_handle_int(oidp, &input, 0, req);
 6349 
 6350         if ((error) || (req->newptr == NULL))
 6351                 return (error);
 6352 
 6353         switch (input) {
 6354                 case e1000_fc_rx_pause:
 6355                 case e1000_fc_tx_pause:
 6356                 case e1000_fc_full:
 6357                 case e1000_fc_none:
 6358                         adapter->hw.fc.requested_mode = input;
 6359                         adapter->fc = input;
 6360                         break;
 6361                 default:
 6362                         /* Do nothing */
 6363                         return (error);
 6364         }
 6365 
 6366         adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
 6367         e1000_force_mac_fc(&adapter->hw);
 6368         /* XXX TODO: update DROP_EN on each RX queue if appropriate */
 6369         return (error);
 6370 }
 6371 
 6372 /*
 6373 ** Manage DMA Coalesce:
 6374 ** Control values:
 6375 **      0/1 - off/on
 6376 **      Legal timer values are:
 6377 **      250,500,1000-10000 in thousands
 6378 */
 6379 static int
 6380 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
 6381 {
 6382         struct adapter *adapter = (struct adapter *) arg1;
 6383         int             error;
 6384 
 6385         error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
 6386 
 6387         if ((error) || (req->newptr == NULL))
 6388                 return (error);
 6389 
 6390         switch (adapter->dmac) {
 6391                 case 0:
 6392                         /* Disabling */
 6393                         break;
 6394                 case 1: /* Just enable and use default */
 6395                         adapter->dmac = 1000;
 6396                         break;
 6397                 case 250:
 6398                 case 500:
 6399                 case 1000:
 6400                 case 2000:
 6401                 case 3000:
 6402                 case 4000:
 6403                 case 5000:
 6404                 case 6000:
 6405                 case 7000:
 6406                 case 8000:
 6407                 case 9000:
 6408                 case 10000:
 6409                         /* Legal values - allow */
 6410                         break;
 6411                 default:
 6412                         /* Do nothing, illegal value */
 6413                         adapter->dmac = 0;
 6414                         return (EINVAL);
 6415         }
 6416         /* Reinit the interface */
 6417         igb_init(adapter);
 6418         return (error);
 6419 }
 6420 
 6421 /*
 6422 ** Manage Energy Efficient Ethernet:
 6423 ** Control values:
 6424 **     0/1 - enabled/disabled
 6425 */
 6426 static int
 6427 igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
 6428 {
 6429         struct adapter  *adapter = (struct adapter *) arg1;
 6430         int             error, value;
 6431 
 6432         value = adapter->hw.dev_spec._82575.eee_disable;
 6433         error = sysctl_handle_int(oidp, &value, 0, req);
 6434         if (error || req->newptr == NULL)
 6435                 return (error);
 6436         IGB_CORE_LOCK(adapter);
 6437         adapter->hw.dev_spec._82575.eee_disable = (value != 0);
 6438         igb_init_locked(adapter);
 6439         IGB_CORE_UNLOCK(adapter);
 6440         return (0);
 6441 }

Cache object: a222c9535963381610a3e6d0abd5474f


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.