The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/e1000/if_igb.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /******************************************************************************
    2 
    3   Copyright (c) 2001-2011, Intel Corporation 
    4   All rights reserved.
    5   
    6   Redistribution and use in source and binary forms, with or without 
    7   modification, are permitted provided that the following conditions are met:
    8   
    9    1. Redistributions of source code must retain the above copyright notice, 
   10       this list of conditions and the following disclaimer.
   11   
   12    2. Redistributions in binary form must reproduce the above copyright 
   13       notice, this list of conditions and the following disclaimer in the 
   14       documentation and/or other materials provided with the distribution.
   15   
   16    3. Neither the name of the Intel Corporation nor the names of its 
   17       contributors may be used to endorse or promote products derived from 
   18       this software without specific prior written permission.
   19   
   20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
   22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
   23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
   24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
   25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
   26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
   27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
   28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
   29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   30   POSSIBILITY OF SUCH DAMAGE.
   31 
   32 ******************************************************************************/
   33 /*$FreeBSD: releng/9.0/sys/dev/e1000/if_igb.c 223831 2011-07-07 00:46:50Z jfv $*/
   34 
   35 
   36 #ifdef HAVE_KERNEL_OPTION_HEADERS
   37 #include "opt_device_polling.h"
   38 #include "opt_inet.h"
   39 #include "opt_inet6.h"
   40 #include "opt_altq.h"
   41 #endif
   42 
   43 #include <sys/param.h>
   44 #include <sys/systm.h>
   45 #if __FreeBSD_version >= 800000
   46 #include <sys/buf_ring.h>
   47 #endif
   48 #include <sys/bus.h>
   49 #include <sys/endian.h>
   50 #include <sys/kernel.h>
   51 #include <sys/kthread.h>
   52 #include <sys/malloc.h>
   53 #include <sys/mbuf.h>
   54 #include <sys/module.h>
   55 #include <sys/rman.h>
   56 #include <sys/socket.h>
   57 #include <sys/sockio.h>
   58 #include <sys/sysctl.h>
   59 #include <sys/taskqueue.h>
   60 #include <sys/eventhandler.h>
   61 #include <sys/pcpu.h>
   62 #include <sys/smp.h>
   63 #include <machine/smp.h>
   64 #include <machine/bus.h>
   65 #include <machine/resource.h>
   66 
   67 #include <net/bpf.h>
   68 #include <net/ethernet.h>
   69 #include <net/if.h>
   70 #include <net/if_arp.h>
   71 #include <net/if_dl.h>
   72 #include <net/if_media.h>
   73 
   74 #include <net/if_types.h>
   75 #include <net/if_vlan_var.h>
   76 
   77 #include <netinet/in_systm.h>
   78 #include <netinet/in.h>
   79 #include <netinet/if_ether.h>
   80 #include <netinet/ip.h>
   81 #include <netinet/ip6.h>
   82 #include <netinet/tcp.h>
   83 #include <netinet/tcp_lro.h>
   84 #include <netinet/udp.h>
   85 
   86 #include <machine/in_cksum.h>
   87 #include <dev/led/led.h>
   88 #include <dev/pci/pcivar.h>
   89 #include <dev/pci/pcireg.h>
   90 
   91 #include "e1000_api.h"
   92 #include "e1000_82575.h"
   93 #include "if_igb.h"
   94 
   95 /*********************************************************************
   96  *  Set this to one to display debug statistics
   97  *********************************************************************/
   98 int     igb_display_debug_stats = 0;
   99 
  100 /*********************************************************************
  101  *  Driver version:
  102  *********************************************************************/
  103 char igb_driver_version[] = "version - 2.2.5";
  104 
  105 
  106 /*********************************************************************
  107  *  PCI Device ID Table
  108  *
  109  *  Used by probe to select devices to load on
  110  *  Last field stores an index into e1000_strings
  111  *  Last entry must be all 0s
  112  *
  113  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  114  *********************************************************************/
  115 
  116 static igb_vendor_info_t igb_vendor_info_array[] =
  117 {
  118         { 0x8086, E1000_DEV_ID_82575EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
  119         { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
  120                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  121         { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
  122                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  123         { 0x8086, E1000_DEV_ID_82576,           PCI_ANY_ID, PCI_ANY_ID, 0},
  124         { 0x8086, E1000_DEV_ID_82576_NS,        PCI_ANY_ID, PCI_ANY_ID, 0},
  125         { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
  126         { 0x8086, E1000_DEV_ID_82576_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
  127         { 0x8086, E1000_DEV_ID_82576_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
  128         { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
  129                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  130         { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
  131                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  132         { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2,
  133                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  134         { 0x8086, E1000_DEV_ID_82576_VF,        PCI_ANY_ID, PCI_ANY_ID, 0},
  135         { 0x8086, E1000_DEV_ID_82580_COPPER,    PCI_ANY_ID, PCI_ANY_ID, 0},
  136         { 0x8086, E1000_DEV_ID_82580_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
  137         { 0x8086, E1000_DEV_ID_82580_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
  138         { 0x8086, E1000_DEV_ID_82580_SGMII,     PCI_ANY_ID, PCI_ANY_ID, 0},
  139         { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL,
  140                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  141         { 0x8086, E1000_DEV_ID_82580_QUAD_FIBER,
  142                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  143         { 0x8086, E1000_DEV_ID_DH89XXCC_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
  144         { 0x8086, E1000_DEV_ID_DH89XXCC_SGMII,  PCI_ANY_ID, PCI_ANY_ID, 0},
  145         { 0x8086, E1000_DEV_ID_DH89XXCC_SFP,    PCI_ANY_ID, PCI_ANY_ID, 0},
  146         { 0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE,
  147                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  148         { 0x8086, E1000_DEV_ID_I350_COPPER,     PCI_ANY_ID, PCI_ANY_ID, 0},
  149         { 0x8086, E1000_DEV_ID_I350_FIBER,      PCI_ANY_ID, PCI_ANY_ID, 0},
  150         { 0x8086, E1000_DEV_ID_I350_SERDES,     PCI_ANY_ID, PCI_ANY_ID, 0},
  151         { 0x8086, E1000_DEV_ID_I350_SGMII,      PCI_ANY_ID, PCI_ANY_ID, 0},
  152         { 0x8086, E1000_DEV_ID_I350_VF,         PCI_ANY_ID, PCI_ANY_ID, 0},
  153         /* required last entry */
  154         { 0, 0, 0, 0, 0}
  155 };
  156 
  157 /*********************************************************************
  158  *  Table of branding strings for all supported NICs.
  159  *********************************************************************/
  160 
  161 static char *igb_strings[] = {
  162         "Intel(R) PRO/1000 Network Connection"
  163 };
  164 
  165 /*********************************************************************
  166  *  Function prototypes
  167  *********************************************************************/
  168 static int      igb_probe(device_t);
  169 static int      igb_attach(device_t);
  170 static int      igb_detach(device_t);
  171 static int      igb_shutdown(device_t);
  172 static int      igb_suspend(device_t);
  173 static int      igb_resume(device_t);
  174 #if __FreeBSD_version >= 800000
  175 static int      igb_mq_start(struct ifnet *, struct mbuf *);
  176 static int      igb_mq_start_locked(struct ifnet *,
  177                     struct tx_ring *, struct mbuf *);
  178 static void     igb_qflush(struct ifnet *);
  179 static void     igb_deferred_mq_start(void *, int);
  180 #else
  181 static void     igb_start(struct ifnet *);
  182 static void     igb_start_locked(struct tx_ring *, struct ifnet *ifp);
  183 #endif
  184 static int      igb_ioctl(struct ifnet *, u_long, caddr_t);
  185 static void     igb_init(void *);
  186 static void     igb_init_locked(struct adapter *);
  187 static void     igb_stop(void *);
  188 static void     igb_media_status(struct ifnet *, struct ifmediareq *);
  189 static int      igb_media_change(struct ifnet *);
  190 static void     igb_identify_hardware(struct adapter *);
  191 static int      igb_allocate_pci_resources(struct adapter *);
  192 static int      igb_allocate_msix(struct adapter *);
  193 static int      igb_allocate_legacy(struct adapter *);
  194 static int      igb_setup_msix(struct adapter *);
  195 static void     igb_free_pci_resources(struct adapter *);
  196 static void     igb_local_timer(void *);
  197 static void     igb_reset(struct adapter *);
  198 static int      igb_setup_interface(device_t, struct adapter *);
  199 static int      igb_allocate_queues(struct adapter *);
  200 static void     igb_configure_queues(struct adapter *);
  201 
  202 static int      igb_allocate_transmit_buffers(struct tx_ring *);
  203 static void     igb_setup_transmit_structures(struct adapter *);
  204 static void     igb_setup_transmit_ring(struct tx_ring *);
  205 static void     igb_initialize_transmit_units(struct adapter *);
  206 static void     igb_free_transmit_structures(struct adapter *);
  207 static void     igb_free_transmit_buffers(struct tx_ring *);
  208 
  209 static int      igb_allocate_receive_buffers(struct rx_ring *);
  210 static int      igb_setup_receive_structures(struct adapter *);
  211 static int      igb_setup_receive_ring(struct rx_ring *);
  212 static void     igb_initialize_receive_units(struct adapter *);
  213 static void     igb_free_receive_structures(struct adapter *);
  214 static void     igb_free_receive_buffers(struct rx_ring *);
  215 static void     igb_free_receive_ring(struct rx_ring *);
  216 
  217 static void     igb_enable_intr(struct adapter *);
  218 static void     igb_disable_intr(struct adapter *);
  219 static void     igb_update_stats_counters(struct adapter *);
  220 static bool     igb_txeof(struct tx_ring *);
  221 
  222 static __inline void igb_rx_discard(struct rx_ring *, int);
  223 static __inline void igb_rx_input(struct rx_ring *,
  224                     struct ifnet *, struct mbuf *, u32);
  225 
  226 static bool     igb_rxeof(struct igb_queue *, int, int *);
  227 static void     igb_rx_checksum(u32, struct mbuf *, u32);
  228 static int      igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
  229 static bool     igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
  230 static void     igb_set_promisc(struct adapter *);
  231 static void     igb_disable_promisc(struct adapter *);
  232 static void     igb_set_multi(struct adapter *);
  233 static void     igb_update_link_status(struct adapter *);
  234 static void     igb_refresh_mbufs(struct rx_ring *, int);
  235 
  236 static void     igb_register_vlan(void *, struct ifnet *, u16);
  237 static void     igb_unregister_vlan(void *, struct ifnet *, u16);
  238 static void     igb_setup_vlan_hw_support(struct adapter *);
  239 
  240 static int      igb_xmit(struct tx_ring *, struct mbuf **);
  241 static int      igb_dma_malloc(struct adapter *, bus_size_t,
  242                     struct igb_dma_alloc *, int);
  243 static void     igb_dma_free(struct adapter *, struct igb_dma_alloc *);
  244 static int      igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS);
  245 static void     igb_print_nvm_info(struct adapter *);
  246 static int      igb_is_valid_ether_addr(u8 *);
  247 static void     igb_add_hw_stats(struct adapter *);
  248 
  249 static void     igb_vf_init_stats(struct adapter *);
  250 static void     igb_update_vf_stats_counters(struct adapter *);
  251 
  252 /* Management and WOL Support */
  253 static void     igb_init_manageability(struct adapter *);
  254 static void     igb_release_manageability(struct adapter *);
  255 static void     igb_get_hw_control(struct adapter *);
  256 static void     igb_release_hw_control(struct adapter *);
  257 static void     igb_enable_wakeup(device_t);
  258 static void     igb_led_func(void *, int);
  259 
  260 static int      igb_irq_fast(void *);
  261 static void     igb_msix_que(void *);
  262 static void     igb_msix_link(void *);
  263 static void     igb_handle_que(void *context, int pending);
  264 static void     igb_handle_link(void *context, int pending);
  265 
  266 static void     igb_set_sysctl_value(struct adapter *, const char *,
  267                     const char *, int *, int);
  268 static int      igb_set_flowcntl(SYSCTL_HANDLER_ARGS);
  269 static int      igb_sysctl_dmac(SYSCTL_HANDLER_ARGS);
  270 
  271 #ifdef DEVICE_POLLING
  272 static poll_handler_t igb_poll;
  273 #endif /* POLLING */
  274 
  275 /*********************************************************************
  276  *  FreeBSD Device Interface Entry Points
  277  *********************************************************************/
  278 
  279 static device_method_t igb_methods[] = {
  280         /* Device interface */
  281         DEVMETHOD(device_probe, igb_probe),
  282         DEVMETHOD(device_attach, igb_attach),
  283         DEVMETHOD(device_detach, igb_detach),
  284         DEVMETHOD(device_shutdown, igb_shutdown),
  285         DEVMETHOD(device_suspend, igb_suspend),
  286         DEVMETHOD(device_resume, igb_resume),
  287         {0, 0}
  288 };
  289 
  290 static driver_t igb_driver = {
  291         "igb", igb_methods, sizeof(struct adapter),
  292 };
  293 
  294 static devclass_t igb_devclass;
  295 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
  296 MODULE_DEPEND(igb, pci, 1, 1, 1);
  297 MODULE_DEPEND(igb, ether, 1, 1, 1);
  298 
  299 /*********************************************************************
  300  *  Tunable default values.
  301  *********************************************************************/
  302 
  303 SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters");
  304 
  305 /* Descriptor defaults */
  306 static int igb_rxd = IGB_DEFAULT_RXD;
  307 static int igb_txd = IGB_DEFAULT_TXD;
  308 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
  309 TUNABLE_INT("hw.igb.txd", &igb_txd);
  310 SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0,
  311     "Number of receive descriptors per queue");
  312 SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0,
  313     "Number of transmit descriptors per queue");
  314 
  315 /*
  316 ** AIM: Adaptive Interrupt Moderation
  317 ** which means that the interrupt rate
  318 ** is varied over time based on the
  319 ** traffic for that interrupt vector
  320 */
  321 static int igb_enable_aim = TRUE;
  322 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
  323 SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RW, &igb_enable_aim, 0,
  324     "Enable adaptive interrupt moderation");
  325 
  326 /*
  327  * MSIX should be the default for best performance,
  328  * but this allows it to be forced off for testing.
  329  */         
  330 static int igb_enable_msix = 1;
  331 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
  332 SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0,
  333     "Enable MSI-X interrupts");
  334 
  335 /*
  336 ** Tuneable Interrupt rate
  337 */
  338 static int igb_max_interrupt_rate = 8000;
  339 TUNABLE_INT("hw.igb.max_interrupt_rate", &igb_max_interrupt_rate);
  340 SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
  341     &igb_max_interrupt_rate, 0, "Maximum interrupts per second");
  342 
  343 /*
  344 ** Header split causes the packet header to
  345 ** be dma'd to a seperate mbuf from the payload.
  346 ** this can have memory alignment benefits. But
  347 ** another plus is that small packets often fit
  348 ** into the header and thus use no cluster. Its
  349 ** a very workload dependent type feature.
  350 */
  351 static bool igb_header_split = FALSE;
  352 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
  353 SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0,
  354     "Enable receive mbuf header split");
  355 
  356 /*
  357 ** This will autoconfigure based on
  358 ** the number of CPUs if left at 0.
  359 */
  360 static int igb_num_queues = 0;
  361 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
  362 SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0,
  363     "Number of queues to configure, 0 indicates autoconfigure");
  364 
  365 /* How many packets rxeof tries to clean at a time */
  366 static int igb_rx_process_limit = 100;
  367 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
  368 SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
  369     &igb_rx_process_limit, 0,
  370     "Maximum number of received packets to process at a time, -1 means unlimited");
  371 
  372 /*********************************************************************
  373  *  Device identification routine
  374  *
  375  *  igb_probe determines if the driver should be loaded on
  376  *  adapter based on PCI vendor/device id of the adapter.
  377  *
  378  *  return BUS_PROBE_DEFAULT on success, positive on failure
  379  *********************************************************************/
  380 
  381 static int
  382 igb_probe(device_t dev)
  383 {
  384         char            adapter_name[60];
  385         uint16_t        pci_vendor_id = 0;
  386         uint16_t        pci_device_id = 0;
  387         uint16_t        pci_subvendor_id = 0;
  388         uint16_t        pci_subdevice_id = 0;
  389         igb_vendor_info_t *ent;
  390 
  391         INIT_DEBUGOUT("igb_probe: begin");
  392 
  393         pci_vendor_id = pci_get_vendor(dev);
  394         if (pci_vendor_id != IGB_VENDOR_ID)
  395                 return (ENXIO);
  396 
  397         pci_device_id = pci_get_device(dev);
  398         pci_subvendor_id = pci_get_subvendor(dev);
  399         pci_subdevice_id = pci_get_subdevice(dev);
  400 
  401         ent = igb_vendor_info_array;
  402         while (ent->vendor_id != 0) {
  403                 if ((pci_vendor_id == ent->vendor_id) &&
  404                     (pci_device_id == ent->device_id) &&
  405 
  406                     ((pci_subvendor_id == ent->subvendor_id) ||
  407                     (ent->subvendor_id == PCI_ANY_ID)) &&
  408 
  409                     ((pci_subdevice_id == ent->subdevice_id) ||
  410                     (ent->subdevice_id == PCI_ANY_ID))) {
  411                         sprintf(adapter_name, "%s %s",
  412                                 igb_strings[ent->index],
  413                                 igb_driver_version);
  414                         device_set_desc_copy(dev, adapter_name);
  415                         return (BUS_PROBE_DEFAULT);
  416                 }
  417                 ent++;
  418         }
  419 
  420         return (ENXIO);
  421 }
  422 
  423 /*********************************************************************
  424  *  Device initialization routine
  425  *
  426  *  The attach entry point is called when the driver is being loaded.
  427  *  This routine identifies the type of hardware, allocates all resources
  428  *  and initializes the hardware.
  429  *
  430  *  return 0 on success, positive on failure
  431  *********************************************************************/
  432 
  433 static int
  434 igb_attach(device_t dev)
  435 {
  436         struct adapter  *adapter;
  437         int             error = 0;
  438         u16             eeprom_data;
  439 
  440         INIT_DEBUGOUT("igb_attach: begin");
  441 
  442         if (resource_disabled("igb", device_get_unit(dev))) {
  443                 device_printf(dev, "Disabled by device hint\n");
  444                 return (ENXIO);
  445         }
  446 
  447         adapter = device_get_softc(dev);
  448         adapter->dev = adapter->osdep.dev = dev;
  449         IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
  450 
  451         /* SYSCTL stuff */
  452         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  453             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  454             OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
  455             igb_sysctl_nvm_info, "I", "NVM Information");
  456 
  457         igb_set_sysctl_value(adapter, "enable_aim",
  458             "Interrupt Moderation", &adapter->enable_aim,
  459             igb_enable_aim);
  460 
  461         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  462             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  463             OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW,
  464             adapter, 0, igb_set_flowcntl, "I", "Flow Control");
  465 
  466         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
  467 
  468         /* Determine hardware and mac info */
  469         igb_identify_hardware(adapter);
  470 
  471         /* Setup PCI resources */
  472         if (igb_allocate_pci_resources(adapter)) {
  473                 device_printf(dev, "Allocation of PCI resources failed\n");
  474                 error = ENXIO;
  475                 goto err_pci;
  476         }
  477 
  478         /* Do Shared Code initialization */
  479         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
  480                 device_printf(dev, "Setup of Shared code failed\n");
  481                 error = ENXIO;
  482                 goto err_pci;
  483         }
  484 
  485         e1000_get_bus_info(&adapter->hw);
  486 
  487         /* Sysctl for limiting the amount of work done in the taskqueue */
  488         igb_set_sysctl_value(adapter, "rx_processing_limit",
  489             "max number of rx packets to process",
  490             &adapter->rx_process_limit, igb_rx_process_limit);
  491 
  492         /*
  493          * Validate number of transmit and receive descriptors. It
  494          * must not exceed hardware maximum, and must be multiple
  495          * of E1000_DBA_ALIGN.
  496          */
  497         if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
  498             (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
  499                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
  500                     IGB_DEFAULT_TXD, igb_txd);
  501                 adapter->num_tx_desc = IGB_DEFAULT_TXD;
  502         } else
  503                 adapter->num_tx_desc = igb_txd;
  504         if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
  505             (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
  506                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
  507                     IGB_DEFAULT_RXD, igb_rxd);
  508                 adapter->num_rx_desc = IGB_DEFAULT_RXD;
  509         } else
  510                 adapter->num_rx_desc = igb_rxd;
  511 
  512         adapter->hw.mac.autoneg = DO_AUTO_NEG;
  513         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
  514         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
  515 
  516         /* Copper options */
  517         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
  518                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
  519                 adapter->hw.phy.disable_polarity_correction = FALSE;
  520                 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
  521         }
  522 
  523         /*
  524          * Set the frame limits assuming
  525          * standard ethernet sized frames.
  526          */
  527         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
  528         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
  529 
  530         /*
  531         ** Allocate and Setup Queues
  532         */
  533         if (igb_allocate_queues(adapter)) {
  534                 error = ENOMEM;
  535                 goto err_pci;
  536         }
  537 
  538         /* Allocate the appropriate stats memory */
  539         if (adapter->vf_ifp) {
  540                 adapter->stats =
  541                     (struct e1000_vf_stats *)malloc(sizeof \
  542                     (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
  543                 igb_vf_init_stats(adapter);
  544         } else
  545                 adapter->stats =
  546                     (struct e1000_hw_stats *)malloc(sizeof \
  547                     (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO);
  548         if (adapter->stats == NULL) {
  549                 device_printf(dev, "Can not allocate stats memory\n");
  550                 error = ENOMEM;
  551                 goto err_late;
  552         }
  553 
  554         /* Allocate multicast array memory. */
  555         adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN *
  556             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
  557         if (adapter->mta == NULL) {
  558                 device_printf(dev, "Can not allocate multicast setup array\n");
  559                 error = ENOMEM;
  560                 goto err_late;
  561         }
  562 
  563         /* Some adapter-specific advanced features */
  564         if (adapter->hw.mac.type >= e1000_i350) {
  565                 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  566                     SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  567                     OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW,
  568                     adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce");
  569                 igb_set_sysctl_value(adapter, "eee_disabled",
  570                     "enable Energy Efficient Ethernet",
  571                     &adapter->hw.dev_spec._82575.eee_disable,
  572                     TRUE);
  573                 e1000_set_eee_i350(&adapter->hw);
  574         }
  575 
  576         /*
  577         ** Start from a known state, this is
  578         ** important in reading the nvm and
  579         ** mac from that.
  580         */
  581         e1000_reset_hw(&adapter->hw);
  582 
  583         /* Make sure we have a good EEPROM before we read from it */
  584         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
  585                 /*
  586                 ** Some PCI-E parts fail the first check due to
  587                 ** the link being in sleep state, call it again,
  588                 ** if it fails a second time its a real issue.
  589                 */
  590                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
  591                         device_printf(dev,
  592                             "The EEPROM Checksum Is Not Valid\n");
  593                         error = EIO;
  594                         goto err_late;
  595                 }
  596         }
  597 
  598         /*
  599         ** Copy the permanent MAC address out of the EEPROM
  600         */
  601         if (e1000_read_mac_addr(&adapter->hw) < 0) {
  602                 device_printf(dev, "EEPROM read error while reading MAC"
  603                     " address\n");
  604                 error = EIO;
  605                 goto err_late;
  606         }
  607         /* Check its sanity */
  608         if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
  609                 device_printf(dev, "Invalid MAC address\n");
  610                 error = EIO;
  611                 goto err_late;
  612         }
  613 
  614         /* 
  615         ** Configure Interrupts
  616         */
  617         if ((adapter->msix > 1) && (igb_enable_msix))
  618                 error = igb_allocate_msix(adapter);
  619         else /* MSI or Legacy */
  620                 error = igb_allocate_legacy(adapter);
  621         if (error)
  622                 goto err_late;
  623 
  624         /* Setup OS specific network interface */
  625         if (igb_setup_interface(dev, adapter) != 0)
  626                 goto err_late;
  627 
  628         /* Now get a good starting state */
  629         igb_reset(adapter);
  630 
  631         /* Initialize statistics */
  632         igb_update_stats_counters(adapter);
  633 
  634         adapter->hw.mac.get_link_status = 1;
  635         igb_update_link_status(adapter);
  636 
  637         /* Indicate SOL/IDER usage */
  638         if (e1000_check_reset_block(&adapter->hw))
  639                 device_printf(dev,
  640                     "PHY reset is blocked due to SOL/IDER session.\n");
  641 
  642         /* Determine if we have to control management hardware */
  643         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
  644 
  645         /*
  646          * Setup Wake-on-Lan
  647          */
  648         /* APME bit in EEPROM is mapped to WUC.APME */
  649         eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
  650         if (eeprom_data)
  651                 adapter->wol = E1000_WUFC_MAG;
  652 
  653         /* Register for VLAN events */
  654         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
  655              igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  656         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
  657              igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  658 
  659         igb_add_hw_stats(adapter);
  660 
  661         /* Tell the stack that the interface is not active */
  662         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
  663 
  664         adapter->led_dev = led_create(igb_led_func, adapter,
  665             device_get_nameunit(dev));
  666 
  667         INIT_DEBUGOUT("igb_attach: end");
  668 
  669         return (0);
  670 
  671 err_late:
  672         igb_detach(dev);
  673         igb_free_transmit_structures(adapter);
  674         igb_free_receive_structures(adapter);
  675         igb_release_hw_control(adapter);
  676         if (adapter->ifp != NULL)
  677                 if_free(adapter->ifp);
  678 err_pci:
  679         igb_free_pci_resources(adapter);
  680         free(adapter->mta, M_DEVBUF);
  681         IGB_CORE_LOCK_DESTROY(adapter);
  682 
  683         return (error);
  684 }
  685 
  686 /*********************************************************************
  687  *  Device removal routine
  688  *
  689  *  The detach entry point is called when the driver is being removed.
  690  *  This routine stops the adapter and deallocates all the resources
  691  *  that were allocated for driver operation.
  692  *
  693  *  return 0 on success, positive on failure
  694  *********************************************************************/
  695 
  696 static int
  697 igb_detach(device_t dev)
  698 {
  699         struct adapter  *adapter = device_get_softc(dev);
  700         struct ifnet    *ifp = adapter->ifp;
  701 
  702         INIT_DEBUGOUT("igb_detach: begin");
  703 
  704         /* Make sure VLANS are not using driver */
  705         if (adapter->ifp->if_vlantrunk != NULL) {
  706                 device_printf(dev,"Vlan in use, detach first\n");
  707                 return (EBUSY);
  708         }
  709 
  710         ether_ifdetach(adapter->ifp);
  711 
  712         if (adapter->led_dev != NULL)
  713                 led_destroy(adapter->led_dev);
  714 
  715 #ifdef DEVICE_POLLING
  716         if (ifp->if_capenable & IFCAP_POLLING)
  717                 ether_poll_deregister(ifp);
  718 #endif
  719 
  720         IGB_CORE_LOCK(adapter);
  721         adapter->in_detach = 1;
  722         igb_stop(adapter);
  723         IGB_CORE_UNLOCK(adapter);
  724 
  725         e1000_phy_hw_reset(&adapter->hw);
  726 
  727         /* Give control back to firmware */
  728         igb_release_manageability(adapter);
  729         igb_release_hw_control(adapter);
  730 
  731         if (adapter->wol) {
  732                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
  733                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
  734                 igb_enable_wakeup(dev);
  735         }
  736 
  737         /* Unregister VLAN events */
  738         if (adapter->vlan_attach != NULL)
  739                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
  740         if (adapter->vlan_detach != NULL)
  741                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
  742 
  743         callout_drain(&adapter->timer);
  744 
  745         igb_free_pci_resources(adapter);
  746         bus_generic_detach(dev);
  747         if_free(ifp);
  748 
  749         igb_free_transmit_structures(adapter);
  750         igb_free_receive_structures(adapter);
  751         if (adapter->mta != NULL)
  752                 free(adapter->mta, M_DEVBUF);
  753 
  754         IGB_CORE_LOCK_DESTROY(adapter);
  755 
  756         return (0);
  757 }
  758 
  759 /*********************************************************************
  760  *
  761  *  Shutdown entry point
  762  *
  763  **********************************************************************/
  764 
  765 static int
  766 igb_shutdown(device_t dev)
  767 {
  768         return igb_suspend(dev);
  769 }
  770 
  771 /*
  772  * Suspend/resume device methods.
  773  */
  774 static int
  775 igb_suspend(device_t dev)
  776 {
  777         struct adapter *adapter = device_get_softc(dev);
  778 
  779         IGB_CORE_LOCK(adapter);
  780 
  781         igb_stop(adapter);
  782 
  783         igb_release_manageability(adapter);
  784         igb_release_hw_control(adapter);
  785 
  786         if (adapter->wol) {
  787                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
  788                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
  789                 igb_enable_wakeup(dev);
  790         }
  791 
  792         IGB_CORE_UNLOCK(adapter);
  793 
  794         return bus_generic_suspend(dev);
  795 }
  796 
  797 static int
  798 igb_resume(device_t dev)
  799 {
  800         struct adapter *adapter = device_get_softc(dev);
  801         struct ifnet *ifp = adapter->ifp;
  802 #if __FreeBSD_version >= 800000
  803         struct tx_ring *txr = adapter->tx_rings;
  804 #endif
  805 
  806         IGB_CORE_LOCK(adapter);
  807         igb_init_locked(adapter);
  808         igb_init_manageability(adapter);
  809 
  810         if ((ifp->if_flags & IFF_UP) &&
  811             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
  812 #if __FreeBSD_version < 800000
  813                 igb_start(ifp);
  814 #else
  815                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
  816                         IGB_TX_LOCK(txr);
  817                         if (!drbr_empty(ifp, txr->br))
  818                                 igb_mq_start_locked(ifp, txr, NULL);
  819                         IGB_TX_UNLOCK(txr);
  820                 }
  821 #endif
  822         }
  823 
  824         IGB_CORE_UNLOCK(adapter);
  825 
  826         return bus_generic_resume(dev);
  827 }
  828 
  829 
  830 #if __FreeBSD_version < 800000
  831 /*********************************************************************
  832  *  Transmit entry point
  833  *
  834  *  igb_start is called by the stack to initiate a transmit.
  835  *  The driver will remain in this routine as long as there are
  836  *  packets to transmit and transmit resources are available.
  837  *  In case resources are not available stack is notified and
  838  *  the packet is requeued.
  839  **********************************************************************/
  840 
  841 static void
  842 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
  843 {
  844         struct adapter  *adapter = ifp->if_softc;
  845         struct mbuf     *m_head;
  846 
  847         IGB_TX_LOCK_ASSERT(txr);
  848 
  849         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
  850             IFF_DRV_RUNNING)
  851                 return;
  852         if (!adapter->link_active)
  853                 return;
  854 
  855         /* Call cleanup if number of TX descriptors low */
  856         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
  857                 igb_txeof(txr);
  858 
  859         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
  860                 if (txr->tx_avail <= IGB_MAX_SCATTER) {
  861                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
  862                         break;
  863                 }
  864                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
  865                 if (m_head == NULL)
  866                         break;
  867                 /*
  868                  *  Encapsulation can modify our pointer, and or make it
  869                  *  NULL on failure.  In that event, we can't requeue.
  870                  */
  871                 if (igb_xmit(txr, &m_head)) {
  872                         if (m_head == NULL)
  873                                 break;
  874                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
  875                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
  876                         break;
  877                 }
  878 
  879                 /* Send a copy of the frame to the BPF listener */
  880                 ETHER_BPF_MTAP(ifp, m_head);
  881 
  882                 /* Set watchdog on */
  883                 txr->watchdog_time = ticks;
  884                 txr->queue_status = IGB_QUEUE_WORKING;
  885         }
  886 }
  887  
  888 /*
  889  * Legacy TX driver routine, called from the
  890  * stack, always uses tx[0], and spins for it.
  891  * Should not be used with multiqueue tx
  892  */
  893 static void
  894 igb_start(struct ifnet *ifp)
  895 {
  896         struct adapter  *adapter = ifp->if_softc;
  897         struct tx_ring  *txr = adapter->tx_rings;
  898 
  899         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  900                 IGB_TX_LOCK(txr);
  901                 igb_start_locked(txr, ifp);
  902                 IGB_TX_UNLOCK(txr);
  903         }
  904         return;
  905 }
  906 
  907 #else /* __FreeBSD_version >= 800000 */
  908 /*
  909 ** Multiqueue Transmit driver
  910 **
  911 */
  912 static int
  913 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
  914 {
  915         struct adapter          *adapter = ifp->if_softc;
  916         struct igb_queue        *que;
  917         struct tx_ring          *txr;
  918         int                     i = 0, err = 0;
  919 
  920         /* Which queue to use */
  921         if ((m->m_flags & M_FLOWID) != 0)
  922                 i = m->m_pkthdr.flowid % adapter->num_queues;
  923 
  924         txr = &adapter->tx_rings[i];
  925         que = &adapter->queues[i];
  926 
  927         if (IGB_TX_TRYLOCK(txr)) {
  928                 err = igb_mq_start_locked(ifp, txr, m);
  929                 IGB_TX_UNLOCK(txr);
  930         } else {
  931                 err = drbr_enqueue(ifp, txr->br, m);
  932                 taskqueue_enqueue(que->tq, &txr->txq_task);
  933         }
  934 
  935         return (err);
  936 }
  937 
  938 static int
  939 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
  940 {
  941         struct adapter  *adapter = txr->adapter;
  942         struct mbuf     *next;
  943         int             err = 0, enq;
  944 
  945         IGB_TX_LOCK_ASSERT(txr);
  946 
  947         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
  948             IFF_DRV_RUNNING || adapter->link_active == 0) {
  949                 if (m != NULL)
  950                         err = drbr_enqueue(ifp, txr->br, m);
  951                 return (err);
  952         }
  953 
  954         enq = 0;
  955         if (m == NULL) {
  956                 next = drbr_dequeue(ifp, txr->br);
  957         } else if (drbr_needs_enqueue(ifp, txr->br)) {
  958                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
  959                         return (err);
  960                 next = drbr_dequeue(ifp, txr->br);
  961         } else
  962                 next = m;
  963 
  964         /* Process the queue */
  965         while (next != NULL) {
  966                 if ((err = igb_xmit(txr, &next)) != 0) {
  967                         if (next != NULL)
  968                                 err = drbr_enqueue(ifp, txr->br, next);
  969                         break;
  970                 }
  971                 enq++;
  972                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
  973                 ETHER_BPF_MTAP(ifp, next);
  974                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
  975                         break;
  976                 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD)
  977                         igb_txeof(txr);
  978                 if (txr->tx_avail <= IGB_MAX_SCATTER) {
  979                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
  980                         break;
  981                 }
  982                 next = drbr_dequeue(ifp, txr->br);
  983         }
  984         if (enq > 0) {
  985                 /* Set the watchdog */
  986                 txr->queue_status = IGB_QUEUE_WORKING;
  987                 txr->watchdog_time = ticks;
  988         }
  989         return (err);
  990 }
  991 
  992 /*
  993  * Called from a taskqueue to drain queued transmit packets.
  994  */
  995 static void
  996 igb_deferred_mq_start(void *arg, int pending)
  997 {
  998         struct tx_ring *txr = arg;
  999         struct adapter *adapter = txr->adapter;
 1000         struct ifnet *ifp = adapter->ifp;
 1001 
 1002         IGB_TX_LOCK(txr);
 1003         if (!drbr_empty(ifp, txr->br))
 1004                 igb_mq_start_locked(ifp, txr, NULL);
 1005         IGB_TX_UNLOCK(txr);
 1006 }
 1007 
 1008 /*
 1009 ** Flush all ring buffers
 1010 */
 1011 static void
 1012 igb_qflush(struct ifnet *ifp)
 1013 {
 1014         struct adapter  *adapter = ifp->if_softc;
 1015         struct tx_ring  *txr = adapter->tx_rings;
 1016         struct mbuf     *m;
 1017 
 1018         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1019                 IGB_TX_LOCK(txr);
 1020                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 1021                         m_freem(m);
 1022                 IGB_TX_UNLOCK(txr);
 1023         }
 1024         if_qflush(ifp);
 1025 }
 1026 #endif /* __FreeBSD_version < 800000 */
 1027 
 1028 /*********************************************************************
 1029  *  Ioctl entry point
 1030  *
 1031  *  igb_ioctl is called when the user wants to configure the
 1032  *  interface.
 1033  *
 1034  *  return 0 on success, positive on failure
 1035  **********************************************************************/
 1036 
 1037 static int
 1038 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 1039 {
 1040         struct adapter  *adapter = ifp->if_softc;
 1041         struct ifreq    *ifr = (struct ifreq *)data;
 1042 #if defined(INET) || defined(INET6)
 1043         struct ifaddr   *ifa = (struct ifaddr *)data;
 1044 #endif
 1045         bool            avoid_reset = FALSE;
 1046         int             error = 0;
 1047 
 1048         if (adapter->in_detach)
 1049                 return (error);
 1050 
 1051         switch (command) {
 1052         case SIOCSIFADDR:
 1053 #ifdef INET
 1054                 if (ifa->ifa_addr->sa_family == AF_INET)
 1055                         avoid_reset = TRUE;
 1056 #endif
 1057 #ifdef INET6
 1058                 if (ifa->ifa_addr->sa_family == AF_INET6)
 1059                         avoid_reset = TRUE;
 1060 #endif
 1061                 /*
 1062                 ** Calling init results in link renegotiation,
 1063                 ** so we avoid doing it when possible.
 1064                 */
 1065                 if (avoid_reset) {
 1066                         ifp->if_flags |= IFF_UP;
 1067                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 1068                                 igb_init(adapter);
 1069 #ifdef INET
 1070                         if (!(ifp->if_flags & IFF_NOARP))
 1071                                 arp_ifinit(ifp, ifa);
 1072 #endif
 1073                 } else
 1074                         error = ether_ioctl(ifp, command, data);
 1075                 break;
 1076         case SIOCSIFMTU:
 1077             {
 1078                 int max_frame_size;
 1079 
 1080                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 1081 
 1082                 IGB_CORE_LOCK(adapter);
 1083                 max_frame_size = 9234;
 1084                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
 1085                     ETHER_CRC_LEN) {
 1086                         IGB_CORE_UNLOCK(adapter);
 1087                         error = EINVAL;
 1088                         break;
 1089                 }
 1090 
 1091                 ifp->if_mtu = ifr->ifr_mtu;
 1092                 adapter->max_frame_size =
 1093                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 1094                 igb_init_locked(adapter);
 1095                 IGB_CORE_UNLOCK(adapter);
 1096                 break;
 1097             }
 1098         case SIOCSIFFLAGS:
 1099                 IOCTL_DEBUGOUT("ioctl rcv'd:\
 1100                     SIOCSIFFLAGS (Set Interface Flags)");
 1101                 IGB_CORE_LOCK(adapter);
 1102                 if (ifp->if_flags & IFF_UP) {
 1103                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 1104                                 if ((ifp->if_flags ^ adapter->if_flags) &
 1105                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 1106                                         igb_disable_promisc(adapter);
 1107                                         igb_set_promisc(adapter);
 1108                                 }
 1109                         } else
 1110                                 igb_init_locked(adapter);
 1111                 } else
 1112                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1113                                 igb_stop(adapter);
 1114                 adapter->if_flags = ifp->if_flags;
 1115                 IGB_CORE_UNLOCK(adapter);
 1116                 break;
 1117         case SIOCADDMULTI:
 1118         case SIOCDELMULTI:
 1119                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 1120                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1121                         IGB_CORE_LOCK(adapter);
 1122                         igb_disable_intr(adapter);
 1123                         igb_set_multi(adapter);
 1124 #ifdef DEVICE_POLLING
 1125                         if (!(ifp->if_capenable & IFCAP_POLLING))
 1126 #endif
 1127                                 igb_enable_intr(adapter);
 1128                         IGB_CORE_UNLOCK(adapter);
 1129                 }
 1130                 break;
 1131         case SIOCSIFMEDIA:
 1132                 /*
 1133                 ** As the speed/duplex settings are being
 1134                 ** changed, we need toreset the PHY.
 1135                 */
 1136                 adapter->hw.phy.reset_disable = FALSE;
 1137                 /* Check SOL/IDER usage */
 1138                 IGB_CORE_LOCK(adapter);
 1139                 if (e1000_check_reset_block(&adapter->hw)) {
 1140                         IGB_CORE_UNLOCK(adapter);
 1141                         device_printf(adapter->dev, "Media change is"
 1142                             " blocked due to SOL/IDER session.\n");
 1143                         break;
 1144                 }
 1145                 IGB_CORE_UNLOCK(adapter);
 1146         case SIOCGIFMEDIA:
 1147                 IOCTL_DEBUGOUT("ioctl rcv'd: \
 1148                     SIOCxIFMEDIA (Get/Set Interface Media)");
 1149                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 1150                 break;
 1151         case SIOCSIFCAP:
 1152             {
 1153                 int mask, reinit;
 1154 
 1155                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 1156                 reinit = 0;
 1157                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 1158 #ifdef DEVICE_POLLING
 1159                 if (mask & IFCAP_POLLING) {
 1160                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
 1161                                 error = ether_poll_register(igb_poll, ifp);
 1162                                 if (error)
 1163                                         return (error);
 1164                                 IGB_CORE_LOCK(adapter);
 1165                                 igb_disable_intr(adapter);
 1166                                 ifp->if_capenable |= IFCAP_POLLING;
 1167                                 IGB_CORE_UNLOCK(adapter);
 1168                         } else {
 1169                                 error = ether_poll_deregister(ifp);
 1170                                 /* Enable interrupt even in error case */
 1171                                 IGB_CORE_LOCK(adapter);
 1172                                 igb_enable_intr(adapter);
 1173                                 ifp->if_capenable &= ~IFCAP_POLLING;
 1174                                 IGB_CORE_UNLOCK(adapter);
 1175                         }
 1176                 }
 1177 #endif
 1178                 if (mask & IFCAP_HWCSUM) {
 1179                         ifp->if_capenable ^= IFCAP_HWCSUM;
 1180                         reinit = 1;
 1181                 }
 1182                 if (mask & IFCAP_TSO4) {
 1183                         ifp->if_capenable ^= IFCAP_TSO4;
 1184                         reinit = 1;
 1185                 }
 1186                 if (mask & IFCAP_VLAN_HWTAGGING) {
 1187                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 1188                         reinit = 1;
 1189                 }
 1190                 if (mask & IFCAP_VLAN_HWFILTER) {
 1191                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 1192                         reinit = 1;
 1193                 }
 1194                 if (mask & IFCAP_VLAN_HWTSO) {
 1195                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
 1196                         reinit = 1;
 1197                 }
 1198                 if (mask & IFCAP_LRO) {
 1199                         ifp->if_capenable ^= IFCAP_LRO;
 1200                         reinit = 1;
 1201                 }
 1202                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
 1203                         igb_init(adapter);
 1204                 VLAN_CAPABILITIES(ifp);
 1205                 break;
 1206             }
 1207 
 1208         default:
 1209                 error = ether_ioctl(ifp, command, data);
 1210                 break;
 1211         }
 1212 
 1213         return (error);
 1214 }
 1215 
 1216 
 1217 /*********************************************************************
 1218  *  Init entry point
 1219  *
 1220  *  This routine is used in two ways. It is used by the stack as
 1221  *  init entry point in network interface structure. It is also used
 1222  *  by the driver as a hw/sw initialization routine to get to a
 1223  *  consistent state.
 1224  *
 1225  *  return 0 on success, positive on failure
 1226  **********************************************************************/
 1227 
 1228 static void
 1229 igb_init_locked(struct adapter *adapter)
 1230 {
 1231         struct ifnet    *ifp = adapter->ifp;
 1232         device_t        dev = adapter->dev;
 1233 
 1234         INIT_DEBUGOUT("igb_init: begin");
 1235 
 1236         IGB_CORE_LOCK_ASSERT(adapter);
 1237 
 1238         igb_disable_intr(adapter);
 1239         callout_stop(&adapter->timer);
 1240 
 1241         /* Get the latest mac address, User can use a LAA */
 1242         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
 1243               ETHER_ADDR_LEN);
 1244 
 1245         /* Put the address into the Receive Address Array */
 1246         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 1247 
 1248         igb_reset(adapter);
 1249         igb_update_link_status(adapter);
 1250 
 1251         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 1252 
 1253         /* Set hardware offload abilities */
 1254         ifp->if_hwassist = 0;
 1255         if (ifp->if_capenable & IFCAP_TXCSUM) {
 1256                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 1257 #if __FreeBSD_version >= 800000
 1258                 if (adapter->hw.mac.type == e1000_82576)
 1259                         ifp->if_hwassist |= CSUM_SCTP;
 1260 #endif
 1261         }
 1262 
 1263         if (ifp->if_capenable & IFCAP_TSO4)
 1264                 ifp->if_hwassist |= CSUM_TSO;
 1265 
 1266         /* Configure for OS presence */
 1267         igb_init_manageability(adapter);
 1268 
 1269         /* Prepare transmit descriptors and buffers */
 1270         igb_setup_transmit_structures(adapter);
 1271         igb_initialize_transmit_units(adapter);
 1272 
 1273         /* Setup Multicast table */
 1274         igb_set_multi(adapter);
 1275 
 1276         /*
 1277         ** Figure out the desired mbuf pool
 1278         ** for doing jumbo/packetsplit
 1279         */
 1280         if (adapter->max_frame_size <= 2048)
 1281                 adapter->rx_mbuf_sz = MCLBYTES;
 1282         else if (adapter->max_frame_size <= 4096)
 1283                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 1284         else
 1285                 adapter->rx_mbuf_sz = MJUM9BYTES;
 1286 
 1287         /* Prepare receive descriptors and buffers */
 1288         if (igb_setup_receive_structures(adapter)) {
 1289                 device_printf(dev, "Could not setup receive structures\n");
 1290                 return;
 1291         }
 1292         igb_initialize_receive_units(adapter);
 1293 
 1294         /* Enable VLAN support */
 1295         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
 1296                 igb_setup_vlan_hw_support(adapter);
 1297                                 
 1298         /* Don't lose promiscuous settings */
 1299         igb_set_promisc(adapter);
 1300 
 1301         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 1302         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 1303 
 1304         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 1305         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
 1306 
 1307         if (adapter->msix > 1) /* Set up queue routing */
 1308                 igb_configure_queues(adapter);
 1309 
 1310         /* this clears any pending interrupts */
 1311         E1000_READ_REG(&adapter->hw, E1000_ICR);
 1312 #ifdef DEVICE_POLLING
 1313         /*
 1314          * Only enable interrupts if we are not polling, make sure
 1315          * they are off otherwise.
 1316          */
 1317         if (ifp->if_capenable & IFCAP_POLLING)
 1318                 igb_disable_intr(adapter);
 1319         else
 1320 #endif /* DEVICE_POLLING */
 1321         {
 1322                 igb_enable_intr(adapter);
 1323                 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
 1324         }
 1325 
 1326         /* Set Energy Efficient Ethernet */
 1327         e1000_set_eee_i350(&adapter->hw);
 1328 
 1329         /* Don't reset the phy next time init gets called */
 1330         adapter->hw.phy.reset_disable = TRUE;
 1331 }
 1332 
 1333 static void
 1334 igb_init(void *arg)
 1335 {
 1336         struct adapter *adapter = arg;
 1337 
 1338         IGB_CORE_LOCK(adapter);
 1339         igb_init_locked(adapter);
 1340         IGB_CORE_UNLOCK(adapter);
 1341 }
 1342 
 1343 
 1344 static void
 1345 igb_handle_que(void *context, int pending)
 1346 {
 1347         struct igb_queue *que = context;
 1348         struct adapter *adapter = que->adapter;
 1349         struct tx_ring *txr = que->txr;
 1350         struct ifnet    *ifp = adapter->ifp;
 1351 
 1352         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1353                 bool    more;
 1354 
 1355                 more = igb_rxeof(que, -1, NULL);
 1356 
 1357                 IGB_TX_LOCK(txr);
 1358                 if (igb_txeof(txr))
 1359                         more = TRUE;
 1360 #if __FreeBSD_version >= 800000
 1361                 if (!drbr_empty(ifp, txr->br))
 1362                         igb_mq_start_locked(ifp, txr, NULL);
 1363 #else
 1364                 igb_start_locked(txr, ifp);
 1365 #endif
 1366                 IGB_TX_UNLOCK(txr);
 1367                 if (more || (ifp->if_drv_flags & IFF_DRV_OACTIVE)) {
 1368                         taskqueue_enqueue(que->tq, &que->que_task);
 1369                         return;
 1370                 }
 1371         }
 1372 
 1373 #ifdef DEVICE_POLLING
 1374         if (ifp->if_capenable & IFCAP_POLLING)
 1375                 return;
 1376 #endif
 1377         /* Reenable this interrupt */
 1378         if (que->eims)
 1379                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
 1380         else
 1381                 igb_enable_intr(adapter);
 1382 }
 1383 
 1384 /* Deal with link in a sleepable context */
 1385 static void
 1386 igb_handle_link(void *context, int pending)
 1387 {
 1388         struct adapter *adapter = context;
 1389 
 1390         adapter->hw.mac.get_link_status = 1;
 1391         igb_update_link_status(adapter);
 1392 }
 1393 
 1394 /*********************************************************************
 1395  *
 1396  *  MSI/Legacy Deferred
 1397  *  Interrupt Service routine  
 1398  *
 1399  *********************************************************************/
 1400 static int
 1401 igb_irq_fast(void *arg)
 1402 {
 1403         struct adapter          *adapter = arg;
 1404         struct igb_queue        *que = adapter->queues;
 1405         u32                     reg_icr;
 1406 
 1407 
 1408         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1409 
 1410         /* Hot eject?  */
 1411         if (reg_icr == 0xffffffff)
 1412                 return FILTER_STRAY;
 1413 
 1414         /* Definitely not our interrupt.  */
 1415         if (reg_icr == 0x0)
 1416                 return FILTER_STRAY;
 1417 
 1418         if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
 1419                 return FILTER_STRAY;
 1420 
 1421         /*
 1422          * Mask interrupts until the taskqueue is finished running.  This is
 1423          * cheap, just assume that it is needed.  This also works around the
 1424          * MSI message reordering errata on certain systems.
 1425          */
 1426         igb_disable_intr(adapter);
 1427         taskqueue_enqueue(que->tq, &que->que_task);
 1428 
 1429         /* Link status change */
 1430         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
 1431                 taskqueue_enqueue(que->tq, &adapter->link_task);
 1432 
 1433         if (reg_icr & E1000_ICR_RXO)
 1434                 adapter->rx_overruns++;
 1435         return FILTER_HANDLED;
 1436 }
 1437 
 1438 #ifdef DEVICE_POLLING
 1439 /*********************************************************************
 1440  *
 1441  *  Legacy polling routine : if using this code you MUST be sure that
 1442  *  multiqueue is not defined, ie, set igb_num_queues to 1.
 1443  *
 1444  *********************************************************************/
 1445 #if __FreeBSD_version >= 800000
 1446 #define POLL_RETURN_COUNT(a) (a)
 1447 static int
 1448 #else
 1449 #define POLL_RETURN_COUNT(a)
 1450 static void
 1451 #endif
 1452 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 1453 {
 1454         struct adapter          *adapter = ifp->if_softc;
 1455         struct igb_queue        *que = adapter->queues;
 1456         struct tx_ring          *txr = adapter->tx_rings;
 1457         u32                     reg_icr, rx_done = 0;
 1458         u32                     loop = IGB_MAX_LOOP;
 1459         bool                    more;
 1460 
 1461         IGB_CORE_LOCK(adapter);
 1462         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 1463                 IGB_CORE_UNLOCK(adapter);
 1464                 return POLL_RETURN_COUNT(rx_done);
 1465         }
 1466 
 1467         if (cmd == POLL_AND_CHECK_STATUS) {
 1468                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1469                 /* Link status change */
 1470                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
 1471                         igb_handle_link(adapter, 0);
 1472 
 1473                 if (reg_icr & E1000_ICR_RXO)
 1474                         adapter->rx_overruns++;
 1475         }
 1476         IGB_CORE_UNLOCK(adapter);
 1477 
 1478         igb_rxeof(que, count, &rx_done);
 1479 
 1480         IGB_TX_LOCK(txr);
 1481         do {
 1482                 more = igb_txeof(txr);
 1483         } while (loop-- && more);
 1484 #if __FreeBSD_version >= 800000
 1485         if (!drbr_empty(ifp, txr->br))
 1486                 igb_mq_start_locked(ifp, txr, NULL);
 1487 #else
 1488         igb_start_locked(txr, ifp);
 1489 #endif
 1490         IGB_TX_UNLOCK(txr);
 1491         return POLL_RETURN_COUNT(rx_done);
 1492 }
 1493 #endif /* DEVICE_POLLING */
 1494 
 1495 /*********************************************************************
 1496  *
 1497  *  MSIX TX Interrupt Service routine
 1498  *
 1499  **********************************************************************/
 1500 static void
 1501 igb_msix_que(void *arg)
 1502 {
 1503         struct igb_queue *que = arg;
 1504         struct adapter *adapter = que->adapter;
 1505         struct tx_ring *txr = que->txr;
 1506         struct rx_ring *rxr = que->rxr;
 1507         u32             newitr = 0;
 1508         bool            more_tx, more_rx;
 1509 
 1510         E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
 1511         ++que->irqs;
 1512 
 1513         IGB_TX_LOCK(txr);
 1514         more_tx = igb_txeof(txr);
 1515         IGB_TX_UNLOCK(txr);
 1516 
 1517         more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL);
 1518 
 1519         if (adapter->enable_aim == FALSE)
 1520                 goto no_calc;
 1521         /*
 1522         ** Do Adaptive Interrupt Moderation:
 1523         **  - Write out last calculated setting
 1524         **  - Calculate based on average size over
 1525         **    the last interval.
 1526         */
 1527         if (que->eitr_setting)
 1528                 E1000_WRITE_REG(&adapter->hw,
 1529                     E1000_EITR(que->msix), que->eitr_setting);
 1530  
 1531         que->eitr_setting = 0;
 1532 
 1533         /* Idle, do nothing */
 1534         if ((txr->bytes == 0) && (rxr->bytes == 0))
 1535                 goto no_calc;
 1536                                 
 1537         /* Used half Default if sub-gig */
 1538         if (adapter->link_speed != 1000)
 1539                 newitr = IGB_DEFAULT_ITR / 2;
 1540         else {
 1541                 if ((txr->bytes) && (txr->packets))
 1542                         newitr = txr->bytes/txr->packets;
 1543                 if ((rxr->bytes) && (rxr->packets))
 1544                         newitr = max(newitr,
 1545                             (rxr->bytes / rxr->packets));
 1546                 newitr += 24; /* account for hardware frame, crc */
 1547                 /* set an upper boundary */
 1548                 newitr = min(newitr, 3000);
 1549                 /* Be nice to the mid range */
 1550                 if ((newitr > 300) && (newitr < 1200))
 1551                         newitr = (newitr / 3);
 1552                 else
 1553                         newitr = (newitr / 2);
 1554         }
 1555         newitr &= 0x7FFC;  /* Mask invalid bits */
 1556         if (adapter->hw.mac.type == e1000_82575)
 1557                 newitr |= newitr << 16;
 1558         else
 1559                 newitr |= E1000_EITR_CNT_IGNR;
 1560                  
 1561         /* save for next interrupt */
 1562         que->eitr_setting = newitr;
 1563 
 1564         /* Reset state */
 1565         txr->bytes = 0;
 1566         txr->packets = 0;
 1567         rxr->bytes = 0;
 1568         rxr->packets = 0;
 1569 
 1570 no_calc:
 1571         /* Schedule a clean task if needed*/
 1572         if (more_tx || more_rx ||
 1573             (adapter->ifp->if_drv_flags & IFF_DRV_OACTIVE))
 1574                 taskqueue_enqueue(que->tq, &que->que_task);
 1575         else
 1576                 /* Reenable this interrupt */
 1577                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
 1578         return;
 1579 }
 1580 
 1581 
 1582 /*********************************************************************
 1583  *
 1584  *  MSIX Link Interrupt Service routine
 1585  *
 1586  **********************************************************************/
 1587 
 1588 static void
 1589 igb_msix_link(void *arg)
 1590 {
 1591         struct adapter  *adapter = arg;
 1592         u32             icr;
 1593 
 1594         ++adapter->link_irq;
 1595         icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1596         if (!(icr & E1000_ICR_LSC))
 1597                 goto spurious;
 1598         igb_handle_link(adapter, 0);
 1599 
 1600 spurious:
 1601         /* Rearm */
 1602         E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
 1603         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
 1604         return;
 1605 }
 1606 
 1607 
 1608 /*********************************************************************
 1609  *
 1610  *  Media Ioctl callback
 1611  *
 1612  *  This routine is called whenever the user queries the status of
 1613  *  the interface using ifconfig.
 1614  *
 1615  **********************************************************************/
 1616 static void
 1617 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 1618 {
 1619         struct adapter *adapter = ifp->if_softc;
 1620         u_char fiber_type = IFM_1000_SX;
 1621 
 1622         INIT_DEBUGOUT("igb_media_status: begin");
 1623 
 1624         IGB_CORE_LOCK(adapter);
 1625         igb_update_link_status(adapter);
 1626 
 1627         ifmr->ifm_status = IFM_AVALID;
 1628         ifmr->ifm_active = IFM_ETHER;
 1629 
 1630         if (!adapter->link_active) {
 1631                 IGB_CORE_UNLOCK(adapter);
 1632                 return;
 1633         }
 1634 
 1635         ifmr->ifm_status |= IFM_ACTIVE;
 1636 
 1637         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 1638             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes))
 1639                 ifmr->ifm_active |= fiber_type | IFM_FDX;
 1640         else {
 1641                 switch (adapter->link_speed) {
 1642                 case 10:
 1643                         ifmr->ifm_active |= IFM_10_T;
 1644                         break;
 1645                 case 100:
 1646                         ifmr->ifm_active |= IFM_100_TX;
 1647                         break;
 1648                 case 1000:
 1649                         ifmr->ifm_active |= IFM_1000_T;
 1650                         break;
 1651                 }
 1652                 if (adapter->link_duplex == FULL_DUPLEX)
 1653                         ifmr->ifm_active |= IFM_FDX;
 1654                 else
 1655                         ifmr->ifm_active |= IFM_HDX;
 1656         }
 1657         IGB_CORE_UNLOCK(adapter);
 1658 }
 1659 
 1660 /*********************************************************************
 1661  *
 1662  *  Media Ioctl callback
 1663  *
 1664  *  This routine is called when the user changes speed/duplex using
 1665  *  media/mediopt option with ifconfig.
 1666  *
 1667  **********************************************************************/
 1668 static int
 1669 igb_media_change(struct ifnet *ifp)
 1670 {
 1671         struct adapter *adapter = ifp->if_softc;
 1672         struct ifmedia  *ifm = &adapter->media;
 1673 
 1674         INIT_DEBUGOUT("igb_media_change: begin");
 1675 
 1676         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 1677                 return (EINVAL);
 1678 
 1679         IGB_CORE_LOCK(adapter);
 1680         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 1681         case IFM_AUTO:
 1682                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1683                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 1684                 break;
 1685         case IFM_1000_LX:
 1686         case IFM_1000_SX:
 1687         case IFM_1000_T:
 1688                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1689                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
 1690                 break;
 1691         case IFM_100_TX:
 1692                 adapter->hw.mac.autoneg = FALSE;
 1693                 adapter->hw.phy.autoneg_advertised = 0;
 1694                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1695                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
 1696                 else
 1697                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
 1698                 break;
 1699         case IFM_10_T:
 1700                 adapter->hw.mac.autoneg = FALSE;
 1701                 adapter->hw.phy.autoneg_advertised = 0;
 1702                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1703                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
 1704                 else
 1705                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
 1706                 break;
 1707         default:
 1708                 device_printf(adapter->dev, "Unsupported media type\n");
 1709         }
 1710 
 1711         igb_init_locked(adapter);
 1712         IGB_CORE_UNLOCK(adapter);
 1713 
 1714         return (0);
 1715 }
 1716 
 1717 
 1718 /*********************************************************************
 1719  *
 1720  *  This routine maps the mbufs to Advanced TX descriptors.
 1721  *  used by the 82575 adapter.
 1722  *  
 1723  **********************************************************************/
 1724 
 1725 static int
 1726 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 1727 {
 1728         struct adapter          *adapter = txr->adapter;
 1729         bus_dma_segment_t       segs[IGB_MAX_SCATTER];
 1730         bus_dmamap_t            map;
 1731         struct igb_tx_buffer    *tx_buffer, *tx_buffer_mapped;
 1732         union e1000_adv_tx_desc *txd = NULL;
 1733         struct mbuf             *m_head;
 1734         u32                     olinfo_status = 0, cmd_type_len = 0;
 1735         int                     nsegs, i, j, error, first, last = 0;
 1736         u32                     hdrlen = 0;
 1737 
 1738         m_head = *m_headp;
 1739 
 1740 
 1741         /* Set basic descriptor constants */
 1742         cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
 1743         cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
 1744         if (m_head->m_flags & M_VLANTAG)
 1745                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
 1746 
 1747         /*
 1748          * Map the packet for DMA.
 1749          *
 1750          * Capture the first descriptor index,
 1751          * this descriptor will have the index
 1752          * of the EOP which is the only one that
 1753          * now gets a DONE bit writeback.
 1754          */
 1755         first = txr->next_avail_desc;
 1756         tx_buffer = &txr->tx_buffers[first];
 1757         tx_buffer_mapped = tx_buffer;
 1758         map = tx_buffer->map;
 1759 
 1760         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
 1761             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 1762 
 1763         if (error == EFBIG) {
 1764                 struct mbuf *m;
 1765 
 1766                 m = m_defrag(*m_headp, M_DONTWAIT);
 1767                 if (m == NULL) {
 1768                         adapter->mbuf_defrag_failed++;
 1769                         m_freem(*m_headp);
 1770                         *m_headp = NULL;
 1771                         return (ENOBUFS);
 1772                 }
 1773                 *m_headp = m;
 1774 
 1775                 /* Try it again */
 1776                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
 1777                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 1778 
 1779                 if (error == ENOMEM) {
 1780                         adapter->no_tx_dma_setup++;
 1781                         return (error);
 1782                 } else if (error != 0) {
 1783                         adapter->no_tx_dma_setup++;
 1784                         m_freem(*m_headp);
 1785                         *m_headp = NULL;
 1786                         return (error);
 1787                 }
 1788         } else if (error == ENOMEM) {
 1789                 adapter->no_tx_dma_setup++;
 1790                 return (error);
 1791         } else if (error != 0) {
 1792                 adapter->no_tx_dma_setup++;
 1793                 m_freem(*m_headp);
 1794                 *m_headp = NULL;
 1795                 return (error);
 1796         }
 1797 
 1798         /* Check again to be sure we have enough descriptors */
 1799         if (nsegs > (txr->tx_avail - 2)) {
 1800                 txr->no_desc_avail++;
 1801                 bus_dmamap_unload(txr->txtag, map);
 1802                 return (ENOBUFS);
 1803         }
 1804         m_head = *m_headp;
 1805 
 1806         /*
 1807          * Set up the context descriptor:
 1808          * used when any hardware offload is done.
 1809          * This includes CSUM, VLAN, and TSO. It
 1810          * will use the first descriptor.
 1811          */
 1812         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
 1813                 if (igb_tso_setup(txr, m_head, &hdrlen)) {
 1814                         cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
 1815                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
 1816                         olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 1817                 } else
 1818                         return (ENXIO); 
 1819         } else if (igb_tx_ctx_setup(txr, m_head))
 1820                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 1821 
 1822         /* Calculate payload length */
 1823         olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
 1824             << E1000_ADVTXD_PAYLEN_SHIFT);
 1825 
 1826         /* 82575 needs the queue index added */
 1827         if (adapter->hw.mac.type == e1000_82575)
 1828                 olinfo_status |= txr->me << 4;
 1829 
 1830         /* Set up our transmit descriptors */
 1831         i = txr->next_avail_desc;
 1832         for (j = 0; j < nsegs; j++) {
 1833                 bus_size_t seg_len;
 1834                 bus_addr_t seg_addr;
 1835 
 1836                 tx_buffer = &txr->tx_buffers[i];
 1837                 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
 1838                 seg_addr = segs[j].ds_addr;
 1839                 seg_len  = segs[j].ds_len;
 1840 
 1841                 txd->read.buffer_addr = htole64(seg_addr);
 1842                 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len);
 1843                 txd->read.olinfo_status = htole32(olinfo_status);
 1844                 last = i;
 1845                 if (++i == adapter->num_tx_desc)
 1846                         i = 0;
 1847                 tx_buffer->m_head = NULL;
 1848                 tx_buffer->next_eop = -1;
 1849         }
 1850 
 1851         txr->next_avail_desc = i;
 1852         txr->tx_avail -= nsegs;
 1853 
 1854         tx_buffer->m_head = m_head;
 1855         tx_buffer_mapped->map = tx_buffer->map;
 1856         tx_buffer->map = map;
 1857         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
 1858 
 1859         /*
 1860          * Last Descriptor of Packet
 1861          * needs End Of Packet (EOP)
 1862          * and Report Status (RS)
 1863          */
 1864         txd->read.cmd_type_len |=
 1865             htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS);
 1866         /*
 1867          * Keep track in the first buffer which
 1868          * descriptor will be written back
 1869          */
 1870         tx_buffer = &txr->tx_buffers[first];
 1871         tx_buffer->next_eop = last;
 1872         txr->watchdog_time = ticks;
 1873 
 1874         /*
 1875          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
 1876          * that this frame is available to transmit.
 1877          */
 1878         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 1879             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 1880         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
 1881         ++txr->tx_packets;
 1882 
 1883         return (0);
 1884 
 1885 }
 1886 
 1887 static void
 1888 igb_set_promisc(struct adapter *adapter)
 1889 {
 1890         struct ifnet    *ifp = adapter->ifp;
 1891         struct e1000_hw *hw = &adapter->hw;
 1892         u32             reg;
 1893 
 1894         if (adapter->vf_ifp) {
 1895                 e1000_promisc_set_vf(hw, e1000_promisc_enabled);
 1896                 return;
 1897         }
 1898 
 1899         reg = E1000_READ_REG(hw, E1000_RCTL);
 1900         if (ifp->if_flags & IFF_PROMISC) {
 1901                 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
 1902                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 1903         } else if (ifp->if_flags & IFF_ALLMULTI) {
 1904                 reg |= E1000_RCTL_MPE;
 1905                 reg &= ~E1000_RCTL_UPE;
 1906                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 1907         }
 1908 }
 1909 
 1910 static void
 1911 igb_disable_promisc(struct adapter *adapter)
 1912 {
 1913         struct e1000_hw *hw = &adapter->hw;
 1914         u32             reg;
 1915 
 1916         if (adapter->vf_ifp) {
 1917                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
 1918                 return;
 1919         }
 1920         reg = E1000_READ_REG(hw, E1000_RCTL);
 1921         reg &=  (~E1000_RCTL_UPE);
 1922         reg &=  (~E1000_RCTL_MPE);
 1923         E1000_WRITE_REG(hw, E1000_RCTL, reg);
 1924 }
 1925 
 1926 
 1927 /*********************************************************************
 1928  *  Multicast Update
 1929  *
 1930  *  This routine is called whenever multicast address list is updated.
 1931  *
 1932  **********************************************************************/
 1933 
 1934 static void
 1935 igb_set_multi(struct adapter *adapter)
 1936 {
 1937         struct ifnet    *ifp = adapter->ifp;
 1938         struct ifmultiaddr *ifma;
 1939         u32 reg_rctl = 0;
 1940         u8  *mta;
 1941 
 1942         int mcnt = 0;
 1943 
 1944         IOCTL_DEBUGOUT("igb_set_multi: begin");
 1945 
 1946         mta = adapter->mta;
 1947         bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN *
 1948             MAX_NUM_MULTICAST_ADDRESSES);
 1949 
 1950 #if __FreeBSD_version < 800000
 1951         IF_ADDR_LOCK(ifp);
 1952 #else
 1953         if_maddr_rlock(ifp);
 1954 #endif
 1955         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 1956                 if (ifma->ifma_addr->sa_family != AF_LINK)
 1957                         continue;
 1958 
 1959                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
 1960                         break;
 1961 
 1962                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 1963                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
 1964                 mcnt++;
 1965         }
 1966 #if __FreeBSD_version < 800000
 1967         IF_ADDR_UNLOCK(ifp);
 1968 #else
 1969         if_maddr_runlock(ifp);
 1970 #endif
 1971 
 1972         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
 1973                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 1974                 reg_rctl |= E1000_RCTL_MPE;
 1975                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 1976         } else
 1977                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
 1978 }
 1979 
 1980 
 1981 /*********************************************************************
 1982  *  Timer routine:
 1983  *      This routine checks for link status,
 1984  *      updates statistics, and does the watchdog.
 1985  *
 1986  **********************************************************************/
 1987 
 1988 static void
 1989 igb_local_timer(void *arg)
 1990 {
 1991         struct adapter          *adapter = arg;
 1992         device_t                dev = adapter->dev;
 1993         struct tx_ring          *txr = adapter->tx_rings;
 1994 
 1995 
 1996         IGB_CORE_LOCK_ASSERT(adapter);
 1997 
 1998         igb_update_link_status(adapter);
 1999         igb_update_stats_counters(adapter);
 2000 
 2001         /* 
 2002         ** If flow control has paused us since last checking
 2003         ** it invalidates the watchdog timing, so dont run it.
 2004         */
 2005         if (adapter->pause_frames) {
 2006                 adapter->pause_frames = 0;
 2007                 goto out;
 2008         }
 2009 
 2010         /*
 2011         ** Watchdog: check for time since any descriptor was cleaned
 2012         */
 2013         for (int i = 0; i < adapter->num_queues; i++, txr++)
 2014                 if (txr->queue_status == IGB_QUEUE_HUNG) 
 2015                         goto timeout;
 2016 out:
 2017         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 2018 #ifndef DEVICE_POLLING
 2019         /* Schedule all queue interrupts - deadlock protection */
 2020         E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask);
 2021 #endif
 2022         return;
 2023 
 2024 timeout:
 2025         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
 2026         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
 2027             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
 2028             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
 2029         device_printf(dev,"TX(%d) desc avail = %d,"
 2030             "Next TX to Clean = %d\n",
 2031             txr->me, txr->tx_avail, txr->next_to_clean);
 2032         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 2033         adapter->watchdog_events++;
 2034         igb_init_locked(adapter);
 2035 }
 2036 
 2037 static void
 2038 igb_update_link_status(struct adapter *adapter)
 2039 {
 2040         struct e1000_hw *hw = &adapter->hw;
 2041         struct ifnet *ifp = adapter->ifp;
 2042         device_t dev = adapter->dev;
 2043         struct tx_ring *txr = adapter->tx_rings;
 2044         u32 link_check, thstat, ctrl;
 2045 
 2046         link_check = thstat = ctrl = 0;
 2047 
 2048         /* Get the cached link value or read for real */
 2049         switch (hw->phy.media_type) {
 2050         case e1000_media_type_copper:
 2051                 if (hw->mac.get_link_status) {
 2052                         /* Do the work to read phy */
 2053                         e1000_check_for_link(hw);
 2054                         link_check = !hw->mac.get_link_status;
 2055                 } else
 2056                         link_check = TRUE;
 2057                 break;
 2058         case e1000_media_type_fiber:
 2059                 e1000_check_for_link(hw);
 2060                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
 2061                                  E1000_STATUS_LU);
 2062                 break;
 2063         case e1000_media_type_internal_serdes:
 2064                 e1000_check_for_link(hw);
 2065                 link_check = adapter->hw.mac.serdes_has_link;
 2066                 break;
 2067         /* VF device is type_unknown */
 2068         case e1000_media_type_unknown:
 2069                 e1000_check_for_link(hw);
 2070                 link_check = !hw->mac.get_link_status;
 2071                 /* Fall thru */
 2072         default:
 2073                 break;
 2074         }
 2075 
 2076         /* Check for thermal downshift or shutdown */
 2077         if (hw->mac.type == e1000_i350) {
 2078                 thstat = E1000_READ_REG(hw, E1000_THSTAT);
 2079                 ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT);
 2080         }
 2081 
 2082         /* Now we check if a transition has happened */
 2083         if (link_check && (adapter->link_active == 0)) {
 2084                 e1000_get_speed_and_duplex(&adapter->hw, 
 2085                     &adapter->link_speed, &adapter->link_duplex);
 2086                 if (bootverbose)
 2087                         device_printf(dev, "Link is up %d Mbps %s\n",
 2088                             adapter->link_speed,
 2089                             ((adapter->link_duplex == FULL_DUPLEX) ?
 2090                             "Full Duplex" : "Half Duplex"));
 2091                 adapter->link_active = 1;
 2092                 ifp->if_baudrate = adapter->link_speed * 1000000;
 2093                 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
 2094                     (thstat & E1000_THSTAT_LINK_THROTTLE))
 2095                         device_printf(dev, "Link: thermal downshift\n");
 2096                 /* This can sleep */
 2097                 if_link_state_change(ifp, LINK_STATE_UP);
 2098         } else if (!link_check && (adapter->link_active == 1)) {
 2099                 ifp->if_baudrate = adapter->link_speed = 0;
 2100                 adapter->link_duplex = 0;
 2101                 if (bootverbose)
 2102                         device_printf(dev, "Link is Down\n");
 2103                 if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) &&
 2104                     (thstat & E1000_THSTAT_PWR_DOWN))
 2105                         device_printf(dev, "Link: thermal shutdown\n");
 2106                 adapter->link_active = 0;
 2107                 /* This can sleep */
 2108                 if_link_state_change(ifp, LINK_STATE_DOWN);
 2109                 /* Turn off watchdogs */
 2110                 for (int i = 0; i < adapter->num_queues; i++, txr++)
 2111                         txr->queue_status = IGB_QUEUE_IDLE;
 2112         }
 2113 }
 2114 
 2115 /*********************************************************************
 2116  *
 2117  *  This routine disables all traffic on the adapter by issuing a
 2118  *  global reset on the MAC and deallocates TX/RX buffers.
 2119  *
 2120  **********************************************************************/
 2121 
 2122 static void
 2123 igb_stop(void *arg)
 2124 {
 2125         struct adapter  *adapter = arg;
 2126         struct ifnet    *ifp = adapter->ifp;
 2127         struct tx_ring *txr = adapter->tx_rings;
 2128 
 2129         IGB_CORE_LOCK_ASSERT(adapter);
 2130 
 2131         INIT_DEBUGOUT("igb_stop: begin");
 2132 
 2133         igb_disable_intr(adapter);
 2134 
 2135         callout_stop(&adapter->timer);
 2136 
 2137         /* Tell the stack that the interface is no longer active */
 2138         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 2139 
 2140         /* Unarm watchdog timer. */
 2141         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 2142                 IGB_TX_LOCK(txr);
 2143                 txr->queue_status = IGB_QUEUE_IDLE;
 2144                 IGB_TX_UNLOCK(txr);
 2145         }
 2146 
 2147         e1000_reset_hw(&adapter->hw);
 2148         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
 2149 
 2150         e1000_led_off(&adapter->hw);
 2151         e1000_cleanup_led(&adapter->hw);
 2152 }
 2153 
 2154 
 2155 /*********************************************************************
 2156  *
 2157  *  Determine hardware revision.
 2158  *
 2159  **********************************************************************/
 2160 static void
 2161 igb_identify_hardware(struct adapter *adapter)
 2162 {
 2163         device_t dev = adapter->dev;
 2164 
 2165         /* Make sure our PCI config space has the necessary stuff set */
 2166         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
 2167         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
 2168             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
 2169                 INIT_DEBUGOUT("Memory Access and/or Bus Master "
 2170                     "bits were not set!\n");
 2171                 adapter->hw.bus.pci_cmd_word |=
 2172                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
 2173                 pci_write_config(dev, PCIR_COMMAND,
 2174                     adapter->hw.bus.pci_cmd_word, 2);
 2175         }
 2176 
 2177         /* Save off the information about this board */
 2178         adapter->hw.vendor_id = pci_get_vendor(dev);
 2179         adapter->hw.device_id = pci_get_device(dev);
 2180         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
 2181         adapter->hw.subsystem_vendor_id =
 2182             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 2183         adapter->hw.subsystem_device_id =
 2184             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 2185 
 2186         /* Set MAC type early for PCI setup */
 2187         e1000_set_mac_type(&adapter->hw);
 2188 
 2189         /* Are we a VF device? */
 2190         if ((adapter->hw.mac.type == e1000_vfadapt) ||
 2191             (adapter->hw.mac.type == e1000_vfadapt_i350))
 2192                 adapter->vf_ifp = 1;
 2193         else
 2194                 adapter->vf_ifp = 0;
 2195 }
 2196 
 2197 static int
 2198 igb_allocate_pci_resources(struct adapter *adapter)
 2199 {
 2200         device_t        dev = adapter->dev;
 2201         int             rid;
 2202 
 2203         rid = PCIR_BAR(0);
 2204         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 2205             &rid, RF_ACTIVE);
 2206         if (adapter->pci_mem == NULL) {
 2207                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 2208                 return (ENXIO);
 2209         }
 2210         adapter->osdep.mem_bus_space_tag =
 2211             rman_get_bustag(adapter->pci_mem);
 2212         adapter->osdep.mem_bus_space_handle =
 2213             rman_get_bushandle(adapter->pci_mem);
 2214         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
 2215 
 2216         adapter->num_queues = 1; /* Defaults for Legacy or MSI */
 2217 
 2218         /* This will setup either MSI/X or MSI */
 2219         adapter->msix = igb_setup_msix(adapter);
 2220         adapter->hw.back = &adapter->osdep;
 2221 
 2222         return (0);
 2223 }
 2224 
 2225 /*********************************************************************
 2226  *
 2227  *  Setup the Legacy or MSI Interrupt handler
 2228  *
 2229  **********************************************************************/
 2230 static int
 2231 igb_allocate_legacy(struct adapter *adapter)
 2232 {
 2233         device_t                dev = adapter->dev;
 2234         struct igb_queue        *que = adapter->queues;
 2235         struct tx_ring          *txr = adapter->tx_rings;
 2236         int                     error, rid = 0;
 2237 
 2238         /* Turn off all interrupts */
 2239         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
 2240 
 2241         /* MSI RID is 1 */
 2242         if (adapter->msix == 1)
 2243                 rid = 1;
 2244 
 2245         /* We allocate a single interrupt resource */
 2246         adapter->res = bus_alloc_resource_any(dev,
 2247             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2248         if (adapter->res == NULL) {
 2249                 device_printf(dev, "Unable to allocate bus resource: "
 2250                     "interrupt\n");
 2251                 return (ENXIO);
 2252         }
 2253 
 2254 #if __FreeBSD_version >= 800000
 2255         TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr);
 2256 #endif
 2257 
 2258         /*
 2259          * Try allocating a fast interrupt and the associated deferred
 2260          * processing contexts.
 2261          */
 2262         TASK_INIT(&que->que_task, 0, igb_handle_que, que);
 2263         /* Make tasklet for deferred link handling */
 2264         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
 2265         que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
 2266             taskqueue_thread_enqueue, &que->tq);
 2267         taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq",
 2268             device_get_nameunit(adapter->dev));
 2269         if ((error = bus_setup_intr(dev, adapter->res,
 2270             INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
 2271             adapter, &adapter->tag)) != 0) {
 2272                 device_printf(dev, "Failed to register fast interrupt "
 2273                             "handler: %d\n", error);
 2274                 taskqueue_free(que->tq);
 2275                 que->tq = NULL;
 2276                 return (error);
 2277         }
 2278 
 2279         return (0);
 2280 }
 2281 
 2282 
 2283 /*********************************************************************
 2284  *
 2285  *  Setup the MSIX Queue Interrupt handlers: 
 2286  *
 2287  **********************************************************************/
 2288 static int
 2289 igb_allocate_msix(struct adapter *adapter)
 2290 {
 2291         device_t                dev = adapter->dev;
 2292         struct igb_queue        *que = adapter->queues;
 2293         int                     error, rid, vector = 0;
 2294 
 2295 
 2296         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
 2297                 rid = vector +1;
 2298                 que->res = bus_alloc_resource_any(dev,
 2299                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2300                 if (que->res == NULL) {
 2301                         device_printf(dev,
 2302                             "Unable to allocate bus resource: "
 2303                             "MSIX Queue Interrupt\n");
 2304                         return (ENXIO);
 2305                 }
 2306                 error = bus_setup_intr(dev, que->res,
 2307                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2308                     igb_msix_que, que, &que->tag);
 2309                 if (error) {
 2310                         que->res = NULL;
 2311                         device_printf(dev, "Failed to register Queue handler");
 2312                         return (error);
 2313                 }
 2314 #if __FreeBSD_version >= 800504
 2315                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
 2316 #endif
 2317                 que->msix = vector;
 2318                 if (adapter->hw.mac.type == e1000_82575)
 2319                         que->eims = E1000_EICR_TX_QUEUE0 << i;
 2320                 else
 2321                         que->eims = 1 << vector;
 2322                 /*
 2323                 ** Bind the msix vector, and thus the
 2324                 ** rings to the corresponding cpu.
 2325                 */
 2326                 if (adapter->num_queues > 1)
 2327                         bus_bind_intr(dev, que->res, i);
 2328 #if __FreeBSD_version >= 800000
 2329                 TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start,
 2330                     que->txr);
 2331 #endif
 2332                 /* Make tasklet for deferred handling */
 2333                 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
 2334                 que->tq = taskqueue_create("igb_que", M_NOWAIT,
 2335                     taskqueue_thread_enqueue, &que->tq);
 2336                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
 2337                     device_get_nameunit(adapter->dev));
 2338         }
 2339 
 2340         /* And Link */
 2341         rid = vector + 1;
 2342         adapter->res = bus_alloc_resource_any(dev,
 2343             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2344         if (adapter->res == NULL) {
 2345                 device_printf(dev,
 2346                     "Unable to allocate bus resource: "
 2347                     "MSIX Link Interrupt\n");
 2348                 return (ENXIO);
 2349         }
 2350         if ((error = bus_setup_intr(dev, adapter->res,
 2351             INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2352             igb_msix_link, adapter, &adapter->tag)) != 0) {
 2353                 device_printf(dev, "Failed to register Link handler");
 2354                 return (error);
 2355         }
 2356 #if __FreeBSD_version >= 800504
 2357         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
 2358 #endif
 2359         adapter->linkvec = vector;
 2360 
 2361         return (0);
 2362 }
 2363 
 2364 
 2365 static void
 2366 igb_configure_queues(struct adapter *adapter)
 2367 {
 2368         struct  e1000_hw        *hw = &adapter->hw;
 2369         struct  igb_queue       *que;
 2370         u32                     tmp, ivar = 0, newitr = 0;
 2371 
 2372         /* First turn on RSS capability */
 2373         if (adapter->hw.mac.type != e1000_82575)
 2374                 E1000_WRITE_REG(hw, E1000_GPIE,
 2375                     E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
 2376                     E1000_GPIE_PBA | E1000_GPIE_NSICR);
 2377 
 2378         /* Turn on MSIX */
 2379         switch (adapter->hw.mac.type) {
 2380         case e1000_82580:
 2381         case e1000_i350:
 2382         case e1000_vfadapt:
 2383         case e1000_vfadapt_i350:
 2384                 /* RX entries */
 2385                 for (int i = 0; i < adapter->num_queues; i++) {
 2386                         u32 index = i >> 1;
 2387                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2388                         que = &adapter->queues[i];
 2389                         if (i & 1) {
 2390                                 ivar &= 0xFF00FFFF;
 2391                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
 2392                         } else {
 2393                                 ivar &= 0xFFFFFF00;
 2394                                 ivar |= que->msix | E1000_IVAR_VALID;
 2395                         }
 2396                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2397                 }
 2398                 /* TX entries */
 2399                 for (int i = 0; i < adapter->num_queues; i++) {
 2400                         u32 index = i >> 1;
 2401                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2402                         que = &adapter->queues[i];
 2403                         if (i & 1) {
 2404                                 ivar &= 0x00FFFFFF;
 2405                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
 2406                         } else {
 2407                                 ivar &= 0xFFFF00FF;
 2408                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
 2409                         }
 2410                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2411                         adapter->que_mask |= que->eims;
 2412                 }
 2413 
 2414                 /* And for the link interrupt */
 2415                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 2416                 adapter->link_mask = 1 << adapter->linkvec;
 2417                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 2418                 break;
 2419         case e1000_82576:
 2420                 /* RX entries */
 2421                 for (int i = 0; i < adapter->num_queues; i++) {
 2422                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2423                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2424                         que = &adapter->queues[i];
 2425                         if (i < 8) {
 2426                                 ivar &= 0xFFFFFF00;
 2427                                 ivar |= que->msix | E1000_IVAR_VALID;
 2428                         } else {
 2429                                 ivar &= 0xFF00FFFF;
 2430                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
 2431                         }
 2432                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2433                         adapter->que_mask |= que->eims;
 2434                 }
 2435                 /* TX entries */
 2436                 for (int i = 0; i < adapter->num_queues; i++) {
 2437                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2438                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2439                         que = &adapter->queues[i];
 2440                         if (i < 8) {
 2441                                 ivar &= 0xFFFF00FF;
 2442                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
 2443                         } else {
 2444                                 ivar &= 0x00FFFFFF;
 2445                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
 2446                         }
 2447                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2448                         adapter->que_mask |= que->eims;
 2449                 }
 2450 
 2451                 /* And for the link interrupt */
 2452                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 2453                 adapter->link_mask = 1 << adapter->linkvec;
 2454                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 2455                 break;
 2456 
 2457         case e1000_82575:
 2458                 /* enable MSI-X support*/
 2459                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 2460                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 2461                 /* Auto-Mask interrupts upon ICR read. */
 2462                 tmp |= E1000_CTRL_EXT_EIAME;
 2463                 tmp |= E1000_CTRL_EXT_IRCA;
 2464                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 2465 
 2466                 /* Queues */
 2467                 for (int i = 0; i < adapter->num_queues; i++) {
 2468                         que = &adapter->queues[i];
 2469                         tmp = E1000_EICR_RX_QUEUE0 << i;
 2470                         tmp |= E1000_EICR_TX_QUEUE0 << i;
 2471                         que->eims = tmp;
 2472                         E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
 2473                             i, que->eims);
 2474                         adapter->que_mask |= que->eims;
 2475                 }
 2476 
 2477                 /* Link */
 2478                 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
 2479                     E1000_EIMS_OTHER);
 2480                 adapter->link_mask |= E1000_EIMS_OTHER;
 2481         default:
 2482                 break;
 2483         }
 2484 
 2485         /* Set the starting interrupt rate */
 2486         if (igb_max_interrupt_rate > 0)
 2487                 newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC;
 2488 
 2489         if (hw->mac.type == e1000_82575)
 2490                 newitr |= newitr << 16;
 2491         else
 2492                 newitr |= E1000_EITR_CNT_IGNR;
 2493 
 2494         for (int i = 0; i < adapter->num_queues; i++) {
 2495                 que = &adapter->queues[i];
 2496                 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
 2497         }
 2498 
 2499         return;
 2500 }
 2501 
 2502 
 2503 static void
 2504 igb_free_pci_resources(struct adapter *adapter)
 2505 {
 2506         struct          igb_queue *que = adapter->queues;
 2507         device_t        dev = adapter->dev;
 2508         int             rid;
 2509 
 2510         /*
 2511         ** There is a slight possibility of a failure mode
 2512         ** in attach that will result in entering this function
 2513         ** before interrupt resources have been initialized, and
 2514         ** in that case we do not want to execute the loops below
 2515         ** We can detect this reliably by the state of the adapter
 2516         ** res pointer.
 2517         */
 2518         if (adapter->res == NULL)
 2519                 goto mem;
 2520 
 2521         /*
 2522          * First release all the interrupt resources:
 2523          */
 2524         for (int i = 0; i < adapter->num_queues; i++, que++) {
 2525                 rid = que->msix + 1;
 2526                 if (que->tag != NULL) {
 2527                         bus_teardown_intr(dev, que->res, que->tag);
 2528                         que->tag = NULL;
 2529                 }
 2530                 if (que->res != NULL)
 2531                         bus_release_resource(dev,
 2532                             SYS_RES_IRQ, rid, que->res);
 2533         }
 2534 
 2535         /* Clean the Legacy or Link interrupt last */
 2536         if (adapter->linkvec) /* we are doing MSIX */
 2537                 rid = adapter->linkvec + 1;
 2538         else
 2539                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
 2540 
 2541         que = adapter->queues;
 2542         if (adapter->tag != NULL) {
 2543                 taskqueue_drain(que->tq, &adapter->link_task);
 2544                 bus_teardown_intr(dev, adapter->res, adapter->tag);
 2545                 adapter->tag = NULL;
 2546         }
 2547         if (adapter->res != NULL)
 2548                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
 2549 
 2550         for (int i = 0; i < adapter->num_queues; i++, que++) {
 2551                 if (que->tq != NULL) {
 2552 #if __FreeBSD_version >= 800000
 2553                         taskqueue_drain(que->tq, &que->txr->txq_task);
 2554 #endif
 2555                         taskqueue_drain(que->tq, &que->que_task);
 2556                         taskqueue_free(que->tq);
 2557                 }
 2558         }
 2559 mem:
 2560         if (adapter->msix)
 2561                 pci_release_msi(dev);
 2562 
 2563         if (adapter->msix_mem != NULL)
 2564                 bus_release_resource(dev, SYS_RES_MEMORY,
 2565                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
 2566 
 2567         if (adapter->pci_mem != NULL)
 2568                 bus_release_resource(dev, SYS_RES_MEMORY,
 2569                     PCIR_BAR(0), adapter->pci_mem);
 2570 
 2571 }
 2572 
 2573 /*
 2574  * Setup Either MSI/X or MSI
 2575  */
 2576 static int
 2577 igb_setup_msix(struct adapter *adapter)
 2578 {
 2579         device_t dev = adapter->dev;
 2580         int rid, want, queues, msgs;
 2581 
 2582         /* tuneable override */
 2583         if (igb_enable_msix == 0)
 2584                 goto msi;
 2585 
 2586         /* First try MSI/X */
 2587         rid = PCIR_BAR(IGB_MSIX_BAR);
 2588         adapter->msix_mem = bus_alloc_resource_any(dev,
 2589             SYS_RES_MEMORY, &rid, RF_ACTIVE);
 2590         if (!adapter->msix_mem) {
 2591                 /* May not be enabled */
 2592                 device_printf(adapter->dev,
 2593                     "Unable to map MSIX table \n");
 2594                 goto msi;
 2595         }
 2596 
 2597         msgs = pci_msix_count(dev); 
 2598         if (msgs == 0) { /* system has msix disabled */
 2599                 bus_release_resource(dev, SYS_RES_MEMORY,
 2600                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
 2601                 adapter->msix_mem = NULL;
 2602                 goto msi;
 2603         }
 2604 
 2605         /* Figure out a reasonable auto config value */
 2606         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
 2607 
 2608         /* Manual override */
 2609         if (igb_num_queues != 0)
 2610                 queues = igb_num_queues;
 2611         if (queues > 8)  /* max queues */
 2612                 queues = 8;
 2613 
 2614         /* Can have max of 4 queues on 82575 */
 2615         if ((adapter->hw.mac.type == e1000_82575) && (queues > 4))
 2616                 queues = 4;
 2617 
 2618         /* Limit the VF devices to one queue */
 2619         if (adapter->vf_ifp)
 2620                 queues = 1;
 2621 
 2622         /*
 2623         ** One vector (RX/TX pair) per queue
 2624         ** plus an additional for Link interrupt
 2625         */
 2626         want = queues + 1;
 2627         if (msgs >= want)
 2628                 msgs = want;
 2629         else {
 2630                 device_printf(adapter->dev,
 2631                     "MSIX Configuration Problem, "
 2632                     "%d vectors configured, but %d queues wanted!\n",
 2633                     msgs, want);
 2634                 return (ENXIO);
 2635         }
 2636         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
 2637                 device_printf(adapter->dev,
 2638                     "Using MSIX interrupts with %d vectors\n", msgs);
 2639                 adapter->num_queues = queues;
 2640                 return (msgs);
 2641         }
 2642 msi:
 2643         msgs = pci_msi_count(dev);
 2644         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
 2645                 device_printf(adapter->dev,"Using MSI interrupt\n");
 2646         return (msgs);
 2647 }
 2648 
 2649 /*********************************************************************
 2650  *
 2651  *  Set up an fresh starting state
 2652  *
 2653  **********************************************************************/
 2654 static void
 2655 igb_reset(struct adapter *adapter)
 2656 {
 2657         device_t        dev = adapter->dev;
 2658         struct e1000_hw *hw = &adapter->hw;
 2659         struct e1000_fc_info *fc = &hw->fc;
 2660         struct ifnet    *ifp = adapter->ifp;
 2661         u32             pba = 0;
 2662         u16             hwm;
 2663 
 2664         INIT_DEBUGOUT("igb_reset: begin");
 2665 
 2666         /* Let the firmware know the OS is in control */
 2667         igb_get_hw_control(adapter);
 2668 
 2669         /*
 2670          * Packet Buffer Allocation (PBA)
 2671          * Writing PBA sets the receive portion of the buffer
 2672          * the remainder is used for the transmit buffer.
 2673          */
 2674         switch (hw->mac.type) {
 2675         case e1000_82575:
 2676                 pba = E1000_PBA_32K;
 2677                 break;
 2678         case e1000_82576:
 2679         case e1000_vfadapt:
 2680                 pba = E1000_READ_REG(hw, E1000_RXPBS);
 2681                 pba &= E1000_RXPBS_SIZE_MASK_82576;
 2682                 break;
 2683         case e1000_82580:
 2684         case e1000_i350:
 2685         case e1000_vfadapt_i350:
 2686                 pba = E1000_READ_REG(hw, E1000_RXPBS);
 2687                 pba = e1000_rxpbs_adjust_82580(pba);
 2688                 break;
 2689                 pba = E1000_PBA_35K;
 2690         default:
 2691                 break;
 2692         }
 2693 
 2694         /* Special needs in case of Jumbo frames */
 2695         if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
 2696                 u32 tx_space, min_tx, min_rx;
 2697                 pba = E1000_READ_REG(hw, E1000_PBA);
 2698                 tx_space = pba >> 16;
 2699                 pba &= 0xffff;
 2700                 min_tx = (adapter->max_frame_size +
 2701                     sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
 2702                 min_tx = roundup2(min_tx, 1024);
 2703                 min_tx >>= 10;
 2704                 min_rx = adapter->max_frame_size;
 2705                 min_rx = roundup2(min_rx, 1024);
 2706                 min_rx >>= 10;
 2707                 if (tx_space < min_tx &&
 2708                     ((min_tx - tx_space) < pba)) {
 2709                         pba = pba - (min_tx - tx_space);
 2710                         /*
 2711                          * if short on rx space, rx wins
 2712                          * and must trump tx adjustment
 2713                          */
 2714                         if (pba < min_rx)
 2715                                 pba = min_rx;
 2716                 }
 2717                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 2718         }
 2719 
 2720         INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
 2721 
 2722         /*
 2723          * These parameters control the automatic generation (Tx) and
 2724          * response (Rx) to Ethernet PAUSE frames.
 2725          * - High water mark should allow for at least two frames to be
 2726          *   received after sending an XOFF.
 2727          * - Low water mark works best when it is very near the high water mark.
 2728          *   This allows the receiver to restart by sending XON when it has
 2729          *   drained a bit.
 2730          */
 2731         hwm = min(((pba << 10) * 9 / 10),
 2732             ((pba << 10) - 2 * adapter->max_frame_size));
 2733 
 2734         if (hw->mac.type < e1000_82576) {
 2735                 fc->high_water = hwm & 0xFFF8;  /* 8-byte granularity */
 2736                 fc->low_water = fc->high_water - 8;
 2737         } else {
 2738                 fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
 2739                 fc->low_water = fc->high_water - 16;
 2740         }
 2741 
 2742         fc->pause_time = IGB_FC_PAUSE_TIME;
 2743         fc->send_xon = TRUE;
 2744         if (fc->requested_mode)
 2745                 fc->current_mode = fc->requested_mode;
 2746         else
 2747                 fc->current_mode = e1000_fc_full;
 2748 
 2749         adapter->fc = fc->current_mode;
 2750 
 2751         /* Issue a global reset */
 2752         e1000_reset_hw(hw);
 2753         E1000_WRITE_REG(hw, E1000_WUC, 0);
 2754 
 2755         if (e1000_init_hw(hw) < 0)
 2756                 device_printf(dev, "Hardware Initialization Failed\n");
 2757 
 2758         /* Setup DMA Coalescing */
 2759         if (hw->mac.type == e1000_i350) {
 2760                 u32 reg = ~E1000_DMACR_DMAC_EN;
 2761 
 2762                 if (adapter->dmac == 0) { /* Disabling it */
 2763                         E1000_WRITE_REG(hw, E1000_DMACR, reg);
 2764                         goto reset_out;
 2765                 }
 2766 
 2767                 hwm = (pba - 4) << 10;
 2768                 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
 2769                     & E1000_DMACR_DMACTHR_MASK);
 2770 
 2771                 /* transition to L0x or L1 if available..*/
 2772                 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
 2773 
 2774                 /* timer = value in adapter->dmac in 32usec intervals */
 2775                 reg |= (adapter->dmac >> 5);
 2776                 E1000_WRITE_REG(hw, E1000_DMACR, reg);
 2777 
 2778                 /* No lower threshold */
 2779                 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
 2780 
 2781                 /* set hwm to PBA -  2 * max frame size */
 2782                 E1000_WRITE_REG(hw, E1000_FCRTC, hwm);
 2783 
 2784                 /* Set the interval before transition */
 2785                 reg = E1000_READ_REG(hw, E1000_DMCTLX);
 2786                 reg |= 0x800000FF; /* 255 usec */
 2787                 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
 2788 
 2789                 /* free space in tx packet buffer to wake from DMA coal */
 2790                 E1000_WRITE_REG(hw, E1000_DMCTXTH,
 2791                     (20480 - (2 * adapter->max_frame_size)) >> 6);
 2792 
 2793                 /* make low power state decision controlled by DMA coal */
 2794                 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
 2795                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
 2796                     reg | E1000_PCIEMISC_LX_DECISION);
 2797                 device_printf(dev, "DMA Coalescing enabled\n");
 2798         }
 2799 
 2800 reset_out:
 2801         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 2802         e1000_get_phy_info(hw);
 2803         e1000_check_for_link(hw);
 2804         return;
 2805 }
 2806 
 2807 /*********************************************************************
 2808  *
 2809  *  Setup networking device structure and register an interface.
 2810  *
 2811  **********************************************************************/
 2812 static int
 2813 igb_setup_interface(device_t dev, struct adapter *adapter)
 2814 {
 2815         struct ifnet   *ifp;
 2816 
 2817         INIT_DEBUGOUT("igb_setup_interface: begin");
 2818 
 2819         ifp = adapter->ifp = if_alloc(IFT_ETHER);
 2820         if (ifp == NULL) {
 2821                 device_printf(dev, "can not allocate ifnet structure\n");
 2822                 return (-1);
 2823         }
 2824         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 2825         ifp->if_mtu = ETHERMTU;
 2826         ifp->if_init =  igb_init;
 2827         ifp->if_softc = adapter;
 2828         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 2829         ifp->if_ioctl = igb_ioctl;
 2830 #if __FreeBSD_version >= 800000
 2831         ifp->if_transmit = igb_mq_start;
 2832         ifp->if_qflush = igb_qflush;
 2833 #else
 2834         ifp->if_start = igb_start;
 2835 #endif
 2836         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
 2837         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
 2838         IFQ_SET_READY(&ifp->if_snd);
 2839 
 2840         ether_ifattach(ifp, adapter->hw.mac.addr);
 2841 
 2842         ifp->if_capabilities = ifp->if_capenable = 0;
 2843 
 2844         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
 2845         ifp->if_capabilities |= IFCAP_TSO4;
 2846         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
 2847         ifp->if_capenable = ifp->if_capabilities;
 2848 
 2849         /* Don't enable LRO by default */
 2850         ifp->if_capabilities |= IFCAP_LRO;
 2851 
 2852 #ifdef DEVICE_POLLING
 2853         ifp->if_capabilities |= IFCAP_POLLING;
 2854 #endif
 2855 
 2856         /*
 2857          * Tell the upper layer(s) we
 2858          * support full VLAN capability.
 2859          */
 2860         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 2861         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
 2862                              |  IFCAP_VLAN_HWTSO
 2863                              |  IFCAP_VLAN_MTU;
 2864         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING
 2865                           |  IFCAP_VLAN_HWTSO
 2866                           |  IFCAP_VLAN_MTU;
 2867 
 2868         /*
 2869         ** Don't turn this on by default, if vlans are
 2870         ** created on another pseudo device (eg. lagg)
 2871         ** then vlan events are not passed thru, breaking
 2872         ** operation, but with HW FILTER off it works. If
 2873         ** using vlans directly on the igb driver you can
 2874         ** enable this and get full hardware tag filtering.
 2875         */
 2876         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
 2877 
 2878         /*
 2879          * Specify the media types supported by this adapter and register
 2880          * callbacks to update media and link information
 2881          */
 2882         ifmedia_init(&adapter->media, IFM_IMASK,
 2883             igb_media_change, igb_media_status);
 2884         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 2885             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
 2886                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 
 2887                             0, NULL);
 2888                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 2889         } else {
 2890                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
 2891                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
 2892                             0, NULL);
 2893                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
 2894                             0, NULL);
 2895                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
 2896                             0, NULL);
 2897                 if (adapter->hw.phy.type != e1000_phy_ife) {
 2898                         ifmedia_add(&adapter->media,
 2899                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
 2900                         ifmedia_add(&adapter->media,
 2901                                 IFM_ETHER | IFM_1000_T, 0, NULL);
 2902                 }
 2903         }
 2904         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 2905         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
 2906         return (0);
 2907 }
 2908 
 2909 
 2910 /*
 2911  * Manage DMA'able memory.
 2912  */
 2913 static void
 2914 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 2915 {
 2916         if (error)
 2917                 return;
 2918         *(bus_addr_t *) arg = segs[0].ds_addr;
 2919 }
 2920 
 2921 static int
 2922 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
 2923         struct igb_dma_alloc *dma, int mapflags)
 2924 {
 2925         int error;
 2926 
 2927         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
 2928                                 IGB_DBA_ALIGN, 0,       /* alignment, bounds */
 2929                                 BUS_SPACE_MAXADDR,      /* lowaddr */
 2930                                 BUS_SPACE_MAXADDR,      /* highaddr */
 2931                                 NULL, NULL,             /* filter, filterarg */
 2932                                 size,                   /* maxsize */
 2933                                 1,                      /* nsegments */
 2934                                 size,                   /* maxsegsize */
 2935                                 0,                      /* flags */
 2936                                 NULL,                   /* lockfunc */
 2937                                 NULL,                   /* lockarg */
 2938                                 &dma->dma_tag);
 2939         if (error) {
 2940                 device_printf(adapter->dev,
 2941                     "%s: bus_dma_tag_create failed: %d\n",
 2942                     __func__, error);
 2943                 goto fail_0;
 2944         }
 2945 
 2946         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 2947             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
 2948         if (error) {
 2949                 device_printf(adapter->dev,
 2950                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
 2951                     __func__, (uintmax_t)size, error);
 2952                 goto fail_2;
 2953         }
 2954 
 2955         dma->dma_paddr = 0;
 2956         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 2957             size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
 2958         if (error || dma->dma_paddr == 0) {
 2959                 device_printf(adapter->dev,
 2960                     "%s: bus_dmamap_load failed: %d\n",
 2961                     __func__, error);
 2962                 goto fail_3;
 2963         }
 2964 
 2965         return (0);
 2966 
 2967 fail_3:
 2968         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 2969 fail_2:
 2970         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 2971         bus_dma_tag_destroy(dma->dma_tag);
 2972 fail_0:
 2973         dma->dma_map = NULL;
 2974         dma->dma_tag = NULL;
 2975 
 2976         return (error);
 2977 }
 2978 
 2979 static void
 2980 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
 2981 {
 2982         if (dma->dma_tag == NULL)
 2983                 return;
 2984         if (dma->dma_map != NULL) {
 2985                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
 2986                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 2987                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 2988                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 2989                 dma->dma_map = NULL;
 2990         }
 2991         bus_dma_tag_destroy(dma->dma_tag);
 2992         dma->dma_tag = NULL;
 2993 }
 2994 
 2995 
 2996 /*********************************************************************
 2997  *
 2998  *  Allocate memory for the transmit and receive rings, and then
 2999  *  the descriptors associated with each, called only once at attach.
 3000  *
 3001  **********************************************************************/
 3002 static int
 3003 igb_allocate_queues(struct adapter *adapter)
 3004 {
 3005         device_t dev = adapter->dev;
 3006         struct igb_queue        *que = NULL;
 3007         struct tx_ring          *txr = NULL;
 3008         struct rx_ring          *rxr = NULL;
 3009         int rsize, tsize, error = E1000_SUCCESS;
 3010         int txconf = 0, rxconf = 0;
 3011 
 3012         /* First allocate the top level queue structs */
 3013         if (!(adapter->queues =
 3014             (struct igb_queue *) malloc(sizeof(struct igb_queue) *
 3015             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3016                 device_printf(dev, "Unable to allocate queue memory\n");
 3017                 error = ENOMEM;
 3018                 goto fail;
 3019         }
 3020 
 3021         /* Next allocate the TX ring struct memory */
 3022         if (!(adapter->tx_rings =
 3023             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
 3024             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3025                 device_printf(dev, "Unable to allocate TX ring memory\n");
 3026                 error = ENOMEM;
 3027                 goto tx_fail;
 3028         }
 3029 
 3030         /* Now allocate the RX */
 3031         if (!(adapter->rx_rings =
 3032             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
 3033             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3034                 device_printf(dev, "Unable to allocate RX ring memory\n");
 3035                 error = ENOMEM;
 3036                 goto rx_fail;
 3037         }
 3038 
 3039         tsize = roundup2(adapter->num_tx_desc *
 3040             sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
 3041         /*
 3042          * Now set up the TX queues, txconf is needed to handle the
 3043          * possibility that things fail midcourse and we need to
 3044          * undo memory gracefully
 3045          */ 
 3046         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
 3047                 /* Set up some basics */
 3048                 txr = &adapter->tx_rings[i];
 3049                 txr->adapter = adapter;
 3050                 txr->me = i;
 3051 
 3052                 /* Initialize the TX lock */
 3053                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
 3054                     device_get_nameunit(dev), txr->me);
 3055                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
 3056 
 3057                 if (igb_dma_malloc(adapter, tsize,
 3058                         &txr->txdma, BUS_DMA_NOWAIT)) {
 3059                         device_printf(dev,
 3060                             "Unable to allocate TX Descriptor memory\n");
 3061                         error = ENOMEM;
 3062                         goto err_tx_desc;
 3063                 }
 3064                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
 3065                 bzero((void *)txr->tx_base, tsize);
 3066 
 3067                 /* Now allocate transmit buffers for the ring */
 3068                 if (igb_allocate_transmit_buffers(txr)) {
 3069                         device_printf(dev,
 3070                             "Critical Failure setting up transmit buffers\n");
 3071                         error = ENOMEM;
 3072                         goto err_tx_desc;
 3073                 }
 3074 #if __FreeBSD_version >= 800000
 3075                 /* Allocate a buf ring */
 3076                 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF,
 3077                     M_WAITOK, &txr->tx_mtx);
 3078 #endif
 3079         }
 3080 
 3081         /*
 3082          * Next the RX queues...
 3083          */ 
 3084         rsize = roundup2(adapter->num_rx_desc *
 3085             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 3086         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
 3087                 rxr = &adapter->rx_rings[i];
 3088                 rxr->adapter = adapter;
 3089                 rxr->me = i;
 3090 
 3091                 /* Initialize the RX lock */
 3092                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
 3093                     device_get_nameunit(dev), txr->me);
 3094                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
 3095 
 3096                 if (igb_dma_malloc(adapter, rsize,
 3097                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
 3098                         device_printf(dev,
 3099                             "Unable to allocate RxDescriptor memory\n");
 3100                         error = ENOMEM;
 3101                         goto err_rx_desc;
 3102                 }
 3103                 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
 3104                 bzero((void *)rxr->rx_base, rsize);
 3105 
 3106                 /* Allocate receive buffers for the ring*/
 3107                 if (igb_allocate_receive_buffers(rxr)) {
 3108                         device_printf(dev,
 3109                             "Critical Failure setting up receive buffers\n");
 3110                         error = ENOMEM;
 3111                         goto err_rx_desc;
 3112                 }
 3113         }
 3114 
 3115         /*
 3116         ** Finally set up the queue holding structs
 3117         */
 3118         for (int i = 0; i < adapter->num_queues; i++) {
 3119                 que = &adapter->queues[i];
 3120                 que->adapter = adapter;
 3121                 que->txr = &adapter->tx_rings[i];
 3122                 que->rxr = &adapter->rx_rings[i];
 3123         }
 3124 
 3125         return (0);
 3126 
 3127 err_rx_desc:
 3128         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
 3129                 igb_dma_free(adapter, &rxr->rxdma);
 3130 err_tx_desc:
 3131         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
 3132                 igb_dma_free(adapter, &txr->txdma);
 3133         free(adapter->rx_rings, M_DEVBUF);
 3134 rx_fail:
 3135 #if __FreeBSD_version >= 800000
 3136         buf_ring_free(txr->br, M_DEVBUF);
 3137 #endif
 3138         free(adapter->tx_rings, M_DEVBUF);
 3139 tx_fail:
 3140         free(adapter->queues, M_DEVBUF);
 3141 fail:
 3142         return (error);
 3143 }
 3144 
 3145 /*********************************************************************
 3146  *
 3147  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
 3148  *  the information needed to transmit a packet on the wire. This is
 3149  *  called only once at attach, setup is done every reset.
 3150  *
 3151  **********************************************************************/
 3152 static int
 3153 igb_allocate_transmit_buffers(struct tx_ring *txr)
 3154 {
 3155         struct adapter *adapter = txr->adapter;
 3156         device_t dev = adapter->dev;
 3157         struct igb_tx_buffer *txbuf;
 3158         int error, i;
 3159 
 3160         /*
 3161          * Setup DMA descriptor areas.
 3162          */
 3163         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 3164                                1, 0,                    /* alignment, bounds */
 3165                                BUS_SPACE_MAXADDR,       /* lowaddr */
 3166                                BUS_SPACE_MAXADDR,       /* highaddr */
 3167                                NULL, NULL,              /* filter, filterarg */
 3168                                IGB_TSO_SIZE,            /* maxsize */
 3169                                IGB_MAX_SCATTER,         /* nsegments */
 3170                                PAGE_SIZE,               /* maxsegsize */
 3171                                0,                       /* flags */
 3172                                NULL,                    /* lockfunc */
 3173                                NULL,                    /* lockfuncarg */
 3174                                &txr->txtag))) {
 3175                 device_printf(dev,"Unable to allocate TX DMA tag\n");
 3176                 goto fail;
 3177         }
 3178 
 3179         if (!(txr->tx_buffers =
 3180             (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) *
 3181             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3182                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 3183                 error = ENOMEM;
 3184                 goto fail;
 3185         }
 3186 
 3187         /* Create the descriptor buffer dma maps */
 3188         txbuf = txr->tx_buffers;
 3189         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 3190                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
 3191                 if (error != 0) {
 3192                         device_printf(dev, "Unable to create TX DMA map\n");
 3193                         goto fail;
 3194                 }
 3195         }
 3196 
 3197         return 0;
 3198 fail:
 3199         /* We free all, it handles case where we are in the middle */
 3200         igb_free_transmit_structures(adapter);
 3201         return (error);
 3202 }
 3203 
 3204 /*********************************************************************
 3205  *
 3206  *  Initialize a transmit ring.
 3207  *
 3208  **********************************************************************/
 3209 static void
 3210 igb_setup_transmit_ring(struct tx_ring *txr)
 3211 {
 3212         struct adapter *adapter = txr->adapter;
 3213         struct igb_tx_buffer *txbuf;
 3214         int i;
 3215 
 3216         /* Clear the old descriptor contents */
 3217         IGB_TX_LOCK(txr);
 3218         bzero((void *)txr->tx_base,
 3219               (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
 3220         /* Reset indices */
 3221         txr->next_avail_desc = 0;
 3222         txr->next_to_clean = 0;
 3223 
 3224         /* Free any existing tx buffers. */
 3225         txbuf = txr->tx_buffers;
 3226         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 3227                 if (txbuf->m_head != NULL) {
 3228                         bus_dmamap_sync(txr->txtag, txbuf->map,
 3229                             BUS_DMASYNC_POSTWRITE);
 3230                         bus_dmamap_unload(txr->txtag, txbuf->map);
 3231                         m_freem(txbuf->m_head);
 3232                         txbuf->m_head = NULL;
 3233                 }
 3234                 /* clear the watch index */
 3235                 txbuf->next_eop = -1;
 3236         }
 3237 
 3238         /* Set number of descriptors available */
 3239         txr->tx_avail = adapter->num_tx_desc;
 3240 
 3241         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3242             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 3243         IGB_TX_UNLOCK(txr);
 3244 }
 3245 
 3246 /*********************************************************************
 3247  *
 3248  *  Initialize all transmit rings.
 3249  *
 3250  **********************************************************************/
 3251 static void
 3252 igb_setup_transmit_structures(struct adapter *adapter)
 3253 {
 3254         struct tx_ring *txr = adapter->tx_rings;
 3255 
 3256         for (int i = 0; i < adapter->num_queues; i++, txr++)
 3257                 igb_setup_transmit_ring(txr);
 3258 
 3259         return;
 3260 }
 3261 
 3262 /*********************************************************************
 3263  *
 3264  *  Enable transmit unit.
 3265  *
 3266  **********************************************************************/
 3267 static void
 3268 igb_initialize_transmit_units(struct adapter *adapter)
 3269 {
 3270         struct tx_ring  *txr = adapter->tx_rings;
 3271         struct e1000_hw *hw = &adapter->hw;
 3272         u32             tctl, txdctl;
 3273 
 3274         INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
 3275         tctl = txdctl = 0;
 3276 
 3277         /* Setup the Tx Descriptor Rings */
 3278         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 3279                 u64 bus_addr = txr->txdma.dma_paddr;
 3280 
 3281                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
 3282                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
 3283                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
 3284                     (uint32_t)(bus_addr >> 32));
 3285                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
 3286                     (uint32_t)bus_addr);
 3287 
 3288                 /* Setup the HW Tx Head and Tail descriptor pointers */
 3289                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
 3290                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
 3291 
 3292                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
 3293                     E1000_READ_REG(hw, E1000_TDBAL(i)),
 3294                     E1000_READ_REG(hw, E1000_TDLEN(i)));
 3295 
 3296                 txr->queue_status = IGB_QUEUE_IDLE;
 3297 
 3298                 txdctl |= IGB_TX_PTHRESH;
 3299                 txdctl |= IGB_TX_HTHRESH << 8;
 3300                 txdctl |= IGB_TX_WTHRESH << 16;
 3301                 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
 3302                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
 3303         }
 3304 
 3305         if (adapter->vf_ifp)
 3306                 return;
 3307 
 3308         e1000_config_collision_dist(hw);
 3309 
 3310         /* Program the Transmit Control Register */
 3311         tctl = E1000_READ_REG(hw, E1000_TCTL);
 3312         tctl &= ~E1000_TCTL_CT;
 3313         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
 3314                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
 3315 
 3316         /* This write will effectively turn on the transmit unit. */
 3317         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
 3318 }
 3319 
 3320 /*********************************************************************
 3321  *
 3322  *  Free all transmit rings.
 3323  *
 3324  **********************************************************************/
 3325 static void
 3326 igb_free_transmit_structures(struct adapter *adapter)
 3327 {
 3328         struct tx_ring *txr = adapter->tx_rings;
 3329 
 3330         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 3331                 IGB_TX_LOCK(txr);
 3332                 igb_free_transmit_buffers(txr);
 3333                 igb_dma_free(adapter, &txr->txdma);
 3334                 IGB_TX_UNLOCK(txr);
 3335                 IGB_TX_LOCK_DESTROY(txr);
 3336         }
 3337         free(adapter->tx_rings, M_DEVBUF);
 3338 }
 3339 
 3340 /*********************************************************************
 3341  *
 3342  *  Free transmit ring related data structures.
 3343  *
 3344  **********************************************************************/
 3345 static void
 3346 igb_free_transmit_buffers(struct tx_ring *txr)
 3347 {
 3348         struct adapter *adapter = txr->adapter;
 3349         struct igb_tx_buffer *tx_buffer;
 3350         int             i;
 3351 
 3352         INIT_DEBUGOUT("free_transmit_ring: begin");
 3353 
 3354         if (txr->tx_buffers == NULL)
 3355                 return;
 3356 
 3357         tx_buffer = txr->tx_buffers;
 3358         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
 3359                 if (tx_buffer->m_head != NULL) {
 3360                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
 3361                             BUS_DMASYNC_POSTWRITE);
 3362                         bus_dmamap_unload(txr->txtag,
 3363                             tx_buffer->map);
 3364                         m_freem(tx_buffer->m_head);
 3365                         tx_buffer->m_head = NULL;
 3366                         if (tx_buffer->map != NULL) {
 3367                                 bus_dmamap_destroy(txr->txtag,
 3368                                     tx_buffer->map);
 3369                                 tx_buffer->map = NULL;
 3370                         }
 3371                 } else if (tx_buffer->map != NULL) {
 3372                         bus_dmamap_unload(txr->txtag,
 3373                             tx_buffer->map);
 3374                         bus_dmamap_destroy(txr->txtag,
 3375                             tx_buffer->map);
 3376                         tx_buffer->map = NULL;
 3377                 }
 3378         }
 3379 #if __FreeBSD_version >= 800000
 3380         if (txr->br != NULL)
 3381                 buf_ring_free(txr->br, M_DEVBUF);
 3382 #endif
 3383         if (txr->tx_buffers != NULL) {
 3384                 free(txr->tx_buffers, M_DEVBUF);
 3385                 txr->tx_buffers = NULL;
 3386         }
 3387         if (txr->txtag != NULL) {
 3388                 bus_dma_tag_destroy(txr->txtag);
 3389                 txr->txtag = NULL;
 3390         }
 3391         return;
 3392 }
 3393 
 3394 /**********************************************************************
 3395  *
 3396  *  Setup work for hardware segmentation offload (TSO)
 3397  *
 3398  **********************************************************************/
 3399 static boolean_t
 3400 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
 3401 {
 3402         struct adapter *adapter = txr->adapter;
 3403         struct e1000_adv_tx_context_desc *TXD;
 3404         struct igb_tx_buffer        *tx_buffer;
 3405         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 3406         u32 mss_l4len_idx = 0;
 3407         u16 vtag = 0;
 3408         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
 3409         struct ether_vlan_header *eh;
 3410         struct ip *ip;
 3411         struct tcphdr *th;
 3412 
 3413 
 3414         /*
 3415          * Determine where frame payload starts.
 3416          * Jump over vlan headers if already present
 3417          */
 3418         eh = mtod(mp, struct ether_vlan_header *);
 3419         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
 3420                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3421         else
 3422                 ehdrlen = ETHER_HDR_LEN;
 3423 
 3424         /* Ensure we have at least the IP+TCP header in the first mbuf. */
 3425         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
 3426                 return FALSE;
 3427 
 3428         /* Only supports IPV4 for now */
 3429         ctxd = txr->next_avail_desc;
 3430         tx_buffer = &txr->tx_buffers[ctxd];
 3431         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3432 
 3433         ip = (struct ip *)(mp->m_data + ehdrlen);
 3434         if (ip->ip_p != IPPROTO_TCP)
 3435                 return FALSE;   /* 0 */
 3436         ip->ip_sum = 0;
 3437         ip_hlen = ip->ip_hl << 2;
 3438         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 3439         th->th_sum = in_pseudo(ip->ip_src.s_addr,
 3440             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 3441         tcp_hlen = th->th_off << 2;
 3442         /*
 3443          * Calculate header length, this is used
 3444          * in the transmit desc in igb_xmit
 3445          */
 3446         *hdrlen = ehdrlen + ip_hlen + tcp_hlen;
 3447 
 3448         /* VLAN MACLEN IPLEN */
 3449         if (mp->m_flags & M_VLANTAG) {
 3450                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3451                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3452         }
 3453 
 3454         vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT);
 3455         vlan_macip_lens |= ip_hlen;
 3456         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
 3457 
 3458         /* ADV DTYPE TUCMD */
 3459         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3460         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3461         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3462         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
 3463 
 3464         /* MSS L4LEN IDX */
 3465         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
 3466         mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
 3467         /* 82575 needs the queue index added */
 3468         if (adapter->hw.mac.type == e1000_82575)
 3469                 mss_l4len_idx |= txr->me << 4;
 3470         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 3471 
 3472         TXD->seqnum_seed = htole32(0);
 3473         tx_buffer->m_head = NULL;
 3474         tx_buffer->next_eop = -1;
 3475 
 3476         if (++ctxd == adapter->num_tx_desc)
 3477                 ctxd = 0;
 3478 
 3479         txr->tx_avail--;
 3480         txr->next_avail_desc = ctxd;
 3481         return TRUE;
 3482 }
 3483 
 3484 
 3485 /*********************************************************************
 3486  *
 3487  *  Context Descriptor setup for VLAN or CSUM
 3488  *
 3489  **********************************************************************/
 3490 
 3491 static bool
 3492 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 3493 {
 3494         struct adapter *adapter = txr->adapter;
 3495         struct e1000_adv_tx_context_desc *TXD;
 3496         struct igb_tx_buffer        *tx_buffer;
 3497         u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx;
 3498         struct ether_vlan_header *eh;
 3499         struct ip *ip = NULL;
 3500         struct ip6_hdr *ip6;
 3501         int  ehdrlen, ctxd, ip_hlen = 0;
 3502         u16     etype, vtag = 0;
 3503         u8      ipproto = 0;
 3504         bool    offload = TRUE;
 3505 
 3506         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
 3507                 offload = FALSE;
 3508 
 3509         vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0;
 3510         ctxd = txr->next_avail_desc;
 3511         tx_buffer = &txr->tx_buffers[ctxd];
 3512         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3513 
 3514         /*
 3515         ** In advanced descriptors the vlan tag must 
 3516         ** be placed into the context descriptor, thus
 3517         ** we need to be here just for that setup.
 3518         */
 3519         if (mp->m_flags & M_VLANTAG) {
 3520                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3521                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3522         } else if (offload == FALSE)
 3523                 return FALSE;
 3524 
 3525         /*
 3526          * Determine where frame payload starts.
 3527          * Jump over vlan headers if already present,
 3528          * helpful for QinQ too.
 3529          */
 3530         eh = mtod(mp, struct ether_vlan_header *);
 3531         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 3532                 etype = ntohs(eh->evl_proto);
 3533                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3534         } else {
 3535                 etype = ntohs(eh->evl_encap_proto);
 3536                 ehdrlen = ETHER_HDR_LEN;
 3537         }
 3538 
 3539         /* Set the ether header length */
 3540         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
 3541 
 3542         switch (etype) {
 3543                 case ETHERTYPE_IP:
 3544                         ip = (struct ip *)(mp->m_data + ehdrlen);
 3545                         ip_hlen = ip->ip_hl << 2;
 3546                         if (mp->m_len < ehdrlen + ip_hlen) {
 3547                                 offload = FALSE;
 3548                                 break;
 3549                         }
 3550                         ipproto = ip->ip_p;
 3551                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3552                         break;
 3553                 case ETHERTYPE_IPV6:
 3554                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3555                         ip_hlen = sizeof(struct ip6_hdr);
 3556                         ipproto = ip6->ip6_nxt;
 3557                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
 3558                         break;
 3559                 default:
 3560                         offload = FALSE;
 3561                         break;
 3562         }
 3563 
 3564         vlan_macip_lens |= ip_hlen;
 3565         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3566 
 3567         switch (ipproto) {
 3568                 case IPPROTO_TCP:
 3569                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
 3570                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3571                         break;
 3572                 case IPPROTO_UDP:
 3573                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
 3574                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
 3575                         break;
 3576 #if __FreeBSD_version >= 800000
 3577                 case IPPROTO_SCTP:
 3578                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
 3579                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
 3580                         break;
 3581 #endif
 3582                 default:
 3583                         offload = FALSE;
 3584                         break;
 3585         }
 3586 
 3587         /* 82575 needs the queue index added */
 3588         if (adapter->hw.mac.type == e1000_82575)
 3589                 mss_l4len_idx = txr->me << 4;
 3590 
 3591         /* Now copy bits into descriptor */
 3592         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
 3593         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
 3594         TXD->seqnum_seed = htole32(0);
 3595         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 3596 
 3597         tx_buffer->m_head = NULL;
 3598         tx_buffer->next_eop = -1;
 3599 
 3600         /* We've consumed the first desc, adjust counters */
 3601         if (++ctxd == adapter->num_tx_desc)
 3602                 ctxd = 0;
 3603         txr->next_avail_desc = ctxd;
 3604         --txr->tx_avail;
 3605 
 3606         return (offload);
 3607 }
 3608 
 3609 
 3610 /**********************************************************************
 3611  *
 3612  *  Examine each tx_buffer in the used queue. If the hardware is done
 3613  *  processing the packet then free associated resources. The
 3614  *  tx_buffer is put back on the free queue.
 3615  *
 3616  *  TRUE return means there's work in the ring to clean, FALSE its empty.
 3617  **********************************************************************/
 3618 static bool
 3619 igb_txeof(struct tx_ring *txr)
 3620 {
 3621         struct adapter  *adapter = txr->adapter;
 3622         int first, last, done, processed;
 3623         struct igb_tx_buffer *tx_buffer;
 3624         struct e1000_tx_desc   *tx_desc, *eop_desc;
 3625         struct ifnet   *ifp = adapter->ifp;
 3626 
 3627         IGB_TX_LOCK_ASSERT(txr);
 3628 
 3629         if (txr->tx_avail == adapter->num_tx_desc) {
 3630                 txr->queue_status = IGB_QUEUE_IDLE;
 3631                 return FALSE;
 3632         }
 3633 
 3634         processed = 0;
 3635         first = txr->next_to_clean;
 3636         tx_desc = &txr->tx_base[first];
 3637         tx_buffer = &txr->tx_buffers[first];
 3638         last = tx_buffer->next_eop;
 3639         eop_desc = &txr->tx_base[last];
 3640 
 3641         /*
 3642          * What this does is get the index of the
 3643          * first descriptor AFTER the EOP of the 
 3644          * first packet, that way we can do the
 3645          * simple comparison on the inner while loop.
 3646          */
 3647         if (++last == adapter->num_tx_desc)
 3648                 last = 0;
 3649         done = last;
 3650 
 3651         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3652             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 3653 
 3654         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
 3655                 /* We clean the range of the packet */
 3656                 while (first != done) {
 3657                         tx_desc->upper.data = 0;
 3658                         tx_desc->lower.data = 0;
 3659                         tx_desc->buffer_addr = 0;
 3660                         ++txr->tx_avail;
 3661                         ++processed;
 3662 
 3663                         if (tx_buffer->m_head) {
 3664                                 txr->bytes +=
 3665                                     tx_buffer->m_head->m_pkthdr.len;
 3666                                 bus_dmamap_sync(txr->txtag,
 3667                                     tx_buffer->map,
 3668                                     BUS_DMASYNC_POSTWRITE);
 3669                                 bus_dmamap_unload(txr->txtag,
 3670                                     tx_buffer->map);
 3671 
 3672                                 m_freem(tx_buffer->m_head);
 3673                                 tx_buffer->m_head = NULL;
 3674                         }
 3675                         tx_buffer->next_eop = -1;
 3676                         txr->watchdog_time = ticks;
 3677 
 3678                         if (++first == adapter->num_tx_desc)
 3679                                 first = 0;
 3680 
 3681                         tx_buffer = &txr->tx_buffers[first];
 3682                         tx_desc = &txr->tx_base[first];
 3683                 }
 3684                 ++txr->packets;
 3685                 ++ifp->if_opackets;
 3686                 /* See if we can continue to the next packet */
 3687                 last = tx_buffer->next_eop;
 3688                 if (last != -1) {
 3689                         eop_desc = &txr->tx_base[last];
 3690                         /* Get new done point */
 3691                         if (++last == adapter->num_tx_desc) last = 0;
 3692                         done = last;
 3693                 } else
 3694                         break;
 3695         }
 3696         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3697             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 3698 
 3699         txr->next_to_clean = first;
 3700 
 3701         /*
 3702         ** Watchdog calculation, we know there's
 3703         ** work outstanding or the first return
 3704         ** would have been taken, so none processed
 3705         ** for too long indicates a hang.
 3706         */
 3707         if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG))
 3708                 txr->queue_status = IGB_QUEUE_HUNG;
 3709 
 3710         /*
 3711          * If we have a minimum free, clear IFF_DRV_OACTIVE
 3712          * to tell the stack that it is OK to send packets.
 3713          */
 3714         if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) {                
 3715                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 3716                 /* All clean, turn off the watchdog */
 3717                 if (txr->tx_avail == adapter->num_tx_desc) {
 3718                         txr->queue_status = IGB_QUEUE_IDLE;
 3719                         return (FALSE);
 3720                 }
 3721         }
 3722         return (TRUE);
 3723 }
 3724 
 3725 /*********************************************************************
 3726  *
 3727  *  Refresh mbuf buffers for RX descriptor rings
 3728  *   - now keeps its own state so discards due to resource
 3729  *     exhaustion are unnecessary, if an mbuf cannot be obtained
 3730  *     it just returns, keeping its placeholder, thus it can simply
 3731  *     be recalled to try again.
 3732  *
 3733  **********************************************************************/
 3734 static void
 3735 igb_refresh_mbufs(struct rx_ring *rxr, int limit)
 3736 {
 3737         struct adapter          *adapter = rxr->adapter;
 3738         bus_dma_segment_t       hseg[1];
 3739         bus_dma_segment_t       pseg[1];
 3740         struct igb_rx_buf       *rxbuf;
 3741         struct mbuf             *mh, *mp;
 3742         int                     i, j, nsegs, error;
 3743         bool                    refreshed = FALSE;
 3744 
 3745         i = j = rxr->next_to_refresh;
 3746         /*
 3747         ** Get one descriptor beyond
 3748         ** our work mark to control
 3749         ** the loop.
 3750         */
 3751         if (++j == adapter->num_rx_desc)
 3752                 j = 0;
 3753 
 3754         while (j != limit) {
 3755                 rxbuf = &rxr->rx_buffers[i];
 3756                 /* No hdr mbuf used with header split off */
 3757                 if (rxr->hdr_split == FALSE)
 3758                         goto no_split;
 3759                 if (rxbuf->m_head == NULL) {
 3760                         mh = m_gethdr(M_DONTWAIT, MT_DATA);
 3761                         if (mh == NULL)
 3762                                 goto update;
 3763                 } else
 3764                         mh = rxbuf->m_head;
 3765 
 3766                 mh->m_pkthdr.len = mh->m_len = MHLEN;
 3767                 mh->m_len = MHLEN;
 3768                 mh->m_flags |= M_PKTHDR;
 3769                 /* Get the memory mapping */
 3770                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
 3771                     rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT);
 3772                 if (error != 0) {
 3773                         printf("Refresh mbufs: hdr dmamap load"
 3774                             " failure - %d\n", error);
 3775                         m_free(mh);
 3776                         rxbuf->m_head = NULL;
 3777                         goto update;
 3778                 }
 3779                 rxbuf->m_head = mh;
 3780                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 3781                     BUS_DMASYNC_PREREAD);
 3782                 rxr->rx_base[i].read.hdr_addr =
 3783                     htole64(hseg[0].ds_addr);
 3784 no_split:
 3785                 if (rxbuf->m_pack == NULL) {
 3786                         mp = m_getjcl(M_DONTWAIT, MT_DATA,
 3787                             M_PKTHDR, adapter->rx_mbuf_sz);
 3788                         if (mp == NULL)
 3789                                 goto update;
 3790                 } else
 3791                         mp = rxbuf->m_pack;
 3792 
 3793                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
 3794                 /* Get the memory mapping */
 3795                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
 3796                     rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT);
 3797                 if (error != 0) {
 3798                         printf("Refresh mbufs: payload dmamap load"
 3799                             " failure - %d\n", error);
 3800                         m_free(mp);
 3801                         rxbuf->m_pack = NULL;
 3802                         goto update;
 3803                 }
 3804                 rxbuf->m_pack = mp;
 3805                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 3806                     BUS_DMASYNC_PREREAD);
 3807                 rxr->rx_base[i].read.pkt_addr =
 3808                     htole64(pseg[0].ds_addr);
 3809                 refreshed = TRUE; /* I feel wefreshed :) */
 3810 
 3811                 i = j; /* our next is precalculated */
 3812                 rxr->next_to_refresh = i;
 3813                 if (++j == adapter->num_rx_desc)
 3814                         j = 0;
 3815         }
 3816 update:
 3817         if (refreshed) /* update tail */
 3818                 E1000_WRITE_REG(&adapter->hw,
 3819                     E1000_RDT(rxr->me), rxr->next_to_refresh);
 3820         return;
 3821 }
 3822 
 3823 
 3824 /*********************************************************************
 3825  *
 3826  *  Allocate memory for rx_buffer structures. Since we use one
 3827  *  rx_buffer per received packet, the maximum number of rx_buffer's
 3828  *  that we'll need is equal to the number of receive descriptors
 3829  *  that we've allocated.
 3830  *
 3831  **********************************************************************/
 3832 static int
 3833 igb_allocate_receive_buffers(struct rx_ring *rxr)
 3834 {
 3835         struct  adapter         *adapter = rxr->adapter;
 3836         device_t                dev = adapter->dev;
 3837         struct igb_rx_buf       *rxbuf;
 3838         int                     i, bsize, error;
 3839 
 3840         bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
 3841         if (!(rxr->rx_buffers =
 3842             (struct igb_rx_buf *) malloc(bsize,
 3843             M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3844                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
 3845                 error = ENOMEM;
 3846                 goto fail;
 3847         }
 3848 
 3849         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 3850                                    1, 0,                /* alignment, bounds */
 3851                                    BUS_SPACE_MAXADDR,   /* lowaddr */
 3852                                    BUS_SPACE_MAXADDR,   /* highaddr */
 3853                                    NULL, NULL,          /* filter, filterarg */
 3854                                    MSIZE,               /* maxsize */
 3855                                    1,                   /* nsegments */
 3856                                    MSIZE,               /* maxsegsize */
 3857                                    0,                   /* flags */
 3858                                    NULL,                /* lockfunc */
 3859                                    NULL,                /* lockfuncarg */
 3860                                    &rxr->htag))) {
 3861                 device_printf(dev, "Unable to create RX DMA tag\n");
 3862                 goto fail;
 3863         }
 3864 
 3865         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
 3866                                    1, 0,                /* alignment, bounds */
 3867                                    BUS_SPACE_MAXADDR,   /* lowaddr */
 3868                                    BUS_SPACE_MAXADDR,   /* highaddr */
 3869                                    NULL, NULL,          /* filter, filterarg */
 3870                                    MJUM9BYTES,          /* maxsize */
 3871                                    1,                   /* nsegments */
 3872                                    MJUM9BYTES,          /* maxsegsize */
 3873                                    0,                   /* flags */
 3874                                    NULL,                /* lockfunc */
 3875                                    NULL,                /* lockfuncarg */
 3876                                    &rxr->ptag))) {
 3877                 device_printf(dev, "Unable to create RX payload DMA tag\n");
 3878                 goto fail;
 3879         }
 3880 
 3881         for (i = 0; i < adapter->num_rx_desc; i++) {
 3882                 rxbuf = &rxr->rx_buffers[i];
 3883                 error = bus_dmamap_create(rxr->htag,
 3884                     BUS_DMA_NOWAIT, &rxbuf->hmap);
 3885                 if (error) {
 3886                         device_printf(dev,
 3887                             "Unable to create RX head DMA maps\n");
 3888                         goto fail;
 3889                 }
 3890                 error = bus_dmamap_create(rxr->ptag,
 3891                     BUS_DMA_NOWAIT, &rxbuf->pmap);
 3892                 if (error) {
 3893                         device_printf(dev,
 3894                             "Unable to create RX packet DMA maps\n");
 3895                         goto fail;
 3896                 }
 3897         }
 3898 
 3899         return (0);
 3900 
 3901 fail:
 3902         /* Frees all, but can handle partial completion */
 3903         igb_free_receive_structures(adapter);
 3904         return (error);
 3905 }
 3906 
 3907 
 3908 static void
 3909 igb_free_receive_ring(struct rx_ring *rxr)
 3910 {
 3911         struct  adapter         *adapter = rxr->adapter;
 3912         struct igb_rx_buf       *rxbuf;
 3913 
 3914 
 3915         for (int i = 0; i < adapter->num_rx_desc; i++) {
 3916                 rxbuf = &rxr->rx_buffers[i];
 3917                 if (rxbuf->m_head != NULL) {
 3918                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 3919                             BUS_DMASYNC_POSTREAD);
 3920                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 3921                         rxbuf->m_head->m_flags |= M_PKTHDR;
 3922                         m_freem(rxbuf->m_head);
 3923                 }
 3924                 if (rxbuf->m_pack != NULL) {
 3925                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 3926                             BUS_DMASYNC_POSTREAD);
 3927                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 3928                         rxbuf->m_pack->m_flags |= M_PKTHDR;
 3929                         m_freem(rxbuf->m_pack);
 3930                 }
 3931                 rxbuf->m_head = NULL;
 3932                 rxbuf->m_pack = NULL;
 3933         }
 3934 }
 3935 
 3936 
 3937 /*********************************************************************
 3938  *
 3939  *  Initialize a receive ring and its buffers.
 3940  *
 3941  **********************************************************************/
 3942 static int
 3943 igb_setup_receive_ring(struct rx_ring *rxr)
 3944 {
 3945         struct  adapter         *adapter;
 3946         struct  ifnet           *ifp;
 3947         device_t                dev;
 3948         struct igb_rx_buf       *rxbuf;
 3949         bus_dma_segment_t       pseg[1], hseg[1];
 3950         struct lro_ctrl         *lro = &rxr->lro;
 3951         int                     rsize, nsegs, error = 0;
 3952 
 3953         adapter = rxr->adapter;
 3954         dev = adapter->dev;
 3955         ifp = adapter->ifp;
 3956 
 3957         /* Clear the ring contents */
 3958         IGB_RX_LOCK(rxr);
 3959         rsize = roundup2(adapter->num_rx_desc *
 3960             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 3961         bzero((void *)rxr->rx_base, rsize);
 3962 
 3963         /*
 3964         ** Free current RX buffer structures and their mbufs
 3965         */
 3966         igb_free_receive_ring(rxr);
 3967 
 3968         /* Configure for header split? */
 3969         if (igb_header_split)
 3970                 rxr->hdr_split = TRUE;
 3971 
 3972         /* Now replenish the ring mbufs */
 3973         for (int j = 0; j < adapter->num_rx_desc; ++j) {
 3974                 struct mbuf     *mh, *mp;
 3975 
 3976                 rxbuf = &rxr->rx_buffers[j];
 3977                 if (rxr->hdr_split == FALSE)
 3978                         goto skip_head;
 3979 
 3980                 /* First the header */
 3981                 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA);
 3982                 if (rxbuf->m_head == NULL) {
 3983                         error = ENOBUFS;
 3984                         goto fail;
 3985                 }
 3986                 m_adj(rxbuf->m_head, ETHER_ALIGN);
 3987                 mh = rxbuf->m_head;
 3988                 mh->m_len = mh->m_pkthdr.len = MHLEN;
 3989                 mh->m_flags |= M_PKTHDR;
 3990                 /* Get the memory mapping */
 3991                 error = bus_dmamap_load_mbuf_sg(rxr->htag,
 3992                     rxbuf->hmap, rxbuf->m_head, hseg,
 3993                     &nsegs, BUS_DMA_NOWAIT);
 3994                 if (error != 0) /* Nothing elegant to do here */
 3995                         goto fail;
 3996                 bus_dmamap_sync(rxr->htag,
 3997                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
 3998                 /* Update descriptor */
 3999                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
 4000 
 4001 skip_head:
 4002                 /* Now the payload cluster */
 4003                 rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA,
 4004                     M_PKTHDR, adapter->rx_mbuf_sz);
 4005                 if (rxbuf->m_pack == NULL) {
 4006                         error = ENOBUFS;
 4007                         goto fail;
 4008                 }
 4009                 mp = rxbuf->m_pack;
 4010                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
 4011                 /* Get the memory mapping */
 4012                 error = bus_dmamap_load_mbuf_sg(rxr->ptag,
 4013                     rxbuf->pmap, mp, pseg,
 4014                     &nsegs, BUS_DMA_NOWAIT);
 4015                 if (error != 0)
 4016                         goto fail;
 4017                 bus_dmamap_sync(rxr->ptag,
 4018                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
 4019                 /* Update descriptor */
 4020                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
 4021         }
 4022 
 4023         /* Setup our descriptor indices */
 4024         rxr->next_to_check = 0;
 4025         rxr->next_to_refresh = adapter->num_rx_desc - 1;
 4026         rxr->lro_enabled = FALSE;
 4027         rxr->rx_split_packets = 0;
 4028         rxr->rx_bytes = 0;
 4029 
 4030         rxr->fmp = NULL;
 4031         rxr->lmp = NULL;
 4032         rxr->discard = FALSE;
 4033 
 4034         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4035             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4036 
 4037         /*
 4038         ** Now set up the LRO interface, we
 4039         ** also only do head split when LRO
 4040         ** is enabled, since so often they
 4041         ** are undesireable in similar setups.
 4042         */
 4043         if (ifp->if_capenable & IFCAP_LRO) {
 4044                 error = tcp_lro_init(lro);
 4045                 if (error) {
 4046                         device_printf(dev, "LRO Initialization failed!\n");
 4047                         goto fail;
 4048                 }
 4049                 INIT_DEBUGOUT("RX LRO Initialized\n");
 4050                 rxr->lro_enabled = TRUE;
 4051                 lro->ifp = adapter->ifp;
 4052         }
 4053 
 4054         IGB_RX_UNLOCK(rxr);
 4055         return (0);
 4056 
 4057 fail:
 4058         igb_free_receive_ring(rxr);
 4059         IGB_RX_UNLOCK(rxr);
 4060         return (error);
 4061 }
 4062 
 4063 
 4064 /*********************************************************************
 4065  *
 4066  *  Initialize all receive rings.
 4067  *
 4068  **********************************************************************/
 4069 static int
 4070 igb_setup_receive_structures(struct adapter *adapter)
 4071 {
 4072         struct rx_ring *rxr = adapter->rx_rings;
 4073         int i;
 4074 
 4075         for (i = 0; i < adapter->num_queues; i++, rxr++)
 4076                 if (igb_setup_receive_ring(rxr))
 4077                         goto fail;
 4078 
 4079         return (0);
 4080 fail:
 4081         /*
 4082          * Free RX buffers allocated so far, we will only handle
 4083          * the rings that completed, the failing case will have
 4084          * cleaned up for itself. 'i' is the endpoint.
 4085          */
 4086         for (int j = 0; j > i; ++j) {
 4087                 rxr = &adapter->rx_rings[i];
 4088                 IGB_RX_LOCK(rxr);
 4089                 igb_free_receive_ring(rxr);
 4090                 IGB_RX_UNLOCK(rxr);
 4091         }
 4092 
 4093         return (ENOBUFS);
 4094 }
 4095 
 4096 /*********************************************************************
 4097  *
 4098  *  Enable receive unit.
 4099  *
 4100  **********************************************************************/
 4101 static void
 4102 igb_initialize_receive_units(struct adapter *adapter)
 4103 {
 4104         struct rx_ring  *rxr = adapter->rx_rings;
 4105         struct ifnet    *ifp = adapter->ifp;
 4106         struct e1000_hw *hw = &adapter->hw;
 4107         u32             rctl, rxcsum, psize, srrctl = 0;
 4108 
 4109         INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
 4110 
 4111         /*
 4112          * Make sure receives are disabled while setting
 4113          * up the descriptor ring
 4114          */
 4115         rctl = E1000_READ_REG(hw, E1000_RCTL);
 4116         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
 4117 
 4118         /*
 4119         ** Set up for header split
 4120         */
 4121         if (igb_header_split) {
 4122                 /* Use a standard mbuf for the header */
 4123                 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
 4124                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
 4125         } else
 4126                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
 4127 
 4128         /*
 4129         ** Set up for jumbo frames
 4130         */
 4131         if (ifp->if_mtu > ETHERMTU) {
 4132                 rctl |= E1000_RCTL_LPE;
 4133                 if (adapter->rx_mbuf_sz == MJUMPAGESIZE) {
 4134                         srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4135                         rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
 4136                 } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) {
 4137                         srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4138                         rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX;
 4139                 }
 4140                 /* Set maximum packet len */
 4141                 psize = adapter->max_frame_size;
 4142                 /* are we on a vlan? */
 4143                 if (adapter->ifp->if_vlantrunk != NULL)
 4144                         psize += VLAN_TAG_SIZE;
 4145                 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
 4146         } else {
 4147                 rctl &= ~E1000_RCTL_LPE;
 4148                 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 4149                 rctl |= E1000_RCTL_SZ_2048;
 4150         }
 4151 
 4152         /* Setup the Base and Length of the Rx Descriptor Rings */
 4153         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 4154                 u64 bus_addr = rxr->rxdma.dma_paddr;
 4155                 u32 rxdctl;
 4156 
 4157                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
 4158                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
 4159                 E1000_WRITE_REG(hw, E1000_RDBAH(i),
 4160                     (uint32_t)(bus_addr >> 32));
 4161                 E1000_WRITE_REG(hw, E1000_RDBAL(i),
 4162                     (uint32_t)bus_addr);
 4163                 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
 4164                 /* Enable this Queue */
 4165                 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
 4166                 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
 4167                 rxdctl &= 0xFFF00000;
 4168                 rxdctl |= IGB_RX_PTHRESH;
 4169                 rxdctl |= IGB_RX_HTHRESH << 8;
 4170                 rxdctl |= IGB_RX_WTHRESH << 16;
 4171                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
 4172         }
 4173 
 4174         /*
 4175         ** Setup for RX MultiQueue
 4176         */
 4177         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
 4178         if (adapter->num_queues >1) {
 4179                 u32 random[10], mrqc, shift = 0;
 4180                 union igb_reta {
 4181                         u32 dword;
 4182                         u8  bytes[4];
 4183                 } reta;
 4184 
 4185                 arc4rand(&random, sizeof(random), 0);
 4186                 if (adapter->hw.mac.type == e1000_82575)
 4187                         shift = 6;
 4188                 /* Warning FM follows */
 4189                 for (int i = 0; i < 128; i++) {
 4190                         reta.bytes[i & 3] =
 4191                             (i % adapter->num_queues) << shift;
 4192                         if ((i & 3) == 3)
 4193                                 E1000_WRITE_REG(hw,
 4194                                     E1000_RETA(i >> 2), reta.dword);
 4195                 }
 4196                 /* Now fill in hash table */
 4197                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
 4198                 for (int i = 0; i < 10; i++)
 4199                         E1000_WRITE_REG_ARRAY(hw,
 4200                             E1000_RSSRK(0), i, random[i]);
 4201 
 4202                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
 4203                     E1000_MRQC_RSS_FIELD_IPV4_TCP);
 4204                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
 4205                     E1000_MRQC_RSS_FIELD_IPV6_TCP);
 4206                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
 4207                     E1000_MRQC_RSS_FIELD_IPV6_UDP);
 4208                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
 4209                     E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
 4210 
 4211                 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
 4212 
 4213                 /*
 4214                 ** NOTE: Receive Full-Packet Checksum Offload 
 4215                 ** is mutually exclusive with Multiqueue. However
 4216                 ** this is not the same as TCP/IP checksums which
 4217                 ** still work.
 4218                 */
 4219                 rxcsum |= E1000_RXCSUM_PCSD;
 4220 #if __FreeBSD_version >= 800000
 4221                 /* For SCTP Offload */
 4222                 if ((hw->mac.type == e1000_82576)
 4223                     && (ifp->if_capenable & IFCAP_RXCSUM))
 4224                         rxcsum |= E1000_RXCSUM_CRCOFL;
 4225 #endif
 4226         } else {
 4227                 /* Non RSS setup */
 4228                 if (ifp->if_capenable & IFCAP_RXCSUM) {
 4229                         rxcsum |= E1000_RXCSUM_IPPCSE;
 4230 #if __FreeBSD_version >= 800000
 4231                         if (adapter->hw.mac.type == e1000_82576)
 4232                                 rxcsum |= E1000_RXCSUM_CRCOFL;
 4233 #endif
 4234                 } else
 4235                         rxcsum &= ~E1000_RXCSUM_TUOFL;
 4236         }
 4237         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
 4238 
 4239         /* Setup the Receive Control Register */
 4240         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
 4241         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
 4242                    E1000_RCTL_RDMTS_HALF |
 4243                    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
 4244         /* Strip CRC bytes. */
 4245         rctl |= E1000_RCTL_SECRC;
 4246         /* Make sure VLAN Filters are off */
 4247         rctl &= ~E1000_RCTL_VFE;
 4248         /* Don't store bad packets */
 4249         rctl &= ~E1000_RCTL_SBP;
 4250 
 4251         /* Enable Receives */
 4252         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 4253 
 4254         /*
 4255          * Setup the HW Rx Head and Tail Descriptor Pointers
 4256          *   - needs to be after enable
 4257          */
 4258         for (int i = 0; i < adapter->num_queues; i++) {
 4259                 rxr = &adapter->rx_rings[i];
 4260                 E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check);
 4261                 E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh);
 4262         }
 4263         return;
 4264 }
 4265 
 4266 /*********************************************************************
 4267  *
 4268  *  Free receive rings.
 4269  *
 4270  **********************************************************************/
 4271 static void
 4272 igb_free_receive_structures(struct adapter *adapter)
 4273 {
 4274         struct rx_ring *rxr = adapter->rx_rings;
 4275 
 4276         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 4277                 struct lro_ctrl *lro = &rxr->lro;
 4278                 igb_free_receive_buffers(rxr);
 4279                 tcp_lro_free(lro);
 4280                 igb_dma_free(adapter, &rxr->rxdma);
 4281         }
 4282 
 4283         free(adapter->rx_rings, M_DEVBUF);
 4284 }
 4285 
 4286 /*********************************************************************
 4287  *
 4288  *  Free receive ring data structures.
 4289  *
 4290  **********************************************************************/
 4291 static void
 4292 igb_free_receive_buffers(struct rx_ring *rxr)
 4293 {
 4294         struct adapter          *adapter = rxr->adapter;
 4295         struct igb_rx_buf       *rxbuf;
 4296         int i;
 4297 
 4298         INIT_DEBUGOUT("free_receive_structures: begin");
 4299 
 4300         /* Cleanup any existing buffers */
 4301         if (rxr->rx_buffers != NULL) {
 4302                 for (i = 0; i < adapter->num_rx_desc; i++) {
 4303                         rxbuf = &rxr->rx_buffers[i];
 4304                         if (rxbuf->m_head != NULL) {
 4305                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
 4306                                     BUS_DMASYNC_POSTREAD);
 4307                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
 4308                                 rxbuf->m_head->m_flags |= M_PKTHDR;
 4309                                 m_freem(rxbuf->m_head);
 4310                         }
 4311                         if (rxbuf->m_pack != NULL) {
 4312                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
 4313                                     BUS_DMASYNC_POSTREAD);
 4314                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
 4315                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
 4316                                 m_freem(rxbuf->m_pack);
 4317                         }
 4318                         rxbuf->m_head = NULL;
 4319                         rxbuf->m_pack = NULL;
 4320                         if (rxbuf->hmap != NULL) {
 4321                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
 4322                                 rxbuf->hmap = NULL;
 4323                         }
 4324                         if (rxbuf->pmap != NULL) {
 4325                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
 4326                                 rxbuf->pmap = NULL;
 4327                         }
 4328                 }
 4329                 if (rxr->rx_buffers != NULL) {
 4330                         free(rxr->rx_buffers, M_DEVBUF);
 4331                         rxr->rx_buffers = NULL;
 4332                 }
 4333         }
 4334 
 4335         if (rxr->htag != NULL) {
 4336                 bus_dma_tag_destroy(rxr->htag);
 4337                 rxr->htag = NULL;
 4338         }
 4339         if (rxr->ptag != NULL) {
 4340                 bus_dma_tag_destroy(rxr->ptag);
 4341                 rxr->ptag = NULL;
 4342         }
 4343 }
 4344 
 4345 static __inline void
 4346 igb_rx_discard(struct rx_ring *rxr, int i)
 4347 {
 4348         struct igb_rx_buf       *rbuf;
 4349 
 4350         rbuf = &rxr->rx_buffers[i];
 4351 
 4352         /* Partially received? Free the chain */
 4353         if (rxr->fmp != NULL) {
 4354                 rxr->fmp->m_flags |= M_PKTHDR;
 4355                 m_freem(rxr->fmp);
 4356                 rxr->fmp = NULL;
 4357                 rxr->lmp = NULL;
 4358         }
 4359 
 4360         /*
 4361         ** With advanced descriptors the writeback
 4362         ** clobbers the buffer addrs, so its easier
 4363         ** to just free the existing mbufs and take
 4364         ** the normal refresh path to get new buffers
 4365         ** and mapping.
 4366         */
 4367         if (rbuf->m_head) {
 4368                 m_free(rbuf->m_head);
 4369                 rbuf->m_head = NULL;
 4370         }
 4371 
 4372         if (rbuf->m_pack) {
 4373                 m_free(rbuf->m_pack);
 4374                 rbuf->m_pack = NULL;
 4375         }
 4376 
 4377         return;
 4378 }
 4379 
 4380 static __inline void
 4381 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
 4382 {
 4383 
 4384         /*
 4385          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
 4386          * should be computed by hardware. Also it should not have VLAN tag in
 4387          * ethernet header.
 4388          */
 4389         if (rxr->lro_enabled &&
 4390             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
 4391             (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
 4392             (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
 4393             (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
 4394             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 
 4395             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
 4396                 /*
 4397                  * Send to the stack if:
 4398                  **  - LRO not enabled, or
 4399                  **  - no LRO resources, or
 4400                  **  - lro enqueue fails
 4401                  */
 4402                 if (rxr->lro.lro_cnt != 0)
 4403                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
 4404                                 return;
 4405         }
 4406         IGB_RX_UNLOCK(rxr);
 4407         (*ifp->if_input)(ifp, m);
 4408         IGB_RX_LOCK(rxr);
 4409 }
 4410 
 4411 /*********************************************************************
 4412  *
 4413  *  This routine executes in interrupt context. It replenishes
 4414  *  the mbufs in the descriptor and sends data which has been
 4415  *  dma'ed into host memory to upper layer.
 4416  *
 4417  *  We loop at most count times if count is > 0, or until done if
 4418  *  count < 0.
 4419  *
 4420  *  Return TRUE if more to clean, FALSE otherwise
 4421  *********************************************************************/
 4422 static bool
 4423 igb_rxeof(struct igb_queue *que, int count, int *done)
 4424 {
 4425         struct adapter          *adapter = que->adapter;
 4426         struct rx_ring          *rxr = que->rxr;
 4427         struct ifnet            *ifp = adapter->ifp;
 4428         struct lro_ctrl         *lro = &rxr->lro;
 4429         struct lro_entry        *queued;
 4430         int                     i, processed = 0, rxdone = 0;
 4431         u32                     ptype, staterr = 0;
 4432         union e1000_adv_rx_desc *cur;
 4433 
 4434         IGB_RX_LOCK(rxr);
 4435         /* Sync the ring. */
 4436         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4437             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 4438 
 4439         /* Main clean loop */
 4440         for (i = rxr->next_to_check; count != 0;) {
 4441                 struct mbuf             *sendmp, *mh, *mp;
 4442                 struct igb_rx_buf       *rxbuf;
 4443                 u16                     hlen, plen, hdr, vtag;
 4444                 bool                    eop = FALSE;
 4445  
 4446                 cur = &rxr->rx_base[i];
 4447                 staterr = le32toh(cur->wb.upper.status_error);
 4448                 if ((staterr & E1000_RXD_STAT_DD) == 0)
 4449                         break;
 4450                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 4451                         break;
 4452                 count--;
 4453                 sendmp = mh = mp = NULL;
 4454                 cur->wb.upper.status_error = 0;
 4455                 rxbuf = &rxr->rx_buffers[i];
 4456                 plen = le16toh(cur->wb.upper.length);
 4457                 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
 4458                 if ((adapter->hw.mac.type == e1000_i350) &&
 4459                     (staterr & E1000_RXDEXT_STATERR_LB))
 4460                         vtag = be16toh(cur->wb.upper.vlan);
 4461                 else
 4462                         vtag = le16toh(cur->wb.upper.vlan);
 4463                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
 4464                 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
 4465 
 4466                 /* Make sure all segments of a bad packet are discarded */
 4467                 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) ||
 4468                     (rxr->discard)) {
 4469                         ifp->if_ierrors++;
 4470                         ++rxr->rx_discarded;
 4471                         if (!eop) /* Catch subsequent segs */
 4472                                 rxr->discard = TRUE;
 4473                         else
 4474                                 rxr->discard = FALSE;
 4475                         igb_rx_discard(rxr, i);
 4476                         goto next_desc;
 4477                 }
 4478 
 4479                 /*
 4480                 ** The way the hardware is configured to
 4481                 ** split, it will ONLY use the header buffer
 4482                 ** when header split is enabled, otherwise we
 4483                 ** get normal behavior, ie, both header and
 4484                 ** payload are DMA'd into the payload buffer.
 4485                 **
 4486                 ** The fmp test is to catch the case where a
 4487                 ** packet spans multiple descriptors, in that
 4488                 ** case only the first header is valid.
 4489                 */
 4490                 if (rxr->hdr_split && rxr->fmp == NULL) {
 4491                         hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
 4492                             E1000_RXDADV_HDRBUFLEN_SHIFT;
 4493                         if (hlen > IGB_HDR_BUF)
 4494                                 hlen = IGB_HDR_BUF;
 4495                         mh = rxr->rx_buffers[i].m_head;
 4496                         mh->m_len = hlen;
 4497                         /* clear buf pointer for refresh */
 4498                         rxbuf->m_head = NULL;
 4499                         /*
 4500                         ** Get the payload length, this
 4501                         ** could be zero if its a small
 4502                         ** packet.
 4503                         */
 4504                         if (plen > 0) {
 4505                                 mp = rxr->rx_buffers[i].m_pack;
 4506                                 mp->m_len = plen;
 4507                                 mh->m_next = mp;
 4508                                 /* clear buf pointer */
 4509                                 rxbuf->m_pack = NULL;
 4510                                 rxr->rx_split_packets++;
 4511                         }
 4512                 } else {
 4513                         /*
 4514                         ** Either no header split, or a
 4515                         ** secondary piece of a fragmented
 4516                         ** split packet.
 4517                         */
 4518                         mh = rxr->rx_buffers[i].m_pack;
 4519                         mh->m_len = plen;
 4520                         /* clear buf info for refresh */
 4521                         rxbuf->m_pack = NULL;
 4522                 }
 4523 
 4524                 ++processed; /* So we know when to refresh */
 4525 
 4526                 /* Initial frame - setup */
 4527                 if (rxr->fmp == NULL) {
 4528                         mh->m_pkthdr.len = mh->m_len;
 4529                         /* Save the head of the chain */
 4530                         rxr->fmp = mh;
 4531                         rxr->lmp = mh;
 4532                         if (mp != NULL) {
 4533                                 /* Add payload if split */
 4534                                 mh->m_pkthdr.len += mp->m_len;
 4535                                 rxr->lmp = mh->m_next;
 4536                         }
 4537                 } else {
 4538                         /* Chain mbuf's together */
 4539                         rxr->lmp->m_next = mh;
 4540                         rxr->lmp = rxr->lmp->m_next;
 4541                         rxr->fmp->m_pkthdr.len += mh->m_len;
 4542                 }
 4543 
 4544                 if (eop) {
 4545                         rxr->fmp->m_pkthdr.rcvif = ifp;
 4546                         ifp->if_ipackets++;
 4547                         rxr->rx_packets++;
 4548                         /* capture data for AIM */
 4549                         rxr->packets++;
 4550                         rxr->bytes += rxr->fmp->m_pkthdr.len;
 4551                         rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
 4552 
 4553                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
 4554                                 igb_rx_checksum(staterr, rxr->fmp, ptype);
 4555 
 4556                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
 4557                             (staterr & E1000_RXD_STAT_VP) != 0) {
 4558                                 rxr->fmp->m_pkthdr.ether_vtag = vtag;
 4559                                 rxr->fmp->m_flags |= M_VLANTAG;
 4560                         }
 4561 #if __FreeBSD_version >= 800000
 4562                         rxr->fmp->m_pkthdr.flowid = que->msix;
 4563                         rxr->fmp->m_flags |= M_FLOWID;
 4564 #endif
 4565                         sendmp = rxr->fmp;
 4566                         /* Make sure to set M_PKTHDR. */
 4567                         sendmp->m_flags |= M_PKTHDR;
 4568                         rxr->fmp = NULL;
 4569                         rxr->lmp = NULL;
 4570                 }
 4571 
 4572 next_desc:
 4573                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4574                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4575 
 4576                 /* Advance our pointers to the next descriptor. */
 4577                 if (++i == adapter->num_rx_desc)
 4578                         i = 0;
 4579                 /*
 4580                 ** Send to the stack or LRO
 4581                 */
 4582                 if (sendmp != NULL) {
 4583                         rxr->next_to_check = i;
 4584                         igb_rx_input(rxr, ifp, sendmp, ptype);
 4585                         i = rxr->next_to_check;
 4586                         rxdone++;
 4587                 }
 4588 
 4589                 /* Every 8 descriptors we go to refresh mbufs */
 4590                 if (processed == 8) {
 4591                         igb_refresh_mbufs(rxr, i);
 4592                         processed = 0;
 4593                 }
 4594         }
 4595 
 4596         /* Catch any remainders */
 4597         if (igb_rx_unrefreshed(rxr))
 4598                 igb_refresh_mbufs(rxr, i);
 4599 
 4600         rxr->next_to_check = i;
 4601 
 4602         /*
 4603          * Flush any outstanding LRO work
 4604          */
 4605         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
 4606                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
 4607                 tcp_lro_flush(lro, queued);
 4608         }
 4609 
 4610         if (done != NULL)
 4611                 *done = rxdone;
 4612 
 4613         IGB_RX_UNLOCK(rxr);
 4614         return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE);
 4615 }
 4616 
 4617 /*********************************************************************
 4618  *
 4619  *  Verify that the hardware indicated that the checksum is valid.
 4620  *  Inform the stack about the status of checksum so that stack
 4621  *  doesn't spend time verifying the checksum.
 4622  *
 4623  *********************************************************************/
 4624 static void
 4625 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
 4626 {
 4627         u16 status = (u16)staterr;
 4628         u8  errors = (u8) (staterr >> 24);
 4629         int sctp;
 4630 
 4631         /* Ignore Checksum bit is set */
 4632         if (status & E1000_RXD_STAT_IXSM) {
 4633                 mp->m_pkthdr.csum_flags = 0;
 4634                 return;
 4635         }
 4636 
 4637         if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
 4638             (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
 4639                 sctp = 1;
 4640         else
 4641                 sctp = 0;
 4642         if (status & E1000_RXD_STAT_IPCS) {
 4643                 /* Did it pass? */
 4644                 if (!(errors & E1000_RXD_ERR_IPE)) {
 4645                         /* IP Checksum Good */
 4646                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
 4647                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
 4648                 } else
 4649                         mp->m_pkthdr.csum_flags = 0;
 4650         }
 4651 
 4652         if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
 4653                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
 4654 #if __FreeBSD_version >= 800000
 4655                 if (sctp) /* reassign */
 4656                         type = CSUM_SCTP_VALID;
 4657 #endif
 4658                 /* Did it pass? */
 4659                 if (!(errors & E1000_RXD_ERR_TCPE)) {
 4660                         mp->m_pkthdr.csum_flags |= type;
 4661                         if (sctp == 0)
 4662                                 mp->m_pkthdr.csum_data = htons(0xffff);
 4663                 }
 4664         }
 4665         return;
 4666 }
 4667 
 4668 /*
 4669  * This routine is run via an vlan
 4670  * config EVENT
 4671  */
 4672 static void
 4673 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 4674 {
 4675         struct adapter  *adapter = ifp->if_softc;
 4676         u32             index, bit;
 4677 
 4678         if (ifp->if_softc !=  arg)   /* Not our event */
 4679                 return;
 4680 
 4681         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 4682                 return;
 4683 
 4684         IGB_CORE_LOCK(adapter);
 4685         index = (vtag >> 5) & 0x7F;
 4686         bit = vtag & 0x1F;
 4687         adapter->shadow_vfta[index] |= (1 << bit);
 4688         ++adapter->num_vlans;
 4689         /* Change hw filter setting */
 4690         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
 4691                 igb_setup_vlan_hw_support(adapter);
 4692         IGB_CORE_UNLOCK(adapter);
 4693 }
 4694 
 4695 /*
 4696  * This routine is run via an vlan
 4697  * unconfig EVENT
 4698  */
 4699 static void
 4700 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 4701 {
 4702         struct adapter  *adapter = ifp->if_softc;
 4703         u32             index, bit;
 4704 
 4705         if (ifp->if_softc !=  arg)
 4706                 return;
 4707 
 4708         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 4709                 return;
 4710 
 4711         IGB_CORE_LOCK(adapter);
 4712         index = (vtag >> 5) & 0x7F;
 4713         bit = vtag & 0x1F;
 4714         adapter->shadow_vfta[index] &= ~(1 << bit);
 4715         --adapter->num_vlans;
 4716         /* Change hw filter setting */
 4717         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
 4718                 igb_setup_vlan_hw_support(adapter);
 4719         IGB_CORE_UNLOCK(adapter);
 4720 }
 4721 
 4722 static void
 4723 igb_setup_vlan_hw_support(struct adapter *adapter)
 4724 {
 4725         struct e1000_hw *hw = &adapter->hw;
 4726         struct ifnet    *ifp = adapter->ifp;
 4727         u32             reg;
 4728 
 4729         if (adapter->vf_ifp) {
 4730                 e1000_rlpml_set_vf(hw,
 4731                     adapter->max_frame_size + VLAN_TAG_SIZE);
 4732                 return;
 4733         }
 4734 
 4735         reg = E1000_READ_REG(hw, E1000_CTRL);
 4736         reg |= E1000_CTRL_VME;
 4737         E1000_WRITE_REG(hw, E1000_CTRL, reg);
 4738 
 4739         /* Enable the Filter Table */
 4740         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
 4741                 reg = E1000_READ_REG(hw, E1000_RCTL);
 4742                 reg &= ~E1000_RCTL_CFIEN;
 4743                 reg |= E1000_RCTL_VFE;
 4744                 E1000_WRITE_REG(hw, E1000_RCTL, reg);
 4745         }
 4746 
 4747         /* Update the frame size */
 4748         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
 4749             adapter->max_frame_size + VLAN_TAG_SIZE);
 4750 
 4751         /* Don't bother with table if no vlans */
 4752         if ((adapter->num_vlans == 0) ||
 4753             ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0))
 4754                 return;
 4755         /*
 4756         ** A soft reset zero's out the VFTA, so
 4757         ** we need to repopulate it now.
 4758         */
 4759         for (int i = 0; i < IGB_VFTA_SIZE; i++)
 4760                 if (adapter->shadow_vfta[i] != 0) {
 4761                         if (adapter->vf_ifp)
 4762                                 e1000_vfta_set_vf(hw,
 4763                                     adapter->shadow_vfta[i], TRUE);
 4764                         else
 4765                                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
 4766                                  i, adapter->shadow_vfta[i]);
 4767                 }
 4768 }
 4769 
 4770 static void
 4771 igb_enable_intr(struct adapter *adapter)
 4772 {
 4773         /* With RSS set up what to auto clear */
 4774         if (adapter->msix_mem) {
 4775                 u32 mask = (adapter->que_mask | adapter->link_mask);
 4776                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask);
 4777                 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask);
 4778                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask);
 4779                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 4780                     E1000_IMS_LSC);
 4781         } else {
 4782                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 4783                     IMS_ENABLE_MASK);
 4784         }
 4785         E1000_WRITE_FLUSH(&adapter->hw);
 4786 
 4787         return;
 4788 }
 4789 
 4790 static void
 4791 igb_disable_intr(struct adapter *adapter)
 4792 {
 4793         if (adapter->msix_mem) {
 4794                 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
 4795                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
 4796         } 
 4797         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
 4798         E1000_WRITE_FLUSH(&adapter->hw);
 4799         return;
 4800 }
 4801 
 4802 /*
 4803  * Bit of a misnomer, what this really means is
 4804  * to enable OS management of the system... aka
 4805  * to disable special hardware management features 
 4806  */
 4807 static void
 4808 igb_init_manageability(struct adapter *adapter)
 4809 {
 4810         if (adapter->has_manage) {
 4811                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
 4812                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 4813 
 4814                 /* disable hardware interception of ARP */
 4815                 manc &= ~(E1000_MANC_ARP_EN);
 4816 
 4817                 /* enable receiving management packets to the host */
 4818                 manc |= E1000_MANC_EN_MNG2HOST;
 4819                 manc2h |= 1 << 5;  /* Mng Port 623 */
 4820                 manc2h |= 1 << 6;  /* Mng Port 664 */
 4821                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
 4822                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 4823         }
 4824 }
 4825 
 4826 /*
 4827  * Give control back to hardware management
 4828  * controller if there is one.
 4829  */
 4830 static void
 4831 igb_release_manageability(struct adapter *adapter)
 4832 {
 4833         if (adapter->has_manage) {
 4834                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 4835 
 4836                 /* re-enable hardware interception of ARP */
 4837                 manc |= E1000_MANC_ARP_EN;
 4838                 manc &= ~E1000_MANC_EN_MNG2HOST;
 4839 
 4840                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 4841         }
 4842 }
 4843 
 4844 /*
 4845  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 4846  * For ASF and Pass Through versions of f/w this means that
 4847  * the driver is loaded. 
 4848  *
 4849  */
 4850 static void
 4851 igb_get_hw_control(struct adapter *adapter)
 4852 {
 4853         u32 ctrl_ext;
 4854 
 4855         if (adapter->vf_ifp)
 4856                 return;
 4857 
 4858         /* Let firmware know the driver has taken over */
 4859         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 4860         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 4861             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
 4862 }
 4863 
 4864 /*
 4865  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 4866  * For ASF and Pass Through versions of f/w this means that the
 4867  * driver is no longer loaded.
 4868  *
 4869  */
 4870 static void
 4871 igb_release_hw_control(struct adapter *adapter)
 4872 {
 4873         u32 ctrl_ext;
 4874 
 4875         if (adapter->vf_ifp)
 4876                 return;
 4877 
 4878         /* Let firmware taken over control of h/w */
 4879         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 4880         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 4881             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
 4882 }
 4883 
 4884 static int
 4885 igb_is_valid_ether_addr(uint8_t *addr)
 4886 {
 4887         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
 4888 
 4889         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
 4890                 return (FALSE);
 4891         }
 4892 
 4893         return (TRUE);
 4894 }
 4895 
 4896 
 4897 /*
 4898  * Enable PCI Wake On Lan capability
 4899  */
 4900 static void
 4901 igb_enable_wakeup(device_t dev)
 4902 {
 4903         u16     cap, status;
 4904         u8      id;
 4905 
 4906         /* First find the capabilities pointer*/
 4907         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
 4908         /* Read the PM Capabilities */
 4909         id = pci_read_config(dev, cap, 1);
 4910         if (id != PCIY_PMG)     /* Something wrong */
 4911                 return;
 4912         /* OK, we have the power capabilities, so
 4913            now get the status register */
 4914         cap += PCIR_POWER_STATUS;
 4915         status = pci_read_config(dev, cap, 2);
 4916         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
 4917         pci_write_config(dev, cap, status, 2);
 4918         return;
 4919 }
 4920 
 4921 static void
 4922 igb_led_func(void *arg, int onoff)
 4923 {
 4924         struct adapter  *adapter = arg;
 4925 
 4926         IGB_CORE_LOCK(adapter);
 4927         if (onoff) {
 4928                 e1000_setup_led(&adapter->hw);
 4929                 e1000_led_on(&adapter->hw);
 4930         } else {
 4931                 e1000_led_off(&adapter->hw);
 4932                 e1000_cleanup_led(&adapter->hw);
 4933         }
 4934         IGB_CORE_UNLOCK(adapter);
 4935 }
 4936 
 4937 /**********************************************************************
 4938  *
 4939  *  Update the board statistics counters.
 4940  *
 4941  **********************************************************************/
 4942 static void
 4943 igb_update_stats_counters(struct adapter *adapter)
 4944 {
 4945         struct ifnet            *ifp;
 4946         struct e1000_hw         *hw = &adapter->hw;
 4947         struct e1000_hw_stats   *stats;
 4948 
 4949         /* 
 4950         ** The virtual function adapter has only a
 4951         ** small controlled set of stats, do only 
 4952         ** those and return.
 4953         */
 4954         if (adapter->vf_ifp) {
 4955                 igb_update_vf_stats_counters(adapter);
 4956                 return;
 4957         }
 4958 
 4959         stats = (struct e1000_hw_stats  *)adapter->stats;
 4960 
 4961         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
 4962            (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
 4963                 stats->symerrs +=
 4964                     E1000_READ_REG(hw,E1000_SYMERRS);
 4965                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
 4966         }
 4967 
 4968         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
 4969         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
 4970         stats->scc += E1000_READ_REG(hw, E1000_SCC);
 4971         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
 4972 
 4973         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
 4974         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
 4975         stats->colc += E1000_READ_REG(hw, E1000_COLC);
 4976         stats->dc += E1000_READ_REG(hw, E1000_DC);
 4977         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
 4978         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
 4979         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
 4980         /*
 4981         ** For watchdog management we need to know if we have been
 4982         ** paused during the last interval, so capture that here.
 4983         */ 
 4984         adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
 4985         stats->xoffrxc += adapter->pause_frames;
 4986         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
 4987         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
 4988         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
 4989         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
 4990         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
 4991         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
 4992         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
 4993         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
 4994         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
 4995         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
 4996         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
 4997         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
 4998 
 4999         /* For the 64-bit byte counters the low dword must be read first. */
 5000         /* Both registers clear on the read of the high dword */
 5001 
 5002         stats->gorc += E1000_READ_REG(hw, E1000_GORCL) +
 5003             ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32);
 5004         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) +
 5005             ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32);
 5006 
 5007         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
 5008         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
 5009         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
 5010         stats->roc += E1000_READ_REG(hw, E1000_ROC);
 5011         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
 5012 
 5013         stats->tor += E1000_READ_REG(hw, E1000_TORH);
 5014         stats->tot += E1000_READ_REG(hw, E1000_TOTH);
 5015 
 5016         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
 5017         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
 5018         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
 5019         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
 5020         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
 5021         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
 5022         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
 5023         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
 5024         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
 5025         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
 5026 
 5027         /* Interrupt Counts */
 5028 
 5029         stats->iac += E1000_READ_REG(hw, E1000_IAC);
 5030         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
 5031         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
 5032         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
 5033         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
 5034         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
 5035         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
 5036         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
 5037         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
 5038 
 5039         /* Host to Card Statistics */
 5040 
 5041         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
 5042         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
 5043         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
 5044         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
 5045         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
 5046         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
 5047         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
 5048         stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) +
 5049             ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32));
 5050         stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) +
 5051             ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32));
 5052         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
 5053         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
 5054         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
 5055 
 5056         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
 5057         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
 5058         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
 5059         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
 5060         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
 5061         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
 5062 
 5063         ifp = adapter->ifp;
 5064         ifp->if_collisions = stats->colc;
 5065 
 5066         /* Rx Errors */
 5067         ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc +
 5068             stats->crcerrs + stats->algnerrc +
 5069             stats->ruc + stats->roc + stats->mpc + stats->cexterr;
 5070 
 5071         /* Tx Errors */
 5072         ifp->if_oerrors = stats->ecol +
 5073             stats->latecol + adapter->watchdog_events;
 5074 
 5075         /* Driver specific counters */
 5076         adapter->device_control = E1000_READ_REG(hw, E1000_CTRL);
 5077         adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL);
 5078         adapter->int_mask = E1000_READ_REG(hw, E1000_IMS);
 5079         adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS);
 5080         adapter->packet_buf_alloc_tx =
 5081             ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16);
 5082         adapter->packet_buf_alloc_rx =
 5083             (E1000_READ_REG(hw, E1000_PBA) & 0xffff);
 5084 }
 5085 
 5086 
 5087 /**********************************************************************
 5088  *
 5089  *  Initialize the VF board statistics counters.
 5090  *
 5091  **********************************************************************/
 5092 static void
 5093 igb_vf_init_stats(struct adapter *adapter)
 5094 {
 5095         struct e1000_hw *hw = &adapter->hw;
 5096         struct e1000_vf_stats   *stats;
 5097 
 5098         stats = (struct e1000_vf_stats  *)adapter->stats;
 5099         if (stats == NULL)
 5100                 return;
 5101         stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC);
 5102         stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC);
 5103         stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC);
 5104         stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC);
 5105         stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC);
 5106 }
 5107  
 5108 /**********************************************************************
 5109  *
 5110  *  Update the VF board statistics counters.
 5111  *
 5112  **********************************************************************/
 5113 static void
 5114 igb_update_vf_stats_counters(struct adapter *adapter)
 5115 {
 5116         struct e1000_hw *hw = &adapter->hw;
 5117         struct e1000_vf_stats   *stats;
 5118 
 5119         if (adapter->link_speed == 0)
 5120                 return;
 5121 
 5122         stats = (struct e1000_vf_stats  *)adapter->stats;
 5123 
 5124         UPDATE_VF_REG(E1000_VFGPRC,
 5125             stats->last_gprc, stats->gprc);
 5126         UPDATE_VF_REG(E1000_VFGORC,
 5127             stats->last_gorc, stats->gorc);
 5128         UPDATE_VF_REG(E1000_VFGPTC,
 5129             stats->last_gptc, stats->gptc);
 5130         UPDATE_VF_REG(E1000_VFGOTC,
 5131             stats->last_gotc, stats->gotc);
 5132         UPDATE_VF_REG(E1000_VFMPRC,
 5133             stats->last_mprc, stats->mprc);
 5134 }
 5135 
 5136 /* Export a single 32-bit register via a read-only sysctl. */
 5137 static int
 5138 igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS)
 5139 {
 5140         struct adapter *adapter;
 5141         u_int val;
 5142 
 5143         adapter = oidp->oid_arg1;
 5144         val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2);
 5145         return (sysctl_handle_int(oidp, &val, 0, req));
 5146 }
 5147 
 5148 /*
 5149 **  Tuneable interrupt rate handler
 5150 */
 5151 static int
 5152 igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
 5153 {
 5154         struct igb_queue        *que = ((struct igb_queue *)oidp->oid_arg1);
 5155         int                     error;
 5156         u32                     reg, usec, rate;
 5157                         
 5158         reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix));
 5159         usec = ((reg & 0x7FFC) >> 2);
 5160         if (usec > 0)
 5161                 rate = 1000000 / usec;
 5162         else
 5163                 rate = 0;
 5164         error = sysctl_handle_int(oidp, &rate, 0, req);
 5165         if (error || !req->newptr)
 5166                 return error;
 5167         return 0;
 5168 }
 5169 
 5170 /*
 5171  * Add sysctl variables, one per statistic, to the system.
 5172  */
 5173 static void
 5174 igb_add_hw_stats(struct adapter *adapter)
 5175 {
 5176         device_t dev = adapter->dev;
 5177 
 5178         struct tx_ring *txr = adapter->tx_rings;
 5179         struct rx_ring *rxr = adapter->rx_rings;
 5180 
 5181         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
 5182         struct sysctl_oid *tree = device_get_sysctl_tree(dev);
 5183         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
 5184         struct e1000_hw_stats *stats = adapter->stats;
 5185 
 5186         struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node;
 5187         struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list;
 5188 
 5189 #define QUEUE_NAME_LEN 32
 5190         char namebuf[QUEUE_NAME_LEN];
 5191 
 5192         /* Driver Statistics */
 5193         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "link_irq", 
 5194                         CTLFLAG_RD, &adapter->link_irq, 0,
 5195                         "Link MSIX IRQ Handled");
 5196         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 
 5197                         CTLFLAG_RD, &adapter->dropped_pkts,
 5198                         "Driver dropped packets");
 5199         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", 
 5200                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
 5201                         "Driver tx dma failure in xmit");
 5202         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns",
 5203                         CTLFLAG_RD, &adapter->rx_overruns,
 5204                         "RX overruns");
 5205         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts",
 5206                         CTLFLAG_RD, &adapter->watchdog_events,
 5207                         "Watchdog timeouts");
 5208 
 5209         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", 
 5210                         CTLFLAG_RD, &adapter->device_control,
 5211                         "Device Control Register");
 5212         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", 
 5213                         CTLFLAG_RD, &adapter->rx_control,
 5214                         "Receiver Control Register");
 5215         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", 
 5216                         CTLFLAG_RD, &adapter->int_mask,
 5217                         "Interrupt Mask");
 5218         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", 
 5219                         CTLFLAG_RD, &adapter->eint_mask,
 5220                         "Extended Interrupt Mask");
 5221         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", 
 5222                         CTLFLAG_RD, &adapter->packet_buf_alloc_tx,
 5223                         "Transmit Buffer Packet Allocation");
 5224         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", 
 5225                         CTLFLAG_RD, &adapter->packet_buf_alloc_rx,
 5226                         "Receive Buffer Packet Allocation");
 5227         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water",
 5228                         CTLFLAG_RD, &adapter->hw.fc.high_water, 0,
 5229                         "Flow Control High Watermark");
 5230         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 
 5231                         CTLFLAG_RD, &adapter->hw.fc.low_water, 0,
 5232                         "Flow Control Low Watermark");
 5233 
 5234         for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
 5235                 struct lro_ctrl *lro = &rxr->lro;
 5236 
 5237                 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
 5238                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
 5239                                             CTLFLAG_RD, NULL, "Queue Name");
 5240                 queue_list = SYSCTL_CHILDREN(queue_node);
 5241 
 5242                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", 
 5243                                 CTLFLAG_RD, &adapter->queues[i],
 5244                                 sizeof(&adapter->queues[i]),
 5245                                 igb_sysctl_interrupt_rate_handler,
 5246                                 "IU", "Interrupt Rate");
 5247 
 5248                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 
 5249                                 CTLFLAG_RD, adapter, E1000_TDH(txr->me),
 5250                                 igb_sysctl_reg_handler, "IU",
 5251                                 "Transmit Descriptor Head");
 5252                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 
 5253                                 CTLFLAG_RD, adapter, E1000_TDT(txr->me),
 5254                                 igb_sysctl_reg_handler, "IU",
 5255                                 "Transmit Descriptor Tail");
 5256                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 
 5257                                 CTLFLAG_RD, &txr->no_desc_avail,
 5258                                 "Queue No Descriptor Available");
 5259                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets",
 5260                                 CTLFLAG_RD, &txr->tx_packets,
 5261                                 "Queue Packets Transmitted");
 5262 
 5263                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 
 5264                                 CTLFLAG_RD, adapter, E1000_RDH(rxr->me),
 5265                                 igb_sysctl_reg_handler, "IU",
 5266                                 "Receive Descriptor Head");
 5267                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 
 5268                                 CTLFLAG_RD, adapter, E1000_RDT(rxr->me),
 5269                                 igb_sysctl_reg_handler, "IU",
 5270                                 "Receive Descriptor Tail");
 5271                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets",
 5272                                 CTLFLAG_RD, &rxr->rx_packets,
 5273                                 "Queue Packets Received");
 5274                 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
 5275                                 CTLFLAG_RD, &rxr->rx_bytes,
 5276                                 "Queue Bytes Received");
 5277                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued",
 5278                                 CTLFLAG_RD, &lro->lro_queued, 0,
 5279                                 "LRO Queued");
 5280                 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed",
 5281                                 CTLFLAG_RD, &lro->lro_flushed, 0,
 5282                                 "LRO Flushed");
 5283         }
 5284 
 5285         /* MAC stats get their own sub node */
 5286 
 5287         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 
 5288                                     CTLFLAG_RD, NULL, "MAC Statistics");
 5289         stat_list = SYSCTL_CHILDREN(stat_node);
 5290 
 5291         /*
 5292         ** VF adapter has a very limited set of stats
 5293         ** since its not managing the metal, so to speak.
 5294         */
 5295         if (adapter->vf_ifp) {
 5296         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
 5297                         CTLFLAG_RD, &stats->gprc,
 5298                         "Good Packets Received");
 5299         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 5300                         CTLFLAG_RD, &stats->gptc,
 5301                         "Good Packets Transmitted");
 5302         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 
 5303                         CTLFLAG_RD, &stats->gorc, 
 5304                         "Good Octets Received"); 
 5305         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 
 5306                         CTLFLAG_RD, &stats->gotc, 
 5307                         "Good Octets Transmitted"); 
 5308         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
 5309                         CTLFLAG_RD, &stats->mprc,
 5310                         "Multicast Packets Received");
 5311                 return;
 5312         }
 5313 
 5314         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", 
 5315                         CTLFLAG_RD, &stats->ecol,
 5316                         "Excessive collisions");
 5317         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", 
 5318                         CTLFLAG_RD, &stats->scc,
 5319                         "Single collisions");
 5320         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 
 5321                         CTLFLAG_RD, &stats->mcc,
 5322                         "Multiple collisions");
 5323         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", 
 5324                         CTLFLAG_RD, &stats->latecol,
 5325                         "Late collisions");
 5326         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", 
 5327                         CTLFLAG_RD, &stats->colc,
 5328                         "Collision Count");
 5329         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors",
 5330                         CTLFLAG_RD, &stats->symerrs,
 5331                         "Symbol Errors");
 5332         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors",
 5333                         CTLFLAG_RD, &stats->sec,
 5334                         "Sequence Errors");
 5335         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count",
 5336                         CTLFLAG_RD, &stats->dc,
 5337                         "Defer Count");
 5338         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets",
 5339                         CTLFLAG_RD, &stats->mpc,
 5340                         "Missed Packets");
 5341         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff",
 5342                         CTLFLAG_RD, &stats->rnbc,
 5343                         "Receive No Buffers");
 5344         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize",
 5345                         CTLFLAG_RD, &stats->ruc,
 5346                         "Receive Undersize");
 5347         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
 5348                         CTLFLAG_RD, &stats->rfc,
 5349                         "Fragmented Packets Received ");
 5350         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize",
 5351                         CTLFLAG_RD, &stats->roc,
 5352                         "Oversized Packets Received");
 5353         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber",
 5354                         CTLFLAG_RD, &stats->rjc,
 5355                         "Recevied Jabber");
 5356         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs",
 5357                         CTLFLAG_RD, &stats->rxerrc,
 5358                         "Receive Errors");
 5359         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs",
 5360                         CTLFLAG_RD, &stats->crcerrs,
 5361                         "CRC errors");
 5362         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs",
 5363                         CTLFLAG_RD, &stats->algnerrc,
 5364                         "Alignment Errors");
 5365         /* On 82575 these are collision counts */
 5366         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs",
 5367                         CTLFLAG_RD, &stats->cexterr,
 5368                         "Collision/Carrier extension errors");
 5369         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd",
 5370                         CTLFLAG_RD, &stats->xonrxc,
 5371                         "XON Received");
 5372         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd",
 5373                         CTLFLAG_RD, &stats->xontxc,
 5374                         "XON Transmitted");
 5375         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd",
 5376                         CTLFLAG_RD, &stats->xoffrxc,
 5377                         "XOFF Received");
 5378         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd",
 5379                         CTLFLAG_RD, &stats->xofftxc,
 5380                         "XOFF Transmitted");
 5381         /* Packet Reception Stats */
 5382         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd",
 5383                         CTLFLAG_RD, &stats->tpr,
 5384                         "Total Packets Received ");
 5385         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd",
 5386                         CTLFLAG_RD, &stats->gprc,
 5387                         "Good Packets Received");
 5388         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd",
 5389                         CTLFLAG_RD, &stats->bprc,
 5390                         "Broadcast Packets Received");
 5391         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd",
 5392                         CTLFLAG_RD, &stats->mprc,
 5393                         "Multicast Packets Received");
 5394         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
 5395                         CTLFLAG_RD, &stats->prc64,
 5396                         "64 byte frames received ");
 5397         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
 5398                         CTLFLAG_RD, &stats->prc127,
 5399                         "65-127 byte frames received");
 5400         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
 5401                         CTLFLAG_RD, &stats->prc255,
 5402                         "128-255 byte frames received");
 5403         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
 5404                         CTLFLAG_RD, &stats->prc511,
 5405                         "256-511 byte frames received");
 5406         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
 5407                         CTLFLAG_RD, &stats->prc1023,
 5408                         "512-1023 byte frames received");
 5409         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
 5410                         CTLFLAG_RD, &stats->prc1522,
 5411                         "1023-1522 byte frames received");
 5412         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 
 5413                         CTLFLAG_RD, &stats->gorc, 
 5414                         "Good Octets Received"); 
 5415 
 5416         /* Packet Transmission Stats */
 5417         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", 
 5418                         CTLFLAG_RD, &stats->gotc, 
 5419                         "Good Octets Transmitted"); 
 5420         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
 5421                         CTLFLAG_RD, &stats->tpt,
 5422                         "Total Packets Transmitted");
 5423         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
 5424                         CTLFLAG_RD, &stats->gptc,
 5425                         "Good Packets Transmitted");
 5426         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
 5427                         CTLFLAG_RD, &stats->bptc,
 5428                         "Broadcast Packets Transmitted");
 5429         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
 5430                         CTLFLAG_RD, &stats->mptc,
 5431                         "Multicast Packets Transmitted");
 5432         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
 5433                         CTLFLAG_RD, &stats->ptc64,
 5434                         "64 byte frames transmitted ");
 5435         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
 5436                         CTLFLAG_RD, &stats->ptc127,
 5437                         "65-127 byte frames transmitted");
 5438         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
 5439                         CTLFLAG_RD, &stats->ptc255,
 5440                         "128-255 byte frames transmitted");
 5441         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
 5442                         CTLFLAG_RD, &stats->ptc511,
 5443                         "256-511 byte frames transmitted");
 5444         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
 5445                         CTLFLAG_RD, &stats->ptc1023,
 5446                         "512-1023 byte frames transmitted");
 5447         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
 5448                         CTLFLAG_RD, &stats->ptc1522,
 5449                         "1024-1522 byte frames transmitted");
 5450         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd",
 5451                         CTLFLAG_RD, &stats->tsctc,
 5452                         "TSO Contexts Transmitted");
 5453         SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail",
 5454                         CTLFLAG_RD, &stats->tsctfc,
 5455                         "TSO Contexts Failed");
 5456 
 5457 
 5458         /* Interrupt Stats */
 5459 
 5460         int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 
 5461                                     CTLFLAG_RD, NULL, "Interrupt Statistics");
 5462         int_list = SYSCTL_CHILDREN(int_node);
 5463 
 5464         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts",
 5465                         CTLFLAG_RD, &stats->iac,
 5466                         "Interrupt Assertion Count");
 5467 
 5468         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer",
 5469                         CTLFLAG_RD, &stats->icrxptc,
 5470                         "Interrupt Cause Rx Pkt Timer Expire Count");
 5471 
 5472         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer",
 5473                         CTLFLAG_RD, &stats->icrxatc,
 5474                         "Interrupt Cause Rx Abs Timer Expire Count");
 5475 
 5476         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer",
 5477                         CTLFLAG_RD, &stats->ictxptc,
 5478                         "Interrupt Cause Tx Pkt Timer Expire Count");
 5479 
 5480         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer",
 5481                         CTLFLAG_RD, &stats->ictxatc,
 5482                         "Interrupt Cause Tx Abs Timer Expire Count");
 5483 
 5484         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty",
 5485                         CTLFLAG_RD, &stats->ictxqec,
 5486                         "Interrupt Cause Tx Queue Empty Count");
 5487 
 5488         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh",
 5489                         CTLFLAG_RD, &stats->ictxqmtc,
 5490                         "Interrupt Cause Tx Queue Min Thresh Count");
 5491 
 5492         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh",
 5493                         CTLFLAG_RD, &stats->icrxdmtc,
 5494                         "Interrupt Cause Rx Desc Min Thresh Count");
 5495 
 5496         SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun",
 5497                         CTLFLAG_RD, &stats->icrxoc,
 5498                         "Interrupt Cause Receiver Overrun Count");
 5499 
 5500         /* Host to Card Stats */
 5501 
 5502         host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", 
 5503                                     CTLFLAG_RD, NULL, 
 5504                                     "Host to Card Statistics");
 5505 
 5506         host_list = SYSCTL_CHILDREN(host_node);
 5507 
 5508         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt",
 5509                         CTLFLAG_RD, &stats->cbtmpc,
 5510                         "Circuit Breaker Tx Packet Count");
 5511 
 5512         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard",
 5513                         CTLFLAG_RD, &stats->htdpmc,
 5514                         "Host Transmit Discarded Packets");
 5515 
 5516         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt",
 5517                         CTLFLAG_RD, &stats->rpthc,
 5518                         "Rx Packets To Host");
 5519 
 5520         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts",
 5521                         CTLFLAG_RD, &stats->cbrmpc,
 5522                         "Circuit Breaker Rx Packet Count");
 5523 
 5524         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop",
 5525                         CTLFLAG_RD, &stats->cbrdpc,
 5526                         "Circuit Breaker Rx Dropped Count");
 5527 
 5528         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt",
 5529                         CTLFLAG_RD, &stats->hgptc,
 5530                         "Host Good Packets Tx Count");
 5531 
 5532         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop",
 5533                         CTLFLAG_RD, &stats->htcbdpc,
 5534                         "Host Tx Circuit Breaker Dropped Count");
 5535 
 5536         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes",
 5537                         CTLFLAG_RD, &stats->hgorc,
 5538                         "Host Good Octets Received Count");
 5539 
 5540         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes",
 5541                         CTLFLAG_RD, &stats->hgotc,
 5542                         "Host Good Octets Transmit Count");
 5543 
 5544         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors",
 5545                         CTLFLAG_RD, &stats->lenerrs,
 5546                         "Length Errors");
 5547 
 5548         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt",
 5549                         CTLFLAG_RD, &stats->scvpc,
 5550                         "SerDes/SGMII Code Violation Pkt Count");
 5551 
 5552         SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed",
 5553                         CTLFLAG_RD, &stats->hrmpc,
 5554                         "Header Redirection Missed Packet Count");
 5555 }
 5556 
 5557 
 5558 /**********************************************************************
 5559  *
 5560  *  This routine provides a way to dump out the adapter eeprom,
 5561  *  often a useful debug/service tool. This only dumps the first
 5562  *  32 words, stuff that matters is in that extent.
 5563  *
 5564  **********************************************************************/
 5565 static int
 5566 igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS)
 5567 {
 5568         struct adapter *adapter;
 5569         int error;
 5570         int result;
 5571 
 5572         result = -1;
 5573         error = sysctl_handle_int(oidp, &result, 0, req);
 5574 
 5575         if (error || !req->newptr)
 5576                 return (error);
 5577 
 5578         /*
 5579          * This value will cause a hex dump of the
 5580          * first 32 16-bit words of the EEPROM to
 5581          * the screen.
 5582          */
 5583         if (result == 1) {
 5584                 adapter = (struct adapter *)arg1;
 5585                 igb_print_nvm_info(adapter);
 5586         }
 5587 
 5588         return (error);
 5589 }
 5590 
 5591 static void
 5592 igb_print_nvm_info(struct adapter *adapter)
 5593 {
 5594         u16     eeprom_data;
 5595         int     i, j, row = 0;
 5596 
 5597         /* Its a bit crude, but it gets the job done */
 5598         printf("\nInterface EEPROM Dump:\n");
 5599         printf("Offset\n0x0000  ");
 5600         for (i = 0, j = 0; i < 32; i++, j++) {
 5601                 if (j == 8) { /* Make the offset block */
 5602                         j = 0; ++row;
 5603                         printf("\n0x00%x0  ",row);
 5604                 }
 5605                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
 5606                 printf("%04x ", eeprom_data);
 5607         }
 5608         printf("\n");
 5609 }
 5610 
 5611 static void
 5612 igb_set_sysctl_value(struct adapter *adapter, const char *name,
 5613         const char *description, int *limit, int value)
 5614 {
 5615         *limit = value;
 5616         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
 5617             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
 5618             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
 5619 }
 5620 
 5621 /*
 5622 ** Set flow control using sysctl:
 5623 ** Flow control values:
 5624 **      0 - off
 5625 **      1 - rx pause
 5626 **      2 - tx pause
 5627 **      3 - full
 5628 */
 5629 static int
 5630 igb_set_flowcntl(SYSCTL_HANDLER_ARGS)
 5631 {
 5632         int error;
 5633         struct adapter *adapter = (struct adapter *) arg1;
 5634 
 5635         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
 5636 
 5637         if ((error) || (req->newptr == NULL))
 5638                 return (error);
 5639 
 5640         switch (adapter->fc) {
 5641                 case e1000_fc_rx_pause:
 5642                 case e1000_fc_tx_pause:
 5643                 case e1000_fc_full:
 5644                         adapter->hw.fc.requested_mode = adapter->fc;
 5645                         break;
 5646                 case e1000_fc_none:
 5647                 default:
 5648                         adapter->hw.fc.requested_mode = e1000_fc_none;
 5649         }
 5650 
 5651         adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode;
 5652         e1000_force_mac_fc(&adapter->hw);
 5653         return (error);
 5654 }
 5655 
 5656 /*
 5657 ** Manage DMA Coalesce:
 5658 ** Control values:
 5659 **      0/1 - off/on
 5660 **      Legal timer values are:
 5661 **      250,500,1000-10000 in thousands
 5662 */
 5663 static int
 5664 igb_sysctl_dmac(SYSCTL_HANDLER_ARGS)
 5665 {
 5666         struct adapter *adapter = (struct adapter *) arg1;
 5667         int             error;
 5668 
 5669         error = sysctl_handle_int(oidp, &adapter->dmac, 0, req);
 5670 
 5671         if ((error) || (req->newptr == NULL))
 5672                 return (error);
 5673 
 5674         switch (adapter->dmac) {
 5675                 case 0:
 5676                         /*Disabling */
 5677                         break;
 5678                 case 1: /* Just enable and use default */
 5679                         adapter->dmac = 1000;
 5680                         break;
 5681                 case 250:
 5682                 case 500:
 5683                 case 1000:
 5684                 case 2000:
 5685                 case 3000:
 5686                 case 4000:
 5687                 case 5000:
 5688                 case 6000:
 5689                 case 7000:
 5690                 case 8000:
 5691                 case 9000:
 5692                 case 10000:
 5693                         /* Legal values - allow */
 5694                         break;
 5695                 default:
 5696                         /* Do nothing, illegal value */
 5697                         adapter->dmac = 0;
 5698                         return (error);
 5699         }
 5700         /* Reinit the interface */
 5701         igb_init(adapter);
 5702         return (error);
 5703 }

Cache object: d136c969d0d3549046d89bc45713b833


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.