The Design and Implementation of the FreeBSD Operating System, Second Edition
Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]

FreeBSD/Linux Kernel Cross Reference
sys/dev/e1000/if_igb.c

Version: -  FREEBSD  -  FREEBSD-13-STABLE  -  FREEBSD-13-0  -  FREEBSD-12-STABLE  -  FREEBSD-12-0  -  FREEBSD-11-STABLE  -  FREEBSD-11-0  -  FREEBSD-10-STABLE  -  FREEBSD-10-0  -  FREEBSD-9-STABLE  -  FREEBSD-9-0  -  FREEBSD-8-STABLE  -  FREEBSD-8-0  -  FREEBSD-7-STABLE  -  FREEBSD-7-0  -  FREEBSD-6-STABLE  -  FREEBSD-6-0  -  FREEBSD-5-STABLE  -  FREEBSD-5-0  -  FREEBSD-4-STABLE  -  FREEBSD-3-STABLE  -  FREEBSD22  -  l41  -  OPENBSD  -  linux-2.6  -  MK84  -  PLAN9  -  xnu-8792 
SearchContext: -  none  -  3  -  10 

    1 /******************************************************************************
    2 
    3   Copyright (c) 2001-2009, Intel Corporation 
    4   All rights reserved.
    5   
    6   Redistribution and use in source and binary forms, with or without 
    7   modification, are permitted provided that the following conditions are met:
    8   
    9    1. Redistributions of source code must retain the above copyright notice, 
   10       this list of conditions and the following disclaimer.
   11   
   12    2. Redistributions in binary form must reproduce the above copyright 
   13       notice, this list of conditions and the following disclaimer in the 
   14       documentation and/or other materials provided with the distribution.
   15   
   16    3. Neither the name of the Intel Corporation nor the names of its 
   17       contributors may be used to endorse or promote products derived from 
   18       this software without specific prior written permission.
   19   
   20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
   21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
   22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
   23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
   24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
   25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
   26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
   27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
   28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
   29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   30   POSSIBILITY OF SUCH DAMAGE.
   31 
   32 ******************************************************************************/
   33 /*$FreeBSD: releng/8.0/sys/dev/e1000/if_igb.c 199325 2009-11-16 18:46:33Z jfv $*/
   34 
   35 
   36 #ifdef HAVE_KERNEL_OPTION_HEADERS
   37 #include "opt_device_polling.h"
   38 #include "opt_inet.h"
   39 #endif
   40 
   41 #include <sys/param.h>
   42 #include <sys/systm.h>
   43 #if __FreeBSD_version >= 800000
   44 #include <sys/buf_ring.h>
   45 #endif
   46 #include <sys/bus.h>
   47 #include <sys/endian.h>
   48 #include <sys/kernel.h>
   49 #include <sys/kthread.h>
   50 #include <sys/malloc.h>
   51 #include <sys/mbuf.h>
   52 #include <sys/module.h>
   53 #include <sys/rman.h>
   54 #include <sys/socket.h>
   55 #include <sys/sockio.h>
   56 #include <sys/sysctl.h>
   57 #include <sys/taskqueue.h>
   58 #include <sys/eventhandler.h>
   59 #include <sys/pcpu.h>
   60 #include <sys/smp.h>
   61 #include <machine/smp.h>
   62 #include <machine/bus.h>
   63 #include <machine/resource.h>
   64 
   65 #ifdef IGB_IEEE1588
   66 #include <sys/ieee1588.h>
   67 #endif
   68 
   69 #include <net/bpf.h>
   70 #include <net/ethernet.h>
   71 #include <net/if.h>
   72 #include <net/if_arp.h>
   73 #include <net/if_dl.h>
   74 #include <net/if_media.h>
   75 
   76 #include <net/if_types.h>
   77 #include <net/if_vlan_var.h>
   78 
   79 #include <netinet/in_systm.h>
   80 #include <netinet/in.h>
   81 #include <netinet/if_ether.h>
   82 #include <netinet/ip.h>
   83 #include <netinet/ip6.h>
   84 #include <netinet/tcp.h>
   85 #include <netinet/tcp_lro.h>
   86 #include <netinet/udp.h>
   87 
   88 #include <machine/in_cksum.h>
   89 #include <dev/pci/pcivar.h>
   90 #include <dev/pci/pcireg.h>
   91 
   92 #include "e1000_api.h"
   93 #include "e1000_82575.h"
   94 #include "if_igb.h"
   95 
   96 /*********************************************************************
   97  *  Set this to one to display debug statistics
   98  *********************************************************************/
   99 int     igb_display_debug_stats = 0;
  100 
  101 /*********************************************************************
  102  *  Driver version:
  103  *********************************************************************/
  104 char igb_driver_version[] = "version - 1.7.3";
  105 
  106 
  107 /*********************************************************************
  108  *  PCI Device ID Table
  109  *
  110  *  Used by probe to select devices to load on
  111  *  Last field stores an index into e1000_strings
  112  *  Last entry must be all 0s
  113  *
  114  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  115  *********************************************************************/
  116 
  117 static igb_vendor_info_t igb_vendor_info_array[] =
  118 {
  119         { 0x8086, E1000_DEV_ID_82575EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
  120         { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
  121                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  122         { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
  123                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  124         { 0x8086, E1000_DEV_ID_82576,           PCI_ANY_ID, PCI_ANY_ID, 0},
  125         { 0x8086, E1000_DEV_ID_82576_NS,        PCI_ANY_ID, PCI_ANY_ID, 0},
  126         { 0x8086, E1000_DEV_ID_82576_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
  127         { 0x8086, E1000_DEV_ID_82576_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
  128         { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
  129                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  130         { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
  131                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
  132         /* required last entry */
  133         { 0, 0, 0, 0, 0}
  134 };
  135 
  136 /*********************************************************************
  137  *  Table of branding strings for all supported NICs.
  138  *********************************************************************/
  139 
  140 static char *igb_strings[] = {
  141         "Intel(R) PRO/1000 Network Connection"
  142 };
  143 
  144 /*********************************************************************
  145  *  Function prototypes
  146  *********************************************************************/
  147 static int      igb_probe(device_t);
  148 static int      igb_attach(device_t);
  149 static int      igb_detach(device_t);
  150 static int      igb_shutdown(device_t);
  151 static int      igb_suspend(device_t);
  152 static int      igb_resume(device_t);
  153 static void     igb_start(struct ifnet *);
  154 static void     igb_start_locked(struct tx_ring *, struct ifnet *ifp);
  155 #if __FreeBSD_version >= 800000
  156 static int      igb_mq_start(struct ifnet *, struct mbuf *);
  157 static int      igb_mq_start_locked(struct ifnet *,
  158                     struct tx_ring *, struct mbuf *);
  159 static void     igb_qflush(struct ifnet *);
  160 #endif
  161 static int      igb_ioctl(struct ifnet *, u_long, caddr_t);
  162 static void     igb_watchdog(struct adapter *);
  163 static void     igb_init(void *);
  164 static void     igb_init_locked(struct adapter *);
  165 static void     igb_stop(void *);
  166 static void     igb_media_status(struct ifnet *, struct ifmediareq *);
  167 static int      igb_media_change(struct ifnet *);
  168 static void     igb_identify_hardware(struct adapter *);
  169 static int      igb_allocate_pci_resources(struct adapter *);
  170 static int      igb_allocate_msix(struct adapter *);
  171 static int      igb_allocate_legacy(struct adapter *);
  172 static int      igb_setup_msix(struct adapter *);
  173 static void     igb_free_pci_resources(struct adapter *);
  174 static void     igb_local_timer(void *);
  175 static int      igb_hardware_init(struct adapter *);
  176 static void     igb_setup_interface(device_t, struct adapter *);
  177 static int      igb_allocate_queues(struct adapter *);
  178 static void     igb_configure_queues(struct adapter *);
  179 
  180 static int      igb_allocate_transmit_buffers(struct tx_ring *);
  181 static void     igb_setup_transmit_structures(struct adapter *);
  182 static void     igb_setup_transmit_ring(struct tx_ring *);
  183 static void     igb_initialize_transmit_units(struct adapter *);
  184 static void     igb_free_transmit_structures(struct adapter *);
  185 static void     igb_free_transmit_buffers(struct tx_ring *);
  186 
  187 static int      igb_allocate_receive_buffers(struct rx_ring *);
  188 static int      igb_setup_receive_structures(struct adapter *);
  189 static int      igb_setup_receive_ring(struct rx_ring *);
  190 static void     igb_initialize_receive_units(struct adapter *);
  191 static void     igb_free_receive_structures(struct adapter *);
  192 static void     igb_free_receive_buffers(struct rx_ring *);
  193 
  194 static void     igb_enable_intr(struct adapter *);
  195 static void     igb_disable_intr(struct adapter *);
  196 static void     igb_update_stats_counters(struct adapter *);
  197 static bool     igb_txeof(struct tx_ring *);
  198 static bool     igb_rxeof(struct rx_ring *, int);
  199 static void     igb_rx_checksum(u32, struct mbuf *, bool);
  200 static int      igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
  201 static bool     igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
  202 static void     igb_set_promisc(struct adapter *);
  203 static void     igb_disable_promisc(struct adapter *);
  204 static void     igb_set_multi(struct adapter *);
  205 static void     igb_print_hw_stats(struct adapter *);
  206 static void     igb_update_link_status(struct adapter *);
  207 static int      igb_get_buf(struct rx_ring *, int, u8);
  208 
  209 static void     igb_register_vlan(void *, struct ifnet *, u16);
  210 static void     igb_unregister_vlan(void *, struct ifnet *, u16);
  211 static void     igb_setup_vlan_hw_support(struct adapter *);
  212 
  213 static int      igb_xmit(struct tx_ring *, struct mbuf **);
  214 static int      igb_dma_malloc(struct adapter *, bus_size_t,
  215                     struct igb_dma_alloc *, int);
  216 static void     igb_dma_free(struct adapter *, struct igb_dma_alloc *);
  217 static void     igb_print_debug_info(struct adapter *);
  218 static void     igb_print_nvm_info(struct adapter *);
  219 static int      igb_is_valid_ether_addr(u8 *);
  220 static int      igb_sysctl_stats(SYSCTL_HANDLER_ARGS);
  221 static int      igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
  222 /* Management and WOL Support */
  223 static void     igb_init_manageability(struct adapter *);
  224 static void     igb_release_manageability(struct adapter *);
  225 static void     igb_get_hw_control(struct adapter *);
  226 static void     igb_release_hw_control(struct adapter *);
  227 static void     igb_enable_wakeup(device_t);
  228 
  229 static int      igb_irq_fast(void *);
  230 static void     igb_add_rx_process_limit(struct adapter *, const char *,
  231                     const char *, int *, int);
  232 static void     igb_handle_rxtx(void *context, int pending);
  233 static void     igb_handle_tx(void *context, int pending);
  234 static void     igb_handle_rx(void *context, int pending);
  235 
  236 /* These are MSIX only irq handlers */
  237 static void     igb_msix_rx(void *);
  238 static void     igb_msix_tx(void *);
  239 static void     igb_msix_link(void *);
  240 
  241 /* Adaptive Interrupt Moderation */
  242 static void     igb_update_aim(struct rx_ring *);
  243 
  244 /*********************************************************************
  245  *  FreeBSD Device Interface Entry Points
  246  *********************************************************************/
  247 
  248 static device_method_t igb_methods[] = {
  249         /* Device interface */
  250         DEVMETHOD(device_probe, igb_probe),
  251         DEVMETHOD(device_attach, igb_attach),
  252         DEVMETHOD(device_detach, igb_detach),
  253         DEVMETHOD(device_shutdown, igb_shutdown),
  254         DEVMETHOD(device_suspend, igb_suspend),
  255         DEVMETHOD(device_resume, igb_resume),
  256         {0, 0}
  257 };
  258 
  259 static driver_t igb_driver = {
  260         "igb", igb_methods, sizeof(struct adapter),
  261 };
  262 
  263 static devclass_t igb_devclass;
  264 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
  265 MODULE_DEPEND(igb, pci, 1, 1, 1);
  266 MODULE_DEPEND(igb, ether, 1, 1, 1);
  267 
  268 /*********************************************************************
  269  *  Tunable default values.
  270  *********************************************************************/
  271 
  272 /* Descriptor defaults */
  273 static int igb_rxd = IGB_DEFAULT_RXD;
  274 static int igb_txd = IGB_DEFAULT_TXD;
  275 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
  276 TUNABLE_INT("hw.igb.txd", &igb_txd);
  277 
  278 /*
  279 ** These parameters are used in Adaptive
  280 ** Interrupt Moderation. The value is set
  281 ** into EITR and controls the interrupt
  282 ** frequency. A variable static scheme can
  283 ** be created by changing the assigned value
  284 ** of igb_ave_latency to the desired value,
  285 ** and then set igb_enable_aim to FALSE.
  286 ** This will result in all EITR registers
  287 ** getting set to that value statically.
  288 */
  289 static int igb_enable_aim = TRUE;
  290 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
  291 static int igb_low_latency = IGB_LOW_LATENCY;
  292 TUNABLE_INT("hw.igb.low_latency", &igb_low_latency);
  293 static int igb_ave_latency = IGB_AVE_LATENCY;
  294 TUNABLE_INT("hw.igb.ave_latency", &igb_ave_latency);
  295 static int igb_bulk_latency = IGB_BULK_LATENCY;
  296 TUNABLE_INT("hw.igb.bulk_latency", &igb_bulk_latency);
  297                 
  298 /*
  299 ** This will autoconfigure based on the number
  300 ** of CPUs if set to 0. Only a matched pair of
  301 ** TX and RX rings are allowed.
  302 */
  303 static int igb_num_queues = 1;
  304 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
  305 
  306 /* How many packets rxeof tries to clean at a time */
  307 static int igb_rx_process_limit = 100;
  308 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
  309 
  310 /* Flow control setting - default to FULL */
  311 static int igb_fc_setting = e1000_fc_full;
  312 TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
  313 
  314 /*
  315 ** Shadow VFTA table, this is needed because
  316 ** the real filter table gets cleared during
  317 ** a soft reset and the driver needs to be able
  318 ** to repopulate it.
  319 */
  320 static u32 igb_shadow_vfta[IGB_VFTA_SIZE];
  321 
  322 
  323 /*********************************************************************
  324  *  Device identification routine
  325  *
  326  *  igb_probe determines if the driver should be loaded on
  327  *  adapter based on PCI vendor/device id of the adapter.
  328  *
  329  *  return BUS_PROBE_DEFAULT on success, positive on failure
  330  *********************************************************************/
  331 
  332 static int
  333 igb_probe(device_t dev)
  334 {
  335         char            adapter_name[60];
  336         uint16_t        pci_vendor_id = 0;
  337         uint16_t        pci_device_id = 0;
  338         uint16_t        pci_subvendor_id = 0;
  339         uint16_t        pci_subdevice_id = 0;
  340         igb_vendor_info_t *ent;
  341 
  342         INIT_DEBUGOUT("igb_probe: begin");
  343 
  344         pci_vendor_id = pci_get_vendor(dev);
  345         if (pci_vendor_id != IGB_VENDOR_ID)
  346                 return (ENXIO);
  347 
  348         pci_device_id = pci_get_device(dev);
  349         pci_subvendor_id = pci_get_subvendor(dev);
  350         pci_subdevice_id = pci_get_subdevice(dev);
  351 
  352         ent = igb_vendor_info_array;
  353         while (ent->vendor_id != 0) {
  354                 if ((pci_vendor_id == ent->vendor_id) &&
  355                     (pci_device_id == ent->device_id) &&
  356 
  357                     ((pci_subvendor_id == ent->subvendor_id) ||
  358                     (ent->subvendor_id == PCI_ANY_ID)) &&
  359 
  360                     ((pci_subdevice_id == ent->subdevice_id) ||
  361                     (ent->subdevice_id == PCI_ANY_ID))) {
  362                         sprintf(adapter_name, "%s %s",
  363                                 igb_strings[ent->index],
  364                                 igb_driver_version);
  365                         device_set_desc_copy(dev, adapter_name);
  366                         return (BUS_PROBE_DEFAULT);
  367                 }
  368                 ent++;
  369         }
  370 
  371         return (ENXIO);
  372 }
  373 
  374 /*********************************************************************
  375  *  Device initialization routine
  376  *
  377  *  The attach entry point is called when the driver is being loaded.
  378  *  This routine identifies the type of hardware, allocates all resources
  379  *  and initializes the hardware.
  380  *
  381  *  return 0 on success, positive on failure
  382  *********************************************************************/
  383 
  384 static int
  385 igb_attach(device_t dev)
  386 {
  387         struct adapter  *adapter;
  388         int             error = 0;
  389         u16             eeprom_data;
  390 
  391         INIT_DEBUGOUT("igb_attach: begin");
  392 
  393         adapter = device_get_softc(dev);
  394         adapter->dev = adapter->osdep.dev = dev;
  395         IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
  396 
  397         /* SYSCTL stuff */
  398         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  399             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  400             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
  401             igb_sysctl_debug_info, "I", "Debug Information");
  402 
  403         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
  404             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  405             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
  406             igb_sysctl_stats, "I", "Statistics");
  407 
  408         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
  409             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
  410             OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW,
  411             &igb_fc_setting, 0, "Flow Control");
  412 
  413         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
  414             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  415             OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
  416             &igb_enable_aim, 1, "Interrupt Moderation");
  417 
  418         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
  419             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  420             OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW,
  421             &igb_low_latency, 1, "Low Latency");
  422                 
  423         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
  424             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  425             OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW,
  426             &igb_ave_latency, 1, "Average Latency");
  427 
  428         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
  429             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
  430             OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW,
  431             &igb_bulk_latency, 1, "Bulk Latency");
  432  
  433         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
  434 
  435         /* Determine hardware and mac info */
  436         igb_identify_hardware(adapter);
  437 
  438         /* Setup PCI resources */
  439         if (igb_allocate_pci_resources(adapter)) {
  440                 device_printf(dev, "Allocation of PCI resources failed\n");
  441                 error = ENXIO;
  442                 goto err_pci;
  443         }
  444 
  445         /* Do Shared Code initialization */
  446         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
  447                 device_printf(dev, "Setup of Shared code failed\n");
  448                 error = ENXIO;
  449                 goto err_pci;
  450         }
  451 
  452         e1000_get_bus_info(&adapter->hw);
  453 
  454         /* Sysctls for limiting the amount of work done in the taskqueue */
  455         igb_add_rx_process_limit(adapter, "rx_processing_limit",
  456             "max number of rx packets to process", &adapter->rx_process_limit,
  457             igb_rx_process_limit);
  458 
  459         /*
  460          * Validate number of transmit and receive descriptors. It
  461          * must not exceed hardware maximum, and must be multiple
  462          * of E1000_DBA_ALIGN.
  463          */
  464         if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
  465             (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
  466                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
  467                     IGB_DEFAULT_TXD, igb_txd);
  468                 adapter->num_tx_desc = IGB_DEFAULT_TXD;
  469         } else
  470                 adapter->num_tx_desc = igb_txd;
  471         if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
  472             (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
  473                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
  474                     IGB_DEFAULT_RXD, igb_rxd);
  475                 adapter->num_rx_desc = IGB_DEFAULT_RXD;
  476         } else
  477                 adapter->num_rx_desc = igb_rxd;
  478 
  479         adapter->hw.mac.autoneg = DO_AUTO_NEG;
  480         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
  481         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
  482 
  483         /* Copper options */
  484         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
  485                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
  486                 adapter->hw.phy.disable_polarity_correction = FALSE;
  487                 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
  488         }
  489 
  490         /*
  491          * Set the frame limits assuming
  492          * standard ethernet sized frames.
  493          */
  494         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
  495         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
  496 
  497         /*
  498         ** Allocate and Setup Queues
  499         */
  500         if (igb_allocate_queues(adapter)) {
  501                 error = ENOMEM;
  502                 goto err_pci;
  503         }
  504 
  505         /*
  506         ** Start from a known state, this is
  507         ** important in reading the nvm and
  508         ** mac from that.
  509         */
  510         e1000_reset_hw(&adapter->hw);
  511 
  512         /* Make sure we have a good EEPROM before we read from it */
  513         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
  514                 /*
  515                 ** Some PCI-E parts fail the first check due to
  516                 ** the link being in sleep state, call it again,
  517                 ** if it fails a second time its a real issue.
  518                 */
  519                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
  520                         device_printf(dev,
  521                             "The EEPROM Checksum Is Not Valid\n");
  522                         error = EIO;
  523                         goto err_late;
  524                 }
  525         }
  526 
  527         /*
  528         ** Copy the permanent MAC address out of the EEPROM
  529         */
  530         if (e1000_read_mac_addr(&adapter->hw) < 0) {
  531                 device_printf(dev, "EEPROM read error while reading MAC"
  532                     " address\n");
  533                 error = EIO;
  534                 goto err_late;
  535         }
  536         /* Check its sanity */
  537         if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
  538                 device_printf(dev, "Invalid MAC address\n");
  539                 error = EIO;
  540                 goto err_late;
  541         }
  542 
  543         /* Now Initialize the hardware */
  544         if (igb_hardware_init(adapter)) {
  545                 device_printf(dev, "Unable to initialize the hardware\n");
  546                 error = EIO;
  547                 goto err_late;
  548         }
  549 
  550         /* 
  551         ** Configure Interrupts
  552         */
  553         if (adapter->msix > 1) /* MSIX */
  554                 error = igb_allocate_msix(adapter);
  555         else /* MSI or Legacy */
  556                 error = igb_allocate_legacy(adapter);
  557         if (error)
  558                 goto err_late;
  559 
  560         /* Setup OS specific network interface */
  561         igb_setup_interface(dev, adapter);
  562 
  563 #ifdef IGB_IEEE1588
  564         /*
  565         ** Setup the timer: IEEE 1588 support
  566         */
  567         adapter->cycles.read = igb_read_clock;
  568         adapter->cycles.mask = (u64)-1;
  569         adapter->cycles.mult = 1;
  570         adapter->cycles.shift = IGB_TSYNC_SHIFT;
  571         E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) |
  572             IGB_TSYNC_CYCLE_TIME * IGB_TSYNC_SHIFT);
  573         E1000_WRITE_REG(&adapter->hw, E1000_SYSTIML, 0x00000000);
  574         E1000_WRITE_REG(&adapter->hw, E1000_SYSTIMH, 0xFF800000);
  575 
  576         // JFV - this is not complete yet
  577 #endif
  578 
  579         /* Initialize statistics */
  580         igb_update_stats_counters(adapter);
  581 
  582         adapter->hw.mac.get_link_status = 1;
  583         igb_update_link_status(adapter);
  584 
  585         /* Indicate SOL/IDER usage */
  586         if (e1000_check_reset_block(&adapter->hw))
  587                 device_printf(dev,
  588                     "PHY reset is blocked due to SOL/IDER session.\n");
  589 
  590         /* Determine if we have to control management hardware */
  591         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
  592 
  593         /*
  594          * Setup Wake-on-Lan
  595          */
  596         /* APME bit in EEPROM is mapped to WUC.APME */
  597         eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
  598         if (eeprom_data)
  599                 adapter->wol = E1000_WUFC_MAG;
  600 
  601         /* Register for VLAN events */
  602         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
  603              igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  604         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
  605              igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
  606 
  607         /* Tell the stack that the interface is not active */
  608         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
  609 
  610         INIT_DEBUGOUT("igb_attach: end");
  611 
  612         return (0);
  613 
  614 err_late:
  615         igb_free_transmit_structures(adapter);
  616         igb_free_receive_structures(adapter);
  617         igb_release_hw_control(adapter);
  618 err_pci:
  619         igb_free_pci_resources(adapter);
  620         IGB_CORE_LOCK_DESTROY(adapter);
  621 
  622         return (error);
  623 }
  624 
  625 /*********************************************************************
  626  *  Device removal routine
  627  *
  628  *  The detach entry point is called when the driver is being removed.
  629  *  This routine stops the adapter and deallocates all the resources
  630  *  that were allocated for driver operation.
  631  *
  632  *  return 0 on success, positive on failure
  633  *********************************************************************/
  634 
  635 static int
  636 igb_detach(device_t dev)
  637 {
  638         struct adapter  *adapter = device_get_softc(dev);
  639         struct ifnet    *ifp = adapter->ifp;
  640 
  641         INIT_DEBUGOUT("igb_detach: begin");
  642 
  643         /* Make sure VLANS are not using driver */
  644         if (adapter->ifp->if_vlantrunk != NULL) {
  645                 device_printf(dev,"Vlan in use, detach first\n");
  646                 return (EBUSY);
  647         }
  648 
  649         IGB_CORE_LOCK(adapter);
  650         adapter->in_detach = 1;
  651         igb_stop(adapter);
  652         IGB_CORE_UNLOCK(adapter);
  653 
  654         e1000_phy_hw_reset(&adapter->hw);
  655 
  656         /* Give control back to firmware */
  657         igb_release_manageability(adapter);
  658         igb_release_hw_control(adapter);
  659 
  660         if (adapter->wol) {
  661                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
  662                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
  663                 igb_enable_wakeup(dev);
  664         }
  665 
  666         /* Unregister VLAN events */
  667         if (adapter->vlan_attach != NULL)
  668                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
  669         if (adapter->vlan_detach != NULL)
  670                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
  671 
  672         ether_ifdetach(adapter->ifp);
  673 
  674         callout_drain(&adapter->timer);
  675 
  676         igb_free_pci_resources(adapter);
  677         bus_generic_detach(dev);
  678         if_free(ifp);
  679 
  680         igb_free_transmit_structures(adapter);
  681         igb_free_receive_structures(adapter);
  682 
  683         IGB_CORE_LOCK_DESTROY(adapter);
  684 
  685         return (0);
  686 }
  687 
  688 /*********************************************************************
  689  *
  690  *  Shutdown entry point
  691  *
  692  **********************************************************************/
  693 
  694 static int
  695 igb_shutdown(device_t dev)
  696 {
  697         return igb_suspend(dev);
  698 }
  699 
  700 /*
  701  * Suspend/resume device methods.
  702  */
  703 static int
  704 igb_suspend(device_t dev)
  705 {
  706         struct adapter *adapter = device_get_softc(dev);
  707 
  708         IGB_CORE_LOCK(adapter);
  709 
  710         igb_stop(adapter);
  711 
  712         igb_release_manageability(adapter);
  713         igb_release_hw_control(adapter);
  714 
  715         if (adapter->wol) {
  716                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
  717                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
  718                 igb_enable_wakeup(dev);
  719         }
  720 
  721         IGB_CORE_UNLOCK(adapter);
  722 
  723         return bus_generic_suspend(dev);
  724 }
  725 
  726 static int
  727 igb_resume(device_t dev)
  728 {
  729         struct adapter *adapter = device_get_softc(dev);
  730         struct ifnet *ifp = adapter->ifp;
  731 
  732         IGB_CORE_LOCK(adapter);
  733         igb_init_locked(adapter);
  734         igb_init_manageability(adapter);
  735 
  736         if ((ifp->if_flags & IFF_UP) &&
  737             (ifp->if_drv_flags & IFF_DRV_RUNNING))
  738                 igb_start(ifp);
  739 
  740         IGB_CORE_UNLOCK(adapter);
  741 
  742         return bus_generic_resume(dev);
  743 }
  744 
  745 
  746 /*********************************************************************
  747  *  Transmit entry point
  748  *
  749  *  igb_start is called by the stack to initiate a transmit.
  750  *  The driver will remain in this routine as long as there are
  751  *  packets to transmit and transmit resources are available.
  752  *  In case resources are not available stack is notified and
  753  *  the packet is requeued.
  754  **********************************************************************/
  755 
  756 static void
  757 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
  758 {
  759         struct adapter  *adapter = ifp->if_softc;
  760         struct mbuf     *m_head;
  761 
  762         IGB_TX_LOCK_ASSERT(txr);
  763 
  764         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
  765             IFF_DRV_RUNNING)
  766                 return;
  767         if (!adapter->link_active)
  768                 return;
  769 
  770         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
  771 
  772                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
  773                 if (m_head == NULL)
  774                         break;
  775                 /*
  776                  *  Encapsulation can modify our pointer, and or make it
  777                  *  NULL on failure.  In that event, we can't requeue.
  778                  */
  779                 if (igb_xmit(txr, &m_head)) {
  780                         if (m_head == NULL)
  781                                 break;
  782                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
  783                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
  784                         break;
  785                 }
  786 
  787                 /* Send a copy of the frame to the BPF listener */
  788                 ETHER_BPF_MTAP(ifp, m_head);
  789 
  790                 /* Set timeout in case hardware has problems transmitting. */
  791                 txr->watchdog_timer = IGB_TX_TIMEOUT;
  792         }
  793 }
  794  
  795 /*
  796  * Legacy TX driver routine, called from the
  797  * stack, always uses tx[0], and spins for it.
  798  * Should not be used with multiqueue tx
  799  */
  800 static void
  801 igb_start(struct ifnet *ifp)
  802 {
  803         struct adapter  *adapter = ifp->if_softc;
  804         struct tx_ring  *txr = adapter->tx_rings;
  805 
  806         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  807                 IGB_TX_LOCK(txr);
  808                 igb_start_locked(txr, ifp);
  809                 IGB_TX_UNLOCK(txr);
  810         }
  811         return;
  812 }
  813 
  814 #if __FreeBSD_version >= 800000
  815 /*
  816 ** Multiqueue Transmit driver
  817 **
  818 */
  819 static int
  820 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
  821 {
  822         struct adapter  *adapter = ifp->if_softc;
  823         struct tx_ring  *txr;
  824         int             i = 0, err = 0;
  825 
  826         /* Which queue to use */
  827         if ((m->m_flags & M_FLOWID) != 0)
  828                 i = m->m_pkthdr.flowid % adapter->num_queues;
  829         txr = &adapter->tx_rings[i];
  830 
  831         if (IGB_TX_TRYLOCK(txr)) {
  832                 err = igb_mq_start_locked(ifp, txr, m);
  833                 IGB_TX_UNLOCK(txr);
  834         } else
  835                 err = drbr_enqueue(ifp, txr->br, m);
  836 
  837         return (err);
  838 }
  839 
  840 static int
  841 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
  842 {
  843         struct adapter  *adapter = txr->adapter;
  844         struct mbuf     *next;
  845         int             err = 0;
  846 
  847         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
  848                 err = drbr_enqueue(ifp, txr->br, m);
  849                 return (err);
  850         }
  851 
  852         if (m == NULL) /* Called by tasklet */
  853                 goto process;
  854 
  855         /* If nothing queued go right to xmit */
  856         if (drbr_empty(ifp, txr->br)) {
  857                 if ((err = igb_xmit(txr, &m)) != 0) {
  858                         if (m != NULL)
  859                                 err = drbr_enqueue(ifp, txr->br, m);
  860                         return (err);
  861                 } else {
  862                         /* Success, update stats */
  863                         drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
  864                         /* Send a copy of the frame to the BPF listener */
  865                         ETHER_BPF_MTAP(ifp, m);
  866                         /* Set the watchdog */
  867                         txr->watchdog_timer = IGB_TX_TIMEOUT;
  868                 }
  869 
  870         } else if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
  871                 return (err);
  872 
  873 process:
  874         if (drbr_empty(ifp, txr->br))
  875                 return (err);
  876 
  877         /* Process the queue */
  878         while (TRUE) {
  879                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
  880                         break;
  881                 next = drbr_dequeue(ifp, txr->br);
  882                 if (next == NULL)
  883                         break;
  884                 if ((err = igb_xmit(txr, &next)) != 0) {
  885                         if (next != NULL)
  886                                 err = drbr_enqueue(ifp, txr->br, next);
  887                         break;
  888                 }
  889                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
  890                 ETHER_BPF_MTAP(ifp, next);
  891                 /* Set the watchdog */
  892                 txr->watchdog_timer = IGB_TX_TIMEOUT;
  893         }
  894                 
  895         if (txr->tx_avail <= IGB_TX_OP_THRESHOLD)
  896                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
  897 
  898         return (err);
  899 }
  900 
  901 /*
  902 ** Flush all ring buffers
  903 */
  904 static void
  905 igb_qflush(struct ifnet *ifp)
  906 {
  907         struct adapter  *adapter = ifp->if_softc;
  908         struct tx_ring  *txr = adapter->tx_rings;
  909         struct mbuf     *m;
  910 
  911         for (int i = 0; i < adapter->num_queues; i++, txr++) {
  912                 IGB_TX_LOCK(txr);
  913                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
  914                         m_freem(m);
  915                 IGB_TX_UNLOCK(txr);
  916         }
  917         if_qflush(ifp);
  918 }
  919 #endif /* __FreeBSD_version >= 800000 */
  920 
  921 /*********************************************************************
  922  *  Ioctl entry point
  923  *
  924  *  igb_ioctl is called when the user wants to configure the
  925  *  interface.
  926  *
  927  *  return 0 on success, positive on failure
  928  **********************************************************************/
  929 
  930 static int
  931 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
  932 {
  933         struct adapter  *adapter = ifp->if_softc;
  934         struct ifreq *ifr = (struct ifreq *)data;
  935 #ifdef INET
  936         struct ifaddr *ifa = (struct ifaddr *)data;
  937 #endif
  938         int error = 0;
  939 
  940         if (adapter->in_detach)
  941                 return (error);
  942 
  943         switch (command) {
  944         case SIOCSIFADDR:
  945 #ifdef INET
  946                 if (ifa->ifa_addr->sa_family == AF_INET) {
  947                         /*
  948                          * XXX
  949                          * Since resetting hardware takes a very long time
  950                          * and results in link renegotiation we only
  951                          * initialize the hardware only when it is absolutely
  952                          * required.
  953                          */
  954                         ifp->if_flags |= IFF_UP;
  955                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
  956                                 IGB_CORE_LOCK(adapter);
  957                                 igb_init_locked(adapter);
  958                                 IGB_CORE_UNLOCK(adapter);
  959                         }
  960                         if (!(ifp->if_flags & IFF_NOARP))
  961                                 arp_ifinit(ifp, ifa);
  962                 } else
  963 #endif
  964                         error = ether_ioctl(ifp, command, data);
  965                 break;
  966         case SIOCSIFMTU:
  967             {
  968                 int max_frame_size;
  969 
  970                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
  971 
  972                 IGB_CORE_LOCK(adapter);
  973                 max_frame_size = 9234;
  974                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
  975                     ETHER_CRC_LEN) {
  976                         IGB_CORE_UNLOCK(adapter);
  977                         error = EINVAL;
  978                         break;
  979                 }
  980 
  981                 ifp->if_mtu = ifr->ifr_mtu;
  982                 adapter->max_frame_size =
  983                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
  984                 igb_init_locked(adapter);
  985                 IGB_CORE_UNLOCK(adapter);
  986                 break;
  987             }
  988         case SIOCSIFFLAGS:
  989                 IOCTL_DEBUGOUT("ioctl rcv'd:\
  990                     SIOCSIFFLAGS (Set Interface Flags)");
  991                 IGB_CORE_LOCK(adapter);
  992                 if (ifp->if_flags & IFF_UP) {
  993                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
  994                                 if ((ifp->if_flags ^ adapter->if_flags) &
  995                                     (IFF_PROMISC | IFF_ALLMULTI)) {
  996                                         igb_disable_promisc(adapter);
  997                                         igb_set_promisc(adapter);
  998                                 }
  999                         } else
 1000                                 igb_init_locked(adapter);
 1001                 } else
 1002                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1003                                 igb_stop(adapter);
 1004                 adapter->if_flags = ifp->if_flags;
 1005                 IGB_CORE_UNLOCK(adapter);
 1006                 break;
 1007         case SIOCADDMULTI:
 1008         case SIOCDELMULTI:
 1009                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 1010                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1011                         IGB_CORE_LOCK(adapter);
 1012                         igb_disable_intr(adapter);
 1013                         igb_set_multi(adapter);
 1014                                 igb_enable_intr(adapter);
 1015                         IGB_CORE_UNLOCK(adapter);
 1016                 }
 1017                 break;
 1018         case SIOCSIFMEDIA:
 1019                 /* Check SOL/IDER usage */
 1020                 IGB_CORE_LOCK(adapter);
 1021                 if (e1000_check_reset_block(&adapter->hw)) {
 1022                         IGB_CORE_UNLOCK(adapter);
 1023                         device_printf(adapter->dev, "Media change is"
 1024                             " blocked due to SOL/IDER session.\n");
 1025                         break;
 1026                 }
 1027                 IGB_CORE_UNLOCK(adapter);
 1028         case SIOCGIFMEDIA:
 1029                 IOCTL_DEBUGOUT("ioctl rcv'd: \
 1030                     SIOCxIFMEDIA (Get/Set Interface Media)");
 1031                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 1032                 break;
 1033         case SIOCSIFCAP:
 1034             {
 1035                 int mask, reinit;
 1036 
 1037                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 1038                 reinit = 0;
 1039                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 1040                 if (mask & IFCAP_HWCSUM) {
 1041                         ifp->if_capenable ^= IFCAP_HWCSUM;
 1042                         reinit = 1;
 1043                 }
 1044                 if (mask & IFCAP_TSO4) {
 1045                         ifp->if_capenable ^= IFCAP_TSO4;
 1046                         reinit = 1;
 1047                 }
 1048                 if (mask & IFCAP_VLAN_HWTAGGING) {
 1049                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 1050                         reinit = 1;
 1051                 }
 1052                 if (mask & IFCAP_LRO) {
 1053                         ifp->if_capenable ^= IFCAP_LRO;
 1054                         reinit = 1;
 1055                 }
 1056                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
 1057                         igb_init(adapter);
 1058                 VLAN_CAPABILITIES(ifp);
 1059                 break;
 1060             }
 1061 
 1062 #ifdef IGB_IEEE1588
 1063         /*
 1064         ** IOCTL support for Precision Time (IEEE 1588) Support
 1065         */
 1066         case SIOCSHWTSTAMP:
 1067                 error = igb_hwtstamp_ioctl(adapter, ifp);
 1068                 break;
 1069 #endif
 1070 
 1071         default:
 1072                 error = ether_ioctl(ifp, command, data);
 1073                 break;
 1074         }
 1075 
 1076         return (error);
 1077 }
 1078 
 1079 /*********************************************************************
 1080  *  Watchdog timer:
 1081  *
 1082  *  This routine is called from the local timer every second.
 1083  *  As long as transmit descriptors are being cleaned the value
 1084  *  is non-zero and we do nothing. Reaching 0 indicates a tx hang
 1085  *  and we then reset the device.
 1086  *
 1087  **********************************************************************/
 1088 
 1089 static void
 1090 igb_watchdog(struct adapter *adapter)
 1091 {
 1092         struct tx_ring  *txr = adapter->tx_rings;
 1093         bool            tx_hang = FALSE;
 1094 
 1095         IGB_CORE_LOCK_ASSERT(adapter);
 1096 
 1097         /*
 1098         ** The timer is set to 5 every time start() queues a packet.
 1099         ** Then txeof keeps resetting it as long as it cleans at
 1100         ** least one descriptor.
 1101         ** Finally, anytime all descriptors are clean the timer is
 1102         ** set to 0.
 1103         **
 1104         ** With TX Multiqueue we need to check every queue's timer,
 1105         ** if any time out we do the reset.
 1106         */
 1107         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1108                 IGB_TX_LOCK(txr);
 1109                 if (txr->watchdog_timer == 0 ||
 1110                     (--txr->watchdog_timer)) {
 1111                         IGB_TX_UNLOCK(txr);
 1112                         continue;
 1113                 } else {
 1114                         tx_hang = TRUE;
 1115                         IGB_TX_UNLOCK(txr);
 1116                         break;
 1117                 }
 1118         }
 1119         if (tx_hang == FALSE)
 1120                 return;
 1121 
 1122         /* If we are in this routine because of pause frames, then
 1123          * don't reset the hardware.
 1124          */
 1125         if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
 1126             E1000_STATUS_TXOFF) {
 1127                 txr = adapter->tx_rings; /* reset pointer */
 1128                 for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1129                         IGB_TX_LOCK(txr);
 1130                         txr->watchdog_timer = IGB_TX_TIMEOUT;
 1131                         IGB_TX_UNLOCK(txr);
 1132                 }
 1133                 return;
 1134         }
 1135 
 1136         if (e1000_check_for_link(&adapter->hw) == 0)
 1137                 device_printf(adapter->dev, "watchdog timeout -- resetting\n");
 1138 
 1139         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1140                 device_printf(adapter->dev, "Queue(%d) tdh = %d, tdt = %d\n",
 1141                     i, E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
 1142                     E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
 1143                 device_printf(adapter->dev, "Queue(%d) desc avail = %d,"
 1144                     " Next Desc to Clean = %d\n", i, txr->tx_avail,
 1145                     txr->next_to_clean);
 1146         }
 1147 
 1148         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 1149         adapter->watchdog_events++;
 1150 
 1151         igb_init_locked(adapter);
 1152 }
 1153 
 1154 /*********************************************************************
 1155  *  Init entry point
 1156  *
 1157  *  This routine is used in two ways. It is used by the stack as
 1158  *  init entry point in network interface structure. It is also used
 1159  *  by the driver as a hw/sw initialization routine to get to a
 1160  *  consistent state.
 1161  *
 1162  *  return 0 on success, positive on failure
 1163  **********************************************************************/
 1164 
 1165 static void
 1166 igb_init_locked(struct adapter *adapter)
 1167 {
 1168         struct rx_ring *rxr = adapter->rx_rings;
 1169         struct tx_ring *txr = adapter->tx_rings;
 1170         struct ifnet    *ifp = adapter->ifp;
 1171         device_t        dev = adapter->dev;
 1172         u32             pba = 0;
 1173 
 1174         INIT_DEBUGOUT("igb_init: begin");
 1175 
 1176         IGB_CORE_LOCK_ASSERT(adapter);
 1177 
 1178         igb_stop(adapter);
 1179 
 1180         /*
 1181          * Packet Buffer Allocation (PBA)
 1182          * Writing PBA sets the receive portion of the buffer
 1183          * the remainder is used for the transmit buffer.
 1184          */
 1185         if (adapter->hw.mac.type == e1000_82575) {
 1186                 INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
 1187                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 1188                 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
 1189         }
 1190         
 1191         /* Get the latest mac address, User can use a LAA */
 1192         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
 1193               ETHER_ADDR_LEN);
 1194 
 1195         /* Put the address into the Receive Address Array */
 1196         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 1197 
 1198         /* Initialize the hardware */
 1199         if (igb_hardware_init(adapter)) {
 1200                 device_printf(dev, "Unable to initialize the hardware\n");
 1201                 return;
 1202         }
 1203         igb_update_link_status(adapter);
 1204 
 1205         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 1206 
 1207         /* Set hardware offload abilities */
 1208         ifp->if_hwassist = 0;
 1209         if (ifp->if_capenable & IFCAP_TXCSUM) {
 1210                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
 1211 #if __FreeBSD_version >= 800000
 1212                 if (adapter->hw.mac.type == e1000_82576)
 1213                         ifp->if_hwassist |= CSUM_SCTP;
 1214 #endif
 1215         }
 1216 
 1217         if (ifp->if_capenable & IFCAP_TSO4)
 1218                 ifp->if_hwassist |= CSUM_TSO;
 1219 
 1220         /* Configure for OS presence */
 1221         igb_init_manageability(adapter);
 1222 
 1223         /* Prepare transmit descriptors and buffers */
 1224         igb_setup_transmit_structures(adapter);
 1225         igb_initialize_transmit_units(adapter);
 1226 
 1227         /* Setup Multicast table */
 1228         igb_set_multi(adapter);
 1229 
 1230         /*
 1231         ** Figure out the desired mbuf pool
 1232         ** for doing jumbo/packetsplit
 1233         */
 1234         if (ifp->if_mtu > ETHERMTU)
 1235                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
 1236         else
 1237                 adapter->rx_mbuf_sz = MCLBYTES;
 1238 
 1239         /* Prepare receive descriptors and buffers */
 1240         if (igb_setup_receive_structures(adapter)) {
 1241                 device_printf(dev, "Could not setup receive structures\n");
 1242                 igb_stop(adapter);
 1243                 return;
 1244         }
 1245         igb_initialize_receive_units(adapter);
 1246 
 1247         /* Don't lose promiscuous settings */
 1248         igb_set_promisc(adapter);
 1249 
 1250         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 1251         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 1252 
 1253         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 1254         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
 1255 
 1256         if (adapter->msix > 1) /* Set up queue routing */
 1257                 igb_configure_queues(adapter);
 1258 
 1259         /* Set up VLAN tag offload and filter */
 1260         igb_setup_vlan_hw_support(adapter);
 1261 
 1262         /* Set default RX interrupt moderation */
 1263         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 1264                 E1000_WRITE_REG(&adapter->hw,
 1265                     E1000_EITR(rxr->msix), igb_ave_latency);
 1266                 rxr->eitr_setting = igb_ave_latency;
 1267         }
 1268 
 1269         /* Set TX interrupt rate & reset TX watchdog */
 1270         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 1271                 E1000_WRITE_REG(&adapter->hw, 
 1272                     E1000_EITR(txr->msix), igb_ave_latency);
 1273                 txr->watchdog_timer = FALSE;
 1274         }
 1275 
 1276         {
 1277                 /* this clears any pending interrupts */
 1278                 E1000_READ_REG(&adapter->hw, E1000_ICR);
 1279                 igb_enable_intr(adapter);
 1280                 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
 1281         }
 1282 
 1283         /* Don't reset the phy next time init gets called */
 1284         adapter->hw.phy.reset_disable = TRUE;
 1285 }
 1286 
 1287 static void
 1288 igb_init(void *arg)
 1289 {
 1290         struct adapter *adapter = arg;
 1291 
 1292         IGB_CORE_LOCK(adapter);
 1293         igb_init_locked(adapter);
 1294         IGB_CORE_UNLOCK(adapter);
 1295 }
 1296 
 1297 
 1298 static void
 1299 igb_handle_rxtx(void *context, int pending)
 1300 {
 1301         struct adapter  *adapter = context;
 1302         struct tx_ring  *txr = adapter->tx_rings;
 1303         struct rx_ring  *rxr = adapter->rx_rings;
 1304         struct ifnet    *ifp;
 1305 
 1306         ifp = adapter->ifp;
 1307 
 1308         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1309                 if (igb_rxeof(rxr, adapter->rx_process_limit))
 1310                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
 1311                 IGB_TX_LOCK(txr);
 1312                 igb_txeof(txr);
 1313 
 1314 #if __FreeBSD_version >= 800000
 1315                 if (!drbr_empty(ifp, txr->br))
 1316                         igb_mq_start_locked(ifp, txr, NULL);
 1317 #else
 1318                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1319                         igb_start_locked(txr, ifp);
 1320 #endif
 1321                 IGB_TX_UNLOCK(txr);
 1322         }
 1323 
 1324         igb_enable_intr(adapter);
 1325 }
 1326 
 1327 static void
 1328 igb_handle_rx(void *context, int pending)
 1329 {
 1330         struct rx_ring  *rxr = context;
 1331         struct adapter  *adapter = rxr->adapter;
 1332         struct ifnet    *ifp = adapter->ifp;
 1333             
 1334         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 1335                 if (igb_rxeof(rxr, adapter->rx_process_limit) != 0)
 1336                         /* More to clean, schedule another task */
 1337                         taskqueue_enqueue(adapter->tq, &rxr->rx_task);
 1338                 
 1339 }
 1340 
 1341 static void
 1342 igb_handle_tx(void *context, int pending)
 1343 {
 1344         struct tx_ring  *txr = context;
 1345         struct adapter  *adapter = txr->adapter;
 1346         struct ifnet    *ifp = adapter->ifp;
 1347 
 1348         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 1349                 IGB_TX_LOCK(txr);
 1350                 igb_txeof(txr);
 1351 #if __FreeBSD_version >= 800000
 1352                 if (!drbr_empty(ifp, txr->br))
 1353                         igb_mq_start_locked(ifp, txr, NULL);
 1354 #else
 1355                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 1356                         igb_start_locked(txr, ifp);
 1357 #endif
 1358                 IGB_TX_UNLOCK(txr);
 1359         }
 1360 }
 1361 
 1362 
 1363 /*********************************************************************
 1364  *
 1365  *  MSI/Legacy Deferred
 1366  *  Interrupt Service routine  
 1367  *
 1368  *********************************************************************/
 1369 static int
 1370 igb_irq_fast(void *arg)
 1371 {
 1372         struct adapter  *adapter = arg;
 1373         uint32_t        reg_icr;
 1374 
 1375 
 1376         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1377 
 1378         /* Hot eject?  */
 1379         if (reg_icr == 0xffffffff)
 1380                 return FILTER_STRAY;
 1381 
 1382         /* Definitely not our interrupt.  */
 1383         if (reg_icr == 0x0)
 1384                 return FILTER_STRAY;
 1385 
 1386         if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
 1387                 return FILTER_STRAY;
 1388 
 1389         /*
 1390          * Mask interrupts until the taskqueue is finished running.  This is
 1391          * cheap, just assume that it is needed.  This also works around the
 1392          * MSI message reordering errata on certain systems.
 1393          */
 1394         igb_disable_intr(adapter);
 1395         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
 1396 
 1397         /* Link status change */
 1398         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 1399                 adapter->hw.mac.get_link_status = 1;
 1400                 igb_update_link_status(adapter);
 1401         }
 1402 
 1403         if (reg_icr & E1000_ICR_RXO)
 1404                 adapter->rx_overruns++;
 1405         return FILTER_HANDLED;
 1406 }
 1407 
 1408 
 1409 /*********************************************************************
 1410  *
 1411  *  MSIX TX Interrupt Service routine
 1412  *
 1413  **********************************************************************/
 1414 static void
 1415 igb_msix_tx(void *arg)
 1416 {
 1417         struct tx_ring *txr = arg;
 1418         struct adapter *adapter = txr->adapter;
 1419         u32             loop = IGB_MAX_LOOP;
 1420         bool            more;
 1421 
 1422         ++txr->tx_irq;
 1423         IGB_TX_LOCK(txr);
 1424 
 1425         do {
 1426                 more = igb_txeof(txr);
 1427         } while (loop-- && more);
 1428 
 1429         IGB_TX_UNLOCK(txr);
 1430 
 1431         /* Schedule a clean task */
 1432         taskqueue_enqueue(adapter->tq, &txr->tx_task);
 1433 
 1434         /* Reenable this interrupt */
 1435         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, txr->eims);
 1436         return;
 1437 }
 1438 
 1439 /*********************************************************************
 1440  *
 1441  *  MSIX RX Interrupt Service routine
 1442  *
 1443  **********************************************************************/
 1444 
 1445 static void
 1446 igb_msix_rx(void *arg)
 1447 {
 1448         struct rx_ring *rxr = arg;
 1449         struct adapter *adapter = rxr->adapter;
 1450         u32             loop = IGB_MAX_LOOP;
 1451         bool            more;
 1452 
 1453         ++rxr->rx_irq;
 1454         do {
 1455                 more = igb_rxeof(rxr, adapter->rx_process_limit);
 1456         } while (loop-- && more);
 1457 
 1458         /* Update interrupt rate */
 1459         if (igb_enable_aim == TRUE)
 1460                 igb_update_aim(rxr);
 1461 
 1462         /* Schedule another clean */
 1463         taskqueue_enqueue(adapter->tq, &rxr->rx_task);
 1464 
 1465         /* Reenable this interrupt */
 1466         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxr->eims);
 1467         return;
 1468 }
 1469 
 1470 
 1471 /*********************************************************************
 1472  *
 1473  *  MSIX Link Interrupt Service routine
 1474  *
 1475  **********************************************************************/
 1476 
 1477 static void
 1478 igb_msix_link(void *arg)
 1479 {
 1480         struct adapter  *adapter = arg;
 1481         u32             icr;
 1482 
 1483         ++adapter->link_irq;
 1484         icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
 1485         if (!(icr & E1000_ICR_LSC))
 1486                 goto spurious;
 1487         adapter->hw.mac.get_link_status = 1;
 1488         igb_update_link_status(adapter);
 1489 
 1490 spurious:
 1491         /* Rearm */
 1492         E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
 1493         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
 1494         return;
 1495 }
 1496 
 1497 
 1498 /*
 1499 ** Routine to adjust the RX EITR value based on traffic,
 1500 ** its a simple three state model, but seems to help.
 1501 **
 1502 ** Note that the three EITR values are tuneable using
 1503 ** sysctl in real time. The feature can be effectively
 1504 ** nullified by setting them equal.
 1505 */
 1506 #define BULK_THRESHOLD  10000
 1507 #define AVE_THRESHOLD   1600 
 1508 
 1509 static void
 1510 igb_update_aim(struct rx_ring *rxr)
 1511 {
 1512         struct adapter  *adapter = rxr->adapter;
 1513         u32             olditr, newitr;
 1514 
 1515         /* Update interrupt moderation based on traffic */
 1516         olditr = rxr->eitr_setting;
 1517         newitr = olditr;
 1518 
 1519         /* Idle, don't change setting */
 1520         if (rxr->bytes == 0)
 1521                 return;
 1522 
 1523         if (olditr == igb_low_latency) {
 1524                 if (rxr->bytes > AVE_THRESHOLD)
 1525                         newitr = igb_ave_latency;
 1526         } else if (olditr == igb_ave_latency) {
 1527                 if (rxr->bytes < AVE_THRESHOLD) 
 1528                         newitr = igb_low_latency;
 1529                 else if (rxr->bytes > BULK_THRESHOLD)
 1530                         newitr = igb_bulk_latency;
 1531         } else if (olditr == igb_bulk_latency) {
 1532                 if (rxr->bytes < BULK_THRESHOLD)
 1533                         newitr = igb_ave_latency;
 1534         }
 1535 
 1536         if (olditr != newitr) {
 1537                 /* Change interrupt rate */
 1538                 rxr->eitr_setting = newitr;
 1539                 if (adapter->hw.mac.type == e1000_82575)
 1540                         newitr |= newitr << 16;
 1541                 else
 1542                         newitr |= 0x8000000;
 1543                 E1000_WRITE_REG(&adapter->hw, E1000_EITR(rxr->me), newitr);
 1544         }
 1545 
 1546         rxr->bytes = 0;
 1547         return;
 1548 }
 1549 
 1550 
 1551 /*********************************************************************
 1552  *
 1553  *  Media Ioctl callback
 1554  *
 1555  *  This routine is called whenever the user queries the status of
 1556  *  the interface using ifconfig.
 1557  *
 1558  **********************************************************************/
 1559 static void
 1560 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 1561 {
 1562         struct adapter *adapter = ifp->if_softc;
 1563         u_char fiber_type = IFM_1000_SX;
 1564 
 1565         INIT_DEBUGOUT("igb_media_status: begin");
 1566 
 1567         IGB_CORE_LOCK(adapter);
 1568         igb_update_link_status(adapter);
 1569 
 1570         ifmr->ifm_status = IFM_AVALID;
 1571         ifmr->ifm_active = IFM_ETHER;
 1572 
 1573         if (!adapter->link_active) {
 1574                 IGB_CORE_UNLOCK(adapter);
 1575                 return;
 1576         }
 1577 
 1578         ifmr->ifm_status |= IFM_ACTIVE;
 1579 
 1580         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 1581             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes))
 1582                 ifmr->ifm_active |= fiber_type | IFM_FDX;
 1583         else {
 1584                 switch (adapter->link_speed) {
 1585                 case 10:
 1586                         ifmr->ifm_active |= IFM_10_T;
 1587                         break;
 1588                 case 100:
 1589                         ifmr->ifm_active |= IFM_100_TX;
 1590                         break;
 1591                 case 1000:
 1592                         ifmr->ifm_active |= IFM_1000_T;
 1593                         break;
 1594                 }
 1595                 if (adapter->link_duplex == FULL_DUPLEX)
 1596                         ifmr->ifm_active |= IFM_FDX;
 1597                 else
 1598                         ifmr->ifm_active |= IFM_HDX;
 1599         }
 1600         IGB_CORE_UNLOCK(adapter);
 1601 }
 1602 
 1603 /*********************************************************************
 1604  *
 1605  *  Media Ioctl callback
 1606  *
 1607  *  This routine is called when the user changes speed/duplex using
 1608  *  media/mediopt option with ifconfig.
 1609  *
 1610  **********************************************************************/
 1611 static int
 1612 igb_media_change(struct ifnet *ifp)
 1613 {
 1614         struct adapter *adapter = ifp->if_softc;
 1615         struct ifmedia  *ifm = &adapter->media;
 1616 
 1617         INIT_DEBUGOUT("igb_media_change: begin");
 1618 
 1619         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 1620                 return (EINVAL);
 1621 
 1622         IGB_CORE_LOCK(adapter);
 1623         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 1624         case IFM_AUTO:
 1625                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1626                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 1627                 break;
 1628         case IFM_1000_LX:
 1629         case IFM_1000_SX:
 1630         case IFM_1000_T:
 1631                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
 1632                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
 1633                 break;
 1634         case IFM_100_TX:
 1635                 adapter->hw.mac.autoneg = FALSE;
 1636                 adapter->hw.phy.autoneg_advertised = 0;
 1637                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1638                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
 1639                 else
 1640                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
 1641                 break;
 1642         case IFM_10_T:
 1643                 adapter->hw.mac.autoneg = FALSE;
 1644                 adapter->hw.phy.autoneg_advertised = 0;
 1645                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 1646                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
 1647                 else
 1648                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
 1649                 break;
 1650         default:
 1651                 device_printf(adapter->dev, "Unsupported media type\n");
 1652         }
 1653 
 1654         /* As the speed/duplex settings my have changed we need to
 1655          * reset the PHY.
 1656          */
 1657         adapter->hw.phy.reset_disable = FALSE;
 1658 
 1659         igb_init_locked(adapter);
 1660         IGB_CORE_UNLOCK(adapter);
 1661 
 1662         return (0);
 1663 }
 1664 
 1665 
 1666 /*********************************************************************
 1667  *
 1668  *  This routine maps the mbufs to Advanced TX descriptors.
 1669  *  used by the 82575 adapter.
 1670  *  
 1671  **********************************************************************/
 1672 
 1673 static int
 1674 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 1675 {
 1676         struct adapter          *adapter = txr->adapter;
 1677         bus_dma_segment_t       segs[IGB_MAX_SCATTER];
 1678         bus_dmamap_t            map;
 1679         struct igb_tx_buffer    *tx_buffer, *tx_buffer_mapped;
 1680         union e1000_adv_tx_desc *txd = NULL;
 1681         struct mbuf             *m_head;
 1682         u32                     olinfo_status = 0, cmd_type_len = 0;
 1683         int                     nsegs, i, j, error, first, last = 0;
 1684         u32                     hdrlen = 0;
 1685 
 1686         m_head = *m_headp;
 1687 
 1688 
 1689         /* Set basic descriptor constants */
 1690         cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
 1691         cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
 1692         if (m_head->m_flags & M_VLANTAG)
 1693                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
 1694 
 1695         /*
 1696          * Force a cleanup if number of TX descriptors
 1697          * available hits the threshold
 1698          */
 1699         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) {
 1700                 igb_txeof(txr);
 1701                 /* Now do we at least have a minimal? */
 1702                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
 1703                         txr->no_desc_avail++;
 1704                         return (ENOBUFS);
 1705                 }
 1706         }
 1707 
 1708         /*
 1709          * Map the packet for DMA.
 1710          *
 1711          * Capture the first descriptor index,
 1712          * this descriptor will have the index
 1713          * of the EOP which is the only one that
 1714          * now gets a DONE bit writeback.
 1715          */
 1716         first = txr->next_avail_desc;
 1717         tx_buffer = &txr->tx_buffers[first];
 1718         tx_buffer_mapped = tx_buffer;
 1719         map = tx_buffer->map;
 1720 
 1721         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
 1722             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 1723 
 1724         if (error == EFBIG) {
 1725                 struct mbuf *m;
 1726 
 1727                 m = m_defrag(*m_headp, M_DONTWAIT);
 1728                 if (m == NULL) {
 1729                         adapter->mbuf_defrag_failed++;
 1730                         m_freem(*m_headp);
 1731                         *m_headp = NULL;
 1732                         return (ENOBUFS);
 1733                 }
 1734                 *m_headp = m;
 1735 
 1736                 /* Try it again */
 1737                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
 1738                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
 1739 
 1740                 if (error == ENOMEM) {
 1741                         adapter->no_tx_dma_setup++;
 1742                         return (error);
 1743                 } else if (error != 0) {
 1744                         adapter->no_tx_dma_setup++;
 1745                         m_freem(*m_headp);
 1746                         *m_headp = NULL;
 1747                         return (error);
 1748                 }
 1749         } else if (error == ENOMEM) {
 1750                 adapter->no_tx_dma_setup++;
 1751                 return (error);
 1752         } else if (error != 0) {
 1753                 adapter->no_tx_dma_setup++;
 1754                 m_freem(*m_headp);
 1755                 *m_headp = NULL;
 1756                 return (error);
 1757         }
 1758 
 1759         /* Check again to be sure we have enough descriptors */
 1760         if (nsegs > (txr->tx_avail - 2)) {
 1761                 txr->no_desc_avail++;
 1762                 bus_dmamap_unload(txr->txtag, map);
 1763                 return (ENOBUFS);
 1764         }
 1765         m_head = *m_headp;
 1766 
 1767         /*
 1768          * Set up the context descriptor:
 1769          * used when any hardware offload is done.
 1770          * This includes CSUM, VLAN, and TSO. It
 1771          * will use the first descriptor.
 1772          */
 1773         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
 1774                 if (igb_tso_setup(txr, m_head, &hdrlen)) {
 1775                         cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
 1776                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
 1777                         olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 1778                 } else
 1779                         return (ENXIO); 
 1780         } else if (igb_tx_ctx_setup(txr, m_head))
 1781                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 1782 
 1783 #ifdef IGB_IEEE1588
 1784         /* This is changing soon to an mtag detection */
 1785         if (we detect this mbuf has a TSTAMP mtag)
 1786                 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
 1787 #endif
 1788         /* Calculate payload length */
 1789         olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
 1790             << E1000_ADVTXD_PAYLEN_SHIFT);
 1791 
 1792         /* Set up our transmit descriptors */
 1793         i = txr->next_avail_desc;
 1794         for (j = 0; j < nsegs; j++) {
 1795                 bus_size_t seg_len;
 1796                 bus_addr_t seg_addr;
 1797 
 1798                 tx_buffer = &txr->tx_buffers[i];
 1799                 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
 1800                 seg_addr = segs[j].ds_addr;
 1801                 seg_len  = segs[j].ds_len;
 1802 
 1803                 txd->read.buffer_addr = htole64(seg_addr);
 1804                 txd->read.cmd_type_len = htole32(
 1805                     adapter->txd_cmd | cmd_type_len | seg_len);
 1806                 txd->read.olinfo_status = htole32(olinfo_status);
 1807                 last = i;
 1808                 if (++i == adapter->num_tx_desc)
 1809                         i = 0;
 1810                 tx_buffer->m_head = NULL;
 1811                 tx_buffer->next_eop = -1;
 1812         }
 1813 
 1814         txr->next_avail_desc = i;
 1815         txr->tx_avail -= nsegs;
 1816 
 1817         tx_buffer->m_head = m_head;
 1818         tx_buffer_mapped->map = tx_buffer->map;
 1819         tx_buffer->map = map;
 1820         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
 1821 
 1822         /*
 1823          * Last Descriptor of Packet
 1824          * needs End Of Packet (EOP)
 1825          * and Report Status (RS)
 1826          */
 1827         txd->read.cmd_type_len |=
 1828             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
 1829         /*
 1830          * Keep track in the first buffer which
 1831          * descriptor will be written back
 1832          */
 1833         tx_buffer = &txr->tx_buffers[first];
 1834         tx_buffer->next_eop = last;
 1835 
 1836         /*
 1837          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
 1838          * that this frame is available to transmit.
 1839          */
 1840         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 1841             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 1842         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
 1843         ++txr->tx_packets;
 1844 
 1845         return (0);
 1846 
 1847 }
 1848 
 1849 static void
 1850 igb_set_promisc(struct adapter *adapter)
 1851 {
 1852         struct ifnet    *ifp = adapter->ifp;
 1853         uint32_t        reg_rctl;
 1854 
 1855         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 1856 
 1857         if (ifp->if_flags & IFF_PROMISC) {
 1858                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
 1859                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 1860         } else if (ifp->if_flags & IFF_ALLMULTI) {
 1861                 reg_rctl |= E1000_RCTL_MPE;
 1862                 reg_rctl &= ~E1000_RCTL_UPE;
 1863                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 1864         }
 1865 }
 1866 
 1867 static void
 1868 igb_disable_promisc(struct adapter *adapter)
 1869 {
 1870         uint32_t        reg_rctl;
 1871 
 1872         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 1873 
 1874         reg_rctl &=  (~E1000_RCTL_UPE);
 1875         reg_rctl &=  (~E1000_RCTL_MPE);
 1876         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 1877 }
 1878 
 1879 
 1880 /*********************************************************************
 1881  *  Multicast Update
 1882  *
 1883  *  This routine is called whenever multicast address list is updated.
 1884  *
 1885  **********************************************************************/
 1886 
 1887 static void
 1888 igb_set_multi(struct adapter *adapter)
 1889 {
 1890         struct ifnet    *ifp = adapter->ifp;
 1891         struct ifmultiaddr *ifma;
 1892         u32 reg_rctl = 0;
 1893         u8  mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN];
 1894 
 1895         int mcnt = 0;
 1896 
 1897         IOCTL_DEBUGOUT("igb_set_multi: begin");
 1898 
 1899         if_maddr_rlock(ifp);
 1900         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 1901                 if (ifma->ifma_addr->sa_family != AF_LINK)
 1902                         continue;
 1903 
 1904                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
 1905                         break;
 1906 
 1907                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 1908                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
 1909                 mcnt++;
 1910         }
 1911         if_maddr_runlock(ifp);
 1912 
 1913         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
 1914                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 1915                 reg_rctl |= E1000_RCTL_MPE;
 1916                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 1917         } else
 1918                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
 1919 }
 1920 
 1921 
 1922 /*********************************************************************
 1923  *  Timer routine
 1924  *
 1925  *  This routine checks for link status and updates statistics.
 1926  *
 1927  **********************************************************************/
 1928 
 1929 static void
 1930 igb_local_timer(void *arg)
 1931 {
 1932         struct adapter  *adapter = arg;
 1933         struct ifnet    *ifp = adapter->ifp;
 1934 
 1935         IGB_CORE_LOCK_ASSERT(adapter);
 1936 
 1937         igb_update_link_status(adapter);
 1938         igb_update_stats_counters(adapter);
 1939 
 1940         if (igb_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
 1941                 igb_print_hw_stats(adapter);
 1942 
 1943         /*
 1944          * Each second we check the watchdog to 
 1945          * protect against hardware hangs.
 1946          */
 1947         igb_watchdog(adapter);
 1948 
 1949         /* Trigger an RX interrupt on all queues */
 1950         E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->rx_mask);
 1951  
 1952         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 1953 
 1954 }
 1955 
 1956 static void
 1957 igb_update_link_status(struct adapter *adapter)
 1958 {
 1959         struct e1000_hw *hw = &adapter->hw;
 1960         struct ifnet *ifp = adapter->ifp;
 1961         device_t dev = adapter->dev;
 1962         struct tx_ring *txr = adapter->tx_rings;
 1963         u32 link_check = 0;
 1964 
 1965         /* Get the cached link value or read for real */
 1966         switch (hw->phy.media_type) {
 1967         case e1000_media_type_copper:
 1968                 if (hw->mac.get_link_status) {
 1969                         /* Do the work to read phy */
 1970                         e1000_check_for_link(hw);
 1971                         link_check = !hw->mac.get_link_status;
 1972                 } else
 1973                         link_check = TRUE;
 1974                 break;
 1975         case e1000_media_type_fiber:
 1976                 e1000_check_for_link(hw);
 1977                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
 1978                                  E1000_STATUS_LU);
 1979                 break;
 1980         case e1000_media_type_internal_serdes:
 1981                 e1000_check_for_link(hw);
 1982                 link_check = adapter->hw.mac.serdes_has_link;
 1983                 break;
 1984         default:
 1985         case e1000_media_type_unknown:
 1986                 break;
 1987         }
 1988 
 1989         /* Now we check if a transition has happened */
 1990         if (link_check && (adapter->link_active == 0)) {
 1991                 e1000_get_speed_and_duplex(&adapter->hw, 
 1992                     &adapter->link_speed, &adapter->link_duplex);
 1993                 if (bootverbose)
 1994                         device_printf(dev, "Link is up %d Mbps %s\n",
 1995                             adapter->link_speed,
 1996                             ((adapter->link_duplex == FULL_DUPLEX) ?
 1997                             "Full Duplex" : "Half Duplex"));
 1998                 adapter->link_active = 1;
 1999                 ifp->if_baudrate = adapter->link_speed * 1000000;
 2000                 if_link_state_change(ifp, LINK_STATE_UP);
 2001         } else if (!link_check && (adapter->link_active == 1)) {
 2002                 ifp->if_baudrate = adapter->link_speed = 0;
 2003                 adapter->link_duplex = 0;
 2004                 if (bootverbose)
 2005                         device_printf(dev, "Link is Down\n");
 2006                 adapter->link_active = 0;
 2007                 if_link_state_change(ifp, LINK_STATE_DOWN);
 2008                 /* Turn off watchdogs */
 2009                 for (int i = 0; i < adapter->num_queues; i++, txr++)
 2010                         txr->watchdog_timer = FALSE;
 2011         }
 2012 }
 2013 
 2014 /*********************************************************************
 2015  *
 2016  *  This routine disables all traffic on the adapter by issuing a
 2017  *  global reset on the MAC and deallocates TX/RX buffers.
 2018  *
 2019  **********************************************************************/
 2020 
 2021 static void
 2022 igb_stop(void *arg)
 2023 {
 2024         struct adapter  *adapter = arg;
 2025         struct ifnet    *ifp = adapter->ifp;
 2026 
 2027         IGB_CORE_LOCK_ASSERT(adapter);
 2028 
 2029         INIT_DEBUGOUT("igb_stop: begin");
 2030 
 2031         igb_disable_intr(adapter);
 2032 
 2033         callout_stop(&adapter->timer);
 2034 
 2035         /* Tell the stack that the interface is no longer active */
 2036         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 2037 
 2038         e1000_reset_hw(&adapter->hw);
 2039         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
 2040 }
 2041 
 2042 
 2043 /*********************************************************************
 2044  *
 2045  *  Determine hardware revision.
 2046  *
 2047  **********************************************************************/
 2048 static void
 2049 igb_identify_hardware(struct adapter *adapter)
 2050 {
 2051         device_t dev = adapter->dev;
 2052 
 2053         /* Make sure our PCI config space has the necessary stuff set */
 2054         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
 2055         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
 2056             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
 2057                 device_printf(dev, "Memory Access and/or Bus Master bits "
 2058                     "were not set!\n");
 2059                 adapter->hw.bus.pci_cmd_word |=
 2060                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
 2061                 pci_write_config(dev, PCIR_COMMAND,
 2062                     adapter->hw.bus.pci_cmd_word, 2);
 2063         }
 2064 
 2065         /* Save off the information about this board */
 2066         adapter->hw.vendor_id = pci_get_vendor(dev);
 2067         adapter->hw.device_id = pci_get_device(dev);
 2068         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
 2069         adapter->hw.subsystem_vendor_id =
 2070             pci_read_config(dev, PCIR_SUBVEND_0, 2);
 2071         adapter->hw.subsystem_device_id =
 2072             pci_read_config(dev, PCIR_SUBDEV_0, 2);
 2073 
 2074         /* Do Shared Code Init and Setup */
 2075         if (e1000_set_mac_type(&adapter->hw)) {
 2076                 device_printf(dev, "Setup init failure\n");
 2077                 return;
 2078         }
 2079 }
 2080 
 2081 static int
 2082 igb_allocate_pci_resources(struct adapter *adapter)
 2083 {
 2084         device_t        dev = adapter->dev;
 2085         int             rid;
 2086 
 2087         rid = PCIR_BAR(0);
 2088         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 2089             &rid, RF_ACTIVE);
 2090         if (adapter->pci_mem == NULL) {
 2091                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 2092                 return (ENXIO);
 2093         }
 2094         adapter->osdep.mem_bus_space_tag =
 2095             rman_get_bustag(adapter->pci_mem);
 2096         adapter->osdep.mem_bus_space_handle =
 2097             rman_get_bushandle(adapter->pci_mem);
 2098         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
 2099 
 2100         adapter->num_queues = 1; /* Defaults for Legacy or MSI */
 2101 
 2102         /* This will setup either MSI/X or MSI */
 2103         adapter->msix = igb_setup_msix(adapter);
 2104         adapter->hw.back = &adapter->osdep;
 2105 
 2106         return (0);
 2107 }
 2108 
 2109 /*********************************************************************
 2110  *
 2111  *  Setup the Legacy or MSI Interrupt handler
 2112  *
 2113  **********************************************************************/
 2114 static int
 2115 igb_allocate_legacy(struct adapter *adapter)
 2116 {
 2117         device_t dev = adapter->dev;
 2118         int error, rid = 0;
 2119 
 2120         /* Turn off all interrupts */
 2121         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
 2122 
 2123         /* MSI RID is 1 */
 2124         if (adapter->msix == 1)
 2125                 rid = 1;
 2126 
 2127         /* We allocate a single interrupt resource */
 2128         adapter->res = bus_alloc_resource_any(dev,
 2129             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2130         if (adapter->res == NULL) {
 2131                 device_printf(dev, "Unable to allocate bus resource: "
 2132                     "interrupt\n");
 2133                 return (ENXIO);
 2134         }
 2135 
 2136         /*
 2137          * Try allocating a fast interrupt and the associated deferred
 2138          * processing contexts.
 2139          */
 2140         TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter);
 2141         adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
 2142             taskqueue_thread_enqueue, &adapter->tq);
 2143         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
 2144             device_get_nameunit(adapter->dev));
 2145         if ((error = bus_setup_intr(dev, adapter->res,
 2146             INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
 2147             adapter, &adapter->tag)) != 0) {
 2148                 device_printf(dev, "Failed to register fast interrupt "
 2149                             "handler: %d\n", error);
 2150                 taskqueue_free(adapter->tq);
 2151                 adapter->tq = NULL;
 2152                 return (error);
 2153         }
 2154 
 2155         return (0);
 2156 }
 2157 
 2158 
 2159 /*********************************************************************
 2160  *
 2161  *  Setup the MSIX Interrupt handlers: 
 2162  *
 2163  **********************************************************************/
 2164 static int
 2165 igb_allocate_msix(struct adapter *adapter)
 2166 {
 2167         device_t dev = adapter->dev;
 2168         struct tx_ring *txr = adapter->tx_rings;
 2169         struct rx_ring *rxr = adapter->rx_rings;
 2170         int error, rid, vector = 0;
 2171 
 2172         /*
 2173          * Setup the interrupt handlers
 2174          */
 2175 
 2176         /* TX Setup */
 2177         for (int i = 0; i < adapter->num_queues; i++, vector++, txr++) {
 2178                 rid = vector +1;
 2179                 txr->res = bus_alloc_resource_any(dev,
 2180                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2181                 if (txr->res == NULL) {
 2182                         device_printf(dev,
 2183                             "Unable to allocate bus resource: "
 2184                             "MSIX TX Interrupt\n");
 2185                         return (ENXIO);
 2186                 }
 2187                 error = bus_setup_intr(dev, txr->res,
 2188                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2189                     igb_msix_tx, txr, &txr->tag);
 2190                 if (error) {
 2191                         txr->res = NULL;
 2192                         device_printf(dev, "Failed to register TX handler");
 2193                         return (error);
 2194                 }
 2195                 /* Make tasklet for deferred handling - one per queue */
 2196                 TASK_INIT(&txr->tx_task, 0, igb_handle_tx, txr);
 2197                 txr->msix = vector;
 2198                 if (adapter->hw.mac.type == e1000_82575)
 2199                         txr->eims = E1000_EICR_TX_QUEUE0 << i;
 2200                 else
 2201                         txr->eims = 1 << vector;
 2202                 /*
 2203                 ** Bind the msix vector, and thus the
 2204                 ** ring to the corresponding cpu.
 2205                 */
 2206                 if (adapter->num_queues > 1)
 2207                         bus_bind_intr(dev, txr->res, i);
 2208         }
 2209 
 2210         /* RX Setup */
 2211         for (int i = 0; i < adapter->num_queues; i++, vector++, rxr++) {
 2212                 rid = vector +1;
 2213                 rxr->res = bus_alloc_resource_any(dev,
 2214                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2215                 if (rxr->res == NULL) {
 2216                         device_printf(dev,
 2217                             "Unable to allocate bus resource: "
 2218                             "MSIX RX Interrupt\n");
 2219                         return (ENXIO);
 2220                 }
 2221                 error = bus_setup_intr(dev, rxr->res,
 2222                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2223                     igb_msix_rx, rxr, &rxr->tag);
 2224                 if (error) {
 2225                         rxr->res = NULL;
 2226                         device_printf(dev, "Failed to register RX handler");
 2227                         return (error);
 2228                 }
 2229                 /* Make tasklet for deferred handling - one per queue */
 2230                 TASK_INIT(&rxr->rx_task, 0, igb_handle_rx, rxr);
 2231                 rxr->msix = vector;
 2232                 if (adapter->hw.mac.type == e1000_82575)
 2233                         rxr->eims = E1000_EICR_RX_QUEUE0 << i;
 2234                 else
 2235                         rxr->eims = 1 << vector;
 2236                 /* Get a mask for local timer */
 2237                 adapter->rx_mask |= rxr->eims;
 2238                 /*
 2239                 ** Bind the msix vector, and thus the
 2240                 ** ring to the corresponding cpu.
 2241                 ** Notice that this makes an RX/TX pair
 2242                 ** bound to each CPU, limited by the MSIX
 2243                 ** vectors.
 2244                 */
 2245                 if (adapter->num_queues > 1)
 2246                         bus_bind_intr(dev, rxr->res, i);
 2247         }
 2248 
 2249         /* And Link */
 2250         rid = vector +1;
 2251         adapter->res = bus_alloc_resource_any(dev,
 2252             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
 2253         if (adapter->res == NULL) {
 2254                 device_printf(dev,
 2255                     "Unable to allocate bus resource: "
 2256                     "MSIX Link Interrupt\n");
 2257                 return (ENXIO);
 2258         }
 2259         if ((error = bus_setup_intr(dev, adapter->res,
 2260             INTR_TYPE_NET | INTR_MPSAFE, NULL,
 2261             igb_msix_link, adapter, &adapter->tag)) != 0) {
 2262                 device_printf(dev, "Failed to register Link handler");
 2263                 return (error);
 2264         }
 2265         adapter->linkvec = vector;
 2266         adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
 2267             taskqueue_thread_enqueue, &adapter->tq);
 2268         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
 2269             device_get_nameunit(adapter->dev));
 2270 
 2271         return (0);
 2272 }
 2273 
 2274 
 2275 static void
 2276 igb_configure_queues(struct adapter *adapter)
 2277 {
 2278         struct  e1000_hw *hw = &adapter->hw;
 2279         struct  tx_ring *txr;
 2280         struct  rx_ring *rxr;
 2281 
 2282         /* Turn on MSIX */
 2283         /*
 2284         ** 82576 uses IVARs to route MSI/X
 2285         ** interrupts, its not very intuitive,
 2286         ** study the code carefully :)
 2287         */
 2288         if (adapter->hw.mac.type == e1000_82576) {
 2289                 u32     ivar = 0;
 2290                 /* First turn on the capability */
 2291                 E1000_WRITE_REG(hw, E1000_GPIE,
 2292                     E1000_GPIE_MSIX_MODE |
 2293                     E1000_GPIE_EIAME |
 2294                     E1000_GPIE_PBA | E1000_GPIE_NSICR);
 2295                 /* RX */
 2296                 for (int i = 0; i < adapter->num_queues; i++) {
 2297                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2298                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2299                         rxr = &adapter->rx_rings[i];
 2300                         if (i < 8) {
 2301                                 ivar &= 0xFFFFFF00;
 2302                                 ivar |= rxr->msix | E1000_IVAR_VALID;
 2303                         } else {
 2304                                 ivar &= 0xFF00FFFF;
 2305                                 ivar |= (rxr->msix | E1000_IVAR_VALID) << 16;
 2306                         }
 2307                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2308                         adapter->eims_mask |= rxr->eims;
 2309                 }
 2310                 /* TX */
 2311                 for (int i = 0; i < adapter->num_queues; i++) {
 2312                         u32 index = i & 0x7; /* Each IVAR has two entries */
 2313                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 2314                         txr = &adapter->tx_rings[i];
 2315                         if (i < 8) {
 2316                                 ivar &= 0xFFFF00FF;
 2317                                 ivar |= (txr->msix | E1000_IVAR_VALID) << 8;
 2318                         } else {
 2319                                 ivar &= 0x00FFFFFF;
 2320                                 ivar |= (txr->msix | E1000_IVAR_VALID) << 24;
 2321                         }
 2322                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 2323                         adapter->eims_mask |= txr->eims;
 2324                 }
 2325 
 2326                 /* And for the link interrupt */
 2327                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
 2328                 adapter->link_mask = 1 << adapter->linkvec;
 2329                 adapter->eims_mask |= adapter->link_mask;
 2330                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
 2331         } else
 2332         { /* 82575 */
 2333                 int tmp;
 2334 
 2335                 /* enable MSI-X PBA support*/
 2336                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 2337                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 2338                 /* Auto-Mask interrupts upon ICR read. */
 2339                 tmp |= E1000_CTRL_EXT_EIAME;
 2340                 tmp |= E1000_CTRL_EXT_IRCA;
 2341                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 2342 
 2343                 /* TX */
 2344                 for (int i = 0; i < adapter->num_queues; i++) {
 2345                         txr = &adapter->tx_rings[i];
 2346                         E1000_WRITE_REG(hw, E1000_MSIXBM(txr->msix),
 2347                             txr->eims);
 2348                         adapter->eims_mask |= txr->eims;
 2349                 }
 2350 
 2351                 /* RX */
 2352                 for (int i = 0; i < adapter->num_queues; i++) {
 2353                         rxr = &adapter->rx_rings[i];
 2354                         E1000_WRITE_REG(hw, E1000_MSIXBM(rxr->msix),
 2355                             rxr->eims);
 2356                         adapter->eims_mask |= rxr->eims;
 2357                 }
 2358 
 2359                 /* Link */
 2360                 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
 2361                     E1000_EIMS_OTHER);
 2362                 adapter->link_mask |= E1000_EIMS_OTHER;
 2363                 adapter->eims_mask |= adapter->link_mask;
 2364         }
 2365         return;
 2366 }
 2367 
 2368 
 2369 static void
 2370 igb_free_pci_resources(struct adapter *adapter)
 2371 {
 2372         struct          tx_ring *txr = adapter->tx_rings;
 2373         struct          rx_ring *rxr = adapter->rx_rings;
 2374         device_t        dev = adapter->dev;
 2375         int             rid;
 2376 
 2377         /*
 2378         ** There is a slight possibility of a failure mode
 2379         ** in attach that will result in entering this function
 2380         ** before interrupt resources have been initialized, and
 2381         ** in that case we do not want to execute the loops below
 2382         ** We can detect this reliably by the state of the adapter
 2383         ** res pointer.
 2384         */
 2385         if (adapter->res == NULL)
 2386                 goto mem;
 2387 
 2388         /*
 2389          * First release all the TX/RX interrupt resources:
 2390          */
 2391         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 2392                 rid = txr->msix + 1;
 2393                 if (txr->tag != NULL) {
 2394                         bus_teardown_intr(dev, txr->res, txr->tag);
 2395                         txr->tag = NULL;
 2396                 }
 2397                 if (txr->res != NULL)
 2398                         bus_release_resource(dev, SYS_RES_IRQ, rid, txr->res);
 2399         }
 2400 
 2401         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 2402                 rid = rxr->msix + 1;
 2403                 if (rxr->tag != NULL) {
 2404                         bus_teardown_intr(dev, rxr->res, rxr->tag);
 2405                         rxr->tag = NULL;
 2406                 }
 2407                 if (rxr->res != NULL)
 2408                         bus_release_resource(dev, SYS_RES_IRQ, rid, rxr->res);
 2409         }
 2410 
 2411         /* Clean the Legacy or Link interrupt last */
 2412         if (adapter->linkvec) /* we are doing MSIX */
 2413                 rid = adapter->linkvec + 1;
 2414         else
 2415                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
 2416 
 2417         if (adapter->tag != NULL) {
 2418                 bus_teardown_intr(dev, adapter->res, adapter->tag);
 2419                 adapter->tag = NULL;
 2420         }
 2421         if (adapter->res != NULL)
 2422                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
 2423 
 2424 mem:
 2425         if (adapter->msix)
 2426                 pci_release_msi(dev);
 2427 
 2428         if (adapter->msix_mem != NULL)
 2429                 bus_release_resource(dev, SYS_RES_MEMORY,
 2430                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
 2431 
 2432         if (adapter->pci_mem != NULL)
 2433                 bus_release_resource(dev, SYS_RES_MEMORY,
 2434                     PCIR_BAR(0), adapter->pci_mem);
 2435 
 2436 }
 2437 
 2438 /*
 2439  * Setup Either MSI/X or MSI
 2440  */
 2441 static int
 2442 igb_setup_msix(struct adapter *adapter)
 2443 {
 2444         device_t dev = adapter->dev;
 2445         int rid, want, queues, msgs;
 2446 
 2447         /* First try MSI/X */
 2448         rid = PCIR_BAR(IGB_MSIX_BAR);
 2449         adapter->msix_mem = bus_alloc_resource_any(dev,
 2450             SYS_RES_MEMORY, &rid, RF_ACTIVE);
 2451         if (!adapter->msix_mem) {
 2452                 /* May not be enabled */
 2453                 device_printf(adapter->dev,
 2454                     "Unable to map MSIX table \n");
 2455                 goto msi;
 2456         }
 2457 
 2458         msgs = pci_msix_count(dev); 
 2459         if (msgs == 0) { /* system has msix disabled */
 2460                 bus_release_resource(dev, SYS_RES_MEMORY,
 2461                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
 2462                 adapter->msix_mem = NULL;
 2463                 goto msi;
 2464         }
 2465 
 2466         /* Figure out a reasonable auto config value */
 2467         queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus;
 2468 
 2469         if (igb_num_queues == 0)
 2470                 igb_num_queues = queues;
 2471         /*
 2472         ** Two vectors (RX/TX pair) per queue
 2473         ** plus an additional for Link interrupt
 2474         */
 2475         want = (igb_num_queues * 2) + 1;
 2476         if (msgs >= want)
 2477                 msgs = want;
 2478         else {
 2479                 device_printf(adapter->dev,
 2480                     "MSIX Configuration Problem, "
 2481                     "%d vectors configured, but %d queues wanted!\n",
 2482                     msgs, want);
 2483                 return (ENXIO);
 2484         }
 2485         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
 2486                 device_printf(adapter->dev,
 2487                     "Using MSIX interrupts with %d vectors\n", msgs);
 2488                 adapter->num_queues = igb_num_queues;
 2489                 return (msgs);
 2490         }
 2491 msi:
 2492         msgs = pci_msi_count(dev);
 2493         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
 2494                 device_printf(adapter->dev,"Using MSI interrupt\n");
 2495         return (msgs);
 2496 }
 2497 
 2498 /*********************************************************************
 2499  *
 2500  *  Initialize the hardware to a configuration
 2501  *  as specified by the adapter structure.
 2502  *
 2503  **********************************************************************/
 2504 static int
 2505 igb_hardware_init(struct adapter *adapter)
 2506 {
 2507         device_t        dev = adapter->dev;
 2508         u32             rx_buffer_size;
 2509 
 2510         INIT_DEBUGOUT("igb_hardware_init: begin");
 2511 
 2512         /* Issue a global reset */
 2513         e1000_reset_hw(&adapter->hw);
 2514 
 2515         /* Let the firmware know the OS is in control */
 2516         igb_get_hw_control(adapter);
 2517 
 2518         /*
 2519          * These parameters control the automatic generation (Tx) and
 2520          * response (Rx) to Ethernet PAUSE frames.
 2521          * - High water mark should allow for at least two frames to be
 2522          *   received after sending an XOFF.
 2523          * - Low water mark works best when it is very near the high water mark.
 2524          *   This allows the receiver to restart by sending XON when it has
 2525          *   drained a bit. Here we use an arbitary value of 1500 which will
 2526          *   restart after one full frame is pulled from the buffer. There
 2527          *   could be several smaller frames in the buffer and if so they will
 2528          *   not trigger the XON until their total number reduces the buffer
 2529          *   by 1500.
 2530          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
 2531          */
 2532         if (adapter->hw.mac.type == e1000_82576)
 2533                 rx_buffer_size = ((E1000_READ_REG(&adapter->hw,
 2534                     E1000_RXPBS) & 0xffff) << 10 );
 2535         else
 2536                 rx_buffer_size = ((E1000_READ_REG(&adapter->hw,
 2537                     E1000_PBA) & 0xffff) << 10 );
 2538 
 2539         adapter->hw.fc.high_water = rx_buffer_size -
 2540             roundup2(adapter->max_frame_size, 1024);
 2541         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
 2542 
 2543         adapter->hw.fc.pause_time = IGB_FC_PAUSE_TIME;
 2544         adapter->hw.fc.send_xon = TRUE;
 2545 
 2546         /* Set Flow control, use the tunable location if sane */
 2547         if ((igb_fc_setting >= 0) || (igb_fc_setting < 4))
 2548                 adapter->hw.fc.requested_mode = igb_fc_setting;
 2549         else
 2550                 adapter->hw.fc.requested_mode = e1000_fc_none;
 2551 
 2552         if (e1000_init_hw(&adapter->hw) < 0) {
 2553                 device_printf(dev, "Hardware Initialization Failed\n");
 2554                 return (EIO);
 2555         }
 2556 
 2557         e1000_check_for_link(&adapter->hw);
 2558 
 2559         return (0);
 2560 }
 2561 
 2562 /*********************************************************************
 2563  *
 2564  *  Setup networking device structure and register an interface.
 2565  *
 2566  **********************************************************************/
 2567 static void
 2568 igb_setup_interface(device_t dev, struct adapter *adapter)
 2569 {
 2570         struct ifnet   *ifp;
 2571 
 2572         INIT_DEBUGOUT("igb_setup_interface: begin");
 2573 
 2574         ifp = adapter->ifp = if_alloc(IFT_ETHER);
 2575         if (ifp == NULL)
 2576                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
 2577         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 2578         ifp->if_mtu = ETHERMTU;
 2579         ifp->if_init =  igb_init;
 2580         ifp->if_softc = adapter;
 2581         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 2582         ifp->if_ioctl = igb_ioctl;
 2583         ifp->if_start = igb_start;
 2584 #if __FreeBSD_version >= 800000
 2585         ifp->if_transmit = igb_mq_start;
 2586         ifp->if_qflush = igb_qflush;
 2587 #endif
 2588         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
 2589         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
 2590         IFQ_SET_READY(&ifp->if_snd);
 2591 
 2592         ether_ifattach(ifp, adapter->hw.mac.addr);
 2593 
 2594         ifp->if_capabilities = ifp->if_capenable = 0;
 2595 
 2596         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU;
 2597         ifp->if_capabilities |= IFCAP_TSO4;
 2598         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
 2599         ifp->if_capenable = ifp->if_capabilities;
 2600 
 2601         /*
 2602          * Tell the upper layer(s) we support long frames.
 2603          */
 2604         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 2605         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
 2606         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
 2607 
 2608         /*
 2609          * Specify the media types supported by this adapter and register
 2610          * callbacks to update media and link information
 2611          */
 2612         ifmedia_init(&adapter->media, IFM_IMASK,
 2613             igb_media_change, igb_media_status);
 2614         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
 2615             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
 2616                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 
 2617                             0, NULL);
 2618                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 2619         } else {
 2620                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
 2621                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
 2622                             0, NULL);
 2623                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
 2624                             0, NULL);
 2625                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
 2626                             0, NULL);
 2627                 if (adapter->hw.phy.type != e1000_phy_ife) {
 2628                         ifmedia_add(&adapter->media,
 2629                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
 2630                         ifmedia_add(&adapter->media,
 2631                                 IFM_ETHER | IFM_1000_T, 0, NULL);
 2632                 }
 2633         }
 2634         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 2635         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
 2636 }
 2637 
 2638 
 2639 /*
 2640  * Manage DMA'able memory.
 2641  */
 2642 static void
 2643 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 2644 {
 2645         if (error)
 2646                 return;
 2647         *(bus_addr_t *) arg = segs[0].ds_addr;
 2648 }
 2649 
 2650 static int
 2651 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
 2652         struct igb_dma_alloc *dma, int mapflags)
 2653 {
 2654         int error;
 2655 
 2656         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
 2657                                 1, 0,                   /* alignment, bounds */
 2658                                 BUS_SPACE_MAXADDR,      /* lowaddr */
 2659                                 BUS_SPACE_MAXADDR,      /* highaddr */
 2660                                 NULL, NULL,             /* filter, filterarg */
 2661                                 size,                   /* maxsize */
 2662                                 1,                      /* nsegments */
 2663                                 size,                   /* maxsegsize */
 2664                                 0,                      /* flags */
 2665                                 NULL,                   /* lockfunc */
 2666                                 NULL,                   /* lockarg */
 2667                                 &dma->dma_tag);
 2668         if (error) {
 2669                 device_printf(adapter->dev,
 2670                     "%s: bus_dma_tag_create failed: %d\n",
 2671                     __func__, error);
 2672                 goto fail_0;
 2673         }
 2674 
 2675         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 2676             BUS_DMA_NOWAIT, &dma->dma_map);
 2677         if (error) {
 2678                 device_printf(adapter->dev,
 2679                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
 2680                     __func__, (uintmax_t)size, error);
 2681                 goto fail_2;
 2682         }
 2683 
 2684         dma->dma_paddr = 0;
 2685         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 2686             size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
 2687         if (error || dma->dma_paddr == 0) {
 2688                 device_printf(adapter->dev,
 2689                     "%s: bus_dmamap_load failed: %d\n",
 2690                     __func__, error);
 2691                 goto fail_3;
 2692         }
 2693 
 2694         return (0);
 2695 
 2696 fail_3:
 2697         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 2698 fail_2:
 2699         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 2700         bus_dma_tag_destroy(dma->dma_tag);
 2701 fail_0:
 2702         dma->dma_map = NULL;
 2703         dma->dma_tag = NULL;
 2704 
 2705         return (error);
 2706 }
 2707 
 2708 static void
 2709 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
 2710 {
 2711         if (dma->dma_tag == NULL)
 2712                 return;
 2713         if (dma->dma_map != NULL) {
 2714                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
 2715                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 2716                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 2717                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 2718                 dma->dma_map = NULL;
 2719         }
 2720         bus_dma_tag_destroy(dma->dma_tag);
 2721         dma->dma_tag = NULL;
 2722 }
 2723 
 2724 
 2725 /*********************************************************************
 2726  *
 2727  *  Allocate memory for the transmit and receive rings, and then
 2728  *  the descriptors associated with each, called only once at attach.
 2729  *
 2730  **********************************************************************/
 2731 static int
 2732 igb_allocate_queues(struct adapter *adapter)
 2733 {
 2734         device_t dev = adapter->dev;
 2735         struct tx_ring *txr;
 2736         struct rx_ring *rxr;
 2737         int rsize, tsize, error = E1000_SUCCESS;
 2738         int txconf = 0, rxconf = 0;
 2739 
 2740         /* First allocate the TX ring struct memory */
 2741         if (!(adapter->tx_rings =
 2742             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
 2743             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 2744                 device_printf(dev, "Unable to allocate TX ring memory\n");
 2745                 error = ENOMEM;
 2746                 goto fail;
 2747         }
 2748         txr = adapter->tx_rings;
 2749 
 2750         /* Next allocate the RX */
 2751         if (!(adapter->rx_rings =
 2752             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
 2753             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 2754                 device_printf(dev, "Unable to allocate RX ring memory\n");
 2755                 error = ENOMEM;
 2756                 goto rx_fail;
 2757         }
 2758         rxr = adapter->rx_rings;
 2759 
 2760         tsize = roundup2(adapter->num_tx_desc *
 2761             sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
 2762         /*
 2763          * Now set up the TX queues, txconf is needed to handle the
 2764          * possibility that things fail midcourse and we need to
 2765          * undo memory gracefully
 2766          */ 
 2767         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
 2768                 /* Set up some basics */
 2769                 txr = &adapter->tx_rings[i];
 2770                 txr->adapter = adapter;
 2771                 txr->me = i;
 2772 
 2773                 /* Initialize the TX lock */
 2774                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
 2775                     device_get_nameunit(dev), txr->me);
 2776                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
 2777 
 2778                 if (igb_dma_malloc(adapter, tsize,
 2779                         &txr->txdma, BUS_DMA_NOWAIT)) {
 2780                         device_printf(dev,
 2781                             "Unable to allocate TX Descriptor memory\n");
 2782                         error = ENOMEM;
 2783                         goto err_tx_desc;
 2784                 }
 2785                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
 2786                 bzero((void *)txr->tx_base, tsize);
 2787 
 2788                 /* Now allocate transmit buffers for the ring */
 2789                 if (igb_allocate_transmit_buffers(txr)) {
 2790                         device_printf(dev,
 2791                             "Critical Failure setting up transmit buffers\n");
 2792                         error = ENOMEM;
 2793                         goto err_tx_desc;
 2794                 }
 2795 #if __FreeBSD_version >= 800000
 2796                 /* Allocate a buf ring */
 2797                 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF,
 2798                     M_WAITOK, &txr->tx_mtx);
 2799 #endif
 2800         }
 2801 
 2802         /*
 2803          * Next the RX queues...
 2804          */ 
 2805         rsize = roundup2(adapter->num_rx_desc *
 2806             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 2807         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
 2808                 rxr = &adapter->rx_rings[i];
 2809                 rxr->adapter = adapter;
 2810                 rxr->me = i;
 2811 
 2812                 /* Initialize the RX lock */
 2813                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
 2814                     device_get_nameunit(dev), txr->me);
 2815                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
 2816 
 2817                 if (igb_dma_malloc(adapter, rsize,
 2818                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
 2819                         device_printf(dev,
 2820                             "Unable to allocate RxDescriptor memory\n");
 2821                         error = ENOMEM;
 2822                         goto err_rx_desc;
 2823                 }
 2824                 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
 2825                 bzero((void *)rxr->rx_base, rsize);
 2826 
 2827                 /* Allocate receive buffers for the ring*/
 2828                 if (igb_allocate_receive_buffers(rxr)) {
 2829                         device_printf(dev,
 2830                             "Critical Failure setting up receive buffers\n");
 2831                         error = ENOMEM;
 2832                         goto err_rx_desc;
 2833                 }
 2834         }
 2835 
 2836         return (0);
 2837 
 2838 err_rx_desc:
 2839         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
 2840                 igb_dma_free(adapter, &rxr->rxdma);
 2841 err_tx_desc:
 2842         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
 2843                 igb_dma_free(adapter, &txr->txdma);
 2844         free(adapter->rx_rings, M_DEVBUF);
 2845 rx_fail:
 2846         free(adapter->tx_rings, M_DEVBUF);
 2847 fail:
 2848         return (error);
 2849 }
 2850 
 2851 /*********************************************************************
 2852  *
 2853  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
 2854  *  the information needed to transmit a packet on the wire. This is
 2855  *  called only once at attach, setup is done every reset.
 2856  *
 2857  **********************************************************************/
 2858 static int
 2859 igb_allocate_transmit_buffers(struct tx_ring *txr)
 2860 {
 2861         struct adapter *adapter = txr->adapter;
 2862         device_t dev = adapter->dev;
 2863         struct igb_tx_buffer *txbuf;
 2864         int error, i;
 2865 
 2866         /*
 2867          * Setup DMA descriptor areas.
 2868          */
 2869         if ((error = bus_dma_tag_create(NULL,           /* parent */
 2870                                1, 0,                    /* alignment, bounds */
 2871                                BUS_SPACE_MAXADDR,       /* lowaddr */
 2872                                BUS_SPACE_MAXADDR,       /* highaddr */
 2873                                NULL, NULL,              /* filter, filterarg */
 2874                                IGB_TSO_SIZE,            /* maxsize */
 2875                                IGB_MAX_SCATTER,         /* nsegments */
 2876                                PAGE_SIZE,               /* maxsegsize */
 2877                                0,                       /* flags */
 2878                                NULL,                    /* lockfunc */
 2879                                NULL,                    /* lockfuncarg */
 2880                                &txr->txtag))) {
 2881                 device_printf(dev,"Unable to allocate TX DMA tag\n");
 2882                 goto fail;
 2883         }
 2884 
 2885         if (!(txr->tx_buffers =
 2886             (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) *
 2887             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 2888                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
 2889                 error = ENOMEM;
 2890                 goto fail;
 2891         }
 2892 
 2893         /* Create the descriptor buffer dma maps */
 2894         txbuf = txr->tx_buffers;
 2895         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 2896                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
 2897                 if (error != 0) {
 2898                         device_printf(dev, "Unable to create TX DMA map\n");
 2899                         goto fail;
 2900                 }
 2901         }
 2902 
 2903         return 0;
 2904 fail:
 2905         /* We free all, it handles case where we are in the middle */
 2906         igb_free_transmit_structures(adapter);
 2907         return (error);
 2908 }
 2909 
 2910 /*********************************************************************
 2911  *
 2912  *  Initialize a transmit ring.
 2913  *
 2914  **********************************************************************/
 2915 static void
 2916 igb_setup_transmit_ring(struct tx_ring *txr)
 2917 {
 2918         struct adapter *adapter = txr->adapter;
 2919         struct igb_tx_buffer *txbuf;
 2920         int i;
 2921 
 2922         /* Clear the old descriptor contents */
 2923         bzero((void *)txr->tx_base,
 2924               (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
 2925         /* Reset indices */
 2926         txr->next_avail_desc = 0;
 2927         txr->next_to_clean = 0;
 2928 
 2929         /* Free any existing tx buffers. */
 2930         txbuf = txr->tx_buffers;
 2931         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
 2932                 if (txbuf->m_head != NULL) {
 2933                         bus_dmamap_sync(txr->txtag, txbuf->map,
 2934                             BUS_DMASYNC_POSTWRITE);
 2935                         bus_dmamap_unload(txr->txtag, txbuf->map);
 2936                         m_freem(txbuf->m_head);
 2937                         txbuf->m_head = NULL;
 2938                 }
 2939                 /* clear the watch index */
 2940                 txbuf->next_eop = -1;
 2941         }
 2942 
 2943         /* Set number of descriptors available */
 2944         txr->tx_avail = adapter->num_tx_desc;
 2945 
 2946         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 2947             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 2948 
 2949 }
 2950 
 2951 /*********************************************************************
 2952  *
 2953  *  Initialize all transmit rings.
 2954  *
 2955  **********************************************************************/
 2956 static void
 2957 igb_setup_transmit_structures(struct adapter *adapter)
 2958 {
 2959         struct tx_ring *txr = adapter->tx_rings;
 2960 
 2961         for (int i = 0; i < adapter->num_queues; i++, txr++)
 2962                 igb_setup_transmit_ring(txr);
 2963 
 2964         return;
 2965 }
 2966 
 2967 /*********************************************************************
 2968  *
 2969  *  Enable transmit unit.
 2970  *
 2971  **********************************************************************/
 2972 static void
 2973 igb_initialize_transmit_units(struct adapter *adapter)
 2974 {
 2975         struct tx_ring  *txr = adapter->tx_rings;
 2976         u32             tctl, txdctl;
 2977 
 2978          INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
 2979 
 2980         /* Setup the Base and Length of the Tx Descriptor Rings */
 2981         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 2982                 u64 bus_addr = txr->txdma.dma_paddr;
 2983 
 2984                 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(i),
 2985                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
 2986                 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(i),
 2987                     (uint32_t)(bus_addr >> 32));
 2988                 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(i),
 2989                     (uint32_t)bus_addr);
 2990 
 2991                 /* Setup the HW Tx Head and Tail descriptor pointers */
 2992                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(i), 0);
 2993                 E1000_WRITE_REG(&adapter->hw, E1000_TDH(i), 0);
 2994 
 2995                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
 2996                     E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
 2997                     E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
 2998 
 2999                 /* Setup Transmit Descriptor Base Settings */   
 3000                 adapter->txd_cmd = E1000_TXD_CMD_IFCS;
 3001 
 3002                 txdctl = E1000_READ_REG(&adapter->hw, E1000_TXDCTL(i));
 3003                 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
 3004                 E1000_WRITE_REG(&adapter->hw, E1000_TXDCTL(i), txdctl);
 3005         }
 3006 
 3007         /* Program the Transmit Control Register */
 3008         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
 3009         tctl &= ~E1000_TCTL_CT;
 3010         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
 3011                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
 3012 
 3013         e1000_config_collision_dist(&adapter->hw);
 3014 
 3015         /* This write will effectively turn on the transmit unit. */
 3016         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
 3017 
 3018 }
 3019 
 3020 /*********************************************************************
 3021  *
 3022  *  Free all transmit rings.
 3023  *
 3024  **********************************************************************/
 3025 static void
 3026 igb_free_transmit_structures(struct adapter *adapter)
 3027 {
 3028         struct tx_ring *txr = adapter->tx_rings;
 3029 
 3030         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 3031                 IGB_TX_LOCK(txr);
 3032                 igb_free_transmit_buffers(txr);
 3033                 igb_dma_free(adapter, &txr->txdma);
 3034                 IGB_TX_UNLOCK(txr);
 3035                 IGB_TX_LOCK_DESTROY(txr);
 3036         }
 3037         free(adapter->tx_rings, M_DEVBUF);
 3038 }
 3039 
 3040 /*********************************************************************
 3041  *
 3042  *  Free transmit ring related data structures.
 3043  *
 3044  **********************************************************************/
 3045 static void
 3046 igb_free_transmit_buffers(struct tx_ring *txr)
 3047 {
 3048         struct adapter *adapter = txr->adapter;
 3049         struct igb_tx_buffer *tx_buffer;
 3050         int             i;
 3051 
 3052         INIT_DEBUGOUT("free_transmit_ring: begin");
 3053 
 3054         if (txr->tx_buffers == NULL)
 3055                 return;
 3056 
 3057         tx_buffer = txr->tx_buffers;
 3058         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
 3059                 if (tx_buffer->m_head != NULL) {
 3060                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
 3061                             BUS_DMASYNC_POSTWRITE);
 3062                         bus_dmamap_unload(txr->txtag,
 3063                             tx_buffer->map);
 3064                         m_freem(tx_buffer->m_head);
 3065                         tx_buffer->m_head = NULL;
 3066                         if (tx_buffer->map != NULL) {
 3067                                 bus_dmamap_destroy(txr->txtag,
 3068                                     tx_buffer->map);
 3069                                 tx_buffer->map = NULL;
 3070                         }
 3071                 } else if (tx_buffer->map != NULL) {
 3072                         bus_dmamap_unload(txr->txtag,
 3073                             tx_buffer->map);
 3074                         bus_dmamap_destroy(txr->txtag,
 3075                             tx_buffer->map);
 3076                         tx_buffer->map = NULL;
 3077                 }
 3078         }
 3079 #if __FreeBSD_version >= 800000
 3080         if (txr->br != NULL)
 3081                 buf_ring_free(txr->br, M_DEVBUF);
 3082 #endif
 3083         if (txr->tx_buffers != NULL) {
 3084                 free(txr->tx_buffers, M_DEVBUF);
 3085                 txr->tx_buffers = NULL;
 3086         }
 3087         if (txr->txtag != NULL) {
 3088                 bus_dma_tag_destroy(txr->txtag);
 3089                 txr->txtag = NULL;
 3090         }
 3091         return;
 3092 }
 3093 
 3094 /**********************************************************************
 3095  *
 3096  *  Setup work for hardware segmentation offload (TSO) on
 3097  *  adapters using advanced tx descriptors (82575)
 3098  *
 3099  **********************************************************************/
 3100 static boolean_t
 3101 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
 3102 {
 3103         struct adapter *adapter = txr->adapter;
 3104         struct e1000_adv_tx_context_desc *TXD;
 3105         struct igb_tx_buffer        *tx_buffer;
 3106         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 3107         u32 mss_l4len_idx = 0;
 3108         u16 vtag = 0;
 3109         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
 3110         struct ether_vlan_header *eh;
 3111         struct ip *ip;
 3112         struct tcphdr *th;
 3113 
 3114 
 3115         /*
 3116          * Determine where frame payload starts.
 3117          * Jump over vlan headers if already present
 3118          */
 3119         eh = mtod(mp, struct ether_vlan_header *);
 3120         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
 3121                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3122         else
 3123                 ehdrlen = ETHER_HDR_LEN;
 3124 
 3125         /* Ensure we have at least the IP+TCP header in the first mbuf. */
 3126         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
 3127                 return FALSE;
 3128 
 3129         /* Only supports IPV4 for now */
 3130         ctxd = txr->next_avail_desc;
 3131         tx_buffer = &txr->tx_buffers[ctxd];
 3132         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3133 
 3134         ip = (struct ip *)(mp->m_data + ehdrlen);
 3135         if (ip->ip_p != IPPROTO_TCP)
 3136                 return FALSE;   /* 0 */
 3137         ip->ip_sum = 0;
 3138         ip_hlen = ip->ip_hl << 2;
 3139         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
 3140         th->th_sum = in_pseudo(ip->ip_src.s_addr,
 3141             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 3142         tcp_hlen = th->th_off << 2;
 3143         /*
 3144          * Calculate header length, this is used
 3145          * in the transmit desc in igb_xmit
 3146          */
 3147         *hdrlen = ehdrlen + ip_hlen + tcp_hlen;
 3148 
 3149         /* VLAN MACLEN IPLEN */
 3150         if (mp->m_flags & M_VLANTAG) {
 3151                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3152                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3153         }
 3154 
 3155         vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT);
 3156         vlan_macip_lens |= ip_hlen;
 3157         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
 3158 
 3159         /* ADV DTYPE TUCMD */
 3160         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3161         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3162         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3163         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
 3164 
 3165         /* MSS L4LEN IDX */
 3166         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
 3167         mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
 3168         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 3169 
 3170         TXD->seqnum_seed = htole32(0);
 3171         tx_buffer->m_head = NULL;
 3172         tx_buffer->next_eop = -1;
 3173 
 3174         if (++ctxd == adapter->num_tx_desc)
 3175                 ctxd = 0;
 3176 
 3177         txr->tx_avail--;
 3178         txr->next_avail_desc = ctxd;
 3179         return TRUE;
 3180 }
 3181 
 3182 
 3183 /*********************************************************************
 3184  *
 3185  *  Context Descriptor setup for VLAN or CSUM
 3186  *
 3187  **********************************************************************/
 3188 
 3189 static bool
 3190 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 3191 {
 3192         struct adapter *adapter = txr->adapter;
 3193         struct e1000_adv_tx_context_desc *TXD;
 3194         struct igb_tx_buffer        *tx_buffer;
 3195         uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 3196         struct ether_vlan_header *eh;
 3197         struct ip *ip = NULL;
 3198         struct ip6_hdr *ip6;
 3199         int  ehdrlen, ctxd, ip_hlen = 0;
 3200         u16     etype, vtag = 0;
 3201         u8      ipproto = 0;
 3202         bool    offload = TRUE;
 3203 
 3204         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
 3205                 offload = FALSE;
 3206 
 3207         ctxd = txr->next_avail_desc;
 3208         tx_buffer = &txr->tx_buffers[ctxd];
 3209         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 3210 
 3211         /*
 3212         ** In advanced descriptors the vlan tag must 
 3213         ** be placed into the context descriptor, thus
 3214         ** we need to be here just for that setup.
 3215         */
 3216         if (mp->m_flags & M_VLANTAG) {
 3217                 vtag = htole16(mp->m_pkthdr.ether_vtag);
 3218                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
 3219         } else if (offload == FALSE)
 3220                 return FALSE;
 3221 
 3222         /*
 3223          * Determine where frame payload starts.
 3224          * Jump over vlan headers if already present,
 3225          * helpful for QinQ too.
 3226          */
 3227         eh = mtod(mp, struct ether_vlan_header *);
 3228         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
 3229                 etype = ntohs(eh->evl_proto);
 3230                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
 3231         } else {
 3232                 etype = ntohs(eh->evl_encap_proto);
 3233                 ehdrlen = ETHER_HDR_LEN;
 3234         }
 3235 
 3236         /* Set the ether header length */
 3237         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
 3238 
 3239         switch (etype) {
 3240                 case ETHERTYPE_IP:
 3241                         ip = (struct ip *)(mp->m_data + ehdrlen);
 3242                         ip_hlen = ip->ip_hl << 2;
 3243                         if (mp->m_len < ehdrlen + ip_hlen) {
 3244                                 offload = FALSE;
 3245                                 break;
 3246                         }
 3247                         ipproto = ip->ip_p;
 3248                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 3249                         break;
 3250                 case ETHERTYPE_IPV6:
 3251                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
 3252                         ip_hlen = sizeof(struct ip6_hdr);
 3253                         if (mp->m_len < ehdrlen + ip_hlen)
 3254                                 return (FALSE);
 3255                         ipproto = ip6->ip6_nxt;
 3256                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
 3257                         break;
 3258                 default:
 3259                         offload = FALSE;
 3260                         break;
 3261         }
 3262 
 3263         vlan_macip_lens |= ip_hlen;
 3264         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
 3265 
 3266         switch (ipproto) {
 3267                 case IPPROTO_TCP:
 3268                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
 3269                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
 3270                         break;
 3271                 case IPPROTO_UDP:
 3272                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
 3273                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
 3274                         break;
 3275 #if __FreeBSD_version >= 800000
 3276                 case IPPROTO_SCTP:
 3277                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
 3278                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
 3279                         break;
 3280 #endif
 3281                 default:
 3282                         offload = FALSE;
 3283                         break;
 3284         }
 3285 
 3286         /* Now copy bits into descriptor */
 3287         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
 3288         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
 3289         TXD->seqnum_seed = htole32(0);
 3290         TXD->mss_l4len_idx = htole32(0);
 3291 
 3292         tx_buffer->m_head = NULL;
 3293         tx_buffer->next_eop = -1;
 3294 
 3295         /* We've consumed the first desc, adjust counters */
 3296         if (++ctxd == adapter->num_tx_desc)
 3297                 ctxd = 0;
 3298         txr->next_avail_desc = ctxd;
 3299         --txr->tx_avail;
 3300 
 3301         return (offload);
 3302 }
 3303 
 3304 
 3305 /**********************************************************************
 3306  *
 3307  *  Examine each tx_buffer in the used queue. If the hardware is done
 3308  *  processing the packet then free associated resources. The
 3309  *  tx_buffer is put back on the free queue.
 3310  *
 3311  *  TRUE return means there's work in the ring to clean, FALSE its empty.
 3312  **********************************************************************/
 3313 static bool
 3314 igb_txeof(struct tx_ring *txr)
 3315 {
 3316         struct adapter  *adapter = txr->adapter;
 3317         int first, last, done, num_avail;
 3318         u32     cleaned = 0;
 3319         struct igb_tx_buffer *tx_buffer;
 3320         struct e1000_tx_desc   *tx_desc, *eop_desc;
 3321         struct ifnet   *ifp = adapter->ifp;
 3322 
 3323         IGB_TX_LOCK_ASSERT(txr);
 3324 
 3325         if (txr->tx_avail == adapter->num_tx_desc)
 3326                 return FALSE;
 3327 
 3328         num_avail = txr->tx_avail;
 3329         first = txr->next_to_clean;
 3330         tx_desc = &txr->tx_base[first];
 3331         tx_buffer = &txr->tx_buffers[first];
 3332         last = tx_buffer->next_eop;
 3333         eop_desc = &txr->tx_base[last];
 3334 
 3335         /*
 3336          * What this does is get the index of the
 3337          * first descriptor AFTER the EOP of the 
 3338          * first packet, that way we can do the
 3339          * simple comparison on the inner while loop.
 3340          */
 3341         if (++last == adapter->num_tx_desc)
 3342                 last = 0;
 3343         done = last;
 3344 
 3345         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3346             BUS_DMASYNC_POSTREAD);
 3347 
 3348         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
 3349                 /* We clean the range of the packet */
 3350                 while (first != done) {
 3351                         tx_desc->upper.data = 0;
 3352                         tx_desc->lower.data = 0;
 3353                         tx_desc->buffer_addr = 0;
 3354                         ++num_avail; ++cleaned;
 3355 
 3356                         if (tx_buffer->m_head) {
 3357                                 ifp->if_opackets++;
 3358                                 bus_dmamap_sync(txr->txtag,
 3359                                     tx_buffer->map,
 3360                                     BUS_DMASYNC_POSTWRITE);
 3361                                 bus_dmamap_unload(txr->txtag,
 3362                                     tx_buffer->map);
 3363 
 3364                                 m_freem(tx_buffer->m_head);
 3365                                 tx_buffer->m_head = NULL;
 3366                         }
 3367                         tx_buffer->next_eop = -1;
 3368 
 3369                         if (++first == adapter->num_tx_desc)
 3370                                 first = 0;
 3371 
 3372                         tx_buffer = &txr->tx_buffers[first];
 3373                         tx_desc = &txr->tx_base[first];
 3374                 }
 3375                 /* See if we can continue to the next packet */
 3376                 last = tx_buffer->next_eop;
 3377                 if (last != -1) {
 3378                         eop_desc = &txr->tx_base[last];
 3379                         /* Get new done point */
 3380                         if (++last == adapter->num_tx_desc) last = 0;
 3381                         done = last;
 3382                 } else
 3383                         break;
 3384         }
 3385         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 3386             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 3387 
 3388         txr->next_to_clean = first;
 3389 
 3390         /*
 3391          * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack
 3392          * that it is OK to send packets.
 3393          * If there are no pending descriptors, clear the timeout. Otherwise,
 3394          * if some descriptors have been freed, restart the timeout.
 3395          */
 3396         if (num_avail > IGB_TX_CLEANUP_THRESHOLD) {                
 3397                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 3398                 /* All clean, turn off the timer */
 3399                 if (num_avail == adapter->num_tx_desc) {
 3400                         txr->watchdog_timer = 0;
 3401                         txr->tx_avail = num_avail;
 3402                         return FALSE;
 3403                 }
 3404         }
 3405 
 3406         /* Some cleaned, reset the timer */
 3407         if (cleaned)
 3408                 txr->watchdog_timer = IGB_TX_TIMEOUT;
 3409         txr->tx_avail = num_avail;
 3410         return TRUE;
 3411 }
 3412 
 3413 
 3414 /*********************************************************************
 3415  *
 3416  *  Setup descriptor buffer(s) from system mbuf buffer pools.
 3417  *              i - designates the ring index
 3418  *              clean - tells the function whether to update
 3419  *                      the header, the packet buffer, or both.
 3420  *
 3421  **********************************************************************/
 3422 static int
 3423 igb_get_buf(struct rx_ring *rxr, int i, u8 clean)
 3424 {
 3425         struct adapter          *adapter = rxr->adapter;
 3426         struct mbuf             *mh, *mp;
 3427         bus_dma_segment_t       seg[2];
 3428         bus_dmamap_t            map;
 3429         struct igb_rx_buffer    *rx_buffer;
 3430         int                     error, nsegs;
 3431         int                     merr = 0;
 3432 
 3433 
 3434         rx_buffer = &rxr->rx_buffers[i];
 3435 
 3436         /* First get our header and payload mbuf */
 3437         if (clean & IGB_CLEAN_HEADER) {
 3438                 mh = m_gethdr(M_DONTWAIT, MT_DATA);
 3439                 if (mh == NULL)
 3440                         goto remap;
 3441         } else  /* reuse */
 3442                 mh = rxr->rx_buffers[i].m_head;
 3443 
 3444         mh->m_len = MHLEN;
 3445         mh->m_flags |= M_PKTHDR;
 3446 
 3447         if (clean & IGB_CLEAN_PAYLOAD) {
 3448                 mp = m_getjcl(M_DONTWAIT, MT_DATA,
 3449                     M_PKTHDR, adapter->rx_mbuf_sz);
 3450                 if (mp == NULL)
 3451                         goto remap;
 3452                 mp->m_len = adapter->rx_mbuf_sz;
 3453                 mp->m_flags &= ~M_PKTHDR;
 3454         } else {        /* reusing */
 3455                 mp = rxr->rx_buffers[i].m_pack;
 3456                 mp->m_len = adapter->rx_mbuf_sz;
 3457                 mp->m_flags &= ~M_PKTHDR;
 3458         }
 3459         /*
 3460         ** Need to create a chain for the following
 3461         ** dmamap call at this point.
 3462         */
 3463         mh->m_next = mp;
 3464         mh->m_pkthdr.len = mh->m_len + mp->m_len;
 3465 
 3466         /* Get the memory mapping */
 3467         error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
 3468             rxr->rx_spare_map, mh, seg, &nsegs, BUS_DMA_NOWAIT);
 3469         if (error != 0) {
 3470                 printf("GET BUF: dmamap load failure - %d\n", error);
 3471                 m_free(mh);
 3472                 return (error);
 3473         }
 3474 
 3475         /* Unload old mapping and update buffer struct */
 3476         if (rx_buffer->m_head != NULL)
 3477                         bus_dmamap_unload(rxr->rxtag, rx_buffer->map);
 3478         map = rx_buffer->map;
 3479         rx_buffer->map = rxr->rx_spare_map;
 3480         rxr->rx_spare_map = map;
 3481         rx_buffer->m_head = mh;
 3482         rx_buffer->m_pack = mp;
 3483         bus_dmamap_sync(rxr->rxtag,
 3484             rx_buffer->map, BUS_DMASYNC_PREREAD);
 3485 
 3486         /* Update descriptor */
 3487         rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr);
 3488         rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr);
 3489 
 3490         return (0);
 3491 
 3492         /*
 3493         ** If we get here, we have an mbuf resource
 3494         ** issue, so we discard the incoming packet
 3495         ** and attempt to reuse existing mbufs next
 3496         ** pass thru the ring, but to do so we must
 3497         ** fix up the descriptor which had the address
 3498         ** clobbered with writeback info.
 3499         */
 3500 remap:
 3501         adapter->mbuf_header_failed++;
 3502         merr = ENOBUFS;
 3503         /* Is there a reusable buffer? */
 3504         mh = rxr->rx_buffers[i].m_head;
 3505         if (mh == NULL) /* Nope, init error */
 3506                 return (merr);
 3507         mp = rxr->rx_buffers[i].m_pack;
 3508         if (mp == NULL) /* Nope, init error */
 3509                 return (merr);
 3510         /* Get our old mapping */
 3511         rx_buffer = &rxr->rx_buffers[i];
 3512         error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
 3513             rx_buffer->map, mh, seg, &nsegs, BUS_DMA_NOWAIT);
 3514         if (error != 0) {
 3515                 /* We really have a problem */
 3516                 m_free(mh);
 3517                 return (error);
 3518         }
 3519         /* Now fix the descriptor as needed */
 3520         rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr);
 3521         rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr);
 3522         return (merr);
 3523 }
 3524 
 3525 
 3526 /*********************************************************************
 3527  *
 3528  *  Allocate memory for rx_buffer structures. Since we use one
 3529  *  rx_buffer per received packet, the maximum number of rx_buffer's
 3530  *  that we'll need is equal to the number of receive descriptors
 3531  *  that we've allocated.
 3532  *
 3533  **********************************************************************/
 3534 static int
 3535 igb_allocate_receive_buffers(struct rx_ring *rxr)
 3536 {
 3537         struct  adapter         *adapter = rxr->adapter;
 3538         device_t                dev = adapter->dev;
 3539         struct igb_rx_buffer    *rxbuf;
 3540         int                     i, bsize, error;
 3541 
 3542         bsize = sizeof(struct igb_rx_buffer) * adapter->num_rx_desc;
 3543         if (!(rxr->rx_buffers =
 3544             (struct igb_rx_buffer *) malloc(bsize,
 3545             M_DEVBUF, M_NOWAIT | M_ZERO))) {
 3546                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
 3547                 error = ENOMEM;
 3548                 goto fail;
 3549         }
 3550 
 3551         /*
 3552         ** The tag is made to accomodate the largest buffer size
 3553         ** with packet split (hence the two segments, even though
 3554         ** it may not always use this.
 3555         */
 3556         if ((error = bus_dma_tag_create(NULL,           /* parent */
 3557                                    1, 0,                /* alignment, bounds */
 3558                                    BUS_SPACE_MAXADDR,   /* lowaddr */
 3559                                    BUS_SPACE_MAXADDR,   /* highaddr */
 3560                                    NULL, NULL,          /* filter, filterarg */
 3561                                    MJUM16BYTES,         /* maxsize */
 3562                                    2,                   /* nsegments */
 3563                                    MJUMPAGESIZE,        /* maxsegsize */
 3564                                    0,                   /* flags */
 3565                                    NULL,                /* lockfunc */
 3566                                    NULL,                /* lockfuncarg */
 3567                                    &rxr->rxtag))) {
 3568                 device_printf(dev, "Unable to create RX DMA tag\n");
 3569                 goto fail;
 3570         }
 3571 
 3572         /* Create the spare map (used by getbuf) */
 3573         error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
 3574              &rxr->rx_spare_map);
 3575         if (error) {
 3576                 device_printf(dev,
 3577                     "%s: bus_dmamap_create header spare failed: %d\n",
 3578                     __func__, error);
 3579                 goto fail;
 3580         }
 3581 
 3582         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
 3583                 rxbuf = &rxr->rx_buffers[i];
 3584                 error = bus_dmamap_create(rxr->rxtag,
 3585                     BUS_DMA_NOWAIT, &rxbuf->map);
 3586                 if (error) {
 3587                         device_printf(dev, "Unable to create RX DMA maps\n");
 3588                         goto fail;
 3589                 }
 3590         }
 3591 
 3592         return (0);
 3593 
 3594 fail:
 3595         /* Frees all, but can handle partial completion */
 3596         igb_free_receive_structures(adapter);
 3597         return (error);
 3598 }
 3599 
 3600 /*********************************************************************
 3601  *
 3602  *  Initialize a receive ring and its buffers.
 3603  *
 3604  **********************************************************************/
 3605 static int
 3606 igb_setup_receive_ring(struct rx_ring *rxr)
 3607 {
 3608         struct  adapter         *adapter;
 3609         struct  ifnet           *ifp;
 3610         device_t                dev;
 3611         struct igb_rx_buffer    *rxbuf;
 3612         struct lro_ctrl         *lro = &rxr->lro;
 3613         int                     j, rsize;
 3614 
 3615         adapter = rxr->adapter;
 3616         dev = adapter->dev;
 3617         ifp = adapter->ifp;
 3618         rxr->lro_enabled = FALSE;
 3619         rxr->hdr_split = FALSE;
 3620 
 3621         /* Clear the ring contents */
 3622         rsize = roundup2(adapter->num_rx_desc *
 3623             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 3624         bzero((void *)rxr->rx_base, rsize);
 3625 
 3626         /*
 3627         ** Free current RX buffer structures and their mbufs
 3628         */
 3629         for (int i = 0; i < adapter->num_rx_desc; i++) {
 3630                 rxbuf = &rxr->rx_buffers[i];
 3631                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
 3632                     BUS_DMASYNC_POSTREAD);
 3633                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
 3634                 if (rxbuf->m_head) {
 3635                         rxbuf->m_head->m_next = rxbuf->m_pack;
 3636                         m_freem(rxbuf->m_head);
 3637                 }
 3638                 rxbuf->m_head = NULL;
 3639                 rxbuf->m_pack = NULL;
 3640         }
 3641 
 3642         /* Next replenish the ring */
 3643         for (j = 0; j < adapter->num_rx_desc; j++) {
 3644                 if (igb_get_buf(rxr, j, IGB_CLEAN_BOTH) == ENOBUFS) {
 3645                         rxr->rx_buffers[j].m_head = NULL;
 3646                         rxr->rx_buffers[j].m_pack = NULL;
 3647                         rxr->rx_base[j].read.hdr_addr = 0;
 3648                         rxr->rx_base[j].read.pkt_addr = 0;
 3649                         goto fail;
 3650                 }
 3651         }
 3652 
 3653         /* Setup our descriptor indices */
 3654         rxr->next_to_check = 0;
 3655         rxr->last_cleaned = 0;
 3656 
 3657         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 3658             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 3659 
 3660         /*
 3661         ** Now set up the LRO interface, we
 3662         ** also only do head split when LRO
 3663         ** is enabled, since so often they
 3664         ** are undesireable in similar setups.
 3665         */
 3666         if (ifp->if_capenable & IFCAP_LRO) {
 3667                 int err = tcp_lro_init(lro);
 3668                 if (err) {
 3669                         device_printf(dev,"LRO Initialization failed!\n");
 3670                         goto fail;
 3671                 }
 3672                 INIT_DEBUGOUT("RX LRO Initialized\n");
 3673                 rxr->lro_enabled = TRUE;
 3674                 rxr->hdr_split = TRUE;
 3675                 lro->ifp = adapter->ifp;
 3676         }
 3677 
 3678         return (0);
 3679 fail:
 3680         /*
 3681          * We need to clean up any buffers allocated
 3682          * so far, 'j' is the failing index.
 3683          */
 3684         for (int i = 0; i < j; i++) {
 3685                 rxbuf = &rxr->rx_buffers[i];
 3686                 if (rxbuf->m_head != NULL) {
 3687                         bus_dmamap_sync(rxr->rxtag, rxbuf->map,
 3688                             BUS_DMASYNC_POSTREAD);
 3689                         bus_dmamap_unload(rxr->rxtag, rxbuf->map);
 3690                         m_freem(rxbuf->m_head);
 3691                         rxbuf->m_head = NULL;
 3692                 }
 3693         }
 3694         return (ENOBUFS);
 3695 }
 3696 
 3697 /*********************************************************************
 3698  *
 3699  *  Initialize all receive rings.
 3700  *
 3701  **********************************************************************/
 3702 static int
 3703 igb_setup_receive_structures(struct adapter *adapter)
 3704 {
 3705         struct rx_ring *rxr = adapter->rx_rings;
 3706         int i, j;
 3707 
 3708         for (i = 0; i < adapter->num_queues; i++, rxr++)
 3709                 if (igb_setup_receive_ring(rxr))
 3710                         goto fail;
 3711 
 3712         return (0);
 3713 fail:
 3714         /*
 3715          * Free RX buffers allocated so far, we will only handle
 3716          * the rings that completed, the failing case will have
 3717          * cleaned up for itself. The value of 'i' will be the
 3718          * failed ring so we must pre-decrement it.
 3719          */
 3720         rxr = adapter->rx_rings;
 3721         for (--i; i > 0; i--, rxr++) {
 3722                 for (j = 0; j < adapter->num_rx_desc; j++) {
 3723                         struct igb_rx_buffer *rxbuf;
 3724                         rxbuf = &rxr->rx_buffers[j];
 3725                         if (rxbuf->m_head != NULL) {
 3726                                 bus_dmamap_sync(rxr->rxtag, rxbuf->map,
 3727                                   BUS_DMASYNC_POSTREAD);
 3728                                 bus_dmamap_unload(rxr->rxtag, rxbuf->map);
 3729                                 m_freem(rxbuf->m_head);
 3730                                 rxbuf->m_head = NULL;
 3731                         }
 3732                 }
 3733         }
 3734 
 3735         return (ENOBUFS);
 3736 }
 3737 
 3738 /*********************************************************************
 3739  *
 3740  *  Enable receive unit.
 3741  *
 3742  **********************************************************************/
 3743 static void
 3744 igb_initialize_receive_units(struct adapter *adapter)
 3745 {
 3746         struct rx_ring  *rxr = adapter->rx_rings;
 3747         struct ifnet    *ifp = adapter->ifp;
 3748         u32             rctl, rxcsum, psize, srrctl = 0;
 3749 
 3750         INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
 3751 
 3752         /*
 3753          * Make sure receives are disabled while setting
 3754          * up the descriptor ring
 3755          */
 3756         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 3757         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
 3758 
 3759         /*
 3760         ** Set up for header split
 3761         */
 3762         if (rxr->hdr_split) {
 3763                 /* Use a standard mbuf for the header */
 3764                 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
 3765                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
 3766         } else
 3767                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
 3768 
 3769         /*
 3770         ** Set up for jumbo frames
 3771         */
 3772         if (ifp->if_mtu > ETHERMTU) {
 3773                 rctl |= E1000_RCTL_LPE;
 3774                 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 3775                 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
 3776 
 3777                 /* Set maximum packet len */
 3778                 psize = adapter->max_frame_size;
 3779                 /* are we on a vlan? */
 3780                 if (adapter->ifp->if_vlantrunk != NULL)
 3781                         psize += VLAN_TAG_SIZE;
 3782                 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
 3783         } else {
 3784                 rctl &= ~E1000_RCTL_LPE;
 3785                 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
 3786                 rctl |= E1000_RCTL_SZ_2048;
 3787         }
 3788 
 3789         /* Setup the Base and Length of the Rx Descriptor Rings */
 3790         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 3791                 u64 bus_addr = rxr->rxdma.dma_paddr;
 3792                 u32 rxdctl;
 3793 
 3794                 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(i),
 3795                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
 3796                 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(i),
 3797                     (uint32_t)(bus_addr >> 32));
 3798                 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(i),
 3799                     (uint32_t)bus_addr);
 3800                 E1000_WRITE_REG(&adapter->hw, E1000_SRRCTL(i), srrctl);
 3801                 /* Enable this Queue */
 3802                 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(i));
 3803                 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
 3804                 rxdctl &= 0xFFF00000;
 3805                 rxdctl |= IGB_RX_PTHRESH;
 3806                 rxdctl |= IGB_RX_HTHRESH << 8;
 3807                 rxdctl |= IGB_RX_WTHRESH << 16;
 3808                 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(i), rxdctl);
 3809         }
 3810 
 3811         /*
 3812         ** Setup for RX MultiQueue
 3813         */
 3814         rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
 3815         if (adapter->num_queues >1) {
 3816                 u32 random[10], mrqc, shift = 0;
 3817                 union igb_reta {
 3818                         u32 dword;
 3819                         u8  bytes[4];
 3820                 } reta;
 3821 
 3822                 arc4rand(&random, sizeof(random), 0);
 3823                 if (adapter->hw.mac.type == e1000_82575)
 3824                         shift = 6;
 3825                 /* Warning FM follows */
 3826                 for (int i = 0; i < 128; i++) {
 3827                         reta.bytes[i & 3] =
 3828                             (i % adapter->num_queues) << shift;
 3829                         if ((i & 3) == 3)
 3830                                 E1000_WRITE_REG(&adapter->hw,
 3831                                     E1000_RETA(i >> 2), reta.dword);
 3832                 }
 3833                 /* Now fill in hash table */
 3834                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
 3835                 for (int i = 0; i < 10; i++)
 3836                         E1000_WRITE_REG_ARRAY(&adapter->hw,
 3837                             E1000_RSSRK(0), i, random[i]);
 3838 
 3839                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
 3840                     E1000_MRQC_RSS_FIELD_IPV4_TCP);
 3841                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
 3842                     E1000_MRQC_RSS_FIELD_IPV6_TCP);
 3843                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
 3844                     E1000_MRQC_RSS_FIELD_IPV6_UDP);
 3845                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
 3846                     E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
 3847 
 3848                 E1000_WRITE_REG(&adapter->hw, E1000_MRQC, mrqc);
 3849 
 3850                 /*
 3851                 ** NOTE: Receive Full-Packet Checksum Offload 
 3852                 ** is mutually exclusive with Multiqueue. However
 3853                 ** this is not the same as TCP/IP checksums which
 3854                 ** still work.
 3855                 */
 3856                 rxcsum |= E1000_RXCSUM_PCSD;
 3857 #if __FreeBSD_version >= 800000
 3858                 /* For SCTP Offload */
 3859                 if ((adapter->hw.mac.type == e1000_82576)
 3860                     && (ifp->if_capenable & IFCAP_RXCSUM))
 3861                         rxcsum |= E1000_RXCSUM_CRCOFL;
 3862 #endif
 3863         } else {
 3864                 /* Non RSS setup */
 3865                 if (ifp->if_capenable & IFCAP_RXCSUM) {
 3866                         rxcsum |= E1000_RXCSUM_IPPCSE;
 3867 #if __FreeBSD_version >= 800000
 3868                         if (adapter->hw.mac.type == e1000_82576)
 3869                                 rxcsum |= E1000_RXCSUM_CRCOFL;
 3870 #endif
 3871                 } else
 3872                         rxcsum &= ~E1000_RXCSUM_TUOFL;
 3873         }
 3874         E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
 3875 
 3876         /* Setup the Receive Control Register */
 3877         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
 3878         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
 3879                    E1000_RCTL_RDMTS_HALF |
 3880                    (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
 3881 
 3882         /* Make sure VLAN Filters are off */
 3883         rctl &= ~E1000_RCTL_VFE;
 3884         /* Don't store bad packets */
 3885         rctl &= ~E1000_RCTL_SBP;
 3886 
 3887         /* Enable Receives */
 3888         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
 3889 
 3890         /*
 3891          * Setup the HW Rx Head and Tail Descriptor Pointers
 3892          *   - needs to be after enable
 3893          */
 3894         for (int i = 0; i < adapter->num_queues; i++) {
 3895                 E1000_WRITE_REG(&adapter->hw, E1000_RDH(i), 0);
 3896                 E1000_WRITE_REG(&adapter->hw, E1000_RDT(i),
 3897                      adapter->num_rx_desc - 1);
 3898         }
 3899         return;
 3900 }
 3901 
 3902 /*********************************************************************
 3903  *
 3904  *  Free receive rings.
 3905  *
 3906  **********************************************************************/
 3907 static void
 3908 igb_free_receive_structures(struct adapter *adapter)
 3909 {
 3910         struct rx_ring *rxr = adapter->rx_rings;
 3911 
 3912         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 3913                 struct lro_ctrl *lro = &rxr->lro;
 3914                 igb_free_receive_buffers(rxr);
 3915                 tcp_lro_free(lro);
 3916                 igb_dma_free(adapter, &rxr->rxdma);
 3917         }
 3918 
 3919         free(adapter->rx_rings, M_DEVBUF);
 3920 }
 3921 
 3922 /*********************************************************************
 3923  *
 3924  *  Free receive ring data structures.
 3925  *
 3926  **********************************************************************/
 3927 static void
 3928 igb_free_receive_buffers(struct rx_ring *rxr)
 3929 {
 3930         struct adapter  *adapter = rxr->adapter;
 3931         struct igb_rx_buffer *rx_buffer;
 3932 
 3933         INIT_DEBUGOUT("free_receive_structures: begin");
 3934 
 3935         if (rxr->rx_spare_map) {
 3936                 bus_dmamap_destroy(rxr->rxtag, rxr->rx_spare_map);
 3937                 rxr->rx_spare_map = NULL;
 3938         }
 3939 
 3940         /* Cleanup any existing buffers */
 3941         if (rxr->rx_buffers != NULL) {
 3942                 rx_buffer = &rxr->rx_buffers[0];
 3943                 for (int i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
 3944                         if (rx_buffer->m_head != NULL) {
 3945                                 bus_dmamap_sync(rxr->rxtag, rx_buffer->map,
 3946                                     BUS_DMASYNC_POSTREAD);
 3947                                 bus_dmamap_unload(rxr->rxtag,
 3948                                     rx_buffer->map);
 3949                                 m_freem(rx_buffer->m_head);
 3950                                 rx_buffer->m_head = NULL;
 3951                         } else if (rx_buffer->map != NULL)
 3952                                 bus_dmamap_unload(rxr->rxtag,
 3953                                     rx_buffer->map);
 3954                         if (rx_buffer->map != NULL) {
 3955                                 bus_dmamap_destroy(rxr->rxtag,
 3956                                     rx_buffer->map);
 3957                                 rx_buffer->map = NULL;
 3958                         }
 3959                 }
 3960         }
 3961 
 3962         if (rxr->rx_buffers != NULL) {
 3963                 free(rxr->rx_buffers, M_DEVBUF);
 3964                 rxr->rx_buffers = NULL;
 3965         }
 3966 
 3967         if (rxr->rxtag != NULL) {
 3968                 bus_dma_tag_destroy(rxr->rxtag);
 3969                 rxr->rxtag = NULL;
 3970         }
 3971 }
 3972 /*********************************************************************
 3973  *
 3974  *  This routine executes in interrupt context. It replenishes
 3975  *  the mbufs in the descriptor and sends data which has been
 3976  *  dma'ed into host memory to upper layer.
 3977  *
 3978  *  We loop at most count times if count is > 0, or until done if
 3979  *  count < 0.
 3980  *
 3981  *  Return TRUE if more to clean, FALSE otherwise
 3982  *********************************************************************/
 3983 static bool
 3984 igb_rxeof(struct rx_ring *rxr, int count)
 3985 {
 3986         struct adapter          *adapter = rxr->adapter;
 3987         struct ifnet            *ifp;
 3988         struct lro_ctrl         *lro = &rxr->lro;
 3989         struct lro_entry        *queued;
 3990         int                     i;
 3991         u32                     staterr;
 3992         union e1000_adv_rx_desc *cur;
 3993 
 3994 
 3995         IGB_RX_LOCK(rxr);
 3996         ifp = adapter->ifp;
 3997         i = rxr->next_to_check;
 3998         cur = &rxr->rx_base[i];
 3999         staterr = cur->wb.upper.status_error;
 4000 
 4001         if (!(staterr & E1000_RXD_STAT_DD)) {
 4002                 IGB_RX_UNLOCK(rxr);
 4003                 return FALSE;
 4004         }
 4005 
 4006         /* Sync the ring */
 4007         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4008             BUS_DMASYNC_POSTREAD);
 4009 
 4010         /* Main clean loop */
 4011         while ((staterr & E1000_RXD_STAT_DD) &&
 4012             (count != 0) &&
 4013             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 4014                 struct mbuf *sendmp, *mh, *mp;
 4015                 u16 hlen, plen, hdr, ptype, len_adj, vtag;
 4016                 u8 dopayload, accept_frame, eop;
 4017  
 4018                 accept_frame = 1;
 4019                 hlen = plen = len_adj = vtag = 0;
 4020                 sendmp = mh = mp = NULL;
 4021                 ptype = (u16)(cur->wb.lower.lo_dword.data >> 4);
 4022 
 4023                 /* Sync the buffers */
 4024                 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map,
 4025                             BUS_DMASYNC_POSTREAD);
 4026 
 4027                 /*
 4028                 ** The way the hardware is configured to
 4029                 ** split, it will ONLY use the header buffer
 4030                 ** when header split is enabled, otherwise we
 4031                 ** get normal behavior, ie, both header and
 4032                 ** payload are DMA'd into the payload buffer.
 4033                 **
 4034                 ** The fmp test is to catch the case where a
 4035                 ** packet spans multiple descriptors, in that
 4036                 ** case only the first header is valid.
 4037                 */
 4038                 if ((rxr->hdr_split) && (rxr->fmp == NULL)){
 4039                         hdr = le16toh(cur->
 4040                             wb.lower.lo_dword.hs_rss.hdr_info);
 4041                         hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
 4042                             E1000_RXDADV_HDRBUFLEN_SHIFT;
 4043                         if (hlen > IGB_HDR_BUF)
 4044                                 hlen = IGB_HDR_BUF;
 4045                         plen = le16toh(cur->wb.upper.length);
 4046                         /* Handle the header mbuf */
 4047                         mh = rxr->rx_buffers[i].m_head;
 4048                         mh->m_len = hlen;
 4049                         dopayload = IGB_CLEAN_HEADER;
 4050                         /*
 4051                         ** Get the payload length, this
 4052                         ** could be zero if its a small
 4053                         ** packet.
 4054                         */
 4055                         if (plen) {
 4056                                 mp = rxr->rx_buffers[i].m_pack;
 4057                                 mp->m_len = plen;
 4058                                 mp->m_next = NULL;
 4059                                 mp->m_flags &= ~M_PKTHDR;
 4060                                 mh->m_next = mp;
 4061                                 mh->m_flags |= M_PKTHDR;
 4062                                 dopayload = IGB_CLEAN_BOTH;
 4063                                 rxr->rx_split_packets++;
 4064                         } else {  /* small packets */
 4065                                 mh->m_flags &= ~M_PKTHDR;
 4066                                 mh->m_next = NULL;
 4067                         }
 4068                 } else {
 4069                         /*
 4070                         ** Either no header split, or a
 4071                         ** secondary piece of a fragmented
 4072                         ** split packet.
 4073                         */
 4074                         mh = rxr->rx_buffers[i].m_pack;
 4075                         mh->m_flags |= M_PKTHDR;
 4076                         mh->m_len = le16toh(cur->wb.upper.length);
 4077                         dopayload = IGB_CLEAN_PAYLOAD;
 4078                 }
 4079 
 4080                 if (staterr & E1000_RXD_STAT_EOP) {
 4081                         count--;
 4082                         eop = 1;
 4083                         /*
 4084                         ** Strip CRC and account for frag
 4085                         */
 4086                         if (mp) { 
 4087                                 if (mp->m_len < ETHER_CRC_LEN) {
 4088                                         /* a frag, how much is left? */
 4089                                         len_adj = ETHER_CRC_LEN - mp->m_len;
 4090                                         mp->m_len = 0;
 4091                                 } else
 4092                                         mp->m_len -= ETHER_CRC_LEN;
 4093                         } else { /* not split */
 4094                                 if (mh->m_len < ETHER_CRC_LEN) {
 4095                                         len_adj = ETHER_CRC_LEN - mh->m_len;
 4096                                         mh->m_len = 0;
 4097                                 } else
 4098                                         mh->m_len -= ETHER_CRC_LEN;
 4099                         }
 4100                 } else 
 4101                         eop = 0;
 4102 
 4103                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)
 4104                         accept_frame = 0;
 4105 #ifdef IGB_IEEE1588
 4106         This linux code needs to be converted to work here
 4107         -----------------------------------------------------
 4108                if (unlikely(staterr & E1000_RXD_STAT_TS)) {
 4109                        u64 regval;
 4110                        u64 ns;
 4111 // Create an mtag and set it up
 4112                        struct skb_shared_hwtstamps *shhwtstamps =
 4113                                skb_hwtstamps(skb);
 4114 
 4115                        rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID),
 4116                        "igb: no RX time stamp available for time stamped packet");
 4117                        regval = rd32(E1000_RXSTMPL);
 4118                        regval |= (u64)rd32(E1000_RXSTMPH) << 32;
 4119 // Do time conversion from the register
 4120                        ns = timecounter_cyc2time(&adapter->clock, regval);
 4121                        clocksync_update(&adapter->sync, ns);
 4122                        memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 4123                        shhwtstamps->hwtstamp = ns_to_ktime(ns);
 4124                        shhwtstamps->syststamp =
 4125                                clocksync_hw2sys(&adapter->sync, ns);
 4126                }
 4127 #endif
 4128                 if (accept_frame) {
 4129                         /*
 4130                         ** get_buf will overwrite the writeback
 4131                         ** descriptor so save the VLAN tag now.
 4132                         */
 4133                         vtag = le16toh(cur->wb.upper.vlan);
 4134                         if (igb_get_buf(rxr, i, dopayload) != 0) {
 4135                                 ifp->if_iqdrops++;
 4136                                 goto discard;
 4137                         }
 4138                         /* Initial frame - setup */
 4139                         if (rxr->fmp == NULL) {
 4140                                 mh->m_flags |= M_PKTHDR;
 4141                                 mh->m_pkthdr.len = mh->m_len;
 4142                                 rxr->fmp = mh; /* Store the first mbuf */
 4143                                 rxr->lmp = mh;
 4144                                 if (mp) { /* Add payload if split */
 4145                                         mh->m_pkthdr.len += mp->m_len;
 4146                                         rxr->lmp = mh->m_next;
 4147                                 }
 4148                         } else {
 4149                                 /* Chain mbuf's together */
 4150                                 mh->m_flags &= ~M_PKTHDR;
 4151                                 rxr->lmp->m_next = mh;
 4152                                 rxr->lmp = rxr->lmp->m_next;
 4153                                 rxr->fmp->m_pkthdr.len += mh->m_len;
 4154                                 /* Adjust for CRC frag */
 4155                                 if (len_adj) {
 4156                                         rxr->lmp->m_len -= len_adj;
 4157                                         rxr->fmp->m_pkthdr.len -= len_adj;
 4158                                 }
 4159                         }
 4160 
 4161                         if (eop) {
 4162                                 bool sctp = ((ptype & 0x40) != 0);
 4163                                 rxr->fmp->m_pkthdr.rcvif = ifp;
 4164                                 ifp->if_ipackets++;
 4165                                 rxr->rx_packets++;
 4166                                 /* capture data for AIM */
 4167                                 rxr->bytes += rxr->fmp->m_pkthdr.len;
 4168                                 rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
 4169 
 4170                                 igb_rx_checksum(staterr, rxr->fmp, sctp);
 4171                                 if (staterr & E1000_RXD_STAT_VP) {
 4172                                         rxr->fmp->m_pkthdr.ether_vtag = vtag;
 4173                                         rxr->fmp->m_flags |= M_VLANTAG;
 4174                                 }
 4175 #if __FreeBSD_version >= 800000
 4176                                 rxr->fmp->m_pkthdr.flowid = curcpu;
 4177                                 rxr->fmp->m_flags |= M_FLOWID;
 4178 #endif
 4179                                 sendmp = rxr->fmp;
 4180                                 rxr->fmp = NULL;
 4181                                 rxr->lmp = NULL;
 4182                         }
 4183                 } else {
 4184                         ifp->if_ierrors++;
 4185 discard:
 4186                         /* Reuse loaded DMA map and just update mbuf chain */
 4187                         if (hlen) {
 4188                                 mh = rxr->rx_buffers[i].m_head;
 4189                                 mh->m_len = MHLEN;
 4190                                 mh->m_next = NULL;
 4191                         }
 4192                         mp = rxr->rx_buffers[i].m_pack;
 4193                         mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz;
 4194                         mp->m_data = mp->m_ext.ext_buf;
 4195                         mp->m_next = NULL;
 4196                         if (adapter->max_frame_size <=
 4197                             (MCLBYTES - ETHER_ALIGN))
 4198                                 m_adj(mp, ETHER_ALIGN);
 4199                         if (rxr->fmp != NULL) {
 4200                                 /* handles the whole chain */
 4201                                 m_freem(rxr->fmp);
 4202                                 rxr->fmp = NULL;
 4203                                 rxr->lmp = NULL;
 4204                         }
 4205                         sendmp = NULL;
 4206                 }
 4207 
 4208                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 4209                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 4210 
 4211                 rxr->last_cleaned = i; /* For updating tail */
 4212 
 4213                 /* Advance our pointers to the next descriptor. */
 4214                 if (++i == adapter->num_rx_desc)
 4215                         i = 0;
 4216  
 4217                 /*
 4218                 ** Note that we hold the RX lock thru
 4219                 ** the following call so this ring's
 4220                 ** next_to_check is not gonna change.
 4221                 */
 4222                 if (sendmp != NULL) {
 4223                         /*
 4224                         ** Send to the stack if:
 4225                         **  - LRO not enabled, or
 4226                         **  - no LRO resources, or
 4227                         **  - lro enqueue fails
 4228                         */
 4229                         if ((!rxr->lro_enabled) ||
 4230                             ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0))))
 4231                                 (*ifp->if_input)(ifp, sendmp);
 4232                 }
 4233 
 4234                 /* Get the next descriptor */
 4235                 cur = &rxr->rx_base[i];
 4236                 staterr = cur->wb.upper.status_error;
 4237         }
 4238         rxr->next_to_check = i;
 4239 
 4240         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
 4241         E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), rxr->last_cleaned);
 4242 
 4243         /*
 4244          * Flush any outstanding LRO work
 4245          */
 4246         while (!SLIST_EMPTY(&lro->lro_active)) {
 4247                 queued = SLIST_FIRST(&lro->lro_active);
 4248                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
 4249                 tcp_lro_flush(lro, queued);
 4250         }
 4251 
 4252         IGB_RX_UNLOCK(rxr);
 4253 
 4254         /*
 4255         ** We still have cleaning to do?
 4256         ** Schedule another interrupt if so.
 4257         */
 4258         if (staterr & E1000_RXD_STAT_DD) {
 4259                 E1000_WRITE_REG(&adapter->hw, E1000_EICS, rxr->eims);
 4260                 return TRUE;
 4261         }
 4262 
 4263         return FALSE;
 4264 }
 4265 
 4266 
 4267 /*********************************************************************
 4268  *
 4269  *  Verify that the hardware indicated that the checksum is valid.
 4270  *  Inform the stack about the status of checksum so that stack
 4271  *  doesn't spend time verifying the checksum.
 4272  *
 4273  *********************************************************************/
 4274 static void
 4275 igb_rx_checksum(u32 staterr, struct mbuf *mp, bool sctp)
 4276 {
 4277         u16 status = (u16)staterr;
 4278         u8  errors = (u8) (staterr >> 24);
 4279 
 4280         /* Ignore Checksum bit is set */
 4281         if (status & E1000_RXD_STAT_IXSM) {
 4282                 mp->m_pkthdr.csum_flags = 0;
 4283                 return;
 4284         }
 4285 
 4286         if (status & E1000_RXD_STAT_IPCS) {
 4287                 /* Did it pass? */
 4288                 if (!(errors & E1000_RXD_ERR_IPE)) {
 4289                         /* IP Checksum Good */
 4290                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
 4291                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
 4292                 } else
 4293                         mp->m_pkthdr.csum_flags = 0;
 4294         }
 4295 
 4296         if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
 4297                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
 4298 #if __FreeBSD_version >= 800000
 4299                 if (sctp) /* reassign */
 4300                         type = CSUM_SCTP_VALID;
 4301 #endif
 4302                 /* Did it pass? */
 4303                 if (!(errors & E1000_RXD_ERR_TCPE)) {
 4304                         mp->m_pkthdr.csum_flags |= type;
 4305                         if (!sctp)
 4306                                 mp->m_pkthdr.csum_data = htons(0xffff);
 4307                 }
 4308         }
 4309         return;
 4310 }
 4311 
 4312 /*
 4313  * This routine is run via an vlan
 4314  * config EVENT
 4315  */
 4316 static void
 4317 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 4318 {
 4319         struct adapter  *adapter = ifp->if_softc;
 4320         u32             index, bit;
 4321 
 4322         if (ifp->if_softc !=  arg)   /* Not our event */
 4323                 return;
 4324 
 4325         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 4326                 return;
 4327 
 4328         index = (vtag >> 5) & 0x7F;
 4329         bit = vtag & 0x1F;
 4330         igb_shadow_vfta[index] |= (1 << bit);
 4331         ++adapter->num_vlans;
 4332         /* Re-init to load the changes */
 4333         igb_init(adapter);
 4334 }
 4335 
 4336 /*
 4337  * This routine is run via an vlan
 4338  * unconfig EVENT
 4339  */
 4340 static void
 4341 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
 4342 {
 4343         struct adapter  *adapter = ifp->if_softc;
 4344         u32             index, bit;
 4345 
 4346         if (ifp->if_softc !=  arg)
 4347                 return;
 4348 
 4349         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
 4350                 return;
 4351 
 4352         index = (vtag >> 5) & 0x7F;
 4353         bit = vtag & 0x1F;
 4354         igb_shadow_vfta[index] &= ~(1 << bit);
 4355         --adapter->num_vlans;
 4356         /* Re-init to load the changes */
 4357         igb_init(adapter);
 4358 }
 4359 
 4360 static void
 4361 igb_setup_vlan_hw_support(struct adapter *adapter)
 4362 {
 4363         struct e1000_hw *hw = &adapter->hw;
 4364         u32             reg;
 4365 
 4366         /*
 4367         ** We get here thru init_locked, meaning
 4368         ** a soft reset, this has already cleared
 4369         ** the VFTA and other state, so if there
 4370         ** have been no vlan's registered do nothing.
 4371         */
 4372         if (adapter->num_vlans == 0)
 4373                 return;
 4374 
 4375         /*
 4376         ** A soft reset zero's out the VFTA, so
 4377         ** we need to repopulate it now.
 4378         */
 4379         for (int i = 0; i < IGB_VFTA_SIZE; i++)
 4380                 if (igb_shadow_vfta[i] != 0)
 4381                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
 4382                             i, igb_shadow_vfta[i]);
 4383 
 4384         reg = E1000_READ_REG(hw, E1000_CTRL);
 4385         reg |= E1000_CTRL_VME;
 4386         E1000_WRITE_REG(hw, E1000_CTRL, reg);
 4387 
 4388         /* Enable the Filter Table */
 4389         reg = E1000_READ_REG(hw, E1000_RCTL);
 4390         reg &= ~E1000_RCTL_CFIEN;
 4391         reg |= E1000_RCTL_VFE;
 4392         E1000_WRITE_REG(hw, E1000_RCTL, reg);
 4393 
 4394         /* Update the frame size */
 4395         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
 4396             adapter->max_frame_size + VLAN_TAG_SIZE);
 4397 }
 4398 
 4399 static void
 4400 igb_enable_intr(struct adapter *adapter)
 4401 {
 4402         /* With RSS set up what to auto clear */
 4403         if (adapter->msix_mem) {
 4404                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC,
 4405                     adapter->eims_mask);
 4406                 E1000_WRITE_REG(&adapter->hw, E1000_EIAM,
 4407                     adapter->eims_mask);
 4408                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS,
 4409                     adapter->eims_mask);
 4410                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 4411                     E1000_IMS_LSC);
 4412         } else {
 4413                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
 4414                     IMS_ENABLE_MASK);
 4415         }
 4416         E1000_WRITE_FLUSH(&adapter->hw);
 4417 
 4418         return;
 4419 }
 4420 
 4421 static void
 4422 igb_disable_intr(struct adapter *adapter)
 4423 {
 4424         if (adapter->msix_mem) {
 4425                 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
 4426                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
 4427         } 
 4428         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
 4429         E1000_WRITE_FLUSH(&adapter->hw);
 4430         return;
 4431 }
 4432 
 4433 /*
 4434  * Bit of a misnomer, what this really means is
 4435  * to enable OS management of the system... aka
 4436  * to disable special hardware management features 
 4437  */
 4438 static void
 4439 igb_init_manageability(struct adapter *adapter)
 4440 {
 4441         if (adapter->has_manage) {
 4442                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
 4443                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 4444 
 4445                 /* disable hardware interception of ARP */
 4446                 manc &= ~(E1000_MANC_ARP_EN);
 4447 
 4448                 /* enable receiving management packets to the host */
 4449                 manc |= E1000_MANC_EN_MNG2HOST;
 4450                 manc2h |= 1 << 5;  /* Mng Port 623 */
 4451                 manc2h |= 1 << 6;  /* Mng Port 664 */
 4452                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
 4453                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 4454         }
 4455 }
 4456 
 4457 /*
 4458  * Give control back to hardware management
 4459  * controller if there is one.
 4460  */
 4461 static void
 4462 igb_release_manageability(struct adapter *adapter)
 4463 {
 4464         if (adapter->has_manage) {
 4465                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
 4466 
 4467                 /* re-enable hardware interception of ARP */
 4468                 manc |= E1000_MANC_ARP_EN;
 4469                 manc &= ~E1000_MANC_EN_MNG2HOST;
 4470 
 4471                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 4472         }
 4473 }
 4474 
 4475 /*
 4476  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 4477  * For ASF and Pass Through versions of f/w this means that
 4478  * the driver is loaded. 
 4479  *
 4480  */
 4481 static void
 4482 igb_get_hw_control(struct adapter *adapter)
 4483 {
 4484         u32 ctrl_ext;
 4485 
 4486         /* Let firmware know the driver has taken over */
 4487         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 4488         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 4489             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
 4490 }
 4491 
 4492 /*
 4493  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 4494  * For ASF and Pass Through versions of f/w this means that the
 4495  * driver is no longer loaded.
 4496  *
 4497  */
 4498 static void
 4499 igb_release_hw_control(struct adapter *adapter)
 4500 {
 4501         u32 ctrl_ext;
 4502 
 4503         /* Let firmware taken over control of h/w */
 4504         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
 4505         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
 4506             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
 4507 }
 4508 
 4509 static int
 4510 igb_is_valid_ether_addr(uint8_t *addr)
 4511 {
 4512         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
 4513 
 4514         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
 4515                 return (FALSE);
 4516         }
 4517 
 4518         return (TRUE);
 4519 }
 4520 
 4521 
 4522 /*
 4523  * Enable PCI Wake On Lan capability
 4524  */
 4525 void
 4526 igb_enable_wakeup(device_t dev)
 4527 {
 4528         u16     cap, status;
 4529         u8      id;
 4530 
 4531         /* First find the capabilities pointer*/
 4532         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
 4533         /* Read the PM Capabilities */
 4534         id = pci_read_config(dev, cap, 1);
 4535         if (id != PCIY_PMG)     /* Something wrong */
 4536                 return;
 4537         /* OK, we have the power capabilities, so
 4538            now get the status register */
 4539         cap += PCIR_POWER_STATUS;
 4540         status = pci_read_config(dev, cap, 2);
 4541         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
 4542         pci_write_config(dev, cap, status, 2);
 4543         return;
 4544 }
 4545 
 4546 
 4547 /**********************************************************************
 4548  *
 4549  *  Update the board statistics counters.
 4550  *
 4551  **********************************************************************/
 4552 static void
 4553 igb_update_stats_counters(struct adapter *adapter)
 4554 {
 4555         struct ifnet   *ifp;
 4556 
 4557         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
 4558            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
 4559                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
 4560                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
 4561         }
 4562         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
 4563         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
 4564         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
 4565         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
 4566 
 4567         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
 4568         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
 4569         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
 4570         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
 4571         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
 4572         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
 4573         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
 4574         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
 4575         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
 4576         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
 4577         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
 4578         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
 4579         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
 4580         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
 4581         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
 4582         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
 4583         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
 4584         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
 4585         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
 4586         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
 4587 
 4588         /* For the 64-bit byte counters the low dword must be read first. */
 4589         /* Both registers clear on the read of the high dword */
 4590 
 4591         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
 4592         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
 4593 
 4594         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
 4595         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
 4596         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
 4597         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
 4598         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
 4599 
 4600         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
 4601         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
 4602 
 4603         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
 4604         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
 4605         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
 4606         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
 4607         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
 4608         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
 4609         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
 4610         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
 4611         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
 4612         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
 4613 
 4614         adapter->stats.algnerrc += 
 4615                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
 4616         adapter->stats.rxerrc += 
 4617                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
 4618         adapter->stats.tncrs += 
 4619                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
 4620         adapter->stats.cexterr += 
 4621                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
 4622         adapter->stats.tsctc += 
 4623                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
 4624         adapter->stats.tsctfc += 
 4625                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
 4626         ifp = adapter->ifp;
 4627 
 4628         ifp->if_collisions = adapter->stats.colc;
 4629 
 4630         /* Rx Errors */
 4631         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
 4632             adapter->stats.crcerrs + adapter->stats.algnerrc +
 4633             adapter->stats.ruc + adapter->stats.roc +
 4634             adapter->stats.mpc + adapter->stats.cexterr;
 4635 
 4636         /* Tx Errors */
 4637         ifp->if_oerrors = adapter->stats.ecol +
 4638             adapter->stats.latecol + adapter->watchdog_events;
 4639 }
 4640 
 4641 
 4642 /**********************************************************************
 4643  *
 4644  *  This routine is called only when igb_display_debug_stats is enabled.
 4645  *  This routine provides a way to take a look at important statistics
 4646  *  maintained by the driver and hardware.
 4647  *
 4648  **********************************************************************/
 4649 static void
 4650 igb_print_debug_info(struct adapter *adapter)
 4651 {
 4652         device_t dev = adapter->dev;
 4653         struct rx_ring *rxr = adapter->rx_rings;
 4654         struct tx_ring *txr = adapter->tx_rings;
 4655         uint8_t *hw_addr = adapter->hw.hw_addr;
 4656 
 4657         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
 4658         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
 4659             E1000_READ_REG(&adapter->hw, E1000_CTRL),
 4660             E1000_READ_REG(&adapter->hw, E1000_RCTL));
 4661 
 4662 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
 4663         device_printf(dev, "IMS = 0x%x EIMS = 0x%x \n",
 4664             E1000_READ_REG(&adapter->hw, E1000_IMS),
 4665             E1000_READ_REG(&adapter->hw, E1000_EIMS));
 4666 #endif
 4667 
 4668         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
 4669             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
 4670             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
 4671         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
 4672             adapter->hw.fc.high_water,
 4673             adapter->hw.fc.low_water);
 4674 
 4675         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 4676                 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i,
 4677                     E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
 4678                     E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
 4679                 device_printf(dev, "TX(%d) no descriptors avail event = %lld\n",
 4680                     txr->me, (long long)txr->no_desc_avail);
 4681                 device_printf(dev, "TX(%d) MSIX IRQ Handled = %lld\n", txr->me,
 4682                     (long long)txr->tx_irq);
 4683                 device_printf(dev, "TX(%d) Packets sent = %lld\n", txr->me,
 4684                     (long long)txr->tx_packets);
 4685         }
 4686 
 4687         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
 4688                 struct lro_ctrl *lro = &rxr->lro;
 4689                 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i,
 4690                     E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
 4691                     E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
 4692                 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me,
 4693                     (long long)rxr->rx_packets);
 4694                 device_printf(dev, "RX(%d) Split Packets = %lld\n", rxr->me,
 4695                     (long long)rxr->rx_split_packets);
 4696                 device_printf(dev, "RX(%d) Byte count = %lld\n", rxr->me,
 4697                     (long long)rxr->rx_bytes);
 4698                 device_printf(dev, "RX(%d) MSIX IRQ Handled = %lld\n", rxr->me,
 4699                     (long long)rxr->rx_irq);
 4700                 device_printf(dev,"RX(%d) LRO Queued= %d\n",
 4701                     rxr->me, lro->lro_queued);
 4702                 device_printf(dev,"RX(%d) LRO Flushed= %d\n",
 4703                     rxr->me, lro->lro_flushed);
 4704         }
 4705 
 4706         device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq);
 4707 
 4708         device_printf(dev, "Mbuf defrag failed = %ld\n",
 4709             adapter->mbuf_defrag_failed);
 4710         device_printf(dev, "Std mbuf header failed = %ld\n",
 4711             adapter->mbuf_header_failed);
 4712         device_printf(dev, "Std mbuf packet failed = %ld\n",
 4713             adapter->mbuf_packet_failed);
 4714         device_printf(dev, "Driver dropped packets = %ld\n",
 4715             adapter->dropped_pkts);
 4716         device_printf(dev, "Driver tx dma failure in xmit = %ld\n",
 4717                 adapter->no_tx_dma_setup);
 4718 }
 4719 
 4720 static void
 4721 igb_print_hw_stats(struct adapter *adapter)
 4722 {
 4723         device_t dev = adapter->dev;
 4724 
 4725         device_printf(dev, "Excessive collisions = %lld\n",
 4726             (long long)adapter->stats.ecol);
 4727 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
 4728         device_printf(dev, "Symbol errors = %lld\n",
 4729             (long long)adapter->stats.symerrs);
 4730 #endif
 4731         device_printf(dev, "Sequence errors = %lld\n",
 4732             (long long)adapter->stats.sec);
 4733         device_printf(dev, "Defer count = %lld\n",
 4734             (long long)adapter->stats.dc);
 4735         device_printf(dev, "Missed Packets = %lld\n",
 4736             (long long)adapter->stats.mpc);
 4737         device_printf(dev, "Receive No Buffers = %lld\n",
 4738             (long long)adapter->stats.rnbc);
 4739         /* RLEC is inaccurate on some hardware, calculate our own. */
 4740         device_printf(dev, "Receive Length Errors = %lld\n",
 4741             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
 4742         device_printf(dev, "Receive errors = %lld\n",
 4743             (long long)adapter->stats.rxerrc);
 4744         device_printf(dev, "Crc errors = %lld\n",
 4745             (long long)adapter->stats.crcerrs);
 4746         device_printf(dev, "Alignment errors = %lld\n",
 4747             (long long)adapter->stats.algnerrc);
 4748         /* On 82575 these are collision counts */
 4749         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
 4750             (long long)adapter->stats.cexterr);
 4751         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
 4752         device_printf(dev, "watchdog timeouts = %ld\n",
 4753             adapter->watchdog_events);
 4754         device_printf(dev, "XON Rcvd = %lld\n",
 4755             (long long)adapter->stats.xonrxc);
 4756         device_printf(dev, "XON Xmtd = %lld\n",
 4757             (long long)adapter->stats.xontxc);
 4758         device_printf(dev, "XOFF Rcvd = %lld\n",
 4759             (long long)adapter->stats.xoffrxc);
 4760         device_printf(dev, "XOFF Xmtd = %lld\n",
 4761             (long long)adapter->stats.xofftxc);
 4762         device_printf(dev, "Good Packets Rcvd = %lld\n",
 4763             (long long)adapter->stats.gprc);
 4764         device_printf(dev, "Good Packets Xmtd = %lld\n",
 4765             (long long)adapter->stats.gptc);
 4766         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
 4767             (long long)adapter->stats.tsctc);
 4768         device_printf(dev, "TSO Contexts Failed = %lld\n",
 4769             (long long)adapter->stats.tsctfc);
 4770 }
 4771 
 4772 /**********************************************************************
 4773  *
 4774  *  This routine provides a way to dump out the adapter eeprom,
 4775  *  often a useful debug/service tool. This only dumps the first
 4776  *  32 words, stuff that matters is in that extent.
 4777  *
 4778  **********************************************************************/
 4779 static void
 4780 igb_print_nvm_info(struct adapter *adapter)
 4781 {
 4782         u16     eeprom_data;
 4783         int     i, j, row = 0;
 4784 
 4785         /* Its a bit crude, but it gets the job done */
 4786         printf("\nInterface EEPROM Dump:\n");
 4787         printf("Offset\n0x0000  ");
 4788         for (i = 0, j = 0; i < 32; i++, j++) {
 4789                 if (j == 8) { /* Make the offset block */
 4790                         j = 0; ++row;
 4791                         printf("\n0x00%x0  ",row);
 4792                 }
 4793                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
 4794                 printf("%04x ", eeprom_data);
 4795         }
 4796         printf("\n");
 4797 }
 4798 
 4799 static int
 4800 igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
 4801 {
 4802         struct adapter *adapter;
 4803         int error;
 4804         int result;
 4805 
 4806         result = -1;
 4807         error = sysctl_handle_int(oidp, &result, 0, req);
 4808 
 4809         if (error || !req->newptr)
 4810                 return (error);
 4811 
 4812         if (result == 1) {
 4813                 adapter = (struct adapter *)arg1;
 4814                 igb_print_debug_info(adapter);
 4815         }
 4816         /*
 4817          * This value will cause a hex dump of the
 4818          * first 32 16-bit words of the EEPROM to
 4819          * the screen.
 4820          */
 4821         if (result == 2) {
 4822                 adapter = (struct adapter *)arg1;
 4823                 igb_print_nvm_info(adapter);
 4824         }
 4825 
 4826         return (error);
 4827 }
 4828 
 4829 
 4830 static int
 4831 igb_sysctl_stats(SYSCTL_HANDLER_ARGS)
 4832 {
 4833         struct adapter *adapter;
 4834         int error;
 4835         int result;
 4836 
 4837         result = -1;
 4838         error = sysctl_handle_int(oidp, &result, 0, req);
 4839 
 4840         if (error || !req->newptr)
 4841                 return (error);
 4842 
 4843         if (result == 1) {
 4844                 adapter = (struct adapter *)arg1;
 4845                 igb_print_hw_stats(adapter);
 4846         }
 4847 
 4848         return (error);
 4849 }
 4850 
 4851 static void
 4852 igb_add_rx_process_limit(struct adapter *adapter, const char *name,
 4853         const char *description, int *limit, int value)
 4854 {
 4855         *limit = value;
 4856         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
 4857             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
 4858             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
 4859 }
 4860 
 4861 #ifdef IGB_IEEE1588
 4862 /*
 4863 ** igb_hwtstamp_ioctl - control hardware time stamping
 4864 **
 4865 ** Outgoing time stamping can be enabled and disabled. Play nice and
 4866 ** disable it when requested, although it shouldn't case any overhead
 4867 ** when no packet needs it. At most one packet in the queue may be
 4868 ** marked for time stamping, otherwise it would be impossible to tell
 4869 ** for sure to which packet the hardware time stamp belongs.
 4870 **
 4871 ** Incoming time stamping has to be configured via the hardware
 4872 ** filters. Not all combinations are supported, in particular event
 4873 ** type has to be specified. Matching the kind of event packet is
 4874 ** not supported, with the exception of "all V2 events regardless of
 4875 ** level 2 or 4".
 4876 **
 4877 */
 4878 static int
 4879 igb_hwtstamp_ioctl(struct adapter *adapter, struct ifreq *ifr)
 4880 {
 4881         struct e1000_hw *hw = &adapter->hw;
 4882         struct hwtstamp_ctrl *config;
 4883         u32 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
 4884         u32 tsync_rx_ctl_bit = E1000_TSYNCRXCTL_ENABLED;
 4885         u32 tsync_rx_ctl_type = 0;
 4886         u32 tsync_rx_cfg = 0;
 4887         int is_l4 = 0;
 4888         int is_l2 = 0;
 4889         u16 port = 319; /* PTP */
 4890         u32 regval;
 4891 
 4892         config = (struct hwtstamp_ctrl *) ifr->ifr_data;
 4893 
 4894         /* reserved for future extensions */
 4895         if (config->flags)
 4896                 return (EINVAL);
 4897 
 4898         switch (config->tx_type) {
 4899         case HWTSTAMP_TX_OFF:
 4900                 tsync_tx_ctl_bit = 0;
 4901                 break;
 4902         case HWTSTAMP_TX_ON:
 4903                 tsync_tx_ctl_bit = E1000_TSYNCTXCTL_ENABLED;
 4904                 break;
 4905         default:
 4906                 return (ERANGE);
 4907         }
 4908 
 4909         switch (config->rx_filter) {
 4910         case HWTSTAMP_FILTER_NONE:
 4911                 tsync_rx_ctl_bit = 0;
 4912                 break;
 4913         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 4914         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 4915         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 4916         case HWTSTAMP_FILTER_ALL:
 4917                 /*
 4918                  * register TSYNCRXCFG must be set, therefore it is not
 4919                  * possible to time stamp both Sync and Delay_Req messages
 4920                  * => fall back to time stamping all packets
 4921                  */
 4922                 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_ALL;
 4923                 config->rx_filter = HWTSTAMP_FILTER_ALL;
 4924                 break;
 4925         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 4926                 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
 4927                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
 4928                 is_l4 = 1;
 4929                 break;
 4930         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 4931                 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L4_V1;
 4932                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
 4933                 is_l4 = 1;
 4934                 break;
 4935         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 4936         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 4937                 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
 4938                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
 4939                 is_l2 = 1;
 4940                 is_l4 = 1;
 4941                 config->rx_filter = HWTSTAMP_FILTER_SOME;
 4942                 break;
 4943         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 4944         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 4945                 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
 4946                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
 4947                 is_l2 = 1;
 4948                 is_l4 = 1;
 4949                 config->rx_filter = HWTSTAMP_FILTER_SOME;
 4950                 break;
 4951         case HWTSTAMP_FILTER_PTP_V2_EVENT:
 4952         case HWTSTAMP_FILTER_PTP_V2_SYNC:
 4953         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 4954                 tsync_rx_ctl_type = E1000_TSYNCRXCTL_TYPE_EVENT_V2;
 4955                 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 4956                 is_l2 = 1;
 4957                 break;
 4958         default:
 4959                 return -ERANGE;
 4960         }
 4961 
 4962         /* enable/disable TX */
 4963         regval = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
 4964         regval = (regval & ~E1000_TSYNCTXCTL_ENABLED) | tsync_tx_ctl_bit;
 4965         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, regval);
 4966 
 4967         /* enable/disable RX, define which PTP packets are time stamped */
 4968         regval = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
 4969         regval = (regval & ~E1000_TSYNCRXCTL_ENABLED) | tsync_rx_ctl_bit;
 4970         regval = (regval & ~0xE) | tsync_rx_ctl_type;
 4971         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, regval);
 4972         E1000_WRITE_REG(hw, E1000_TSYNCRXCFG, tsync_rx_cfg);
 4973 
 4974         /*
 4975          * Ethertype Filter Queue Filter[0][15:0] = 0x88F7
 4976          *                                          (Ethertype to filter on)
 4977          * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter)
 4978          * Ethertype Filter Queue Filter[0][30] = 0x1 (Enable Timestamping)
 4979          */
 4980         E1000_WRITE_REG(hw, E1000_ETQF0, is_l2 ? 0x440088f7 : 0);
 4981 
 4982         /* L4 Queue Filter[0]: only filter by source and destination port */
 4983         E1000_WRITE_REG(hw, E1000_SPQF0, htons(port));
 4984         E1000_WRITE_REG(hw, E1000_IMIREXT(0), is_l4 ?
 4985              ((1<<12) | (1<<19) /* bypass size and control flags */) : 0);
 4986         E1000_WRITE_REG(hw, E1000_IMIR(0), is_l4 ?
 4987              (htons(port)
 4988               | (0<<16) /* immediate interrupt disabled */
 4989               | 0 /* (1<<17) bit cleared: do not bypass
 4990                      destination port check */)
 4991                 : 0);
 4992         E1000_WRITE_REG(hw, E1000_FTQF0, is_l4 ?
 4993              (0x11 /* UDP */
 4994               | (1<<15) /* VF not compared */
 4995               | (1<<27) /* Enable Timestamping */
 4996               | (7<<28) /* only source port filter enabled,
 4997                            source/target address and protocol
 4998                            masked */)
 4999              : ((1<<15) | (15<<28) /* all mask bits set = filter not
 5000                                       enabled */));
 5001 
 5002         wrfl();
 5003 
 5004         adapter->hwtstamp_ctrl = config;
 5005 
 5006         /* clear TX/RX time stamp registers, just to be sure */
 5007         regval = E1000_READ_REG(hw, E1000_TXSTMPH);
 5008         regval = E1000_READ_REG(hw, E1000_RXSTMPH);
 5009 
 5010         return (error);
 5011 }
 5012 
 5013 /*
 5014 ** igb_read_clock - read raw cycle counter (to be used by time counter)
 5015 */
 5016 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 5017 {
 5018        struct igb_adapter *adapter =
 5019                container_of(tc, struct igb_adapter, cycles);
 5020        struct e1000_hw *hw = &adapter->hw;
 5021        u64 stamp;
 5022 
 5023        stamp =  E1000_READ_REG(hw, E1000_SYSTIML);
 5024        stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIMH) << 32ULL;
 5025 
 5026        return (stamp);
 5027 }
 5028 
 5029 #endif /* IGB_IEEE1588 */

Cache object: 52215326180db0a3d26fbccb084ac3c0


[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]


This page is part of the FreeBSD/Linux Linux Kernel Cross-Reference, and was automatically generated using a modified version of the LXR engine.