Now available: The Design and Implementation of the FreeBSD Operating System (Second Edition)
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]
FreeBSD/Linux Kernel Cross Reference
sys/dev/e1000/if_igb.c
Version:
- FREEBSD - FREEBSD-13-STABLE - FREEBSD-13-0 - FREEBSD-12-STABLE - FREEBSD-12-0 - FREEBSD-11-STABLE - FREEBSD-11-0 - FREEBSD-10-STABLE - FREEBSD-10-0 - FREEBSD-9-STABLE - FREEBSD-9-0 - FREEBSD-8-STABLE - FREEBSD-8-0 - FREEBSD-7-STABLE - FREEBSD-7-0 - FREEBSD-6-STABLE - FREEBSD-6-0 - FREEBSD-5-STABLE - FREEBSD-5-0 - FREEBSD-4-STABLE - FREEBSD-3-STABLE - FREEBSD22 - l41 - OPENBSD - linux-2.6 - MK84 - PLAN9 - xnu-8792
SearchContext: - none - 3 - 10
SearchContext: - none - 3 - 10
1 /****************************************************************************** 2 3 Copyright (c) 2001-2008, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifdef HAVE_KERNEL_OPTION_HEADERS 36 #include "opt_device_polling.h" 37 #include "opt_inet.h" 38 #endif 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/bus.h> 43 #include <sys/endian.h> 44 #include <sys/kernel.h> 45 #include <sys/kthread.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/module.h> 49 #include <sys/rman.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <sys/taskqueue.h> 54 #include <sys/eventhandler.h> 55 #include <sys/pcpu.h> 56 #ifdef IGB_TIMESYNC 57 #include <sys/ioccom.h> 58 #include <sys/time.h> 59 #endif 60 #include <machine/bus.h> 61 #include <machine/resource.h> 62 63 #include <net/bpf.h> 64 #include <net/ethernet.h> 65 #include <net/if.h> 66 #include <net/if_arp.h> 67 #include <net/if_dl.h> 68 #include <net/if_media.h> 69 70 #include <net/if_types.h> 71 #include <net/if_vlan_var.h> 72 73 #include <netinet/in_systm.h> 74 #include <netinet/in.h> 75 #include <netinet/if_ether.h> 76 #include <netinet/ip.h> 77 #include <netinet/ip6.h> 78 #include <netinet/tcp.h> 79 #include <netinet/tcp_lro.h> 80 #include <netinet/udp.h> 81 82 #include <machine/in_cksum.h> 83 #include <dev/pci/pcivar.h> 84 #include <dev/pci/pcireg.h> 85 86 #include "e1000_api.h" 87 #include "e1000_82575.h" 88 #include "if_igb.h" 89 90 /********************************************************************* 91 * Set this to one to display debug statistics 92 *********************************************************************/ 93 int igb_display_debug_stats = 0; 94 95 /********************************************************************* 96 * Driver version: 97 *********************************************************************/ 98 char igb_driver_version[] = "version - 1.4.1"; 99 100 101 /********************************************************************* 102 * PCI Device ID Table 103 * 104 * Used by probe to select devices to load on 105 * Last field stores an index into e1000_strings 106 * Last entry must be all 0s 107 * 108 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 109 *********************************************************************/ 110 111 static igb_vendor_info_t igb_vendor_info_array[] = 112 { 113 { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 115 PCI_ANY_ID, PCI_ANY_ID, 0}, 116 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 117 PCI_ANY_ID, PCI_ANY_ID, 0}, 118 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0}, 119 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 121 /* required last entry */ 122 { 0, 0, 0, 0, 0} 123 }; 124 125 /********************************************************************* 126 * Table of branding strings for all supported NICs. 127 *********************************************************************/ 128 129 static char *igb_strings[] = { 130 "Intel(R) PRO/1000 Network Connection" 131 }; 132 133 /********************************************************************* 134 * Function prototypes 135 *********************************************************************/ 136 static int igb_probe(device_t); 137 static int igb_attach(device_t); 138 static int igb_detach(device_t); 139 static int igb_shutdown(device_t); 140 static int igb_suspend(device_t); 141 static int igb_resume(device_t); 142 static void igb_start(struct ifnet *); 143 static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); 144 static int igb_ioctl(struct ifnet *, u_long, caddr_t); 145 static void igb_watchdog(struct adapter *); 146 static void igb_init(void *); 147 static void igb_init_locked(struct adapter *); 148 static void igb_stop(void *); 149 static void igb_media_status(struct ifnet *, struct ifmediareq *); 150 static int igb_media_change(struct ifnet *); 151 static void igb_identify_hardware(struct adapter *); 152 static int igb_allocate_pci_resources(struct adapter *); 153 static int igb_allocate_msix(struct adapter *); 154 static int igb_allocate_legacy(struct adapter *); 155 static int igb_setup_msix(struct adapter *); 156 static void igb_free_pci_resources(struct adapter *); 157 static void igb_local_timer(void *); 158 static int igb_hardware_init(struct adapter *); 159 static void igb_setup_interface(device_t, struct adapter *); 160 static int igb_allocate_queues(struct adapter *); 161 static void igb_configure_queues(struct adapter *); 162 163 static int igb_allocate_transmit_buffers(struct tx_ring *); 164 static void igb_setup_transmit_structures(struct adapter *); 165 static void igb_setup_transmit_ring(struct tx_ring *); 166 static void igb_initialize_transmit_units(struct adapter *); 167 static void igb_free_transmit_structures(struct adapter *); 168 static void igb_free_transmit_buffers(struct tx_ring *); 169 170 static int igb_allocate_receive_buffers(struct rx_ring *); 171 static int igb_setup_receive_structures(struct adapter *); 172 static int igb_setup_receive_ring(struct rx_ring *); 173 static void igb_initialize_receive_units(struct adapter *); 174 static void igb_free_receive_structures(struct adapter *); 175 static void igb_free_receive_buffers(struct rx_ring *); 176 177 static void igb_enable_intr(struct adapter *); 178 static void igb_disable_intr(struct adapter *); 179 static void igb_update_stats_counters(struct adapter *); 180 static bool igb_txeof(struct tx_ring *); 181 static bool igb_rxeof(struct rx_ring *, int); 182 #ifndef __NO_STRICT_ALIGNMENT 183 static int igb_fixup_rx(struct rx_ring *); 184 #endif 185 static void igb_rx_checksum(u32, struct mbuf *); 186 static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *); 187 static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 188 static void igb_set_promisc(struct adapter *); 189 static void igb_disable_promisc(struct adapter *); 190 static void igb_set_multi(struct adapter *); 191 static void igb_print_hw_stats(struct adapter *); 192 static void igb_update_link_status(struct adapter *); 193 static int igb_get_buf(struct rx_ring *, int); 194 195 #ifdef IGB_HW_VLAN_SUPPORT 196 static void igb_register_vlan(void *, struct ifnet *, u16); 197 static void igb_unregister_vlan(void *, struct ifnet *, u16); 198 #endif 199 200 static int igb_xmit(struct tx_ring *, struct mbuf **); 201 static int igb_dma_malloc(struct adapter *, bus_size_t, 202 struct igb_dma_alloc *, int); 203 static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); 204 static void igb_print_debug_info(struct adapter *); 205 static void igb_print_nvm_info(struct adapter *); 206 static int igb_is_valid_ether_addr(u8 *); 207 static int igb_sysctl_stats(SYSCTL_HANDLER_ARGS); 208 static int igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 209 /* Management and WOL Support */ 210 static void igb_init_manageability(struct adapter *); 211 static void igb_release_manageability(struct adapter *); 212 static void igb_get_hw_control(struct adapter *); 213 static void igb_release_hw_control(struct adapter *); 214 static void igb_enable_wakeup(device_t); 215 216 #ifdef IGB_TIMESYNC 217 /* Precision Time sync support */ 218 static int igb_tsync_init(struct adapter *); 219 static void igb_tsync_disable(struct adapter *); 220 #endif 221 222 static int igb_irq_fast(void *); 223 static void igb_add_rx_process_limit(struct adapter *, const char *, 224 const char *, int *, int); 225 static void igb_handle_rxtx(void *context, int pending); 226 static void igb_handle_tx(void *context, int pending); 227 static void igb_handle_rx(void *context, int pending); 228 static void igb_handle_link(void *context, int pending); 229 230 /* These are MSIX only irq handlers */ 231 static void igb_msix_rx(void *); 232 static void igb_msix_tx(void *); 233 static void igb_msix_link(void *); 234 235 /* Adaptive Interrupt Moderation */ 236 static void igb_update_aim(struct rx_ring *); 237 238 /********************************************************************* 239 * FreeBSD Device Interface Entry Points 240 *********************************************************************/ 241 242 static device_method_t igb_methods[] = { 243 /* Device interface */ 244 DEVMETHOD(device_probe, igb_probe), 245 DEVMETHOD(device_attach, igb_attach), 246 DEVMETHOD(device_detach, igb_detach), 247 DEVMETHOD(device_shutdown, igb_shutdown), 248 DEVMETHOD(device_suspend, igb_suspend), 249 DEVMETHOD(device_resume, igb_resume), 250 {0, 0} 251 }; 252 253 static driver_t igb_driver = { 254 "igb", igb_methods, sizeof(struct adapter), 255 }; 256 257 static devclass_t igb_devclass; 258 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); 259 MODULE_DEPEND(igb, pci, 1, 1, 1); 260 MODULE_DEPEND(igb, ether, 1, 1, 1); 261 262 /********************************************************************* 263 * Tunable default values. 264 *********************************************************************/ 265 266 /* Descriptor defaults */ 267 static int igb_rxd = IGB_DEFAULT_RXD; 268 static int igb_txd = IGB_DEFAULT_TXD; 269 TUNABLE_INT("hw.igb.rxd", &igb_rxd); 270 TUNABLE_INT("hw.igb.txd", &igb_txd); 271 272 /* 273 ** These parameters are used in Adaptive 274 ** Interrupt Moderation. The value is set 275 ** into EITR and controls the interrupt 276 ** frequency. They can be modified but 277 ** be careful in tuning them. 278 */ 279 static int igb_enable_aim = TRUE; 280 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); 281 static int igb_low_latency = IGB_LOW_LATENCY; 282 TUNABLE_INT("hw.igb.low_latency", &igb_low_latency); 283 static int igb_ave_latency = IGB_AVE_LATENCY; 284 TUNABLE_INT("hw.igb.ave_latency", &igb_ave_latency); 285 static int igb_bulk_latency = IGB_BULK_LATENCY; 286 TUNABLE_INT("hw.igb.bulk_latency", &igb_bulk_latency); 287 288 /* 289 ** IF YOU CHANGE THESE: be sure and change IGB_MSIX_VEC in 290 ** if_igb.h to match. These can be autoconfigured if set to 291 ** 0, it will then be based on number of cpus. 292 */ 293 static int igb_tx_queues = 1; 294 static int igb_rx_queues = 1; 295 TUNABLE_INT("hw.igb.tx_queues", &igb_tx_queues); 296 TUNABLE_INT("hw.igb.rx_queues", &igb_rx_queues); 297 298 /* How many packets rxeof tries to clean at a time */ 299 static int igb_rx_process_limit = 100; 300 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); 301 302 /* Flow control setting - default to none */ 303 static int igb_fc_setting = 0; 304 TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting); 305 306 /* 307 * Should the driver do LRO on the RX end 308 * this can be toggled on the fly, but the 309 * interface must be reset (down/up) for it 310 * to take effect. 311 */ 312 static int igb_enable_lro = 1; 313 TUNABLE_INT("hw.igb.enable_lro", &igb_enable_lro); 314 315 extern int mp_ncpus; 316 /********************************************************************* 317 * Device identification routine 318 * 319 * igb_probe determines if the driver should be loaded on 320 * adapter based on PCI vendor/device id of the adapter. 321 * 322 * return BUS_PROBE_DEFAULT on success, positive on failure 323 *********************************************************************/ 324 325 static int 326 igb_probe(device_t dev) 327 { 328 char adapter_name[60]; 329 uint16_t pci_vendor_id = 0; 330 uint16_t pci_device_id = 0; 331 uint16_t pci_subvendor_id = 0; 332 uint16_t pci_subdevice_id = 0; 333 igb_vendor_info_t *ent; 334 335 INIT_DEBUGOUT("igb_probe: begin"); 336 337 pci_vendor_id = pci_get_vendor(dev); 338 if (pci_vendor_id != IGB_VENDOR_ID) 339 return (ENXIO); 340 341 pci_device_id = pci_get_device(dev); 342 pci_subvendor_id = pci_get_subvendor(dev); 343 pci_subdevice_id = pci_get_subdevice(dev); 344 345 ent = igb_vendor_info_array; 346 while (ent->vendor_id != 0) { 347 if ((pci_vendor_id == ent->vendor_id) && 348 (pci_device_id == ent->device_id) && 349 350 ((pci_subvendor_id == ent->subvendor_id) || 351 (ent->subvendor_id == PCI_ANY_ID)) && 352 353 ((pci_subdevice_id == ent->subdevice_id) || 354 (ent->subdevice_id == PCI_ANY_ID))) { 355 sprintf(adapter_name, "%s %s", 356 igb_strings[ent->index], 357 igb_driver_version); 358 device_set_desc_copy(dev, adapter_name); 359 return (BUS_PROBE_DEFAULT); 360 } 361 ent++; 362 } 363 364 return (ENXIO); 365 } 366 367 /********************************************************************* 368 * Device initialization routine 369 * 370 * The attach entry point is called when the driver is being loaded. 371 * This routine identifies the type of hardware, allocates all resources 372 * and initializes the hardware. 373 * 374 * return 0 on success, positive on failure 375 *********************************************************************/ 376 377 static int 378 igb_attach(device_t dev) 379 { 380 struct adapter *adapter; 381 int error = 0; 382 u16 eeprom_data; 383 384 INIT_DEBUGOUT("igb_attach: begin"); 385 386 adapter = device_get_softc(dev); 387 adapter->dev = adapter->osdep.dev = dev; 388 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 389 390 /* SYSCTL stuff */ 391 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 392 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 393 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 394 igb_sysctl_debug_info, "I", "Debug Information"); 395 396 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 397 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 398 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 399 igb_sysctl_stats, "I", "Statistics"); 400 401 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 402 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 403 OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW, 404 &igb_fc_setting, 0, "Flow Control"); 405 406 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 407 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 408 OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, 409 &igb_enable_lro, 0, "Large Receive Offload"); 410 411 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 412 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 413 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 414 &igb_enable_aim, 1, "Interrupt Moderation"); 415 416 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 417 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 418 OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW, 419 &igb_low_latency, 1, "Low Latency"); 420 421 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 422 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 423 OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW, 424 &igb_ave_latency, 1, "Average Latency"); 425 426 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 427 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 428 OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW, 429 &igb_bulk_latency, 1, "Bulk Latency"); 430 431 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 432 433 /* Determine hardware and mac info */ 434 igb_identify_hardware(adapter); 435 436 /* Setup PCI resources */ 437 if (igb_allocate_pci_resources(adapter)) { 438 device_printf(dev, "Allocation of PCI resources failed\n"); 439 error = ENXIO; 440 goto err_pci; 441 } 442 443 /* Do Shared Code initialization */ 444 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 445 device_printf(dev, "Setup of Shared code failed\n"); 446 error = ENXIO; 447 goto err_pci; 448 } 449 450 e1000_get_bus_info(&adapter->hw); 451 452 /* Sysctls for limiting the amount of work done in the taskqueue */ 453 igb_add_rx_process_limit(adapter, "rx_processing_limit", 454 "max number of rx packets to process", &adapter->rx_process_limit, 455 igb_rx_process_limit); 456 457 /* 458 * Validate number of transmit and receive descriptors. It 459 * must not exceed hardware maximum, and must be multiple 460 * of E1000_DBA_ALIGN. 461 */ 462 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || 463 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { 464 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 465 IGB_DEFAULT_TXD, igb_txd); 466 adapter->num_tx_desc = IGB_DEFAULT_TXD; 467 } else 468 adapter->num_tx_desc = igb_txd; 469 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || 470 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { 471 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 472 IGB_DEFAULT_RXD, igb_rxd); 473 adapter->num_rx_desc = IGB_DEFAULT_RXD; 474 } else 475 adapter->num_rx_desc = igb_rxd; 476 477 adapter->hw.mac.autoneg = DO_AUTO_NEG; 478 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 479 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 480 adapter->rx_buffer_len = 2048; 481 482 /* Copper options */ 483 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 484 adapter->hw.phy.mdix = AUTO_ALL_MODES; 485 adapter->hw.phy.disable_polarity_correction = FALSE; 486 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; 487 } 488 489 /* 490 * Set the frame limits assuming 491 * standard ethernet sized frames. 492 */ 493 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 494 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 495 496 /* 497 ** Allocate and Setup Queues 498 */ 499 if (igb_allocate_queues(adapter)) { 500 error = ENOMEM; 501 goto err_pci; 502 } 503 504 /* Make sure we have a good EEPROM before we read from it */ 505 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 506 /* 507 ** Some PCI-E parts fail the first check due to 508 ** the link being in sleep state, call it again, 509 ** if it fails a second time its a real issue. 510 */ 511 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 512 device_printf(dev, 513 "The EEPROM Checksum Is Not Valid\n"); 514 error = EIO; 515 goto err_late; 516 } 517 } 518 519 /* Initialize the hardware */ 520 if (igb_hardware_init(adapter)) { 521 device_printf(dev, "Unable to initialize the hardware\n"); 522 error = EIO; 523 goto err_late; 524 } 525 526 /* Copy the permanent MAC address out of the EEPROM */ 527 if (e1000_read_mac_addr(&adapter->hw) < 0) { 528 device_printf(dev, "EEPROM read error while reading MAC" 529 " address\n"); 530 error = EIO; 531 goto err_late; 532 } 533 534 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { 535 device_printf(dev, "Invalid MAC address\n"); 536 error = EIO; 537 goto err_late; 538 } 539 540 /* 541 ** Configure Interrupts 542 */ 543 if (adapter->msix > 1) /* MSIX */ 544 error = igb_allocate_msix(adapter); 545 else /* MSI or Legacy */ 546 error = igb_allocate_legacy(adapter); 547 if (error) 548 goto err_late; 549 550 /* Setup OS specific network interface */ 551 igb_setup_interface(dev, adapter); 552 553 /* Initialize statistics */ 554 igb_update_stats_counters(adapter); 555 556 adapter->hw.mac.get_link_status = 1; 557 igb_update_link_status(adapter); 558 559 /* Indicate SOL/IDER usage */ 560 if (e1000_check_reset_block(&adapter->hw)) 561 device_printf(dev, 562 "PHY reset is blocked due to SOL/IDER session.\n"); 563 564 /* Determine if we have to control management hardware */ 565 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 566 567 /* 568 * Setup Wake-on-Lan 569 */ 570 /* APME bit in EEPROM is mapped to WUC.APME */ 571 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; 572 if (eeprom_data) 573 adapter->wol = E1000_WUFC_MAG; 574 575 #ifdef IGB_HW_VLAN_SUPPORT 576 /* Register for VLAN events */ 577 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 578 igb_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 579 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 580 igb_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 581 #endif 582 583 /* Tell the stack that the interface is not active */ 584 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 585 586 INIT_DEBUGOUT("igb_attach: end"); 587 588 return (0); 589 590 err_late: 591 igb_free_transmit_structures(adapter); 592 igb_free_receive_structures(adapter); 593 igb_release_hw_control(adapter); 594 err_pci: 595 igb_free_pci_resources(adapter); 596 IGB_CORE_LOCK_DESTROY(adapter); 597 598 return (error); 599 } 600 601 /********************************************************************* 602 * Device removal routine 603 * 604 * The detach entry point is called when the driver is being removed. 605 * This routine stops the adapter and deallocates all the resources 606 * that were allocated for driver operation. 607 * 608 * return 0 on success, positive on failure 609 *********************************************************************/ 610 611 static int 612 igb_detach(device_t dev) 613 { 614 struct adapter *adapter = device_get_softc(dev); 615 struct ifnet *ifp = adapter->ifp; 616 617 INIT_DEBUGOUT("igb_detach: begin"); 618 619 /* Make sure VLANS are not using driver */ 620 if (adapter->ifp->if_vlantrunk != NULL) { 621 device_printf(dev,"Vlan in use, detach first\n"); 622 return (EBUSY); 623 } 624 625 IGB_CORE_LOCK(adapter); 626 adapter->in_detach = 1; 627 igb_stop(adapter); 628 IGB_CORE_UNLOCK(adapter); 629 630 e1000_phy_hw_reset(&adapter->hw); 631 632 /* Give control back to firmware */ 633 igb_release_manageability(adapter); 634 igb_release_hw_control(adapter); 635 636 if (adapter->wol) { 637 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 638 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 639 igb_enable_wakeup(dev); 640 } 641 642 #ifdef IGB_HW_VLAN_SUPPORT 643 /* Unregister VLAN events */ 644 if (adapter->vlan_attach != NULL) 645 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 646 if (adapter->vlan_detach != NULL) 647 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 648 #endif 649 650 ether_ifdetach(adapter->ifp); 651 652 callout_drain(&adapter->timer); 653 654 igb_free_pci_resources(adapter); 655 bus_generic_detach(dev); 656 if_free(ifp); 657 658 igb_free_transmit_structures(adapter); 659 igb_free_receive_structures(adapter); 660 661 IGB_CORE_LOCK_DESTROY(adapter); 662 663 return (0); 664 } 665 666 /********************************************************************* 667 * 668 * Shutdown entry point 669 * 670 **********************************************************************/ 671 672 static int 673 igb_shutdown(device_t dev) 674 { 675 return igb_suspend(dev); 676 } 677 678 /* 679 * Suspend/resume device methods. 680 */ 681 static int 682 igb_suspend(device_t dev) 683 { 684 struct adapter *adapter = device_get_softc(dev); 685 686 IGB_CORE_LOCK(adapter); 687 688 igb_stop(adapter); 689 690 igb_release_manageability(adapter); 691 igb_release_hw_control(adapter); 692 693 if (adapter->wol) { 694 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 695 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 696 igb_enable_wakeup(dev); 697 } 698 699 IGB_CORE_UNLOCK(adapter); 700 701 return bus_generic_suspend(dev); 702 } 703 704 static int 705 igb_resume(device_t dev) 706 { 707 struct adapter *adapter = device_get_softc(dev); 708 struct ifnet *ifp = adapter->ifp; 709 710 IGB_CORE_LOCK(adapter); 711 igb_init_locked(adapter); 712 igb_init_manageability(adapter); 713 714 if ((ifp->if_flags & IFF_UP) && 715 (ifp->if_drv_flags & IFF_DRV_RUNNING)) 716 igb_start(ifp); 717 718 IGB_CORE_UNLOCK(adapter); 719 720 return bus_generic_resume(dev); 721 } 722 723 724 /********************************************************************* 725 * Transmit entry point 726 * 727 * igb_start is called by the stack to initiate a transmit. 728 * The driver will remain in this routine as long as there are 729 * packets to transmit and transmit resources are available. 730 * In case resources are not available stack is notified and 731 * the packet is requeued. 732 **********************************************************************/ 733 734 static void 735 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) 736 { 737 struct adapter *adapter = ifp->if_softc; 738 struct mbuf *m_head; 739 740 IGB_TX_LOCK_ASSERT(txr); 741 742 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 743 IFF_DRV_RUNNING) 744 return; 745 if (!adapter->link_active) 746 return; 747 748 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 749 750 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 751 if (m_head == NULL) 752 break; 753 /* 754 * Encapsulation can modify our pointer, and or make it 755 * NULL on failure. In that event, we can't requeue. 756 */ 757 if (igb_xmit(txr, &m_head)) { 758 if (m_head == NULL) 759 break; 760 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 761 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 762 break; 763 } 764 765 /* Send a copy of the frame to the BPF listener */ 766 ETHER_BPF_MTAP(ifp, m_head); 767 768 /* Set timeout in case hardware has problems transmitting. */ 769 txr->watchdog_timer = IGB_TX_TIMEOUT; 770 } 771 } 772 773 static void 774 igb_start(struct ifnet *ifp) 775 { 776 struct adapter *adapter = ifp->if_softc; 777 struct tx_ring *txr; 778 u32 queue = 0; 779 780 /* 781 ** This is really just here for testing 782 ** TX multiqueue, ultimately what is 783 ** needed is the flow support in the stack 784 ** and appropriate logic here to deal with 785 ** it. -jfv 786 */ 787 if (adapter->num_tx_queues > 1) 788 queue = (curcpu % adapter->num_tx_queues); 789 790 txr = &adapter->tx_rings[queue]; 791 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 792 IGB_TX_LOCK(txr); 793 igb_start_locked(txr, ifp); 794 IGB_TX_UNLOCK(txr); 795 } 796 } 797 798 /********************************************************************* 799 * Ioctl entry point 800 * 801 * igb_ioctl is called when the user wants to configure the 802 * interface. 803 * 804 * return 0 on success, positive on failure 805 **********************************************************************/ 806 807 static int 808 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 809 { 810 struct adapter *adapter = ifp->if_softc; 811 struct ifreq *ifr = (struct ifreq *)data; 812 struct ifaddr *ifa = (struct ifaddr *)data; 813 int error = 0; 814 815 if (adapter->in_detach) 816 return (error); 817 818 switch (command) { 819 case SIOCSIFADDR: 820 if (ifa->ifa_addr->sa_family == AF_INET) { 821 /* 822 * XXX 823 * Since resetting hardware takes a very long time 824 * and results in link renegotiation we only 825 * initialize the hardware only when it is absolutely 826 * required. 827 */ 828 ifp->if_flags |= IFF_UP; 829 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 830 IGB_CORE_LOCK(adapter); 831 igb_init_locked(adapter); 832 IGB_CORE_UNLOCK(adapter); 833 } 834 arp_ifinit(ifp, ifa); 835 } else 836 error = ether_ioctl(ifp, command, data); 837 break; 838 case SIOCSIFMTU: 839 { 840 int max_frame_size; 841 842 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 843 844 IGB_CORE_LOCK(adapter); 845 max_frame_size = 9234; 846 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 847 ETHER_CRC_LEN) { 848 IGB_CORE_UNLOCK(adapter); 849 error = EINVAL; 850 break; 851 } 852 853 ifp->if_mtu = ifr->ifr_mtu; 854 adapter->max_frame_size = 855 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 856 igb_init_locked(adapter); 857 IGB_CORE_UNLOCK(adapter); 858 break; 859 } 860 case SIOCSIFFLAGS: 861 IOCTL_DEBUGOUT("ioctl rcv'd:\ 862 SIOCSIFFLAGS (Set Interface Flags)"); 863 IGB_CORE_LOCK(adapter); 864 if (ifp->if_flags & IFF_UP) { 865 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 866 if ((ifp->if_flags ^ adapter->if_flags) & 867 (IFF_PROMISC | IFF_ALLMULTI)) { 868 igb_disable_promisc(adapter); 869 igb_set_promisc(adapter); 870 } 871 } else 872 igb_init_locked(adapter); 873 } else 874 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 875 igb_stop(adapter); 876 adapter->if_flags = ifp->if_flags; 877 IGB_CORE_UNLOCK(adapter); 878 break; 879 case SIOCADDMULTI: 880 case SIOCDELMULTI: 881 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 882 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 883 IGB_CORE_LOCK(adapter); 884 igb_disable_intr(adapter); 885 igb_set_multi(adapter); 886 igb_enable_intr(adapter); 887 IGB_CORE_UNLOCK(adapter); 888 } 889 break; 890 case SIOCSIFMEDIA: 891 /* Check SOL/IDER usage */ 892 IGB_CORE_LOCK(adapter); 893 if (e1000_check_reset_block(&adapter->hw)) { 894 IGB_CORE_UNLOCK(adapter); 895 device_printf(adapter->dev, "Media change is" 896 " blocked due to SOL/IDER session.\n"); 897 break; 898 } 899 IGB_CORE_UNLOCK(adapter); 900 case SIOCGIFMEDIA: 901 IOCTL_DEBUGOUT("ioctl rcv'd: \ 902 SIOCxIFMEDIA (Get/Set Interface Media)"); 903 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 904 break; 905 case SIOCSIFCAP: 906 { 907 int mask, reinit; 908 909 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 910 reinit = 0; 911 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 912 if (mask & IFCAP_HWCSUM) { 913 ifp->if_capenable ^= IFCAP_HWCSUM; 914 reinit = 1; 915 } 916 if (mask & IFCAP_TSO4) { 917 ifp->if_capenable ^= IFCAP_TSO4; 918 reinit = 1; 919 } 920 if (mask & IFCAP_VLAN_HWTAGGING) { 921 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 922 reinit = 1; 923 } 924 #ifdef IGB_HW_VLAN_SUPPORT 925 if (mask & IFCAP_VLAN_HWFILTER) { 926 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 927 reinit = 1; 928 } 929 #endif 930 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 931 igb_init(adapter); 932 VLAN_CAPABILITIES(ifp); 933 break; 934 } 935 936 #ifdef IGB_TIMESYNC 937 /* 938 ** IOCTL support for Precision Time (IEEE 1588) Support 939 */ 940 case IGB_TIMESYNC_READTS: 941 { 942 u32 rx_ctl, tx_ctl; 943 struct igb_tsync_read *tdata; 944 945 tdata = (struct igb_tsync_read *) ifr->ifr_data; 946 947 if (tdata->read_current_time) { 948 getnanotime(&tdata->system_time); 949 tdata->network_time = E1000_READ_REG(&adapter->hw, 950 E1000_SYSTIML); 951 tdata->network_time |= 952 (u64)E1000_READ_REG(&adapter->hw, 953 E1000_SYSTIMH ) << 32; 954 } 955 956 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 957 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 958 959 if (rx_ctl & 0x1) { 960 u32 tmp; 961 unsigned char *tmp_cp; 962 963 tdata->rx_valid = 1; 964 tdata->rx_stamp = E1000_READ_REG(&adapter->hw, E1000_RXSTMPL); 965 tdata->rx_stamp |= (u64)E1000_READ_REG(&adapter->hw, 966 E1000_RXSTMPH) << 32; 967 968 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRL); 969 tmp_cp = (unsigned char *) &tmp; 970 tdata->srcid[0] = tmp_cp[0]; 971 tdata->srcid[1] = tmp_cp[1]; 972 tdata->srcid[2] = tmp_cp[2]; 973 tdata->srcid[3] = tmp_cp[3]; 974 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 975 tmp_cp = (unsigned char *) &tmp; 976 tdata->srcid[4] = tmp_cp[0]; 977 tdata->srcid[5] = tmp_cp[1]; 978 tdata->seqid = tmp >> 16; 979 tdata->seqid = htons(tdata->seqid); 980 } else 981 tdata->rx_valid = 0; 982 983 if (tx_ctl & 0x1) { 984 tdata->tx_valid = 1; 985 tdata->tx_stamp = E1000_READ_REG(&adapter->hw, E1000_TXSTMPL); 986 tdata->tx_stamp |= (u64) E1000_READ_REG(&adapter->hw, 987 E1000_TXSTMPH) << 32; 988 } else 989 tdata->tx_valid = 0; 990 991 return (0); 992 } 993 #endif /* IGB_TIMESYNC */ 994 995 default: 996 error = ether_ioctl(ifp, command, data); 997 break; 998 } 999 1000 return (error); 1001 } 1002 1003 /********************************************************************* 1004 * Watchdog timer: 1005 * 1006 * This routine is called from the local timer every second. 1007 * As long as transmit descriptors are being cleaned the value 1008 * is non-zero and we do nothing. Reaching 0 indicates a tx hang 1009 * and we then reset the device. 1010 * 1011 **********************************************************************/ 1012 1013 static void 1014 igb_watchdog(struct adapter *adapter) 1015 { 1016 struct tx_ring *txr = adapter->tx_rings; 1017 bool tx_hang = FALSE; 1018 1019 IGB_CORE_LOCK_ASSERT(adapter); 1020 1021 /* 1022 ** The timer is set to 5 every time start() queues a packet. 1023 ** Then txeof keeps resetting it as long as it cleans at 1024 ** least one descriptor. 1025 ** Finally, anytime all descriptors are clean the timer is 1026 ** set to 0. 1027 ** 1028 ** With TX Multiqueue we need to check every queue's timer, 1029 ** if any time out we do the reset. 1030 */ 1031 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1032 IGB_TX_LOCK(txr); 1033 if (txr->watchdog_timer == 0 || 1034 (--txr->watchdog_timer)) { 1035 IGB_TX_UNLOCK(txr); 1036 continue; 1037 } else { 1038 tx_hang = TRUE; 1039 IGB_TX_UNLOCK(txr); 1040 break; 1041 } 1042 } 1043 if (tx_hang == FALSE) 1044 return; 1045 1046 /* If we are in this routine because of pause frames, then 1047 * don't reset the hardware. 1048 */ 1049 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 1050 E1000_STATUS_TXOFF) { 1051 txr = adapter->tx_rings; /* reset pointer */ 1052 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1053 IGB_TX_LOCK(txr); 1054 txr->watchdog_timer = IGB_TX_TIMEOUT; 1055 IGB_TX_UNLOCK(txr); 1056 } 1057 return; 1058 } 1059 1060 if (e1000_check_for_link(&adapter->hw) == 0) 1061 device_printf(adapter->dev, "watchdog timeout -- resetting\n"); 1062 1063 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1064 device_printf(adapter->dev, "Queue(%d) tdh = %d, tdt = %d\n", 1065 i, E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 1066 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 1067 device_printf(adapter->dev, "Queue(%d) desc avail = %d," 1068 " Next Desc to Clean = %d\n", i, txr->tx_avail, 1069 txr->next_to_clean); 1070 } 1071 1072 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1073 adapter->watchdog_events++; 1074 1075 igb_init_locked(adapter); 1076 } 1077 1078 /********************************************************************* 1079 * Init entry point 1080 * 1081 * This routine is used in two ways. It is used by the stack as 1082 * init entry point in network interface structure. It is also used 1083 * by the driver as a hw/sw initialization routine to get to a 1084 * consistent state. 1085 * 1086 * return 0 on success, positive on failure 1087 **********************************************************************/ 1088 1089 static void 1090 igb_init_locked(struct adapter *adapter) 1091 { 1092 struct rx_ring *rxr = adapter->rx_rings; 1093 struct tx_ring *txr = adapter->tx_rings; 1094 struct ifnet *ifp = adapter->ifp; 1095 device_t dev = adapter->dev; 1096 u32 pba = 0; 1097 1098 INIT_DEBUGOUT("igb_init: begin"); 1099 1100 IGB_CORE_LOCK_ASSERT(adapter); 1101 1102 igb_stop(adapter); 1103 1104 /* 1105 * Packet Buffer Allocation (PBA) 1106 * Writing PBA sets the receive portion of the buffer 1107 * the remainder is used for the transmit buffer. 1108 */ 1109 if (adapter->hw.mac.type == e1000_82575) { 1110 INIT_DEBUGOUT1("igb_init: pba=%dK",pba); 1111 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1112 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1113 } 1114 1115 /* Get the latest mac address, User can use a LAA */ 1116 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1117 ETHER_ADDR_LEN); 1118 1119 /* Put the address into the Receive Address Array */ 1120 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1121 1122 /* Initialize the hardware */ 1123 if (igb_hardware_init(adapter)) { 1124 device_printf(dev, "Unable to initialize the hardware\n"); 1125 return; 1126 } 1127 igb_update_link_status(adapter); 1128 1129 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1130 1131 #ifndef IGB_HW_VLAN_SUPPORT 1132 /* Vlan's enabled but HW Filtering off */ 1133 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1134 u32 ctrl; 1135 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1136 ctrl |= E1000_CTRL_VME; 1137 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1138 } 1139 #endif 1140 1141 /* Set hardware offload abilities */ 1142 ifp->if_hwassist = 0; 1143 if (ifp->if_capenable & IFCAP_TXCSUM) 1144 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1145 if (ifp->if_capenable & IFCAP_TSO4) 1146 ifp->if_hwassist |= CSUM_TSO; 1147 1148 /* Configure for OS presence */ 1149 igb_init_manageability(adapter); 1150 1151 /* Prepare transmit descriptors and buffers */ 1152 igb_setup_transmit_structures(adapter); 1153 igb_initialize_transmit_units(adapter); 1154 1155 /* Setup Multicast table */ 1156 igb_set_multi(adapter); 1157 1158 /* Prepare receive descriptors and buffers */ 1159 if (igb_setup_receive_structures(adapter)) { 1160 device_printf(dev, "Could not setup receive structures\n"); 1161 igb_stop(adapter); 1162 return; 1163 } 1164 igb_initialize_receive_units(adapter); 1165 1166 /* Don't lose promiscuous settings */ 1167 igb_set_promisc(adapter); 1168 1169 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1170 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1171 1172 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1173 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1174 1175 if (adapter->msix > 1) /* Set up queue routing */ 1176 igb_configure_queues(adapter); 1177 1178 /* Set default RX interrupt moderation */ 1179 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 1180 E1000_WRITE_REG(&adapter->hw, 1181 E1000_EITR(rxr->msix), igb_ave_latency); 1182 rxr->eitr_setting = igb_ave_latency; 1183 } 1184 1185 /* Set TX interrupt rate & reset TX watchdog */ 1186 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1187 E1000_WRITE_REG(&adapter->hw, 1188 E1000_EITR(txr->msix), igb_ave_latency); 1189 txr->watchdog_timer = FALSE; 1190 } 1191 1192 /* this clears any pending interrupts */ 1193 E1000_READ_REG(&adapter->hw, E1000_ICR); 1194 igb_enable_intr(adapter); 1195 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); 1196 1197 #ifdef IGB_TIMESYNC 1198 /* Initialize IEEE 1588 Time sync if available */ 1199 if (adapter->hw.mac.type == e1000_82576) 1200 igb_tsync_init(adapter); 1201 #endif 1202 1203 /* Don't reset the phy next time init gets called */ 1204 adapter->hw.phy.reset_disable = TRUE; 1205 } 1206 1207 static void 1208 igb_init(void *arg) 1209 { 1210 struct adapter *adapter = arg; 1211 1212 IGB_CORE_LOCK(adapter); 1213 igb_init_locked(adapter); 1214 IGB_CORE_UNLOCK(adapter); 1215 } 1216 1217 1218 static void 1219 igb_handle_link(void *context, int pending) 1220 { 1221 struct adapter *adapter = context; 1222 struct ifnet *ifp; 1223 1224 ifp = adapter->ifp; 1225 1226 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1227 return; 1228 1229 IGB_CORE_LOCK(adapter); 1230 callout_stop(&adapter->timer); 1231 igb_update_link_status(adapter); 1232 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1233 IGB_CORE_UNLOCK(adapter); 1234 } 1235 1236 static void 1237 igb_handle_rxtx(void *context, int pending) 1238 { 1239 struct adapter *adapter = context; 1240 struct tx_ring *txr = adapter->tx_rings; 1241 struct rx_ring *rxr = adapter->rx_rings; 1242 struct ifnet *ifp; 1243 1244 ifp = adapter->ifp; 1245 1246 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1247 if (igb_rxeof(rxr, adapter->rx_process_limit) != 0) 1248 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1249 IGB_TX_LOCK(txr); 1250 igb_txeof(txr); 1251 1252 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1253 igb_start_locked(txr, ifp); 1254 IGB_TX_UNLOCK(txr); 1255 } 1256 1257 igb_enable_intr(adapter); 1258 } 1259 1260 static void 1261 igb_handle_rx(void *context, int pending) 1262 { 1263 struct rx_ring *rxr = context; 1264 struct adapter *adapter = rxr->adapter; 1265 struct ifnet *ifp = adapter->ifp; 1266 1267 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1268 if (igb_rxeof(rxr, adapter->rx_process_limit) != 0) 1269 /* More to clean, schedule another task */ 1270 taskqueue_enqueue(adapter->tq, &rxr->rx_task); 1271 1272 } 1273 1274 static void 1275 igb_handle_tx(void *context, int pending) 1276 { 1277 struct tx_ring *txr = context; 1278 struct adapter *adapter = txr->adapter; 1279 struct ifnet *ifp = adapter->ifp; 1280 1281 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1282 IGB_TX_LOCK(txr); 1283 igb_txeof(txr); 1284 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1285 igb_start_locked(txr, ifp); 1286 IGB_TX_UNLOCK(txr); 1287 } 1288 } 1289 1290 1291 /********************************************************************* 1292 * 1293 * MSI/Legacy Deferred 1294 * Interrupt Service routine 1295 * 1296 *********************************************************************/ 1297 static int 1298 igb_irq_fast(void *arg) 1299 { 1300 struct adapter *adapter = arg; 1301 uint32_t reg_icr; 1302 1303 1304 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1305 1306 /* Hot eject? */ 1307 if (reg_icr == 0xffffffff) 1308 return FILTER_STRAY; 1309 1310 /* Definitely not our interrupt. */ 1311 if (reg_icr == 0x0) 1312 return FILTER_STRAY; 1313 1314 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1315 return FILTER_STRAY; 1316 1317 /* 1318 * Mask interrupts until the taskqueue is finished running. This is 1319 * cheap, just assume that it is needed. This also works around the 1320 * MSI message reordering errata on certain systems. 1321 */ 1322 igb_disable_intr(adapter); 1323 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1324 1325 /* Link status change */ 1326 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1327 adapter->hw.mac.get_link_status = 1; 1328 taskqueue_enqueue(adapter->tq, &adapter->link_task); 1329 } 1330 1331 if (reg_icr & E1000_ICR_RXO) 1332 adapter->rx_overruns++; 1333 return FILTER_HANDLED; 1334 } 1335 1336 1337 /********************************************************************* 1338 * 1339 * MSIX TX Interrupt Service routine 1340 * 1341 **********************************************************************/ 1342 1343 static void 1344 igb_msix_tx(void *arg) 1345 { 1346 struct tx_ring *txr = arg; 1347 struct adapter *adapter = txr->adapter; 1348 struct ifnet *ifp = adapter->ifp; 1349 1350 ++txr->tx_irq; 1351 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1352 IGB_TX_LOCK(txr); 1353 igb_txeof(txr); 1354 IGB_TX_UNLOCK(txr); 1355 taskqueue_enqueue(adapter->tq, &txr->tx_task); 1356 } 1357 /* Reenable this interrupt */ 1358 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, txr->eims); 1359 return; 1360 } 1361 1362 /********************************************************************* 1363 * 1364 * MSIX RX Interrupt Service routine 1365 * 1366 **********************************************************************/ 1367 1368 static void 1369 igb_msix_rx(void *arg) 1370 { 1371 struct rx_ring *rxr = arg; 1372 struct adapter *adapter = rxr->adapter; 1373 u32 more, loop = 5; 1374 1375 ++rxr->rx_irq; 1376 do { 1377 more = igb_rxeof(rxr, adapter->rx_process_limit); 1378 } while (loop-- || more != 0); 1379 1380 taskqueue_enqueue(adapter->tq, &rxr->rx_task); 1381 1382 /* Update interrupt rate */ 1383 if (igb_enable_aim == TRUE) 1384 igb_update_aim(rxr); 1385 1386 /* Reenable this interrupt */ 1387 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxr->eims); 1388 return; 1389 } 1390 1391 1392 /* 1393 ** Routine to adjust the RX EITR value based on traffic, 1394 ** its a simple three state model, but seems to help. 1395 ** 1396 ** Note that the three EITR values are tuneable using 1397 ** sysctl in real time. The feature can be effectively 1398 ** nullified by setting them equal. 1399 */ 1400 #define BULK_THRESHOLD 10000 1401 #define AVE_THRESHOLD 1600 1402 1403 static void 1404 igb_update_aim(struct rx_ring *rxr) 1405 { 1406 struct adapter *adapter = rxr->adapter; 1407 u32 olditr, newitr; 1408 1409 /* Update interrupt moderation based on traffic */ 1410 olditr = rxr->eitr_setting; 1411 newitr = olditr; 1412 1413 /* Idle, don't change setting */ 1414 if (rxr->bytes == 0) 1415 return; 1416 1417 if (olditr == igb_low_latency) { 1418 if (rxr->bytes > AVE_THRESHOLD) 1419 newitr = igb_ave_latency; 1420 } else if (olditr == igb_ave_latency) { 1421 if (rxr->bytes < AVE_THRESHOLD) 1422 newitr = igb_low_latency; 1423 else if (rxr->bytes > BULK_THRESHOLD) 1424 newitr = igb_bulk_latency; 1425 } else if (olditr == igb_bulk_latency) { 1426 if (rxr->bytes < BULK_THRESHOLD) 1427 newitr = igb_ave_latency; 1428 } 1429 1430 if (olditr != newitr) { 1431 /* Change interrupt rate */ 1432 rxr->eitr_setting = newitr; 1433 E1000_WRITE_REG(&adapter->hw, E1000_EITR(rxr->me), 1434 newitr | (newitr << 16)); 1435 } 1436 1437 rxr->bytes = 0; 1438 return; 1439 } 1440 1441 1442 /********************************************************************* 1443 * 1444 * MSIX Link Interrupt Service routine 1445 * 1446 **********************************************************************/ 1447 1448 static void 1449 igb_msix_link(void *arg) 1450 { 1451 struct adapter *adapter = arg; 1452 u32 icr; 1453 1454 ++adapter->link_irq; 1455 icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1456 if (!(icr & E1000_ICR_LSC)) 1457 goto spurious; 1458 adapter->hw.mac.get_link_status = 1; 1459 taskqueue_enqueue(adapter->tq, &adapter->link_task); 1460 1461 spurious: 1462 /* Rearm */ 1463 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); 1464 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); 1465 return; 1466 } 1467 1468 1469 /********************************************************************* 1470 * 1471 * Media Ioctl callback 1472 * 1473 * This routine is called whenever the user queries the status of 1474 * the interface using ifconfig. 1475 * 1476 **********************************************************************/ 1477 static void 1478 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1479 { 1480 struct adapter *adapter = ifp->if_softc; 1481 u_char fiber_type = IFM_1000_SX; 1482 1483 INIT_DEBUGOUT("igb_media_status: begin"); 1484 1485 IGB_CORE_LOCK(adapter); 1486 igb_update_link_status(adapter); 1487 1488 ifmr->ifm_status = IFM_AVALID; 1489 ifmr->ifm_active = IFM_ETHER; 1490 1491 if (!adapter->link_active) { 1492 IGB_CORE_UNLOCK(adapter); 1493 return; 1494 } 1495 1496 ifmr->ifm_status |= IFM_ACTIVE; 1497 1498 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1499 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) 1500 ifmr->ifm_active |= fiber_type | IFM_FDX; 1501 else { 1502 switch (adapter->link_speed) { 1503 case 10: 1504 ifmr->ifm_active |= IFM_10_T; 1505 break; 1506 case 100: 1507 ifmr->ifm_active |= IFM_100_TX; 1508 break; 1509 case 1000: 1510 ifmr->ifm_active |= IFM_1000_T; 1511 break; 1512 } 1513 if (adapter->link_duplex == FULL_DUPLEX) 1514 ifmr->ifm_active |= IFM_FDX; 1515 else 1516 ifmr->ifm_active |= IFM_HDX; 1517 } 1518 IGB_CORE_UNLOCK(adapter); 1519 } 1520 1521 /********************************************************************* 1522 * 1523 * Media Ioctl callback 1524 * 1525 * This routine is called when the user changes speed/duplex using 1526 * media/mediopt option with ifconfig. 1527 * 1528 **********************************************************************/ 1529 static int 1530 igb_media_change(struct ifnet *ifp) 1531 { 1532 struct adapter *adapter = ifp->if_softc; 1533 struct ifmedia *ifm = &adapter->media; 1534 1535 INIT_DEBUGOUT("igb_media_change: begin"); 1536 1537 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1538 return (EINVAL); 1539 1540 IGB_CORE_LOCK(adapter); 1541 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1542 case IFM_AUTO: 1543 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1544 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1545 break; 1546 case IFM_1000_LX: 1547 case IFM_1000_SX: 1548 case IFM_1000_T: 1549 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1550 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1551 break; 1552 case IFM_100_TX: 1553 adapter->hw.mac.autoneg = FALSE; 1554 adapter->hw.phy.autoneg_advertised = 0; 1555 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1556 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1557 else 1558 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1559 break; 1560 case IFM_10_T: 1561 adapter->hw.mac.autoneg = FALSE; 1562 adapter->hw.phy.autoneg_advertised = 0; 1563 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1564 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1565 else 1566 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1567 break; 1568 default: 1569 device_printf(adapter->dev, "Unsupported media type\n"); 1570 } 1571 1572 /* As the speed/duplex settings my have changed we need to 1573 * reset the PHY. 1574 */ 1575 adapter->hw.phy.reset_disable = FALSE; 1576 1577 igb_init_locked(adapter); 1578 IGB_CORE_UNLOCK(adapter); 1579 1580 return (0); 1581 } 1582 1583 1584 /********************************************************************* 1585 * 1586 * This routine maps the mbufs to Advanced TX descriptors. 1587 * used by the 82575 adapter. 1588 * 1589 **********************************************************************/ 1590 1591 static int 1592 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1593 { 1594 struct adapter *adapter = txr->adapter; 1595 bus_dma_segment_t segs[IGB_MAX_SCATTER]; 1596 bus_dmamap_t map; 1597 struct igb_buffer *tx_buffer, *tx_buffer_mapped; 1598 union e1000_adv_tx_desc *txd = NULL; 1599 struct mbuf *m_head; 1600 u32 olinfo_status = 0, cmd_type_len = 0; 1601 int nsegs, i, j, error, first, last = 0; 1602 u32 hdrlen = 0, offload = 0; 1603 1604 m_head = *m_headp; 1605 1606 1607 /* Set basic descriptor constants */ 1608 cmd_type_len |= E1000_ADVTXD_DTYP_DATA; 1609 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT; 1610 if (m_head->m_flags & M_VLANTAG) 1611 cmd_type_len |= E1000_ADVTXD_DCMD_VLE; 1612 1613 /* 1614 * Force a cleanup if number of TX descriptors 1615 * available hits the threshold 1616 */ 1617 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) { 1618 igb_txeof(txr); 1619 /* Now do we at least have a minimal? */ 1620 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 1621 txr->no_desc_avail++; 1622 return (ENOBUFS); 1623 } 1624 } 1625 1626 /* 1627 * Map the packet for DMA. 1628 * 1629 * Capture the first descriptor index, 1630 * this descriptor will have the index 1631 * of the EOP which is the only one that 1632 * now gets a DONE bit writeback. 1633 */ 1634 first = txr->next_avail_desc; 1635 tx_buffer = &txr->tx_buffers[first]; 1636 tx_buffer_mapped = tx_buffer; 1637 map = tx_buffer->map; 1638 1639 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1640 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1641 1642 if (error == EFBIG) { 1643 struct mbuf *m; 1644 1645 m = m_defrag(*m_headp, M_DONTWAIT); 1646 if (m == NULL) { 1647 adapter->mbuf_alloc_failed++; 1648 m_freem(*m_headp); 1649 *m_headp = NULL; 1650 return (ENOBUFS); 1651 } 1652 *m_headp = m; 1653 1654 /* Try it again */ 1655 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1656 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1657 1658 if (error == ENOMEM) { 1659 adapter->no_tx_dma_setup++; 1660 return (error); 1661 } else if (error != 0) { 1662 adapter->no_tx_dma_setup++; 1663 m_freem(*m_headp); 1664 *m_headp = NULL; 1665 return (error); 1666 } 1667 } else if (error == ENOMEM) { 1668 adapter->no_tx_dma_setup++; 1669 return (error); 1670 } else if (error != 0) { 1671 adapter->no_tx_dma_setup++; 1672 m_freem(*m_headp); 1673 *m_headp = NULL; 1674 return (error); 1675 } 1676 1677 /* Check again to be sure we have enough descriptors */ 1678 if (nsegs > (txr->tx_avail - 2)) { 1679 txr->no_desc_avail++; 1680 bus_dmamap_unload(txr->txtag, map); 1681 return (ENOBUFS); 1682 } 1683 m_head = *m_headp; 1684 1685 /* 1686 * Set up the context descriptor: 1687 * used when any hardware offload is done. 1688 * This includes CSUM, VLAN, and TSO. It 1689 * will use the first descriptor. 1690 */ 1691 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1692 if (igb_tso_setup(txr, m_head, &hdrlen)) { 1693 cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 1694 olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 1695 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1696 } else 1697 return (ENXIO); 1698 } else 1699 /* Do all other context descriptor setup */ 1700 offload = igb_tx_ctx_setup(txr, m_head); 1701 if (offload == TRUE) 1702 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1703 #ifdef IGB_TIMESYNC 1704 if (offload == IGB_TIMESTAMP) 1705 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP; 1706 #endif 1707 /* Calculate payload length */ 1708 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen) 1709 << E1000_ADVTXD_PAYLEN_SHIFT); 1710 1711 /* Set up our transmit descriptors */ 1712 i = txr->next_avail_desc; 1713 for (j = 0; j < nsegs; j++) { 1714 bus_size_t seg_len; 1715 bus_addr_t seg_addr; 1716 1717 tx_buffer = &txr->tx_buffers[i]; 1718 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; 1719 seg_addr = segs[j].ds_addr; 1720 seg_len = segs[j].ds_len; 1721 1722 txd->read.buffer_addr = htole64(seg_addr); 1723 txd->read.cmd_type_len = htole32( 1724 adapter->txd_cmd | cmd_type_len | seg_len); 1725 txd->read.olinfo_status = htole32(olinfo_status); 1726 last = i; 1727 if (++i == adapter->num_tx_desc) 1728 i = 0; 1729 tx_buffer->m_head = NULL; 1730 tx_buffer->next_eop = -1; 1731 } 1732 1733 txr->next_avail_desc = i; 1734 txr->tx_avail -= nsegs; 1735 1736 tx_buffer->m_head = m_head; 1737 tx_buffer_mapped->map = tx_buffer->map; 1738 tx_buffer->map = map; 1739 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1740 1741 /* 1742 * Last Descriptor of Packet 1743 * needs End Of Packet (EOP) 1744 * and Report Status (RS) 1745 */ 1746 txd->read.cmd_type_len |= 1747 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 1748 /* 1749 * Keep track in the first buffer which 1750 * descriptor will be written back 1751 */ 1752 tx_buffer = &txr->tx_buffers[first]; 1753 tx_buffer->next_eop = last; 1754 1755 /* 1756 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1757 * that this frame is available to transmit. 1758 */ 1759 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1760 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1761 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1762 ++txr->tx_packets; 1763 1764 return (0); 1765 1766 } 1767 1768 static void 1769 igb_set_promisc(struct adapter *adapter) 1770 { 1771 struct ifnet *ifp = adapter->ifp; 1772 uint32_t reg_rctl; 1773 1774 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1775 1776 if (ifp->if_flags & IFF_PROMISC) { 1777 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1778 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1779 } else if (ifp->if_flags & IFF_ALLMULTI) { 1780 reg_rctl |= E1000_RCTL_MPE; 1781 reg_rctl &= ~E1000_RCTL_UPE; 1782 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1783 } 1784 } 1785 1786 static void 1787 igb_disable_promisc(struct adapter *adapter) 1788 { 1789 uint32_t reg_rctl; 1790 1791 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1792 1793 reg_rctl &= (~E1000_RCTL_UPE); 1794 reg_rctl &= (~E1000_RCTL_MPE); 1795 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1796 } 1797 1798 1799 /********************************************************************* 1800 * Multicast Update 1801 * 1802 * This routine is called whenever multicast address list is updated. 1803 * 1804 **********************************************************************/ 1805 1806 static void 1807 igb_set_multi(struct adapter *adapter) 1808 { 1809 struct ifnet *ifp = adapter->ifp; 1810 struct ifmultiaddr *ifma; 1811 u32 reg_rctl = 0; 1812 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN]; 1813 1814 int mcnt = 0; 1815 1816 IOCTL_DEBUGOUT("igb_set_multi: begin"); 1817 1818 IF_ADDR_LOCK(ifp); 1819 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1820 if (ifma->ifma_addr->sa_family != AF_LINK) 1821 continue; 1822 1823 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1824 break; 1825 1826 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1827 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 1828 mcnt++; 1829 } 1830 IF_ADDR_UNLOCK(ifp); 1831 1832 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 1833 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1834 reg_rctl |= E1000_RCTL_MPE; 1835 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1836 } else 1837 e1000_update_mc_addr_list(&adapter->hw, mta, 1838 mcnt, 1, adapter->hw.mac.rar_entry_count); 1839 } 1840 1841 1842 /********************************************************************* 1843 * Timer routine 1844 * 1845 * This routine checks for link status and updates statistics. 1846 * 1847 **********************************************************************/ 1848 1849 static void 1850 igb_local_timer(void *arg) 1851 { 1852 struct adapter *adapter = arg; 1853 struct ifnet *ifp = adapter->ifp; 1854 1855 IGB_CORE_LOCK_ASSERT(adapter); 1856 1857 igb_update_link_status(adapter); 1858 igb_update_stats_counters(adapter); 1859 1860 if (igb_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 1861 igb_print_hw_stats(adapter); 1862 1863 /* 1864 * Each second we check the watchdog to 1865 * protect against hardware hangs. 1866 */ 1867 igb_watchdog(adapter); 1868 1869 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1870 1871 } 1872 1873 static void 1874 igb_update_link_status(struct adapter *adapter) 1875 { 1876 struct e1000_hw *hw = &adapter->hw; 1877 struct ifnet *ifp = adapter->ifp; 1878 device_t dev = adapter->dev; 1879 struct tx_ring *txr = adapter->tx_rings; 1880 u32 link_check = 0; 1881 1882 /* Get the cached link value or read for real */ 1883 switch (hw->phy.media_type) { 1884 case e1000_media_type_copper: 1885 if (hw->mac.get_link_status) { 1886 /* Do the work to read phy */ 1887 e1000_check_for_link(hw); 1888 link_check = !hw->mac.get_link_status; 1889 } else 1890 link_check = TRUE; 1891 break; 1892 case e1000_media_type_fiber: 1893 e1000_check_for_link(hw); 1894 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 1895 E1000_STATUS_LU); 1896 break; 1897 case e1000_media_type_internal_serdes: 1898 e1000_check_for_link(hw); 1899 link_check = adapter->hw.mac.serdes_has_link; 1900 break; 1901 default: 1902 case e1000_media_type_unknown: 1903 break; 1904 } 1905 1906 /* Now we check if a transition has happened */ 1907 if (link_check && (adapter->link_active == 0)) { 1908 e1000_get_speed_and_duplex(&adapter->hw, 1909 &adapter->link_speed, &adapter->link_duplex); 1910 if (bootverbose) 1911 device_printf(dev, "Link is up %d Mbps %s\n", 1912 adapter->link_speed, 1913 ((adapter->link_duplex == FULL_DUPLEX) ? 1914 "Full Duplex" : "Half Duplex")); 1915 adapter->link_active = 1; 1916 ifp->if_baudrate = adapter->link_speed * 1000000; 1917 if_link_state_change(ifp, LINK_STATE_UP); 1918 } else if (!link_check && (adapter->link_active == 1)) { 1919 ifp->if_baudrate = adapter->link_speed = 0; 1920 adapter->link_duplex = 0; 1921 if (bootverbose) 1922 device_printf(dev, "Link is Down\n"); 1923 adapter->link_active = 0; 1924 if_link_state_change(ifp, LINK_STATE_DOWN); 1925 /* Turn off watchdogs */ 1926 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 1927 txr->watchdog_timer = FALSE; 1928 } 1929 } 1930 1931 /********************************************************************* 1932 * 1933 * This routine disables all traffic on the adapter by issuing a 1934 * global reset on the MAC and deallocates TX/RX buffers. 1935 * 1936 **********************************************************************/ 1937 1938 static void 1939 igb_stop(void *arg) 1940 { 1941 struct adapter *adapter = arg; 1942 struct ifnet *ifp = adapter->ifp; 1943 1944 IGB_CORE_LOCK_ASSERT(adapter); 1945 1946 INIT_DEBUGOUT("igb_stop: begin"); 1947 1948 igb_disable_intr(adapter); 1949 1950 callout_stop(&adapter->timer); 1951 1952 /* Tell the stack that the interface is no longer active */ 1953 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1954 1955 #ifdef IGB_TIMESYNC 1956 /* Disable IEEE 1588 Time sync */ 1957 if (adapter->hw.mac.type == e1000_82576) 1958 igb_tsync_disable(adapter); 1959 #endif 1960 1961 e1000_reset_hw(&adapter->hw); 1962 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 1963 } 1964 1965 1966 /********************************************************************* 1967 * 1968 * Determine hardware revision. 1969 * 1970 **********************************************************************/ 1971 static void 1972 igb_identify_hardware(struct adapter *adapter) 1973 { 1974 device_t dev = adapter->dev; 1975 1976 /* Make sure our PCI config space has the necessary stuff set */ 1977 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1978 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 1979 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 1980 device_printf(dev, "Memory Access and/or Bus Master bits " 1981 "were not set!\n"); 1982 adapter->hw.bus.pci_cmd_word |= 1983 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 1984 pci_write_config(dev, PCIR_COMMAND, 1985 adapter->hw.bus.pci_cmd_word, 2); 1986 } 1987 1988 /* Save off the information about this board */ 1989 adapter->hw.vendor_id = pci_get_vendor(dev); 1990 adapter->hw.device_id = pci_get_device(dev); 1991 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1992 adapter->hw.subsystem_vendor_id = 1993 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1994 adapter->hw.subsystem_device_id = 1995 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1996 1997 /* Do Shared Code Init and Setup */ 1998 if (e1000_set_mac_type(&adapter->hw)) { 1999 device_printf(dev, "Setup init failure\n"); 2000 return; 2001 } 2002 } 2003 2004 static int 2005 igb_allocate_pci_resources(struct adapter *adapter) 2006 { 2007 device_t dev = adapter->dev; 2008 int rid, error = 0; 2009 2010 rid = PCIR_BAR(0); 2011 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2012 &rid, RF_ACTIVE); 2013 if (adapter->pci_mem == NULL) { 2014 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2015 return (ENXIO); 2016 } 2017 adapter->osdep.mem_bus_space_tag = 2018 rman_get_bustag(adapter->pci_mem); 2019 adapter->osdep.mem_bus_space_handle = 2020 rman_get_bushandle(adapter->pci_mem); 2021 adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle; 2022 2023 /* 2024 ** Init the resource arrays 2025 */ 2026 for (int i = 0; i < IGB_MSIX_VEC; i++) { 2027 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2028 adapter->tag[i] = NULL; 2029 adapter->res[i] = NULL; 2030 } 2031 2032 adapter->num_tx_queues = 1; /* Defaults for Legacy or MSI */ 2033 adapter->num_rx_queues = 1; 2034 2035 /* This will setup either MSI/X or MSI */ 2036 adapter->msix = igb_setup_msix(adapter); 2037 2038 adapter->hw.back = &adapter->osdep; 2039 2040 return (error); 2041 } 2042 2043 /********************************************************************* 2044 * 2045 * Setup the Legacy or MSI Interrupt handler 2046 * 2047 **********************************************************************/ 2048 static int 2049 igb_allocate_legacy(struct adapter *adapter) 2050 { 2051 device_t dev = adapter->dev; 2052 int error; 2053 2054 /* Turn off all interrupts */ 2055 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2056 2057 /* Legacy RID at 0 */ 2058 if (adapter->msix == 0) 2059 adapter->rid[0] = 0; 2060 2061 /* We allocate a single interrupt resource */ 2062 adapter->res[0] = bus_alloc_resource_any(dev, 2063 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 2064 if (adapter->res[0] == NULL) { 2065 device_printf(dev, "Unable to allocate bus resource: " 2066 "interrupt\n"); 2067 return (ENXIO); 2068 } 2069 2070 /* 2071 * Try allocating a fast interrupt and the associated deferred 2072 * processing contexts. 2073 */ 2074 TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter); 2075 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); 2076 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2077 taskqueue_thread_enqueue, &adapter->tq); 2078 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2079 device_get_nameunit(adapter->dev)); 2080 if ((error = bus_setup_intr(dev, adapter->res[0], 2081 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, adapter, 2082 &adapter->tag[0])) != 0) { 2083 device_printf(dev, "Failed to register fast interrupt " 2084 "handler: %d\n", error); 2085 taskqueue_free(adapter->tq); 2086 adapter->tq = NULL; 2087 return (error); 2088 } 2089 2090 return (0); 2091 } 2092 2093 2094 /********************************************************************* 2095 * 2096 * Setup the MSIX Interrupt handlers: 2097 * 2098 **********************************************************************/ 2099 static int 2100 igb_allocate_msix(struct adapter *adapter) 2101 { 2102 device_t dev = adapter->dev; 2103 struct tx_ring *txr = adapter->tx_rings; 2104 struct rx_ring *rxr = adapter->rx_rings; 2105 int error, vector = 0; 2106 2107 /* 2108 * Setup the interrupt handlers 2109 */ 2110 2111 /* TX Setup */ 2112 for (int i = 0; i < adapter->num_tx_queues; i++, vector++, txr++) { 2113 adapter->res[vector] = bus_alloc_resource_any(dev, 2114 SYS_RES_IRQ, &adapter->rid[vector], 2115 RF_SHAREABLE | RF_ACTIVE); 2116 if (adapter->res[vector] == NULL) { 2117 device_printf(dev, 2118 "Unable to allocate bus resource: " 2119 "MSIX TX Interrupt\n"); 2120 return (ENXIO); 2121 } 2122 error = bus_setup_intr(dev, adapter->res[vector], 2123 INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_tx, 2124 txr, &adapter->tag[vector]); 2125 if (error) { 2126 adapter->res[vector] = NULL; 2127 device_printf(dev, "Failed to register TX handler"); 2128 return (error); 2129 } 2130 /* Make tasklet for deferred handling - one per queue */ 2131 TASK_INIT(&txr->tx_task, 0, igb_handle_tx, txr); 2132 if (adapter->hw.mac.type == e1000_82575) { 2133 txr->eims = E1000_EICR_TX_QUEUE0 << i; 2134 /* MSIXBM registers start at 0 */ 2135 txr->msix = adapter->rid[vector] - 1; 2136 } else { 2137 txr->eims = 1 << vector; 2138 txr->msix = vector; 2139 } 2140 } 2141 2142 /* RX Setup */ 2143 for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rxr++) { 2144 adapter->res[vector] = bus_alloc_resource_any(dev, 2145 SYS_RES_IRQ, &adapter->rid[vector], 2146 RF_SHAREABLE | RF_ACTIVE); 2147 if (adapter->res[vector] == NULL) { 2148 device_printf(dev, 2149 "Unable to allocate bus resource: " 2150 "MSIX RX Interrupt\n"); 2151 return (ENXIO); 2152 } 2153 error = bus_setup_intr(dev, adapter->res[vector], 2154 INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_rx, 2155 rxr, &adapter->tag[vector]); 2156 if (error) { 2157 adapter->res[vector] = NULL; 2158 device_printf(dev, "Failed to register RX handler"); 2159 return (error); 2160 } 2161 TASK_INIT(&rxr->rx_task, 0, igb_handle_rx, rxr); 2162 if (adapter->hw.mac.type == e1000_82575) { 2163 rxr->eims = E1000_EICR_RX_QUEUE0 << i; 2164 rxr->msix = adapter->rid[vector] - 1; 2165 } else { 2166 rxr->eims = 1 << vector; 2167 rxr->msix = vector; 2168 } 2169 } 2170 2171 /* And Link */ 2172 adapter->res[vector] = bus_alloc_resource_any(dev, 2173 SYS_RES_IRQ, &adapter->rid[vector], 2174 RF_SHAREABLE | RF_ACTIVE); 2175 if (adapter->res[vector] == NULL) { 2176 device_printf(dev, 2177 "Unable to allocate bus resource: " 2178 "MSIX Link Interrupt\n"); 2179 return (ENXIO); 2180 } 2181 if ((error = bus_setup_intr(dev, adapter->res[vector], 2182 INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_link, 2183 adapter, &adapter->tag[vector])) != 0) { 2184 device_printf(dev, "Failed to register Link handler"); 2185 return (error); 2186 } 2187 if (adapter->hw.mac.type == e1000_82575) 2188 adapter->linkvec = adapter->rid[vector] - 1; 2189 else 2190 adapter->linkvec = vector; 2191 2192 /* Make tasklet for deferred link interrupt handling */ 2193 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); 2194 2195 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2196 taskqueue_thread_enqueue, &adapter->tq); 2197 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2198 device_get_nameunit(adapter->dev)); 2199 2200 return (0); 2201 } 2202 2203 static void 2204 igb_configure_queues(struct adapter *adapter) 2205 { 2206 struct e1000_hw *hw = &adapter->hw; 2207 struct tx_ring *txr; 2208 struct rx_ring *rxr; 2209 2210 /* Turn on MSIX */ 2211 /* 2212 ** 82576 uses IVARs to route MSI/X 2213 ** interrupts, its not very intuitive, 2214 ** study the code carefully :) 2215 */ 2216 if (adapter->hw.mac.type == e1000_82576) { 2217 u32 ivar = 0; 2218 /* First turn on the capability */ 2219 E1000_WRITE_REG(hw, E1000_GPIE, 2220 E1000_GPIE_MSIX_MODE | 2221 E1000_GPIE_EIAME | 2222 E1000_GPIE_PBA | E1000_GPIE_NSICR); 2223 /* RX */ 2224 for (int i = 0; i < adapter->num_rx_queues; i++) { 2225 u32 index = i & 0x7; /* Each IVAR has two entries */ 2226 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2227 rxr = &adapter->rx_rings[i]; 2228 if (i < 8) { 2229 ivar &= 0xFFFFFF00; 2230 ivar |= rxr->msix | E1000_IVAR_VALID; 2231 } else { 2232 ivar &= 0xFF00FFFF; 2233 ivar |= (rxr->msix | E1000_IVAR_VALID) << 16; 2234 } 2235 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2236 adapter->eims_mask |= rxr->eims; 2237 } 2238 /* TX */ 2239 for (int i = 0; i < adapter->num_tx_queues; i++) { 2240 u32 index = i & 0x7; /* Each IVAR has two entries */ 2241 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2242 txr = &adapter->tx_rings[i]; 2243 if (i < 8) { 2244 ivar &= 0xFFFF00FF; 2245 ivar |= (txr->msix | E1000_IVAR_VALID) << 8; 2246 } else { 2247 ivar &= 0x00FFFFFF; 2248 ivar |= (txr->msix | E1000_IVAR_VALID) << 24; 2249 } 2250 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2251 adapter->eims_mask |= txr->eims; 2252 } 2253 2254 /* And for the link interrupt */ 2255 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2256 adapter->link_mask = 1 << adapter->linkvec; 2257 adapter->eims_mask |= adapter->link_mask; 2258 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2259 } else 2260 { /* 82575 */ 2261 int tmp; 2262 2263 /* enable MSI-X PBA support*/ 2264 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 2265 tmp |= E1000_CTRL_EXT_PBA_CLR; 2266 /* Auto-Mask interrupts upon ICR read. */ 2267 tmp |= E1000_CTRL_EXT_EIAME; 2268 tmp |= E1000_CTRL_EXT_IRCA; 2269 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 2270 2271 /* TX */ 2272 for (int i = 0; i < adapter->num_tx_queues; i++) { 2273 txr = &adapter->tx_rings[i]; 2274 E1000_WRITE_REG(hw, E1000_MSIXBM(txr->msix), 2275 txr->eims); 2276 adapter->eims_mask |= txr->eims; 2277 } 2278 2279 /* RX */ 2280 for (int i = 0; i < adapter->num_rx_queues; i++) { 2281 rxr = &adapter->rx_rings[i]; 2282 E1000_WRITE_REG(hw, E1000_MSIXBM(rxr->msix), 2283 rxr->eims); 2284 adapter->eims_mask |= rxr->eims; 2285 } 2286 2287 /* Link */ 2288 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), 2289 E1000_EIMS_OTHER); 2290 adapter->link_mask |= E1000_EIMS_OTHER; 2291 adapter->eims_mask |= adapter->link_mask; 2292 } 2293 return; 2294 } 2295 2296 2297 static void 2298 igb_free_pci_resources(struct adapter *adapter) 2299 { 2300 device_t dev = adapter->dev; 2301 2302 /* Make sure the for loop below runs once */ 2303 if (adapter->msix == 0) 2304 adapter->msix = 1; 2305 2306 /* 2307 * First release all the interrupt resources: 2308 * notice that since these are just kept 2309 * in an array we can do the same logic 2310 * whether its MSIX or just legacy. 2311 */ 2312 for (int i = 0; i < adapter->msix; i++) { 2313 if (adapter->tag[i] != NULL) { 2314 bus_teardown_intr(dev, adapter->res[i], 2315 adapter->tag[i]); 2316 adapter->tag[i] = NULL; 2317 } 2318 if (adapter->res[i] != NULL) { 2319 bus_release_resource(dev, SYS_RES_IRQ, 2320 adapter->rid[i], adapter->res[i]); 2321 } 2322 } 2323 2324 if (adapter->msix) 2325 pci_release_msi(dev); 2326 2327 if (adapter->msix_mem != NULL) 2328 bus_release_resource(dev, SYS_RES_MEMORY, 2329 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2330 2331 if (adapter->pci_mem != NULL) 2332 bus_release_resource(dev, SYS_RES_MEMORY, 2333 PCIR_BAR(0), adapter->pci_mem); 2334 2335 } 2336 2337 /* 2338 * Setup Either MSI/X or MSI 2339 */ 2340 static int 2341 igb_setup_msix(struct adapter *adapter) 2342 { 2343 device_t dev = adapter->dev; 2344 int rid, want, queues, msgs; 2345 2346 /* First try MSI/X */ 2347 rid = PCIR_BAR(IGB_MSIX_BAR); 2348 adapter->msix_mem = bus_alloc_resource_any(dev, 2349 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2350 if (!adapter->msix_mem) { 2351 /* May not be enabled */ 2352 device_printf(adapter->dev, 2353 "Unable to map MSIX table \n"); 2354 goto msi; 2355 } 2356 2357 msgs = pci_msix_count(dev); 2358 if (msgs == 0) { /* system has msix disabled */ 2359 bus_release_resource(dev, SYS_RES_MEMORY, 2360 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2361 adapter->msix_mem = NULL; 2362 goto msi; 2363 } 2364 2365 /* Limit by the number set in header */ 2366 if (msgs > IGB_MSIX_VEC) 2367 msgs = IGB_MSIX_VEC; 2368 2369 /* Figure out a reasonable auto config value */ 2370 queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus; 2371 2372 if (igb_tx_queues == 0) 2373 igb_tx_queues = queues; 2374 if (igb_rx_queues == 0) 2375 igb_rx_queues = queues; 2376 want = igb_tx_queues + igb_rx_queues + 1; 2377 if (msgs >= want) 2378 msgs = want; 2379 else { 2380 device_printf(adapter->dev, 2381 "MSIX Configuration Problem, " 2382 "%d vectors configured, but %d queues wanted!\n", 2383 msgs, want); 2384 return (ENXIO); 2385 } 2386 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2387 device_printf(adapter->dev, 2388 "Using MSIX interrupts with %d vectors\n", msgs); 2389 adapter->num_tx_queues = igb_tx_queues; 2390 adapter->num_rx_queues = igb_rx_queues; 2391 return (msgs); 2392 } 2393 msi: 2394 msgs = pci_msi_count(dev); 2395 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2396 device_printf(adapter->dev,"Using MSI interrupt\n"); 2397 return (msgs); 2398 } 2399 2400 /********************************************************************* 2401 * 2402 * Initialize the hardware to a configuration 2403 * as specified by the adapter structure. 2404 * 2405 **********************************************************************/ 2406 static int 2407 igb_hardware_init(struct adapter *adapter) 2408 { 2409 device_t dev = adapter->dev; 2410 u32 rx_buffer_size; 2411 2412 INIT_DEBUGOUT("igb_hardware_init: begin"); 2413 2414 /* Issue a global reset */ 2415 e1000_reset_hw(&adapter->hw); 2416 2417 /* Let the firmware know the OS is in control */ 2418 igb_get_hw_control(adapter); 2419 2420 /* 2421 * These parameters control the automatic generation (Tx) and 2422 * response (Rx) to Ethernet PAUSE frames. 2423 * - High water mark should allow for at least two frames to be 2424 * received after sending an XOFF. 2425 * - Low water mark works best when it is very near the high water mark. 2426 * This allows the receiver to restart by sending XON when it has 2427 * drained a bit. Here we use an arbitary value of 1500 which will 2428 * restart after one full frame is pulled from the buffer. There 2429 * could be several smaller frames in the buffer and if so they will 2430 * not trigger the XON until their total number reduces the buffer 2431 * by 1500. 2432 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2433 */ 2434 if (adapter->hw.mac.type == e1000_82576) 2435 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, 2436 E1000_RXPBS) & 0xffff) << 10 ); 2437 else 2438 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, 2439 E1000_PBA) & 0xffff) << 10 ); 2440 2441 adapter->hw.fc.high_water = rx_buffer_size - 2442 roundup2(adapter->max_frame_size, 1024); 2443 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 2444 2445 adapter->hw.fc.pause_time = IGB_FC_PAUSE_TIME; 2446 adapter->hw.fc.send_xon = TRUE; 2447 2448 /* Set Flow control, use the tunable location if sane */ 2449 if ((igb_fc_setting >= 0) || (igb_fc_setting < 4)) 2450 adapter->hw.fc.requested_mode = igb_fc_setting; 2451 else 2452 adapter->hw.fc.requested_mode = e1000_fc_none; 2453 2454 if (e1000_init_hw(&adapter->hw) < 0) { 2455 device_printf(dev, "Hardware Initialization Failed\n"); 2456 return (EIO); 2457 } 2458 2459 e1000_check_for_link(&adapter->hw); 2460 2461 return (0); 2462 } 2463 2464 /********************************************************************* 2465 * 2466 * Setup networking device structure and register an interface. 2467 * 2468 **********************************************************************/ 2469 static void 2470 igb_setup_interface(device_t dev, struct adapter *adapter) 2471 { 2472 struct ifnet *ifp; 2473 2474 INIT_DEBUGOUT("igb_setup_interface: begin"); 2475 2476 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2477 if (ifp == NULL) 2478 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2479 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2480 ifp->if_mtu = ETHERMTU; 2481 ifp->if_init = igb_init; 2482 ifp->if_softc = adapter; 2483 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2484 ifp->if_ioctl = igb_ioctl; 2485 ifp->if_start = igb_start; 2486 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2487 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2488 IFQ_SET_READY(&ifp->if_snd); 2489 2490 ether_ifattach(ifp, adapter->hw.mac.addr); 2491 2492 ifp->if_capabilities = ifp->if_capenable = 0; 2493 2494 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2495 ifp->if_capabilities |= IFCAP_TSO4; 2496 ifp->if_capenable = ifp->if_capabilities; 2497 2498 /* 2499 * Tell the upper layer(s) what we support. 2500 */ 2501 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2502 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 2503 ifp->if_capabilities |= IFCAP_VLAN_MTU; 2504 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING; 2505 ifp->if_capenable |= IFCAP_VLAN_MTU; 2506 2507 #ifdef IGB_HW_VLAN_SUPPORT 2508 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2509 ifp->if_capenable |= IFCAP_VLAN_HWFILTER; 2510 #endif 2511 2512 /* 2513 * Specify the media types supported by this adapter and register 2514 * callbacks to update media and link information 2515 */ 2516 ifmedia_init(&adapter->media, IFM_IMASK, 2517 igb_media_change, igb_media_status); 2518 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2519 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2520 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 2521 0, NULL); 2522 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2523 } else { 2524 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2525 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2526 0, NULL); 2527 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2528 0, NULL); 2529 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2530 0, NULL); 2531 if (adapter->hw.phy.type != e1000_phy_ife) { 2532 ifmedia_add(&adapter->media, 2533 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2534 ifmedia_add(&adapter->media, 2535 IFM_ETHER | IFM_1000_T, 0, NULL); 2536 } 2537 } 2538 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2539 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2540 } 2541 2542 2543 /* 2544 * Manage DMA'able memory. 2545 */ 2546 static void 2547 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2548 { 2549 if (error) 2550 return; 2551 *(bus_addr_t *) arg = segs[0].ds_addr; 2552 } 2553 2554 static int 2555 igb_dma_malloc(struct adapter *adapter, bus_size_t size, 2556 struct igb_dma_alloc *dma, int mapflags) 2557 { 2558 int error; 2559 2560 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2561 IGB_DBA_ALIGN, 0, /* alignment, bounds */ 2562 BUS_SPACE_MAXADDR, /* lowaddr */ 2563 BUS_SPACE_MAXADDR, /* highaddr */ 2564 NULL, NULL, /* filter, filterarg */ 2565 size, /* maxsize */ 2566 1, /* nsegments */ 2567 size, /* maxsegsize */ 2568 0, /* flags */ 2569 NULL, /* lockfunc */ 2570 NULL, /* lockarg */ 2571 &dma->dma_tag); 2572 if (error) { 2573 device_printf(adapter->dev, 2574 "%s: bus_dma_tag_create failed: %d\n", 2575 __func__, error); 2576 goto fail_0; 2577 } 2578 2579 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2580 BUS_DMA_NOWAIT, &dma->dma_map); 2581 if (error) { 2582 device_printf(adapter->dev, 2583 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2584 __func__, (uintmax_t)size, error); 2585 goto fail_2; 2586 } 2587 2588 dma->dma_paddr = 0; 2589 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2590 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2591 if (error || dma->dma_paddr == 0) { 2592 device_printf(adapter->dev, 2593 "%s: bus_dmamap_load failed: %d\n", 2594 __func__, error); 2595 goto fail_3; 2596 } 2597 2598 return (0); 2599 2600 fail_3: 2601 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2602 fail_2: 2603 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2604 bus_dma_tag_destroy(dma->dma_tag); 2605 fail_0: 2606 dma->dma_map = NULL; 2607 dma->dma_tag = NULL; 2608 2609 return (error); 2610 } 2611 2612 static void 2613 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) 2614 { 2615 if (dma->dma_tag == NULL) 2616 return; 2617 if (dma->dma_map != NULL) { 2618 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2619 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2620 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2621 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2622 dma->dma_map = NULL; 2623 } 2624 bus_dma_tag_destroy(dma->dma_tag); 2625 dma->dma_tag = NULL; 2626 } 2627 2628 2629 /********************************************************************* 2630 * 2631 * Allocate memory for the transmit and receive rings, and then 2632 * the descriptors associated with each, called only once at attach. 2633 * 2634 **********************************************************************/ 2635 static int 2636 igb_allocate_queues(struct adapter *adapter) 2637 { 2638 device_t dev = adapter->dev; 2639 struct tx_ring *txr; 2640 struct rx_ring *rxr; 2641 int rsize, tsize, error = E1000_SUCCESS; 2642 int txconf = 0, rxconf = 0; 2643 2644 /* First allocate the TX ring struct memory */ 2645 if (!(adapter->tx_rings = 2646 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2647 adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2648 device_printf(dev, "Unable to allocate TX ring memory\n"); 2649 error = ENOMEM; 2650 goto fail; 2651 } 2652 txr = adapter->tx_rings; 2653 2654 /* Next allocate the RX */ 2655 if (!(adapter->rx_rings = 2656 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2657 adapter->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2658 device_printf(dev, "Unable to allocate RX ring memory\n"); 2659 error = ENOMEM; 2660 goto rx_fail; 2661 } 2662 rxr = adapter->rx_rings; 2663 2664 tsize = roundup2(adapter->num_tx_desc * 2665 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); 2666 /* 2667 * Now set up the TX queues, txconf is needed to handle the 2668 * possibility that things fail midcourse and we need to 2669 * undo memory gracefully 2670 */ 2671 for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) { 2672 /* Set up some basics */ 2673 txr = &adapter->tx_rings[i]; 2674 txr->adapter = adapter; 2675 txr->me = i; 2676 2677 /* Initialize the TX lock */ 2678 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2679 device_get_nameunit(dev), txr->me); 2680 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2681 2682 if (igb_dma_malloc(adapter, tsize, 2683 &txr->txdma, BUS_DMA_NOWAIT)) { 2684 device_printf(dev, 2685 "Unable to allocate TX Descriptor memory\n"); 2686 error = ENOMEM; 2687 goto err_tx_desc; 2688 } 2689 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2690 bzero((void *)txr->tx_base, tsize); 2691 2692 /* Now allocate transmit buffers for the ring */ 2693 if (igb_allocate_transmit_buffers(txr)) { 2694 device_printf(dev, 2695 "Critical Failure setting up transmit buffers\n"); 2696 error = ENOMEM; 2697 goto err_tx_desc; 2698 } 2699 2700 } 2701 2702 /* 2703 * Next the RX queues... 2704 */ 2705 rsize = roundup2(adapter->num_rx_desc * 2706 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 2707 for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) { 2708 rxr = &adapter->rx_rings[i]; 2709 rxr->adapter = adapter; 2710 rxr->me = i; 2711 2712 /* Initialize the RX lock */ 2713 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2714 device_get_nameunit(dev), txr->me); 2715 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2716 2717 if (igb_dma_malloc(adapter, rsize, 2718 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2719 device_printf(dev, 2720 "Unable to allocate RxDescriptor memory\n"); 2721 error = ENOMEM; 2722 goto err_rx_desc; 2723 } 2724 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2725 bzero((void *)rxr->rx_base, rsize); 2726 2727 /* Allocate receive buffers for the ring*/ 2728 if (igb_allocate_receive_buffers(rxr)) { 2729 device_printf(dev, 2730 "Critical Failure setting up receive buffers\n"); 2731 error = ENOMEM; 2732 goto err_rx_desc; 2733 } 2734 } 2735 2736 return (0); 2737 2738 err_rx_desc: 2739 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2740 igb_dma_free(adapter, &rxr->rxdma); 2741 err_tx_desc: 2742 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2743 igb_dma_free(adapter, &txr->txdma); 2744 free(adapter->rx_rings, M_DEVBUF); 2745 rx_fail: 2746 free(adapter->tx_rings, M_DEVBUF); 2747 fail: 2748 return (error); 2749 } 2750 2751 /********************************************************************* 2752 * 2753 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2754 * the information needed to transmit a packet on the wire. This is 2755 * called only once at attach, setup is done every reset. 2756 * 2757 **********************************************************************/ 2758 static int 2759 igb_allocate_transmit_buffers(struct tx_ring *txr) 2760 { 2761 struct adapter *adapter = txr->adapter; 2762 device_t dev = adapter->dev; 2763 struct igb_buffer *txbuf; 2764 int error, i; 2765 2766 /* 2767 * Setup DMA descriptor areas. 2768 */ 2769 if ((error = bus_dma_tag_create(NULL, /* parent */ 2770 PAGE_SIZE, 0, /* alignment, bounds */ 2771 BUS_SPACE_MAXADDR, /* lowaddr */ 2772 BUS_SPACE_MAXADDR, /* highaddr */ 2773 NULL, NULL, /* filter, filterarg */ 2774 IGB_TSO_SIZE, /* maxsize */ 2775 IGB_MAX_SCATTER, /* nsegments */ 2776 PAGE_SIZE, /* maxsegsize */ 2777 0, /* flags */ 2778 NULL, /* lockfunc */ 2779 NULL, /* lockfuncarg */ 2780 &txr->txtag))) { 2781 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2782 goto fail; 2783 } 2784 2785 if (!(txr->tx_buffers = 2786 (struct igb_buffer *) malloc(sizeof(struct igb_buffer) * 2787 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2788 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2789 error = ENOMEM; 2790 goto fail; 2791 } 2792 2793 /* Create the descriptor buffer dma maps */ 2794 txbuf = txr->tx_buffers; 2795 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2796 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2797 if (error != 0) { 2798 device_printf(dev, "Unable to create TX DMA map\n"); 2799 goto fail; 2800 } 2801 } 2802 2803 return 0; 2804 fail: 2805 /* We free all, it handles case where we are in the middle */ 2806 igb_free_transmit_structures(adapter); 2807 return (error); 2808 } 2809 2810 /********************************************************************* 2811 * 2812 * Initialize a transmit ring. 2813 * 2814 **********************************************************************/ 2815 static void 2816 igb_setup_transmit_ring(struct tx_ring *txr) 2817 { 2818 struct adapter *adapter = txr->adapter; 2819 struct igb_buffer *txbuf; 2820 int i; 2821 2822 /* Clear the old ring contents */ 2823 bzero((void *)txr->tx_base, 2824 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); 2825 /* Reset indices */ 2826 txr->next_avail_desc = 0; 2827 txr->next_to_clean = 0; 2828 2829 /* Free any existing tx buffers. */ 2830 txbuf = txr->tx_buffers; 2831 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2832 if (txbuf->m_head != NULL) { 2833 bus_dmamap_sync(txr->txtag, txbuf->map, 2834 BUS_DMASYNC_POSTWRITE); 2835 bus_dmamap_unload(txr->txtag, txbuf->map); 2836 m_freem(txbuf->m_head); 2837 txbuf->m_head = NULL; 2838 } 2839 /* clear the watch index */ 2840 txbuf->next_eop = -1; 2841 } 2842 2843 /* Set number of descriptors available */ 2844 txr->tx_avail = adapter->num_tx_desc; 2845 2846 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2847 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2848 2849 } 2850 2851 /********************************************************************* 2852 * 2853 * Initialize all transmit rings. 2854 * 2855 **********************************************************************/ 2856 static void 2857 igb_setup_transmit_structures(struct adapter *adapter) 2858 { 2859 struct tx_ring *txr = adapter->tx_rings; 2860 2861 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 2862 igb_setup_transmit_ring(txr); 2863 2864 return; 2865 } 2866 2867 /********************************************************************* 2868 * 2869 * Enable transmit unit. 2870 * 2871 **********************************************************************/ 2872 static void 2873 igb_initialize_transmit_units(struct adapter *adapter) 2874 { 2875 struct tx_ring *txr = adapter->tx_rings; 2876 u32 tctl, txdctl; 2877 2878 INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); 2879 2880 /* Setup the Base and Length of the Tx Descriptor Rings */ 2881 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2882 u64 bus_addr = txr->txdma.dma_paddr; 2883 2884 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(i), 2885 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 2886 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(i), 2887 (uint32_t)(bus_addr >> 32)); 2888 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(i), 2889 (uint32_t)bus_addr); 2890 2891 /* Setup the HW Tx Head and Tail descriptor pointers */ 2892 E1000_WRITE_REG(&adapter->hw, E1000_TDT(i), 0); 2893 E1000_WRITE_REG(&adapter->hw, E1000_TDH(i), 0); 2894 2895 HW_DEBUGOUT2("Base = %x, Length = %x\n", 2896 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), 2897 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); 2898 2899 /* Setup Transmit Descriptor Base Settings */ 2900 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 2901 2902 txdctl = E1000_READ_REG(&adapter->hw, E1000_TXDCTL(i)); 2903 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 2904 E1000_WRITE_REG(&adapter->hw, E1000_TXDCTL(i), txdctl); 2905 } 2906 2907 /* Program the Transmit Control Register */ 2908 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2909 tctl &= ~E1000_TCTL_CT; 2910 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 2911 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 2912 2913 e1000_config_collision_dist(&adapter->hw); 2914 2915 /* This write will effectively turn on the transmit unit. */ 2916 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2917 2918 } 2919 2920 /********************************************************************* 2921 * 2922 * Free all transmit rings. 2923 * 2924 **********************************************************************/ 2925 static void 2926 igb_free_transmit_structures(struct adapter *adapter) 2927 { 2928 struct tx_ring *txr = adapter->tx_rings; 2929 2930 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2931 IGB_TX_LOCK(txr); 2932 igb_free_transmit_buffers(txr); 2933 igb_dma_free(adapter, &txr->txdma); 2934 IGB_TX_UNLOCK(txr); 2935 IGB_TX_LOCK_DESTROY(txr); 2936 } 2937 free(adapter->tx_rings, M_DEVBUF); 2938 } 2939 2940 /********************************************************************* 2941 * 2942 * Free transmit ring related data structures. 2943 * 2944 **********************************************************************/ 2945 static void 2946 igb_free_transmit_buffers(struct tx_ring *txr) 2947 { 2948 struct adapter *adapter = txr->adapter; 2949 struct igb_buffer *tx_buffer; 2950 int i; 2951 2952 INIT_DEBUGOUT("free_transmit_ring: begin"); 2953 2954 if (txr->tx_buffers == NULL) 2955 return; 2956 2957 tx_buffer = txr->tx_buffers; 2958 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2959 if (tx_buffer->m_head != NULL) { 2960 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2961 BUS_DMASYNC_POSTWRITE); 2962 bus_dmamap_unload(txr->txtag, 2963 tx_buffer->map); 2964 m_freem(tx_buffer->m_head); 2965 tx_buffer->m_head = NULL; 2966 if (tx_buffer->map != NULL) { 2967 bus_dmamap_destroy(txr->txtag, 2968 tx_buffer->map); 2969 tx_buffer->map = NULL; 2970 } 2971 } else if (tx_buffer->map != NULL) { 2972 bus_dmamap_unload(txr->txtag, 2973 tx_buffer->map); 2974 bus_dmamap_destroy(txr->txtag, 2975 tx_buffer->map); 2976 tx_buffer->map = NULL; 2977 } 2978 } 2979 2980 if (txr->tx_buffers != NULL) { 2981 free(txr->tx_buffers, M_DEVBUF); 2982 txr->tx_buffers = NULL; 2983 } 2984 if (txr->txtag != NULL) { 2985 bus_dma_tag_destroy(txr->txtag); 2986 txr->txtag = NULL; 2987 } 2988 return; 2989 } 2990 2991 /********************************************************************** 2992 * 2993 * Setup work for hardware segmentation offload (TSO) on 2994 * adapters using advanced tx descriptors (82575) 2995 * 2996 **********************************************************************/ 2997 static boolean_t 2998 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen) 2999 { 3000 struct adapter *adapter = txr->adapter; 3001 struct e1000_adv_tx_context_desc *TXD; 3002 struct igb_buffer *tx_buffer; 3003 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3004 u32 mss_l4len_idx = 0; 3005 u16 vtag = 0; 3006 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3007 struct ether_vlan_header *eh; 3008 struct ip *ip; 3009 struct tcphdr *th; 3010 3011 3012 /* 3013 * Determine where frame payload starts. 3014 * Jump over vlan headers if already present 3015 */ 3016 eh = mtod(mp, struct ether_vlan_header *); 3017 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3018 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3019 else 3020 ehdrlen = ETHER_HDR_LEN; 3021 3022 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3023 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3024 return FALSE; 3025 3026 /* Only supports IPV4 for now */ 3027 ctxd = txr->next_avail_desc; 3028 tx_buffer = &txr->tx_buffers[ctxd]; 3029 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3030 3031 ip = (struct ip *)(mp->m_data + ehdrlen); 3032 if (ip->ip_p != IPPROTO_TCP) 3033 return FALSE; /* 0 */ 3034 ip->ip_sum = 0; 3035 ip_hlen = ip->ip_hl << 2; 3036 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3037 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3038 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3039 tcp_hlen = th->th_off << 2; 3040 /* 3041 * Calculate header length, this is used 3042 * in the transmit desc in igb_xmit 3043 */ 3044 *hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3045 3046 /* VLAN MACLEN IPLEN */ 3047 if (mp->m_flags & M_VLANTAG) { 3048 vtag = htole16(mp->m_pkthdr.ether_vtag); 3049 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3050 } 3051 3052 vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT); 3053 vlan_macip_lens |= ip_hlen; 3054 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3055 3056 /* ADV DTYPE TUCMD */ 3057 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3058 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3059 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3060 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3061 3062 /* MSS L4LEN IDX */ 3063 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); 3064 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); 3065 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3066 3067 TXD->seqnum_seed = htole32(0); 3068 tx_buffer->m_head = NULL; 3069 tx_buffer->next_eop = -1; 3070 3071 if (++ctxd == adapter->num_tx_desc) 3072 ctxd = 0; 3073 3074 txr->tx_avail--; 3075 txr->next_avail_desc = ctxd; 3076 return TRUE; 3077 } 3078 3079 3080 /********************************************************************* 3081 * 3082 * Context Descriptor setup for VLAN or CSUM 3083 * 3084 **********************************************************************/ 3085 3086 static int 3087 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3088 { 3089 struct adapter *adapter = txr->adapter; 3090 struct e1000_adv_tx_context_desc *TXD; 3091 struct igb_buffer *tx_buffer; 3092 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3093 struct ether_vlan_header *eh; 3094 struct ip *ip = NULL; 3095 struct ip6_hdr *ip6; 3096 int ehdrlen, ip_hlen = 0; 3097 u16 etype; 3098 u8 ipproto = 0; 3099 bool offload = TRUE; 3100 u16 vtag = 0; 3101 3102 int ctxd = txr->next_avail_desc; 3103 tx_buffer = &txr->tx_buffers[ctxd]; 3104 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3105 3106 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 3107 offload = FALSE; /* Only here to handle VLANs */ 3108 /* 3109 ** In advanced descriptors the vlan tag must 3110 ** be placed into the descriptor itself. 3111 */ 3112 if (mp->m_flags & M_VLANTAG) { 3113 vtag = htole16(mp->m_pkthdr.ether_vtag); 3114 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3115 } else if (offload == FALSE) 3116 return FALSE; 3117 /* 3118 * Determine where frame payload starts. 3119 * Jump over vlan headers if already present, 3120 * helpful for QinQ too. 3121 */ 3122 eh = mtod(mp, struct ether_vlan_header *); 3123 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3124 etype = ntohs(eh->evl_proto); 3125 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3126 } else { 3127 etype = ntohs(eh->evl_encap_proto); 3128 ehdrlen = ETHER_HDR_LEN; 3129 } 3130 3131 /* Set the ether header length */ 3132 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 3133 3134 switch (etype) { 3135 case ETHERTYPE_IP: 3136 ip = (struct ip *)(mp->m_data + ehdrlen); 3137 ip_hlen = ip->ip_hl << 2; 3138 if (mp->m_len < ehdrlen + ip_hlen) { 3139 offload = FALSE; 3140 break; 3141 } 3142 ipproto = ip->ip_p; 3143 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3144 break; 3145 case ETHERTYPE_IPV6: 3146 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3147 ip_hlen = sizeof(struct ip6_hdr); 3148 if (mp->m_len < ehdrlen + ip_hlen) 3149 return FALSE; /* failure */ 3150 ipproto = ip6->ip6_nxt; 3151 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 3152 break; 3153 #ifdef IGB_TIMESYNC 3154 case ETHERTYPE_IEEE1588: 3155 offload = IGB_TIMESTAMP; 3156 break; 3157 #endif 3158 default: 3159 offload = FALSE; 3160 break; 3161 } 3162 3163 vlan_macip_lens |= ip_hlen; 3164 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3165 3166 switch (ipproto) { 3167 case IPPROTO_TCP: 3168 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 3169 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3170 break; 3171 case IPPROTO_UDP: 3172 { 3173 #ifdef IGB_TIMESYNC 3174 void *hdr = (caddr_t) ip + ip_hlen; 3175 struct udphdr *uh = (struct udphdr *)hdr; 3176 3177 if (uh->uh_dport == htons(TSYNC_PORT)) 3178 offload = IGB_TIMESTAMP; 3179 #endif 3180 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 3181 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; 3182 break; 3183 } 3184 default: 3185 offload = FALSE; 3186 break; 3187 } 3188 3189 /* Now copy bits into descriptor */ 3190 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3191 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3192 TXD->seqnum_seed = htole32(0); 3193 TXD->mss_l4len_idx = htole32(0); 3194 3195 tx_buffer->m_head = NULL; 3196 tx_buffer->next_eop = -1; 3197 3198 /* We've consumed the first desc, adjust counters */ 3199 if (++ctxd == adapter->num_tx_desc) 3200 ctxd = 0; 3201 txr->next_avail_desc = ctxd; 3202 --txr->tx_avail; 3203 3204 return (offload); 3205 } 3206 3207 3208 /********************************************************************** 3209 * 3210 * Examine each tx_buffer in the used queue. If the hardware is done 3211 * processing the packet then free associated resources. The 3212 * tx_buffer is put back on the free queue. 3213 * 3214 * TRUE return means there's work in the ring to clean, FALSE its empty. 3215 **********************************************************************/ 3216 static bool 3217 igb_txeof(struct tx_ring *txr) 3218 { 3219 struct adapter *adapter = txr->adapter; 3220 int first, last, done, num_avail; 3221 struct igb_buffer *tx_buffer; 3222 struct e1000_tx_desc *tx_desc, *eop_desc; 3223 struct ifnet *ifp = adapter->ifp; 3224 3225 IGB_TX_LOCK_ASSERT(txr); 3226 3227 if (txr->tx_avail == adapter->num_tx_desc) 3228 return FALSE; 3229 3230 num_avail = txr->tx_avail; 3231 first = txr->next_to_clean; 3232 tx_desc = &txr->tx_base[first]; 3233 tx_buffer = &txr->tx_buffers[first]; 3234 last = tx_buffer->next_eop; 3235 eop_desc = &txr->tx_base[last]; 3236 3237 /* 3238 * What this does is get the index of the 3239 * first descriptor AFTER the EOP of the 3240 * first packet, that way we can do the 3241 * simple comparison on the inner while loop. 3242 */ 3243 if (++last == adapter->num_tx_desc) 3244 last = 0; 3245 done = last; 3246 3247 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3248 BUS_DMASYNC_POSTREAD); 3249 3250 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3251 /* We clean the range of the packet */ 3252 while (first != done) { 3253 tx_desc->upper.data = 0; 3254 tx_desc->lower.data = 0; 3255 tx_desc->buffer_addr = 0; 3256 num_avail++; 3257 3258 if (tx_buffer->m_head) { 3259 ifp->if_opackets++; 3260 bus_dmamap_sync(txr->txtag, 3261 tx_buffer->map, 3262 BUS_DMASYNC_POSTWRITE); 3263 bus_dmamap_unload(txr->txtag, 3264 tx_buffer->map); 3265 3266 m_freem(tx_buffer->m_head); 3267 tx_buffer->m_head = NULL; 3268 } 3269 tx_buffer->next_eop = -1; 3270 3271 if (++first == adapter->num_tx_desc) 3272 first = 0; 3273 3274 tx_buffer = &txr->tx_buffers[first]; 3275 tx_desc = &txr->tx_base[first]; 3276 } 3277 /* See if we can continue to the next packet */ 3278 last = tx_buffer->next_eop; 3279 if (last != -1) { 3280 eop_desc = &txr->tx_base[last]; 3281 /* Get new done point */ 3282 if (++last == adapter->num_tx_desc) last = 0; 3283 done = last; 3284 } else 3285 break; 3286 } 3287 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3288 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3289 3290 txr->next_to_clean = first; 3291 3292 /* 3293 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack 3294 * that it is OK to send packets. 3295 * If there are no pending descriptors, clear the timeout. Otherwise, 3296 * if some descriptors have been freed, restart the timeout. 3297 */ 3298 if (num_avail > IGB_TX_CLEANUP_THRESHOLD) { 3299 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3300 /* All clean, turn off the timer */ 3301 if (num_avail == adapter->num_tx_desc) { 3302 txr->watchdog_timer = 0; 3303 txr->tx_avail = num_avail; 3304 return FALSE; 3305 } 3306 /* Some cleaned, reset the timer */ 3307 else if (num_avail != txr->tx_avail) 3308 txr->watchdog_timer = IGB_TX_TIMEOUT; 3309 } 3310 txr->tx_avail = num_avail; 3311 return TRUE; 3312 } 3313 3314 3315 /********************************************************************* 3316 * 3317 * Get a buffer from system mbuf buffer pool. 3318 * 3319 **********************************************************************/ 3320 static int 3321 igb_get_buf(struct rx_ring *rxr, int i) 3322 { 3323 struct adapter *adapter = rxr->adapter; 3324 struct mbuf *m; 3325 bus_dma_segment_t segs[1]; 3326 bus_dmamap_t map; 3327 struct igb_buffer *rx_buffer; 3328 int error, nsegs; 3329 3330 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3331 if (m == NULL) { 3332 adapter->mbuf_cluster_failed++; 3333 return (ENOBUFS); 3334 } 3335 m->m_len = m->m_pkthdr.len = MCLBYTES; 3336 3337 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 3338 m_adj(m, ETHER_ALIGN); 3339 3340 /* 3341 * Using memory from the mbuf cluster pool, invoke the 3342 * bus_dma machinery to arrange the memory mapping. 3343 */ 3344 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3345 rxr->rx_spare_map, m, segs, &nsegs, BUS_DMA_NOWAIT); 3346 if (error != 0) { 3347 m_free(m); 3348 return (error); 3349 } 3350 3351 /* If nsegs is wrong then the stack is corrupt. */ 3352 KASSERT(nsegs == 1, ("Too many segments returned!")); 3353 3354 rx_buffer = &rxr->rx_buffers[i]; 3355 if (rx_buffer->m_head != NULL) 3356 bus_dmamap_unload(rxr->rxtag, rx_buffer->map); 3357 3358 map = rx_buffer->map; 3359 rx_buffer->map = rxr->rx_spare_map; 3360 rxr->rx_spare_map = map; 3361 bus_dmamap_sync(rxr->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); 3362 rx_buffer->m_head = m; 3363 3364 rxr->rx_base[i].read.pkt_addr = htole64(segs[0].ds_addr); 3365 return (0); 3366 } 3367 3368 3369 /********************************************************************* 3370 * 3371 * Allocate memory for rx_buffer structures. Since we use one 3372 * rx_buffer per received packet, the maximum number of rx_buffer's 3373 * that we'll need is equal to the number of receive descriptors 3374 * that we've allocated. 3375 * 3376 **********************************************************************/ 3377 static int 3378 igb_allocate_receive_buffers(struct rx_ring *rxr) 3379 { 3380 struct adapter *adapter = rxr->adapter; 3381 device_t dev = adapter->dev; 3382 struct igb_buffer *rxbuf; 3383 int i, bsize, error; 3384 3385 bsize = sizeof(struct igb_buffer) * adapter->num_rx_desc; 3386 if (!(rxr->rx_buffers = 3387 (struct igb_buffer *) malloc(bsize, 3388 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3389 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3390 error = ENOMEM; 3391 goto fail; 3392 } 3393 3394 if ((error = bus_dma_tag_create(NULL, /* parent */ 3395 PAGE_SIZE, 0, /* alignment, bounds */ 3396 BUS_SPACE_MAXADDR, /* lowaddr */ 3397 BUS_SPACE_MAXADDR, /* highaddr */ 3398 NULL, NULL, /* filter, filterarg */ 3399 MCLBYTES, /* maxsize */ 3400 1, /* nsegments */ 3401 MCLBYTES, /* maxsegsize */ 3402 0, /* flags */ 3403 NULL, /* lockfunc */ 3404 NULL, /* lockfuncarg */ 3405 &rxr->rxtag))) { 3406 device_printf(dev, "Unable to create RX Small DMA tag\n"); 3407 goto fail; 3408 } 3409 3410 /* Create the spare map (used by getbuf) */ 3411 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3412 &rxr->rx_spare_map); 3413 if (error) { 3414 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3415 __func__, error); 3416 goto fail; 3417 } 3418 3419 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3420 rxbuf = &rxr->rx_buffers[i]; 3421 error = bus_dmamap_create(rxr->rxtag, 3422 BUS_DMA_NOWAIT, &rxbuf->map); 3423 if (error) { 3424 device_printf(dev, "Unable to create Small RX DMA map\n"); 3425 goto fail; 3426 } 3427 } 3428 3429 return (0); 3430 3431 fail: 3432 /* Frees all, but can handle partial completion */ 3433 igb_free_receive_structures(adapter); 3434 return (error); 3435 } 3436 3437 /********************************************************************* 3438 * 3439 * Initialize a receive ring and its buffers. 3440 * 3441 **********************************************************************/ 3442 static int 3443 igb_setup_receive_ring(struct rx_ring *rxr) 3444 { 3445 struct adapter *adapter; 3446 device_t dev; 3447 struct igb_buffer *rxbuf; 3448 struct lro_ctrl *lro = &rxr->lro; 3449 int j, rsize; 3450 3451 adapter = rxr->adapter; 3452 dev = adapter->dev; 3453 rsize = roundup2(adapter->num_rx_desc * 3454 sizeof(union e1000_adv_rx_desc), 4096); 3455 /* Clear the ring contents */ 3456 bzero((void *)rxr->rx_base, rsize); 3457 3458 /* 3459 ** Free current RX buffers: the size buffer 3460 ** that is loaded is indicated by the buffer 3461 ** bigbuf value. 3462 */ 3463 for (int i = 0; i < adapter->num_rx_desc; i++) { 3464 rxbuf = &rxr->rx_buffers[i]; 3465 if (rxbuf->m_head != NULL) { 3466 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3467 BUS_DMASYNC_POSTREAD); 3468 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3469 m_freem(rxbuf->m_head); 3470 rxbuf->m_head = NULL; 3471 } 3472 } 3473 3474 for (j = 0; j < adapter->num_rx_desc; j++) { 3475 if (igb_get_buf(rxr, j) == ENOBUFS) { 3476 rxr->rx_buffers[j].m_head = NULL; 3477 rxr->rx_base[j].read.pkt_addr = 0; 3478 goto fail; 3479 } 3480 } 3481 3482 /* Setup our descriptor indices */ 3483 rxr->next_to_check = 0; 3484 rxr->last_cleaned = 0; 3485 3486 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3487 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3488 3489 /* Now set up the LRO interface */ 3490 if (igb_enable_lro) { 3491 int err = tcp_lro_init(lro); 3492 if (err) { 3493 device_printf(dev,"LRO Initialization failed!\n"); 3494 goto fail; 3495 } 3496 INIT_DEBUGOUT("RX LRO Initialized\n"); 3497 lro->ifp = adapter->ifp; 3498 } 3499 3500 return (0); 3501 fail: 3502 /* 3503 * We need to clean up any buffers allocated 3504 * so far, 'j' is the failing index. 3505 */ 3506 for (int i = 0; i < j; i++) { 3507 rxbuf = &rxr->rx_buffers[i]; 3508 if (rxbuf->m_head != NULL) { 3509 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3510 BUS_DMASYNC_POSTREAD); 3511 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3512 m_freem(rxbuf->m_head); 3513 rxbuf->m_head = NULL; 3514 } 3515 } 3516 return (ENOBUFS); 3517 } 3518 3519 /********************************************************************* 3520 * 3521 * Initialize all receive rings. 3522 * 3523 **********************************************************************/ 3524 static int 3525 igb_setup_receive_structures(struct adapter *adapter) 3526 { 3527 struct rx_ring *rxr = adapter->rx_rings; 3528 int j; 3529 3530 for (j = 0; j < adapter->num_rx_queues; j++, rxr++) 3531 if (igb_setup_receive_ring(rxr)) 3532 goto fail; 3533 3534 return (0); 3535 fail: 3536 /* 3537 * Free RX buffers allocated so far, we will only handle 3538 * the rings that completed, the failing case will have 3539 * cleaned up for itself. Clean up til 'j', the failure. 3540 */ 3541 for (int i = 0; i < j; i++) { 3542 rxr = &adapter->rx_rings[i]; 3543 for (int n = 0; n < adapter->num_rx_desc; n++) { 3544 struct igb_buffer *rxbuf; 3545 rxbuf = &rxr->rx_buffers[n]; 3546 if (rxbuf->m_head != NULL) { 3547 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3548 BUS_DMASYNC_POSTREAD); 3549 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3550 m_freem(rxbuf->m_head); 3551 rxbuf->m_head = NULL; 3552 } 3553 } 3554 } 3555 3556 return (ENOBUFS); 3557 } 3558 3559 /********************************************************************* 3560 * 3561 * Enable receive unit. 3562 * 3563 **********************************************************************/ 3564 static void 3565 igb_initialize_receive_units(struct adapter *adapter) 3566 { 3567 struct rx_ring *rxr = adapter->rx_rings; 3568 struct ifnet *ifp = adapter->ifp; 3569 u32 rctl, rxcsum, psize; 3570 3571 INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); 3572 3573 /* 3574 * Make sure receives are disabled while setting 3575 * up the descriptor ring 3576 */ 3577 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 3578 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3579 3580 /* Setup the Base and Length of the Rx Descriptor Rings */ 3581 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3582 u64 bus_addr = rxr->rxdma.dma_paddr; 3583 u32 rxdctl, srrctl; 3584 3585 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(i), 3586 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 3587 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(i), 3588 (uint32_t)(bus_addr >> 32)); 3589 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(i), 3590 (uint32_t)bus_addr); 3591 /* Use Advanced Descriptor type */ 3592 srrctl = E1000_READ_REG(&adapter->hw, E1000_SRRCTL(i)); 3593 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 3594 E1000_WRITE_REG(&adapter->hw, E1000_SRRCTL(i), srrctl); 3595 /* Enable this Queue */ 3596 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(i)); 3597 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 3598 rxdctl &= 0xFFF00000; 3599 rxdctl |= IGB_RX_PTHRESH; 3600 rxdctl |= IGB_RX_HTHRESH << 8; 3601 rxdctl |= IGB_RX_WTHRESH << 16; 3602 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(i), rxdctl); 3603 } 3604 3605 /* 3606 ** Setup for RX MultiQueue 3607 */ 3608 if (adapter->num_rx_queues >1) { 3609 u32 random[10], mrqc, shift = 0; 3610 union igb_reta { 3611 u32 dword; 3612 u8 bytes[4]; 3613 } reta; 3614 3615 arc4rand(&random, sizeof(random), 0); 3616 if (adapter->hw.mac.type == e1000_82575) 3617 shift = 6; 3618 /* Warning FM follows */ 3619 for (int i = 0; i < 128; i++) { 3620 reta.bytes[i & 3] = 3621 (i % adapter->num_rx_queues) << shift; 3622 if ((i & 3) == 3) 3623 E1000_WRITE_REG(&adapter->hw, 3624 E1000_RETA(i & ~3), reta.dword); 3625 } 3626 /* Now fill in hash table */ 3627 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 3628 for (int i = 0; i < 10; i++) 3629 E1000_WRITE_REG_ARRAY(&adapter->hw, 3630 E1000_RSSRK(0), i, random[i]); 3631 3632 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 3633 E1000_MRQC_RSS_FIELD_IPV4_TCP); 3634 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 3635 E1000_MRQC_RSS_FIELD_IPV6_TCP); 3636 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 3637 E1000_MRQC_RSS_FIELD_IPV6_UDP); 3638 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 3639 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3640 3641 E1000_WRITE_REG(&adapter->hw, E1000_MRQC, mrqc); 3642 3643 /* 3644 ** NOTE: Receive Full-Packet Checksum Offload 3645 ** is mutually exclusive with Multiqueue. However 3646 ** this is not the same as TCP/IP checksums which 3647 ** still work. 3648 */ 3649 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 3650 rxcsum |= E1000_RXCSUM_PCSD; 3651 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 3652 } else if (ifp->if_capenable & IFCAP_RXCSUM) { 3653 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 3654 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 3655 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 3656 } 3657 3658 /* Setup the Receive Control Register */ 3659 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3660 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 3661 E1000_RCTL_RDMTS_HALF | 3662 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3663 3664 /* Make sure VLAN Filters are off */ 3665 rctl &= ~E1000_RCTL_VFE; 3666 3667 rctl &= ~E1000_RCTL_SBP; 3668 3669 switch (adapter->rx_buffer_len) { 3670 default: 3671 case 2048: 3672 rctl |= E1000_RCTL_SZ_2048; 3673 break; 3674 case 4096: 3675 rctl |= E1000_RCTL_SZ_4096 | 3676 E1000_RCTL_BSEX | E1000_RCTL_LPE; 3677 break; 3678 case 8192: 3679 rctl |= E1000_RCTL_SZ_8192 | 3680 E1000_RCTL_BSEX | E1000_RCTL_LPE; 3681 break; 3682 case 16384: 3683 rctl |= E1000_RCTL_SZ_16384 | 3684 E1000_RCTL_BSEX | E1000_RCTL_LPE; 3685 break; 3686 } 3687 3688 if (ifp->if_mtu > ETHERMTU) { 3689 /* Set maximum packet len */ 3690 psize = adapter->max_frame_size; 3691 /* are we on a vlan? */ 3692 if (adapter->ifp->if_vlantrunk != NULL) 3693 psize += VLAN_TAG_SIZE; 3694 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); 3695 rctl |= E1000_RCTL_LPE; 3696 } else 3697 rctl &= ~E1000_RCTL_LPE; 3698 3699 /* Enable Receives */ 3700 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 3701 3702 /* 3703 * Setup the HW Rx Head and Tail Descriptor Pointers 3704 * - needs to be after enable 3705 */ 3706 for (int i = 0; i < adapter->num_rx_queues; i++) { 3707 E1000_WRITE_REG(&adapter->hw, E1000_RDH(i), 0); 3708 E1000_WRITE_REG(&adapter->hw, E1000_RDT(i), 3709 adapter->num_rx_desc - 1); 3710 } 3711 return; 3712 } 3713 3714 /********************************************************************* 3715 * 3716 * Free receive rings. 3717 * 3718 **********************************************************************/ 3719 static void 3720 igb_free_receive_structures(struct adapter *adapter) 3721 { 3722 struct rx_ring *rxr = adapter->rx_rings; 3723 3724 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3725 struct lro_ctrl *lro = &rxr->lro; 3726 igb_free_receive_buffers(rxr); 3727 tcp_lro_free(lro); 3728 igb_dma_free(adapter, &rxr->rxdma); 3729 } 3730 3731 free(adapter->rx_rings, M_DEVBUF); 3732 } 3733 3734 /********************************************************************* 3735 * 3736 * Free receive ring data structures. 3737 * 3738 **********************************************************************/ 3739 static void 3740 igb_free_receive_buffers(struct rx_ring *rxr) 3741 { 3742 struct adapter *adapter = rxr->adapter; 3743 struct igb_buffer *rx_buffer; 3744 3745 INIT_DEBUGOUT("free_receive_structures: begin"); 3746 3747 if (rxr->rx_spare_map) { 3748 bus_dmamap_destroy(rxr->rxtag, rxr->rx_spare_map); 3749 rxr->rx_spare_map = NULL; 3750 } 3751 3752 /* Cleanup any existing buffers */ 3753 if (rxr->rx_buffers != NULL) { 3754 rx_buffer = &rxr->rx_buffers[0]; 3755 for (int i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 3756 if (rx_buffer->m_head != NULL) { 3757 bus_dmamap_sync(rxr->rxtag, rx_buffer->map, 3758 BUS_DMASYNC_POSTREAD); 3759 bus_dmamap_unload(rxr->rxtag, 3760 rx_buffer->map); 3761 m_freem(rx_buffer->m_head); 3762 rx_buffer->m_head = NULL; 3763 } else if (rx_buffer->map != NULL) 3764 bus_dmamap_unload(rxr->rxtag, 3765 rx_buffer->map); 3766 if (rx_buffer->map != NULL) { 3767 bus_dmamap_destroy(rxr->rxtag, 3768 rx_buffer->map); 3769 rx_buffer->map = NULL; 3770 } 3771 } 3772 } 3773 3774 if (rxr->rx_buffers != NULL) { 3775 free(rxr->rx_buffers, M_DEVBUF); 3776 rxr->rx_buffers = NULL; 3777 } 3778 3779 if (rxr->rxtag != NULL) { 3780 bus_dma_tag_destroy(rxr->rxtag); 3781 rxr->rxtag = NULL; 3782 } 3783 } 3784 /********************************************************************* 3785 * 3786 * This routine executes in interrupt context. It replenishes 3787 * the mbufs in the descriptor and sends data which has been 3788 * dma'ed into host memory to upper layer. 3789 * 3790 * We loop at most count times if count is > 0, or until done if 3791 * count < 0. 3792 * 3793 * Return TRUE if all clean, FALSE otherwise 3794 *********************************************************************/ 3795 static bool 3796 igb_rxeof(struct rx_ring *rxr, int count) 3797 { 3798 struct adapter *adapter = rxr->adapter; 3799 struct ifnet *ifp; 3800 struct lro_ctrl *lro = &rxr->lro; 3801 struct lro_entry *queued; 3802 struct mbuf *mp; 3803 uint8_t accept_frame = 0; 3804 uint8_t eop = 0; 3805 uint16_t len, desc_len, prev_len_adj; 3806 int i; 3807 u32 staterr; 3808 union e1000_adv_rx_desc *cur; 3809 3810 IGB_RX_LOCK(rxr); 3811 ifp = adapter->ifp; 3812 i = rxr->next_to_check; 3813 cur = &rxr->rx_base[i]; 3814 staterr = cur->wb.upper.status_error; 3815 3816 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3817 BUS_DMASYNC_POSTREAD); 3818 3819 if (!(staterr & E1000_RXD_STAT_DD)) { 3820 IGB_RX_UNLOCK(rxr); 3821 return FALSE; 3822 } 3823 3824 while ((staterr & E1000_RXD_STAT_DD) && 3825 (count != 0) && 3826 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3827 struct mbuf *m = NULL; 3828 3829 mp = rxr->rx_buffers[i].m_head; 3830 /* 3831 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 3832 * needs to access the last received byte in the mbuf. 3833 */ 3834 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map, 3835 BUS_DMASYNC_POSTREAD); 3836 3837 accept_frame = 1; 3838 prev_len_adj = 0; 3839 desc_len = le16toh(cur->wb.upper.length); 3840 if (staterr & E1000_RXD_STAT_EOP) { 3841 count--; 3842 eop = 1; 3843 if (desc_len < ETHER_CRC_LEN) { 3844 len = 0; 3845 prev_len_adj = ETHER_CRC_LEN - desc_len; 3846 } else 3847 len = desc_len - ETHER_CRC_LEN; 3848 } else { 3849 eop = 0; 3850 len = desc_len; 3851 } 3852 3853 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 3854 u32 pkt_len = desc_len; 3855 3856 if (rxr->fmp != NULL) 3857 pkt_len += rxr->fmp->m_pkthdr.len; 3858 3859 accept_frame = 0; 3860 } 3861 3862 if (accept_frame) { 3863 if (igb_get_buf(rxr, i) != 0) { 3864 ifp->if_iqdrops++; 3865 goto discard; 3866 } 3867 3868 /* Assign correct length to the current fragment */ 3869 mp->m_len = len; 3870 3871 if (rxr->fmp == NULL) { 3872 mp->m_pkthdr.len = len; 3873 rxr->fmp = mp; /* Store the first mbuf */ 3874 rxr->lmp = mp; 3875 } else { 3876 /* Chain mbuf's together */ 3877 mp->m_flags &= ~M_PKTHDR; 3878 /* 3879 * Adjust length of previous mbuf in chain if 3880 * we received less than 4 bytes in the last 3881 * descriptor. 3882 */ 3883 if (prev_len_adj > 0) { 3884 rxr->lmp->m_len -= prev_len_adj; 3885 rxr->fmp->m_pkthdr.len -= 3886 prev_len_adj; 3887 } 3888 rxr->lmp->m_next = mp; 3889 rxr->lmp = rxr->lmp->m_next; 3890 rxr->fmp->m_pkthdr.len += len; 3891 } 3892 3893 if (eop) { 3894 rxr->fmp->m_pkthdr.rcvif = ifp; 3895 ifp->if_ipackets++; 3896 rxr->rx_packets++; 3897 rxr->bytes += rxr->fmp->m_pkthdr.len; 3898 rxr->rx_bytes += rxr->bytes; 3899 3900 igb_rx_checksum(staterr, rxr->fmp); 3901 #ifndef __NO_STRICT_ALIGNMENT 3902 if (adapter->max_frame_size > 3903 (MCLBYTES - ETHER_ALIGN) && 3904 igb_fixup_rx(rxr) != 0) 3905 goto skip; 3906 #endif 3907 if (staterr & E1000_RXD_STAT_VP) { 3908 rxr->fmp->m_pkthdr.ether_vtag = 3909 le16toh(cur->wb.upper.vlan); 3910 rxr->fmp->m_flags |= M_VLANTAG; 3911 } 3912 #ifndef __NO_STRICT_ALIGNMENT 3913 skip: 3914 #endif 3915 m = rxr->fmp; 3916 rxr->fmp = NULL; 3917 rxr->lmp = NULL; 3918 } 3919 } else { 3920 ifp->if_ierrors++; 3921 discard: 3922 /* Reuse loaded DMA map and just update mbuf chain */ 3923 mp = rxr->rx_buffers[i].m_head; 3924 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 3925 mp->m_data = mp->m_ext.ext_buf; 3926 mp->m_next = NULL; 3927 if (adapter->max_frame_size <= 3928 (MCLBYTES - ETHER_ALIGN)) 3929 m_adj(mp, ETHER_ALIGN); 3930 if (rxr->fmp != NULL) { 3931 m_freem(rxr->fmp); 3932 rxr->fmp = NULL; 3933 rxr->lmp = NULL; 3934 } 3935 m = NULL; 3936 } 3937 3938 /* Zero out the receive descriptors status. */ 3939 cur->wb.upper.status_error = 0; 3940 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3941 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3942 3943 rxr->last_cleaned = i; /* For updating tail */ 3944 3945 /* Advance our pointers to the next descriptor. */ 3946 if (++i == adapter->num_rx_desc) 3947 i = 0; 3948 3949 if (m != NULL) { 3950 rxr->next_to_check = i; 3951 /* Use LRO if possible */ 3952 if ((!lro->lro_cnt) || (tcp_lro_rx(lro, m, 0))) { 3953 /* Pass up to the stack */ 3954 (*ifp->if_input)(ifp, m); 3955 i = rxr->next_to_check; 3956 } 3957 } 3958 /* Get the next descriptor */ 3959 cur = &rxr->rx_base[i]; 3960 staterr = cur->wb.upper.status_error; 3961 } 3962 rxr->next_to_check = i; 3963 3964 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 3965 E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), rxr->last_cleaned); 3966 3967 /* 3968 * Flush any outstanding LRO work 3969 */ 3970 while (!SLIST_EMPTY(&lro->lro_active)) { 3971 queued = SLIST_FIRST(&lro->lro_active); 3972 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3973 tcp_lro_flush(lro, queued); 3974 } 3975 3976 IGB_RX_UNLOCK(rxr); 3977 3978 if (!((staterr) & E1000_RXD_STAT_DD)) 3979 return FALSE; 3980 3981 return TRUE; 3982 } 3983 3984 #ifndef __NO_STRICT_ALIGNMENT 3985 /* 3986 * When jumbo frames are enabled we should realign entire payload on 3987 * architecures with strict alignment. This is serious design mistake of 8254x 3988 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 3989 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 3990 * payload. On architecures without strict alignment restrictions 8254x still 3991 * performs unaligned memory access which would reduce the performance too. 3992 * To avoid copying over an entire frame to align, we allocate a new mbuf and 3993 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 3994 * existing mbuf chain. 3995 * 3996 * Be aware, best performance of the 8254x is achived only when jumbo frame is 3997 * not used at all on architectures with strict alignment. 3998 */ 3999 static int 4000 igb_fixup_rx(struct rx_ring *rxr) 4001 { 4002 struct adapter *adapter = rxr->adapter; 4003 struct mbuf *m, *n; 4004 int error; 4005 4006 error = 0; 4007 m = rxr->fmp; 4008 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4009 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4010 m->m_data += ETHER_HDR_LEN; 4011 } else { 4012 MGETHDR(n, M_DONTWAIT, MT_DATA); 4013 if (n != NULL) { 4014 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4015 m->m_data += ETHER_HDR_LEN; 4016 m->m_len -= ETHER_HDR_LEN; 4017 n->m_len = ETHER_HDR_LEN; 4018 M_MOVE_PKTHDR(n, m); 4019 n->m_next = m; 4020 rxr->fmp = n; 4021 } else { 4022 adapter->dropped_pkts++; 4023 m_freem(rxr->fmp); 4024 rxr->fmp = NULL; 4025 error = ENOMEM; 4026 } 4027 } 4028 4029 return (error); 4030 } 4031 #endif 4032 4033 /********************************************************************* 4034 * 4035 * Verify that the hardware indicated that the checksum is valid. 4036 * Inform the stack about the status of checksum so that stack 4037 * doesn't spend time verifying the checksum. 4038 * 4039 *********************************************************************/ 4040 static void 4041 igb_rx_checksum(u32 staterr, struct mbuf *mp) 4042 { 4043 u16 status = (u16)staterr; 4044 u8 errors = (u8) (staterr >> 24); 4045 4046 /* Ignore Checksum bit is set */ 4047 if (status & E1000_RXD_STAT_IXSM) { 4048 mp->m_pkthdr.csum_flags = 0; 4049 return; 4050 } 4051 4052 if (status & E1000_RXD_STAT_IPCS) { 4053 /* Did it pass? */ 4054 if (!(errors & E1000_RXD_ERR_IPE)) { 4055 /* IP Checksum Good */ 4056 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4057 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4058 4059 } else 4060 mp->m_pkthdr.csum_flags = 0; 4061 } 4062 4063 if (status & E1000_RXD_STAT_TCPCS) { 4064 /* Did it pass? */ 4065 if (!(errors & E1000_RXD_ERR_TCPE)) { 4066 mp->m_pkthdr.csum_flags |= 4067 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4068 mp->m_pkthdr.csum_data = htons(0xffff); 4069 } 4070 } 4071 return; 4072 } 4073 4074 #ifdef IGB_HW_VLAN_SUPPORT 4075 /* 4076 * This routine is run via an vlan 4077 * config EVENT 4078 */ 4079 static void 4080 igb_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4081 { 4082 struct adapter *adapter = ifp->if_softc; 4083 u32 ctrl, rctl, index, vfta; 4084 4085 /* Shouldn't happen */ 4086 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0) 4087 return; 4088 4089 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4090 ctrl |= E1000_CTRL_VME; 4091 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4092 4093 /* Setup for Hardware Filter */ 4094 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4095 rctl |= E1000_RCTL_VFE; 4096 rctl &= ~E1000_RCTL_CFIEN; 4097 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4098 4099 /* Make entry in the hardware filter table */ 4100 index = ((vtag >> 5) & 0x7F); 4101 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4102 vfta |= (1 << (vtag & 0x1F)); 4103 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4104 4105 /* Update the frame size */ 4106 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4107 adapter->max_frame_size + VLAN_TAG_SIZE); 4108 4109 } 4110 4111 /* 4112 * This routine is run via an vlan 4113 * unconfig EVENT 4114 */ 4115 static void 4116 igb_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4117 { 4118 struct adapter *adapter = ifp->if_softc; 4119 u32 index, vfta; 4120 4121 /* Shouldn't happen */ 4122 if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0) 4123 return; 4124 4125 /* Remove entry in the hardware filter table */ 4126 index = ((vtag >> 5) & 0x7F); 4127 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4128 vfta &= ~(1 << (vtag & 0x1F)); 4129 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4130 /* Have all vlans unregistered? */ 4131 if (adapter->ifp->if_vlantrunk == NULL) { 4132 u32 rctl; 4133 /* Turn off the filter table */ 4134 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4135 rctl &= ~E1000_RCTL_VFE; 4136 rctl |= E1000_RCTL_CFIEN; 4137 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4138 /* Reset the frame size */ 4139 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4140 adapter->max_frame_size); 4141 } 4142 } 4143 #endif /* IGB_HW_VLAN_SUPPORT */ 4144 4145 static void 4146 igb_enable_intr(struct adapter *adapter) 4147 { 4148 /* With RSS set up what to auto clear */ 4149 if (adapter->msix_mem) { 4150 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 4151 adapter->eims_mask); 4152 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, 4153 adapter->eims_mask); 4154 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, 4155 adapter->eims_mask); 4156 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4157 E1000_IMS_LSC); 4158 } else { 4159 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4160 IMS_ENABLE_MASK); 4161 } 4162 E1000_WRITE_FLUSH(&adapter->hw); 4163 4164 return; 4165 } 4166 4167 static void 4168 igb_disable_intr(struct adapter *adapter) 4169 { 4170 if (adapter->msix_mem) { 4171 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); 4172 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); 4173 } 4174 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); 4175 E1000_WRITE_FLUSH(&adapter->hw); 4176 return; 4177 } 4178 4179 /* 4180 * Bit of a misnomer, what this really means is 4181 * to enable OS management of the system... aka 4182 * to disable special hardware management features 4183 */ 4184 static void 4185 igb_init_manageability(struct adapter *adapter) 4186 { 4187 /* A shared code workaround */ 4188 #define E1000_82542_MANC2H E1000_MANC2H 4189 if (adapter->has_manage) { 4190 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4191 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4192 4193 /* disable hardware interception of ARP */ 4194 manc &= ~(E1000_MANC_ARP_EN); 4195 4196 /* enable receiving management packets to the host */ 4197 manc |= E1000_MANC_EN_MNG2HOST; 4198 #define E1000_MNG2HOST_PORT_623 (1 << 5) 4199 #define E1000_MNG2HOST_PORT_664 (1 << 6) 4200 manc2h |= E1000_MNG2HOST_PORT_623; 4201 manc2h |= E1000_MNG2HOST_PORT_664; 4202 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4203 4204 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4205 } 4206 } 4207 4208 /* 4209 * Give control back to hardware management 4210 * controller if there is one. 4211 */ 4212 static void 4213 igb_release_manageability(struct adapter *adapter) 4214 { 4215 if (adapter->has_manage) { 4216 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4217 4218 /* re-enable hardware interception of ARP */ 4219 manc |= E1000_MANC_ARP_EN; 4220 manc &= ~E1000_MANC_EN_MNG2HOST; 4221 4222 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4223 } 4224 } 4225 4226 /* 4227 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 4228 * For ASF and Pass Through versions of f/w this means that 4229 * the driver is loaded. 4230 * 4231 */ 4232 static void 4233 igb_get_hw_control(struct adapter *adapter) 4234 { 4235 u32 ctrl_ext; 4236 4237 /* Let firmware know the driver has taken over */ 4238 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4239 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4240 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4241 } 4242 4243 /* 4244 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 4245 * For ASF and Pass Through versions of f/w this means that the 4246 * driver is no longer loaded. 4247 * 4248 */ 4249 static void 4250 igb_release_hw_control(struct adapter *adapter) 4251 { 4252 u32 ctrl_ext; 4253 4254 /* Let firmware taken over control of h/w */ 4255 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4256 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4257 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4258 } 4259 4260 static int 4261 igb_is_valid_ether_addr(uint8_t *addr) 4262 { 4263 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4264 4265 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4266 return (FALSE); 4267 } 4268 4269 return (TRUE); 4270 } 4271 4272 4273 /* 4274 * Enable PCI Wake On Lan capability 4275 */ 4276 void 4277 igb_enable_wakeup(device_t dev) 4278 { 4279 u16 cap, status; 4280 u8 id; 4281 4282 /* First find the capabilities pointer*/ 4283 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4284 /* Read the PM Capabilities */ 4285 id = pci_read_config(dev, cap, 1); 4286 if (id != PCIY_PMG) /* Something wrong */ 4287 return; 4288 /* OK, we have the power capabilities, so 4289 now get the status register */ 4290 cap += PCIR_POWER_STATUS; 4291 status = pci_read_config(dev, cap, 2); 4292 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4293 pci_write_config(dev, cap, status, 2); 4294 return; 4295 } 4296 4297 4298 /********************************************************************** 4299 * 4300 * Update the board statistics counters. 4301 * 4302 **********************************************************************/ 4303 static void 4304 igb_update_stats_counters(struct adapter *adapter) 4305 { 4306 struct ifnet *ifp; 4307 4308 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4309 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4310 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4311 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 4312 } 4313 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4314 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4315 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4316 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4317 4318 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4319 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4320 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4321 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4322 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4323 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4324 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4325 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4326 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4327 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4328 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4329 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4330 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4331 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4332 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4333 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4334 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4335 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4336 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4337 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4338 4339 /* For the 64-bit byte counters the low dword must be read first. */ 4340 /* Both registers clear on the read of the high dword */ 4341 4342 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4343 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4344 4345 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4346 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4347 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4348 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4349 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4350 4351 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4352 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4353 4354 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4355 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4356 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4357 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4358 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4359 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4360 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4361 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4362 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4363 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4364 4365 adapter->stats.algnerrc += 4366 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4367 adapter->stats.rxerrc += 4368 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4369 adapter->stats.tncrs += 4370 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4371 adapter->stats.cexterr += 4372 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4373 adapter->stats.tsctc += 4374 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4375 adapter->stats.tsctfc += 4376 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4377 ifp = adapter->ifp; 4378 4379 ifp->if_collisions = adapter->stats.colc; 4380 4381 /* Rx Errors */ 4382 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4383 adapter->stats.crcerrs + adapter->stats.algnerrc + 4384 adapter->stats.ruc + adapter->stats.roc + 4385 adapter->stats.mpc + adapter->stats.cexterr; 4386 4387 /* Tx Errors */ 4388 ifp->if_oerrors = adapter->stats.ecol + 4389 adapter->stats.latecol + adapter->watchdog_events; 4390 } 4391 4392 4393 /********************************************************************** 4394 * 4395 * This routine is called only when igb_display_debug_stats is enabled. 4396 * This routine provides a way to take a look at important statistics 4397 * maintained by the driver and hardware. 4398 * 4399 **********************************************************************/ 4400 static void 4401 igb_print_debug_info(struct adapter *adapter) 4402 { 4403 device_t dev = adapter->dev; 4404 struct rx_ring *rxr = adapter->rx_rings; 4405 struct tx_ring *txr = adapter->tx_rings; 4406 uint8_t *hw_addr = adapter->hw.hw_addr; 4407 4408 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4409 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4410 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4411 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4412 4413 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 4414 device_printf(dev, "IMS = 0x%x EIMS = 0x%x \n", 4415 E1000_READ_REG(&adapter->hw, E1000_IMS), 4416 E1000_READ_REG(&adapter->hw, E1000_EIMS)); 4417 #endif 4418 4419 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4420 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4421 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4422 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4423 adapter->hw.fc.high_water, 4424 adapter->hw.fc.low_water); 4425 4426 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 4427 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i, 4428 E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 4429 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 4430 device_printf(dev, "no descriptors avail event = %lld\n", 4431 (long long)txr->no_desc_avail); 4432 device_printf(dev, "TX(%d) MSIX IRQ Handled = %lld\n", txr->me, 4433 (long long)txr->tx_irq); 4434 device_printf(dev, "TX(%d) Packets sent = %lld\n", txr->me, 4435 (long long)txr->tx_packets); 4436 } 4437 4438 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 4439 struct lro_ctrl *lro = &rxr->lro; 4440 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i, 4441 E1000_READ_REG(&adapter->hw, E1000_RDH(i)), 4442 E1000_READ_REG(&adapter->hw, E1000_RDT(i))); 4443 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me, 4444 (long long)rxr->rx_packets); 4445 device_printf(dev, "RX(%d) Byte count = %lld\n", rxr->me, 4446 (long long)rxr->rx_bytes); 4447 device_printf(dev, "RX(%d) MSIX IRQ Handled = %lld\n", rxr->me, 4448 (long long)rxr->rx_irq); 4449 device_printf(dev,"RX(%d) LRO Queued= %d\n", 4450 rxr->me, lro->lro_queued); 4451 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 4452 rxr->me, lro->lro_flushed); 4453 } 4454 4455 device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq); 4456 4457 device_printf(dev, "Std mbuf failed = %ld\n", 4458 adapter->mbuf_alloc_failed); 4459 device_printf(dev, "Std mbuf cluster failed = %ld\n", 4460 adapter->mbuf_cluster_failed); 4461 device_printf(dev, "Driver dropped packets = %ld\n", 4462 adapter->dropped_pkts); 4463 device_printf(dev, "Driver tx dma failure in xmit = %ld\n", 4464 adapter->no_tx_dma_setup); 4465 } 4466 4467 static void 4468 igb_print_hw_stats(struct adapter *adapter) 4469 { 4470 device_t dev = adapter->dev; 4471 4472 device_printf(dev, "Excessive collisions = %lld\n", 4473 (long long)adapter->stats.ecol); 4474 #if (DEBUG_HW > 0) /* Dont output these errors normally */ 4475 device_printf(dev, "Symbol errors = %lld\n", 4476 (long long)adapter->stats.symerrs); 4477 #endif 4478 device_printf(dev, "Sequence errors = %lld\n", 4479 (long long)adapter->stats.sec); 4480 device_printf(dev, "Defer count = %lld\n", 4481 (long long)adapter->stats.dc); 4482 device_printf(dev, "Missed Packets = %lld\n", 4483 (long long)adapter->stats.mpc); 4484 device_printf(dev, "Receive No Buffers = %lld\n", 4485 (long long)adapter->stats.rnbc); 4486 /* RLEC is inaccurate on some hardware, calculate our own. */ 4487 device_printf(dev, "Receive Length Errors = %lld\n", 4488 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4489 device_printf(dev, "Receive errors = %lld\n", 4490 (long long)adapter->stats.rxerrc); 4491 device_printf(dev, "Crc errors = %lld\n", 4492 (long long)adapter->stats.crcerrs); 4493 device_printf(dev, "Alignment errors = %lld\n", 4494 (long long)adapter->stats.algnerrc); 4495 /* On 82575 these are collision counts */ 4496 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4497 (long long)adapter->stats.cexterr); 4498 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 4499 device_printf(dev, "watchdog timeouts = %ld\n", 4500 adapter->watchdog_events); 4501 device_printf(dev, "XON Rcvd = %lld\n", 4502 (long long)adapter->stats.xonrxc); 4503 device_printf(dev, "XON Xmtd = %lld\n", 4504 (long long)adapter->stats.xontxc); 4505 device_printf(dev, "XOFF Rcvd = %lld\n", 4506 (long long)adapter->stats.xoffrxc); 4507 device_printf(dev, "XOFF Xmtd = %lld\n", 4508 (long long)adapter->stats.xofftxc); 4509 device_printf(dev, "Good Packets Rcvd = %lld\n", 4510 (long long)adapter->stats.gprc); 4511 device_printf(dev, "Good Packets Xmtd = %lld\n", 4512 (long long)adapter->stats.gptc); 4513 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 4514 (long long)adapter->stats.tsctc); 4515 device_printf(dev, "TSO Contexts Failed = %lld\n", 4516 (long long)adapter->stats.tsctfc); 4517 } 4518 4519 /********************************************************************** 4520 * 4521 * This routine provides a way to dump out the adapter eeprom, 4522 * often a useful debug/service tool. This only dumps the first 4523 * 32 words, stuff that matters is in that extent. 4524 * 4525 **********************************************************************/ 4526 static void 4527 igb_print_nvm_info(struct adapter *adapter) 4528 { 4529 u16 eeprom_data; 4530 int i, j, row = 0; 4531 4532 /* Its a bit crude, but it gets the job done */ 4533 printf("\nInterface EEPROM Dump:\n"); 4534 printf("Offset\n0x0000 "); 4535 for (i = 0, j = 0; i < 32; i++, j++) { 4536 if (j == 8) { /* Make the offset block */ 4537 j = 0; ++row; 4538 printf("\n0x00%x0 ",row); 4539 } 4540 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4541 printf("%04x ", eeprom_data); 4542 } 4543 printf("\n"); 4544 } 4545 4546 static int 4547 igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4548 { 4549 struct adapter *adapter; 4550 int error; 4551 int result; 4552 4553 result = -1; 4554 error = sysctl_handle_int(oidp, &result, 0, req); 4555 4556 if (error || !req->newptr) 4557 return (error); 4558 4559 if (result == 1) { 4560 adapter = (struct adapter *)arg1; 4561 igb_print_debug_info(adapter); 4562 } 4563 /* 4564 * This value will cause a hex dump of the 4565 * first 32 16-bit words of the EEPROM to 4566 * the screen. 4567 */ 4568 if (result == 2) { 4569 adapter = (struct adapter *)arg1; 4570 igb_print_nvm_info(adapter); 4571 } 4572 4573 return (error); 4574 } 4575 4576 4577 static int 4578 igb_sysctl_stats(SYSCTL_HANDLER_ARGS) 4579 { 4580 struct adapter *adapter; 4581 int error; 4582 int result; 4583 4584 result = -1; 4585 error = sysctl_handle_int(oidp, &result, 0, req); 4586 4587 if (error || !req->newptr) 4588 return (error); 4589 4590 if (result == 1) { 4591 adapter = (struct adapter *)arg1; 4592 igb_print_hw_stats(adapter); 4593 } 4594 4595 return (error); 4596 } 4597 4598 static void 4599 igb_add_rx_process_limit(struct adapter *adapter, const char *name, 4600 const char *description, int *limit, int value) 4601 { 4602 *limit = value; 4603 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4604 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4605 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4606 } 4607 4608 #ifdef IGB_TIMESYNC 4609 /* 4610 * Initialize the Time Sync Feature 4611 */ 4612 static int 4613 igb_tsync_init(struct adapter *adapter) 4614 { 4615 device_t dev = adapter->dev; 4616 u32 tx_ctl, rx_ctl, val; 4617 4618 4619 E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) | 4620 20833/PICOSECS_PER_TICK); 4621 4622 adapter->last_stamp = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 4623 adapter->last_stamp |= (u64)E1000_READ_REG(&adapter->hw, 4624 E1000_SYSTIMH) << 32ULL; 4625 4626 /* Enable the TX side */ 4627 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4628 tx_ctl |= 0x10; 4629 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 4630 E1000_WRITE_FLUSH(&adapter->hw); 4631 4632 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4633 if ((tx_ctl & 0x10) == 0) { 4634 device_printf(dev, "Failed to enable TX timestamping\n"); 4635 return (ENXIO); 4636 } 4637 4638 /* Enable RX */ 4639 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4640 rx_ctl |= 0x10; /* Enable the feature */ 4641 rx_ctl |= 0x04; /* This value turns on Ver 1 and 2 */ 4642 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 4643 4644 /* 4645 * Ethertype Filter Queue Filter[0][15:0] = 0x88F7 (Ethertype) 4646 * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter) 4647 * Ethertype Filter Queue Filter[0][31] = 0x1 (Enable Timestamping) 4648 */ 4649 E1000_WRITE_REG(&adapter->hw, E1000_ETQF(0), 0x440088f7); 4650 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCFG, 0x0); 4651 4652 /* 4653 * Source Port Queue Filter Setup: 4654 * this is for UDP port filtering 4655 */ 4656 E1000_WRITE_REG(&adapter->hw, E1000_SPQF(0), TSYNC_PORT); 4657 /* Protocol = UDP, enable Timestamp, and filter on source/protocol */ 4658 val = (0x11 | (1 << 27) | (6 << 28)); 4659 E1000_WRITE_REG(&adapter->hw, E1000_FTQF(0), val); 4660 4661 E1000_WRITE_FLUSH(&adapter->hw); 4662 4663 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4664 if ((rx_ctl & 0x10) == 0) { 4665 device_printf(dev, "Failed to enable RX timestamping\n"); 4666 return (ENXIO); 4667 } 4668 4669 device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n"); 4670 4671 return (0); 4672 } 4673 4674 /* 4675 * Disable the Time Sync Feature 4676 */ 4677 static void 4678 igb_tsync_disable(struct adapter *adapter) 4679 { 4680 u32 tx_ctl, rx_ctl; 4681 4682 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4683 tx_ctl &= ~0x10; 4684 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 4685 E1000_WRITE_FLUSH(&adapter->hw); 4686 4687 /* Invalidate TX Timestamp */ 4688 E1000_READ_REG(&adapter->hw, E1000_TXSTMPH); 4689 4690 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4691 if (tx_ctl & 0x10) 4692 HW_DEBUGOUT("Failed to disable TX timestamping\n"); 4693 4694 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4695 rx_ctl &= ~0x10; 4696 4697 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 4698 E1000_WRITE_FLUSH(&adapter->hw); 4699 4700 /* Invalidate RX Timestamp */ 4701 E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 4702 4703 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4704 if (rx_ctl & 0x10) 4705 HW_DEBUGOUT("Failed to disable RX timestamping\n"); 4706 4707 return; 4708 } 4709 4710 #endif /* IGB_TIMESYNC */
Cache object: c89e4e424372e17e44bf4a24903eee75
[ source navigation ] [ diff markup ] [ identifier search ] [ freetext search ] [ file search ] [ list types ] [ track identifier ]